diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,149439 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: True + | > Precision: fp16 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 32 + | > Num. of Torch Threads: 24 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + > Restoring from best_model.pth ... + > Restoring Model... + > Partial model initialization... + | > Layer dimention missmatch between model definition and checkpoint: text_encoder.emb.weight + | > 948 / 949 layers are restored. + > Model restored from step 379281 + + > Model has 83060908 parameters + + > EPOCH: 0/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:00:41)  + + --> TIME: 2025-04-18 23:00:49 -- STEP: 18/142 -- GLOBAL_STEP: 379300 + | > loss_disc: 2.588078260421753 (2.454701238208347) + | > loss_disc_real_0: 0.20437856018543243 (0.12804383701748318) + | > loss_disc_real_1: 0.2100633680820465 (0.19779306815730202) + | > loss_disc_real_2: 0.22191160917282104 (0.22524861991405487) + | > loss_disc_real_3: 0.22063730657100677 (0.2295184400346544) + | > loss_disc_real_4: 0.22634541988372803 (0.2280339002609253) + | > loss_disc_real_5: 0.25786858797073364 (0.240282298790084) + | > loss_0: 2.588078260421753 (2.454701238208347) + | > grad_norm_0: tensor(20.8184, device='cuda:0') (tensor(14.3978, device='cuda:0')) + | > loss_gen: 2.1941239833831787 (2.3959125677744546) + | > loss_kl: 3.7698495388031006 (5.513903631104363) + | > loss_feat: 5.284894943237305 (7.12395527627733) + | > loss_mel: 15.984687805175781 (17.92994435628255) + | > loss_duration: 1.927765130996704 (2.3089569674597845) + | > amp_scaler: 256.0 (1962.6666666666667) + | > loss_1: 29.16132164001465 (35.27267371283637) + | > grad_norm_1: tensor(193.9302, device='cuda:0') (tensor(202.4609, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.3424 (0.34912357065412736) + | > loader_time: 0.0053 (0.004794756571451823) + + + --> TIME: 2025-04-18 23:00:58 -- STEP: 43/142 -- GLOBAL_STEP: 379325 + | > loss_disc: 2.3513119220733643 (2.4629780747169665) + | > loss_disc_real_0: 0.11505546420812607 (0.12724336236715314) + | > loss_disc_real_1: 0.19756168127059937 (0.20261492632156194) + | > loss_disc_real_2: 0.24094638228416443 (0.22693567740362744) + | > loss_disc_real_3: 0.196877121925354 (0.22773825671783712) + | > loss_disc_real_4: 0.18802641332149506 (0.22804810245369755) + | > loss_disc_real_5: 0.17387138307094574 (0.23503454514714175) + | > loss_0: 2.3513119220733643 (2.4629780747169665) + | > grad_norm_0: tensor(14.5982, device='cuda:0') (tensor(14.0727, device='cuda:0')) + | > loss_gen: 2.300281286239624 (2.3656960088153216) + | > loss_kl: 3.2583534717559814 (4.287032964617707) + | > loss_feat: 7.249305248260498 (7.068674586540045) + | > loss_mel: 18.409717559814453 (18.02448547718136) + | > loss_duration: 2.061657428741455 (2.1307176240654875) + | > amp_scaler: 256.0 (970.4186046511628) + | > loss_1: 33.27931594848633 (33.87660705211552) + | > grad_norm_1: tensor(340.8602, device='cuda:0') (tensor(265.1089, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.3714 (0.3535254888756331) + | > loader_time: 0.0057 (0.005283971165501794) + + + --> TIME: 2025-04-18 23:01:08 -- STEP: 68/142 -- GLOBAL_STEP: 379350 + | > loss_disc: 2.374164581298828 (2.4638600629918703) + | > loss_disc_real_0: 0.10299266874790192 (0.12739163593334304) + | > loss_disc_real_1: 0.24800501763820648 (0.20533818291390643) + | > loss_disc_real_2: 0.22295093536376953 (0.2254886955899351) + | > loss_disc_real_3: 0.22526849806308746 (0.22826087409082582) + | > loss_disc_real_4: 0.22175469994544983 (0.2294216123135651) + | > loss_disc_real_5: 0.2352406233549118 (0.23403019260834246) + | > loss_0: 2.374164581298828 (2.4638600629918703) + | > grad_norm_0: tensor(10.4838, device='cuda:0') (tensor(13.1358, device='cuda:0')) + | > loss_gen: 2.465850353240967 (2.3688357226988845) + | > loss_kl: 3.109447479248047 (3.8408042893690237) + | > loss_feat: 7.817902088165283 (7.054282496957218) + | > loss_mel: 18.509254455566406 (18.05495170985951) + | > loss_duration: 2.092968463897705 (2.118592001059475) + | > amp_scaler: 256.0 (707.7647058823529) + | > loss_1: 33.99542236328125 (33.43746639700498) + | > grad_norm_1: tensor(320.9363, device='cuda:0') (tensor(257.2138, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.3978 (0.36302990422529335) + | > loader_time: 0.006 (0.005548287840450511) + + + --> TIME: 2025-04-18 23:01:18 -- STEP: 93/142 -- GLOBAL_STEP: 379375 + | > loss_disc: 2.411881685256958 (2.4722493976675044) + | > loss_disc_real_0: 0.0880064144730568 (0.12884504860767756) + | > loss_disc_real_1: 0.20996356010437012 (0.20704117176994205) + | > loss_disc_real_2: 0.17601798474788666 (0.22631963542712633) + | > loss_disc_real_3: 0.25124308466911316 (0.22996643762434682) + | > loss_disc_real_4: 0.21697959303855896 (0.22919021578886176) + | > loss_disc_real_5: 0.26836907863616943 (0.23504311743602957) + | > loss_0: 2.411881685256958 (2.4722493976675044) + | > grad_norm_0: tensor(7.3459, device='cuda:0') (tensor(11.9099, device='cuda:0')) + | > loss_gen: 2.409054756164551 (2.3549106582518538) + | > loss_kl: 2.8355166912078857 (3.6009349284633525) + | > loss_feat: 6.997631072998047 (6.992477688738095) + | > loss_mel: 18.998502731323242 (18.129644732321456) + | > loss_duration: 2.128450393676758 (2.1199342704588355) + | > amp_scaler: 256.0 (586.3225806451615) + | > loss_1: 33.36915588378906 (33.19790251537035) + | > grad_norm_1: tensor(138.0976, device='cuda:0') (tensor(228.5406, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.4028 (0.37331018396603166) + | > loader_time: 0.0067 (0.005837079017393051) + + + --> TIME: 2025-04-18 23:01:29 -- STEP: 118/142 -- GLOBAL_STEP: 379400 + | > loss_disc: 2.3703789710998535 (2.4752905712289315) + | > loss_disc_real_0: 0.1422002613544464 (0.13000752707406626) + | > loss_disc_real_1: 0.22235451638698578 (0.20765069177595238) + | > loss_disc_real_2: 0.2407640516757965 (0.226897017683013) + | > loss_disc_real_3: 0.21720115840435028 (0.2301892205062559) + | > loss_disc_real_4: 0.2587493062019348 (0.2287639216598818) + | > loss_disc_real_5: 0.2086602747440338 (0.2336416500857321) + | > loss_0: 2.3703789710998535 (2.4752905712289315) + | > grad_norm_0: tensor(18.8321, device='cuda:0') (tensor(11.4813, device='cuda:0')) + | > loss_gen: 2.4784762859344482 (2.3485884706852795) + | > loss_kl: 2.8733880519866943 (3.4484066619711418) + | > loss_feat: 7.643556594848633 (6.968243477708202) + | > loss_mel: 18.60491371154785 (18.154322438320865) + | > loss_duration: 2.10707426071167 (2.1204251950070003) + | > amp_scaler: 256.0 (516.3389830508476) + | > loss_1: 33.7074089050293 (33.039986400280974) + | > grad_norm_1: tensor(645.9063, device='cuda:0') (tensor(232.5603, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 0.4493 (0.3836527880975756) + | > loader_time: 0.0073 (0.006089630773511983) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6244258880615234 (2.6244258880615234) + | > loss_disc_real_0: 0.09627295285463333 (0.09627295285463333) + | > loss_disc_real_1: 0.2797205448150635 (0.2797205448150635) + | > loss_disc_real_2: 0.2742309868335724 (0.2742309868335724) + | > loss_disc_real_3: 0.25759270787239075 (0.25759270787239075) + | > loss_disc_real_4: 0.28776371479034424 (0.28776371479034424) + | > loss_disc_real_5: 0.2668397128582001 (0.2668397128582001) + | > loss_0: 2.6244258880615234 (2.6244258880615234) + | > loss_gen: 2.3144454956054688 (2.3144454956054688) + | > loss_kl: 2.9304070472717285 (2.9304070472717285) + | > loss_feat: 5.6079583168029785 (5.6079583168029785) + | > loss_mel: 17.546772003173828 (17.546772003173828) + | > loss_duration: 2.1467769145965576 (2.1467769145965576) + | > loss_1: 30.546358108520508 (30.546358108520508) + + --> STEP: 1 + | > loss_disc: 2.4384377002716064 (2.4384377002716064) + | > loss_disc_real_0: 0.09383835643529892 (0.09383835643529892) + | > loss_disc_real_1: 0.22040334343910217 (0.22040334343910217) + | > loss_disc_real_2: 0.20619738101959229 (0.20619738101959229) + | > loss_disc_real_3: 0.20998668670654297 (0.20998668670654297) + | > loss_disc_real_4: 0.259989470243454 (0.259989470243454) + | > loss_disc_real_5: 0.24093692004680634 (0.24093692004680634) + | > loss_0: 2.4384377002716064 (2.4384377002716064) + | > loss_gen: 2.2251861095428467 (2.2251861095428467) + | > loss_kl: 2.824530839920044 (2.824530839920044) + | > loss_feat: 6.861834526062012 (6.861834526062012) + | > loss_mel: 18.861345291137695 (18.861345291137695) + | > loss_duration: 2.2031614780426025 (2.2031614780426025) + | > loss_1: 32.97605895996094 (32.97605895996094) + + --> STEP: 2 + | > loss_disc: 2.3687524795532227 (2.4035950899124146) + | > loss_disc_real_0: 0.10564521700143814 (0.09974178671836853) + | > loss_disc_real_1: 0.2347661256790161 (0.22758473455905914) + | > loss_disc_real_2: 0.22471989691257477 (0.21545863896608353) + | > loss_disc_real_3: 0.23226028680801392 (0.22112348675727844) + | > loss_disc_real_4: 0.2270534187555313 (0.24352144449949265) + | > loss_disc_real_5: 0.20860940217971802 (0.22477316111326218) + | > loss_0: 2.3687524795532227 (2.4035950899124146) + | > loss_gen: 2.4166228771209717 (2.320904493331909) + | > loss_kl: 2.8021557331085205 (2.8133432865142822) + | > loss_feat: 7.831837177276611 (7.3468358516693115) + | > loss_mel: 18.896455764770508 (18.8789005279541) + | > loss_duration: 2.186276435852051 (2.1947189569473267) + | > loss_1: 34.13334655761719 (33.55470275878906) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035250186920166016 (+0.0) + | > avg_loss_disc: 2.4035950899124146 (+0.0) + | > avg_loss_disc_real_0: 0.09974178671836853 (+0.0) + | > avg_loss_disc_real_1: 0.22758473455905914 (+0.0) + | > avg_loss_disc_real_2: 0.21545863896608353 (+0.0) + | > avg_loss_disc_real_3: 0.22112348675727844 (+0.0) + | > avg_loss_disc_real_4: 0.24352144449949265 (+0.0) + | > avg_loss_disc_real_5: 0.22477316111326218 (+0.0) + | > avg_loss_0: 2.4035950899124146 (+0.0) + | > avg_loss_gen: 2.320904493331909 (+0.0) + | > avg_loss_kl: 2.8133432865142822 (+0.0) + | > avg_loss_feat: 7.3468358516693115 (+0.0) + | > avg_loss_mel: 18.8789005279541 (+0.0) + | > avg_loss_duration: 2.1947189569473267 (+0.0) + | > avg_loss_1: 33.55470275878906 (+0.0) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_379424.pth + + > EPOCH: 1/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:01:42)  + + --> TIME: 2025-04-18 23:01:43 -- STEP: 1/142 -- GLOBAL_STEP: 379425 + | > loss_disc: 2.377432346343994 (2.377432346343994) + | > loss_disc_real_0: 0.10220064967870712 (0.10220064967870712) + | > loss_disc_real_1: 0.21251638233661652 (0.21251638233661652) + | > loss_disc_real_2: 0.23220419883728027 (0.23220419883728027) + | > loss_disc_real_3: 0.20679998397827148 (0.20679998397827148) + | > loss_disc_real_4: 0.2248978614807129 (0.2248978614807129) + | > loss_disc_real_5: 0.2103528380393982 (0.2103528380393982) + | > loss_0: 2.377432346343994 (2.377432346343994) + | > grad_norm_0: tensor(13.7532, device='cuda:0') (tensor(13.7532, device='cuda:0')) + | > loss_gen: 2.4059531688690186 (2.4059531688690186) + | > loss_kl: 2.7288076877593994 (2.7288076877593994) + | > loss_feat: 7.181520462036133 (7.181520462036133) + | > loss_mel: 17.637300491333008 (17.637300491333008) + | > loss_duration: 2.138171434402466 (2.138171434402466) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.09175109863281 (32.09175109863281) + | > grad_norm_1: tensor(530.1816, device='cuda:0') (tensor(530.1816, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.318 (0.3179666996002197) + | > loader_time: 0.0045 (0.004469633102416992) + + + --> TIME: 2025-04-18 23:01:52 -- STEP: 26/142 -- GLOBAL_STEP: 379450 + | > loss_disc: 2.490752696990967 (2.4700002120091367) + | > loss_disc_real_0: 0.10968829691410065 (0.12690595050270742) + | > loss_disc_real_1: 0.15030154585838318 (0.20464684584966072) + | > loss_disc_real_2: 0.23176100850105286 (0.23102171604449934) + | > loss_disc_real_3: 0.25232893228530884 (0.2316840130549211) + | > loss_disc_real_4: 0.25377029180526733 (0.23205556204685798) + | > loss_disc_real_5: 0.2745656669139862 (0.2318461285187648) + | > loss_0: 2.490752696990967 (2.4700002120091367) + | > grad_norm_0: tensor(8.6579, device='cuda:0') (tensor(14.8373, device='cuda:0')) + | > loss_gen: 2.466610908508301 (2.3903759626241827) + | > loss_kl: 2.831240177154541 (2.7941355796960683) + | > loss_feat: 6.889050483703613 (7.16649682705219) + | > loss_mel: 17.937942504882812 (18.21870972560002) + | > loss_duration: 2.1773273944854736 (2.154905520952665) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.30216979980469 (32.724623680114746) + | > grad_norm_1: tensor(478.2562, device='cuda:0') (tensor(336.1901, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.3517 (0.33163882218874424) + | > loader_time: 0.0051 (0.0047867206426767195) + + + --> TIME: 2025-04-18 23:02:01 -- STEP: 51/142 -- GLOBAL_STEP: 379475 + | > loss_disc: 2.548492431640625 (2.4555951333513453) + | > loss_disc_real_0: 0.1745380312204361 (0.13016457592739777) + | > loss_disc_real_1: 0.2510733902454376 (0.20746923194212072) + | > loss_disc_real_2: 0.22570352256298065 (0.22546088637090198) + | > loss_disc_real_3: 0.2257101833820343 (0.22978362441062927) + | > loss_disc_real_4: 0.2197183072566986 (0.22756288828803042) + | > loss_disc_real_5: 0.21944297850131989 (0.22866775884347804) + | > loss_0: 2.548492431640625 (2.4555951333513453) + | > grad_norm_0: tensor(16.5665, device='cuda:0') (tensor(16.9376, device='cuda:0')) + | > loss_gen: 2.3982908725738525 (2.406129570568308) + | > loss_kl: 2.821009635925293 (2.7930216882743086) + | > loss_feat: 6.954890251159668 (7.18610310087017) + | > loss_mel: 17.338863372802734 (18.14301670298856) + | > loss_duration: 2.1515233516693115 (2.154872688592649) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.66457748413086 (32.683143540924675) + | > grad_norm_1: tensor(396.3487, device='cuda:0') (tensor(341.5788, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.3785 (0.3466421155368582) + | > loader_time: 0.0059 (0.005218080445831897) + + + --> TIME: 2025-04-18 23:02:11 -- STEP: 76/142 -- GLOBAL_STEP: 379500 + | > loss_disc: 2.5807242393493652 (2.452610520940078) + | > loss_disc_real_0: 0.20592863857746124 (0.12827966450468492) + | > loss_disc_real_1: 0.21259990334510803 (0.20652104561266146) + | > loss_disc_real_2: 0.2213170826435089 (0.22527658998181946) + | > loss_disc_real_3: 0.21800850331783295 (0.22895521631366328) + | > loss_disc_real_4: 0.23973752558231354 (0.22801183183726512) + | > loss_disc_real_5: 0.2441299706697464 (0.2299009047840771) + | > loss_0: 2.5807242393493652 (2.452610520940078) + | > grad_norm_0: tensor(8.7342, device='cuda:0') (tensor(17.4642, device='cuda:0')) + | > loss_gen: 2.404264211654663 (2.4159332011875354) + | > loss_kl: 2.8937554359436035 (2.7799996388585946) + | > loss_feat: 6.338549613952637 (7.186751616628547) + | > loss_mel: 17.547388076782227 (18.186440844284856) + | > loss_duration: 2.1571335792541504 (2.156464150077418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.34109115600586 (32.725589400843575) + | > grad_norm_1: tensor(211.8185, device='cuda:0') (tensor(347.8948, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.3963 (0.3608829222227399) + | > loader_time: 0.0065 (0.005569793676075183) + + + --> TIME: 2025-04-18 23:02:21 -- STEP: 101/142 -- GLOBAL_STEP: 379525 + | > loss_disc: 2.4164540767669678 (2.461918063683085) + | > loss_disc_real_0: 0.12006392329931259 (0.1278977076901068) + | > loss_disc_real_1: 0.19802811741828918 (0.2059545754501135) + | > loss_disc_real_2: 0.20882616937160492 (0.22642965349230434) + | > loss_disc_real_3: 0.23378773033618927 (0.23005127301900694) + | > loss_disc_real_4: 0.21270717680454254 (0.22845757037106126) + | > loss_disc_real_5: 0.2567059397697449 (0.23140904853249541) + | > loss_0: 2.4164540767669678 (2.461918063683085) + | > grad_norm_0: tensor(6.5199, device='cuda:0') (tensor(15.8030, device='cuda:0')) + | > loss_gen: 2.4193150997161865 (2.3877494642049966) + | > loss_kl: 2.710811138153076 (2.774046812907304) + | > loss_feat: 7.208552837371826 (7.080981174317917) + | > loss_mel: 19.27680206298828 (18.22736526715874) + | > loss_duration: 2.1474249362945557 (2.1570827418034626) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.762908935546875 (32.6272254037385) + | > grad_norm_1: tensor(48.3179, device='cuda:0') (tensor(314.6284, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.4299 (0.3737269038020975) + | > loader_time: 0.0079 (0.006027981786444636) + + + --> TIME: 2025-04-18 23:02:33 -- STEP: 126/142 -- GLOBAL_STEP: 379550 + | > loss_disc: 2.3810317516326904 (2.471589650426592) + | > loss_disc_real_0: 0.12293481826782227 (0.1289751406108576) + | > loss_disc_real_1: 0.1687486171722412 (0.20587673203812706) + | > loss_disc_real_2: 0.22785860300064087 (0.2280777541417924) + | > loss_disc_real_3: 0.24088436365127563 (0.23086649475116577) + | > loss_disc_real_4: 0.25956857204437256 (0.23043977445553218) + | > loss_disc_real_5: 0.27997809648513794 (0.23289842383256035) + | > loss_0: 2.3810317516326904 (2.471589650426592) + | > grad_norm_0: tensor(9.7329, device='cuda:0') (tensor(14.4378, device='cuda:0')) + | > loss_gen: 2.5286929607391357 (2.3718506003183033) + | > loss_kl: 2.699367046356201 (2.7641211937344266) + | > loss_feat: 7.707503795623779 (7.002746453360905) + | > loss_mel: 19.027305603027344 (18.184355538988868) + | > loss_duration: 2.089097499847412 (2.15362452514588) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.05196762084961 (32.47669825478205) + | > grad_norm_1: tensor(178.0746, device='cuda:0') (tensor(283.8492, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.4456 (0.3909749171090505) + | > loader_time: 0.0074 (0.006451627564808679) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5004777908325195 (2.5004777908325195) + | > loss_disc_real_0: 0.21005530655384064 (0.21005530655384064) + | > loss_disc_real_1: 0.21053986251354218 (0.21053986251354218) + | > loss_disc_real_2: 0.21398143470287323 (0.21398143470287323) + | > loss_disc_real_3: 0.2633008360862732 (0.2633008360862732) + | > loss_disc_real_4: 0.23372022807598114 (0.23372022807598114) + | > loss_disc_real_5: 0.2615724205970764 (0.2615724205970764) + | > loss_0: 2.5004777908325195 (2.5004777908325195) + | > loss_gen: 2.633150815963745 (2.633150815963745) + | > loss_kl: 2.779392957687378 (2.779392957687378) + | > loss_feat: 7.044427871704102 (7.044427871704102) + | > loss_mel: 19.479476928710938 (19.479476928710938) + | > loss_duration: 2.1561779975891113 (2.1561779975891113) + | > loss_1: 34.092628479003906 (34.092628479003906) + + --> STEP: 1 + | > loss_disc: 2.3831422328948975 (2.3831422328948975) + | > loss_disc_real_0: 0.20731154084205627 (0.20731154084205627) + | > loss_disc_real_1: 0.22576001286506653 (0.22576001286506653) + | > loss_disc_real_2: 0.19910641014575958 (0.19910641014575958) + | > loss_disc_real_3: 0.19556960463523865 (0.19556960463523865) + | > loss_disc_real_4: 0.18670405447483063 (0.18670405447483063) + | > loss_disc_real_5: 0.22031323611736298 (0.22031323611736298) + | > loss_0: 2.3831422328948975 (2.3831422328948975) + | > loss_gen: 2.556567668914795 (2.556567668914795) + | > loss_kl: 2.7192611694335938 (2.7192611694335938) + | > loss_feat: 7.802090644836426 (7.802090644836426) + | > loss_mel: 19.531322479248047 (19.531322479248047) + | > loss_duration: 2.2193474769592285 (2.2193474769592285) + | > loss_1: 34.828590393066406 (34.828590393066406) + + --> STEP: 2 + | > loss_disc: 2.426805019378662 (2.40497362613678) + | > loss_disc_real_0: 0.23469509184360504 (0.22100331634283066) + | > loss_disc_real_1: 0.21640901267528534 (0.22108451277017593) + | > loss_disc_real_2: 0.22675231099128723 (0.2129293605685234) + | > loss_disc_real_3: 0.25228607654571533 (0.223927840590477) + | > loss_disc_real_4: 0.21583092212677002 (0.20126748830080032) + | > loss_disc_real_5: 0.235630601644516 (0.22797191888093948) + | > loss_0: 2.426805019378662 (2.40497362613678) + | > loss_gen: 2.7210638523101807 (2.638815760612488) + | > loss_kl: 2.636759042739868 (2.678010106086731) + | > loss_feat: 7.608305931091309 (7.705198287963867) + | > loss_mel: 17.746902465820312 (18.63911247253418) + | > loss_duration: 2.1577062606811523 (2.1885268688201904) + | > loss_1: 32.87073516845703 (33.84966278076172) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003633260726928711 (+0.00010824203491210938) + | > avg_loss_disc: 2.40497362613678 (+0.0013785362243652344) + | > avg_loss_disc_real_0: 0.22100331634283066 (+0.12126152962446213) + | > avg_loss_disc_real_1: 0.22108451277017593 (-0.006500221788883209) + | > avg_loss_disc_real_2: 0.2129293605685234 (-0.0025292783975601196) + | > avg_loss_disc_real_3: 0.223927840590477 (+0.0028043538331985474) + | > avg_loss_disc_real_4: 0.20126748830080032 (-0.04225395619869232) + | > avg_loss_disc_real_5: 0.22797191888093948 (+0.003198757767677307) + | > avg_loss_0: 2.40497362613678 (+0.0013785362243652344) + | > avg_loss_gen: 2.638815760612488 (+0.3179112672805786) + | > avg_loss_kl: 2.678010106086731 (-0.13533318042755127) + | > avg_loss_feat: 7.705198287963867 (+0.35836243629455566) + | > avg_loss_mel: 18.63911247253418 (-0.23978805541992188) + | > avg_loss_duration: 2.1885268688201904 (-0.0061920881271362305) + | > avg_loss_1: 33.84966278076172 (+0.29496002197265625) + + + > EPOCH: 2/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:02:42)  + + --> TIME: 2025-04-18 23:02:46 -- STEP: 9/142 -- GLOBAL_STEP: 379575 + | > loss_disc: 2.5475404262542725 (2.5015250047047934) + | > loss_disc_real_0: 0.12530246376991272 (0.1342421289947298) + | > loss_disc_real_1: 0.1978439837694168 (0.21352354685465494) + | > loss_disc_real_2: 0.217403382062912 (0.2326042503118515) + | > loss_disc_real_3: 0.2616194486618042 (0.23205578989452785) + | > loss_disc_real_4: 0.23068980872631073 (0.23689033091068268) + | > loss_disc_real_5: 0.24974268674850464 (0.22996361553668976) + | > loss_0: 2.5475404262542725 (2.5015250047047934) + | > grad_norm_0: tensor(9.9579, device='cuda:0') (tensor(10.5863, device='cuda:0')) + | > loss_gen: 2.2881174087524414 (2.2918454011281333) + | > loss_kl: 2.66745924949646 (2.6838793754577637) + | > loss_feat: 6.880539417266846 (6.962601025899251) + | > loss_mel: 17.510469436645508 (17.94190004136827) + | > loss_duration: 2.143311023712158 (2.1606534322102866) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.48989486694336 (32.04087808397081) + | > grad_norm_1: tensor(179.1226, device='cuda:0') (tensor(168.9653, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.3238 (0.323474645614624) + | > loader_time: 0.0048 (0.00428620974222819) + + + --> TIME: 2025-04-18 23:02:54 -- STEP: 34/142 -- GLOBAL_STEP: 379600 + | > loss_disc: 2.600490093231201 (2.4772653509588802) + | > loss_disc_real_0: 0.17199067771434784 (0.1286014579236508) + | > loss_disc_real_1: 0.20205020904541016 (0.21193017416140614) + | > loss_disc_real_2: 0.22720682621002197 (0.22452598415753422) + | > loss_disc_real_3: 0.23982872068881989 (0.23190049914752736) + | > loss_disc_real_4: 0.22051918506622314 (0.2303279037861263) + | > loss_disc_real_5: 0.2534782886505127 (0.2309779537951245) + | > loss_0: 2.600490093231201 (2.4772653509588802) + | > grad_norm_0: tensor(4.7767, device='cuda:0') (tensor(8.8816, device='cuda:0')) + | > loss_gen: 2.2223565578460693 (2.3324668758055744) + | > loss_kl: 2.7708323001861572 (2.693476719014785) + | > loss_feat: 6.646885871887207 (7.033783968757181) + | > loss_mel: 17.431339263916016 (18.212089650771198) + | > loss_duration: 2.1145639419555664 (2.138904704767116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.185977935791016 (32.41072189106659) + | > grad_norm_1: tensor(87.6941, device='cuda:0') (tensor(186.8065, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.3636 (0.3393913998323329) + | > loader_time: 0.0056 (0.005041991963106043) + + + --> TIME: 2025-04-18 23:03:04 -- STEP: 59/142 -- GLOBAL_STEP: 379625 + | > loss_disc: 2.5697836875915527 (2.466763948990127) + | > loss_disc_real_0: 0.12713342905044556 (0.12881032705812132) + | > loss_disc_real_1: 0.23472893238067627 (0.21011484730041632) + | > loss_disc_real_2: 0.2616131901741028 (0.2252769561137183) + | > loss_disc_real_3: 0.22431179881095886 (0.2282071921784999) + | > loss_disc_real_4: 0.24346749484539032 (0.22779980226088378) + | > loss_disc_real_5: 0.24179910123348236 (0.23033621376853877) + | > loss_0: 2.5697836875915527 (2.466763948990127) + | > grad_norm_0: tensor(12.1597, device='cuda:0') (tensor(8.6019, device='cuda:0')) + | > loss_gen: 2.2450923919677734 (2.3431746838456498) + | > loss_kl: 2.518845796585083 (2.664826740652828) + | > loss_feat: 6.217448711395264 (7.025157791073039) + | > loss_mel: 16.952857971191406 (18.201355659355553) + | > loss_duration: 2.1063802242279053 (2.128280942722902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.040624618530273 (32.36279584593693) + | > grad_norm_1: tensor(80.1222, device='cuda:0') (tensor(171.7596, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.3708 (0.35360861632783536) + | > loader_time: 0.0059 (0.005364773637157375) + + + --> TIME: 2025-04-18 23:03:14 -- STEP: 84/142 -- GLOBAL_STEP: 379650 + | > loss_disc: 2.5974392890930176 (2.4769715241023484) + | > loss_disc_real_0: 0.11961022764444351 (0.1314426186893668) + | > loss_disc_real_1: 0.23969538509845734 (0.21083595337612288) + | > loss_disc_real_2: 0.2393992841243744 (0.22552051980580606) + | > loss_disc_real_3: 0.2394888699054718 (0.22917281516960689) + | > loss_disc_real_4: 0.2574162483215332 (0.22916594449253308) + | > loss_disc_real_5: 0.25330671668052673 (0.23166921060709728) + | > loss_0: 2.5974392890930176 (2.4769715241023484) + | > grad_norm_0: tensor(7.9817, device='cuda:0') (tensor(8.3367, device='cuda:0')) + | > loss_gen: 2.187523126602173 (2.329313635826112) + | > loss_kl: 2.623901605606079 (2.6643217972346704) + | > loss_feat: 5.310033798217773 (6.948460238320487) + | > loss_mel: 14.853730201721191 (18.15298444884164) + | > loss_duration: 2.0854344367980957 (2.115202270802997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.060623168945312 (32.21028253010342) + | > grad_norm_1: tensor(132.6756, device='cuda:0') (tensor(158.8906, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.3908 (0.36354696182977586) + | > loader_time: 0.0064 (0.0056162164324805855) + + + --> TIME: 2025-04-18 23:03:24 -- STEP: 109/142 -- GLOBAL_STEP: 379675 + | > loss_disc: 2.468240737915039 (2.484725626236802) + | > loss_disc_real_0: 0.10818161070346832 (0.1318092006472274) + | > loss_disc_real_1: 0.22750891745090485 (0.21021442402393445) + | > loss_disc_real_2: 0.24253244698047638 (0.2266704348522589) + | > loss_disc_real_3: 0.2731682062149048 (0.23002633548110996) + | > loss_disc_real_4: 0.24442818760871887 (0.23053468258009044) + | > loss_disc_real_5: 0.22067299485206604 (0.23260836822724124) + | > loss_0: 2.468240737915039 (2.484725626236802) + | > grad_norm_0: tensor(10.0624, device='cuda:0') (tensor(9.1829, device='cuda:0')) + | > loss_gen: 2.4029958248138428 (2.326348521293851) + | > loss_kl: 2.5652334690093994 (2.6521172282892627) + | > loss_feat: 6.5097222328186035 (6.919194374609431) + | > loss_mel: 17.536911010742188 (18.08811545590743) + | > loss_duration: 2.026909351348877 (2.0981309610769285) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.041770935058594 (32.08390664616858) + | > grad_norm_1: tensor(306.7320, device='cuda:0') (tensor(183.9520, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.4083 (0.3730171164241406) + | > loader_time: 0.0066 (0.0058361215328951496) + + + --> TIME: 2025-04-18 23:03:35 -- STEP: 134/142 -- GLOBAL_STEP: 379700 + | > loss_disc: 2.4906005859375 (2.485403991457242) + | > loss_disc_real_0: 0.12011346966028214 (0.13299507731154794) + | > loss_disc_real_1: 0.24650724232196808 (0.2102665139445618) + | > loss_disc_real_2: 0.2415861040353775 (0.22689813920366234) + | > loss_disc_real_3: 0.23486992716789246 (0.23003001733502346) + | > loss_disc_real_4: 0.2218187004327774 (0.23055507709730916) + | > loss_disc_real_5: 0.2752159833908081 (0.23244716994353196) + | > loss_0: 2.4906005859375 (2.485403991457242) + | > grad_norm_0: tensor(15.4632, device='cuda:0') (tensor(9.5073, device='cuda:0')) + | > loss_gen: 2.213772773742676 (2.321760309276297) + | > loss_kl: 2.5826449394226074 (2.6375882963636026) + | > loss_feat: 7.1576080322265625 (6.884837182600107) + | > loss_mel: 18.56963348388672 (18.020252939480464) + | > loss_duration: 2.017422676086426 (2.0798596757561403) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.541080474853516 (31.94429843105489) + | > grad_norm_1: tensor(405.6010, device='cuda:0') (tensor(196.0719, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.4317 (0.38352698354578724) + | > loader_time: 0.0071 (0.0060628029837537165) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3401355743408203 (2.3401355743408203) + | > loss_disc_real_0: 0.11106319725513458 (0.11106319725513458) + | > loss_disc_real_1: 0.16857478022575378 (0.16857478022575378) + | > loss_disc_real_2: 0.22067487239837646 (0.22067487239837646) + | > loss_disc_real_3: 0.2905694246292114 (0.2905694246292114) + | > loss_disc_real_4: 0.23252615332603455 (0.23252615332603455) + | > loss_disc_real_5: 0.24300140142440796 (0.24300140142440796) + | > loss_0: 2.3401355743408203 (2.3401355743408203) + | > loss_gen: 2.451429843902588 (2.451429843902588) + | > loss_kl: 2.7906389236450195 (2.7906389236450195) + | > loss_feat: 7.625642776489258 (7.625642776489258) + | > loss_mel: 19.29615592956543 (19.29615592956543) + | > loss_duration: 1.9770278930664062 (1.9770278930664062) + | > loss_1: 34.14089584350586 (34.14089584350586) + + --> STEP: 1 + | > loss_disc: 2.569969415664673 (2.569969415664673) + | > loss_disc_real_0: 0.1984192579984665 (0.1984192579984665) + | > loss_disc_real_1: 0.21180716156959534 (0.21180716156959534) + | > loss_disc_real_2: 0.22856853902339935 (0.22856853902339935) + | > loss_disc_real_3: 0.26914939284324646 (0.26914939284324646) + | > loss_disc_real_4: 0.23132456839084625 (0.23132456839084625) + | > loss_disc_real_5: 0.28211086988449097 (0.28211086988449097) + | > loss_0: 2.569969415664673 (2.569969415664673) + | > loss_gen: 2.221294403076172 (2.221294403076172) + | > loss_kl: 2.52312970161438 (2.52312970161438) + | > loss_feat: 4.742880344390869 (4.742880344390869) + | > loss_mel: 15.539946556091309 (15.539946556091309) + | > loss_duration: 2.024407386779785 (2.024407386779785) + | > loss_1: 27.051658630371094 (27.051658630371094) + + --> STEP: 2 + | > loss_disc: 2.3897132873535156 (2.4798413515090942) + | > loss_disc_real_0: 0.13102614879608154 (0.16472270339727402) + | > loss_disc_real_1: 0.22063510119915009 (0.2162211313843727) + | > loss_disc_real_2: 0.19904810190200806 (0.2138083204627037) + | > loss_disc_real_3: 0.26663151383399963 (0.26789045333862305) + | > loss_disc_real_4: 0.2578102648258209 (0.2445674166083336) + | > loss_disc_real_5: 0.22963839769363403 (0.2558746337890625) + | > loss_0: 2.3897132873535156 (2.4798413515090942) + | > loss_gen: 2.4265294075012207 (2.3239119052886963) + | > loss_kl: 2.5724503993988037 (2.547790050506592) + | > loss_feat: 6.771953582763672 (5.7574169635772705) + | > loss_mel: 18.367372512817383 (16.953659534454346) + | > loss_duration: 1.9689075946807861 (1.9966574907302856) + | > loss_1: 32.10721206665039 (29.579435348510742) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033507347106933594 (-0.00028252601623535156) + | > avg_loss_disc: 2.4798413515090942 (+0.07486772537231445) + | > avg_loss_disc_real_0: 0.16472270339727402 (-0.05628061294555664) + | > avg_loss_disc_real_1: 0.2162211313843727 (-0.004863381385803223) + | > avg_loss_disc_real_2: 0.2138083204627037 (+0.0008789598941802979) + | > avg_loss_disc_real_3: 0.26789045333862305 (+0.04396261274814606) + | > avg_loss_disc_real_4: 0.2445674166083336 (+0.043299928307533264) + | > avg_loss_disc_real_5: 0.2558746337890625 (+0.027902714908123016) + | > avg_loss_0: 2.4798413515090942 (+0.07486772537231445) + | > avg_loss_gen: 2.3239119052886963 (-0.3149038553237915) + | > avg_loss_kl: 2.547790050506592 (-0.13022005558013916) + | > avg_loss_feat: 5.7574169635772705 (-1.9477813243865967) + | > avg_loss_mel: 16.953659534454346 (-1.685452938079834) + | > avg_loss_duration: 1.9966574907302856 (-0.19186937808990479) + | > avg_loss_1: 29.579435348510742 (-4.270227432250977) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_379708.pth + + > EPOCH: 3/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:03:41)  + + --> TIME: 2025-04-18 23:03:47 -- STEP: 17/142 -- GLOBAL_STEP: 379725 + | > loss_disc: 2.2896199226379395 (2.4315569541033577) + | > loss_disc_real_0: 0.15423910319805145 (0.13085153874228983) + | > loss_disc_real_1: 0.21460482478141785 (0.2120608231600593) + | > loss_disc_real_2: 0.2146783322095871 (0.22301856384557836) + | > loss_disc_real_3: 0.1890098601579666 (0.2207858615061816) + | > loss_disc_real_4: 0.2266053706407547 (0.22499506175518036) + | > loss_disc_real_5: 0.20522119104862213 (0.2260123131906285) + | > loss_0: 2.2896199226379395 (2.4315569541033577) + | > grad_norm_0: tensor(8.1137, device='cuda:0') (tensor(7.2752, device='cuda:0')) + | > loss_gen: 2.489410161972046 (2.350781889522777) + | > loss_kl: 2.5280323028564453 (2.494394442614387) + | > loss_feat: 7.662861347198486 (7.0634748234468345) + | > loss_mel: 18.6260986328125 (17.705232059254367) + | > loss_duration: 1.931350827217102 (1.9653223752975464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.237754821777344 (31.57920613008387) + | > grad_norm_1: tensor(191.3425, device='cuda:0') (tensor(155.3801, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.3412 (0.3271344409269445) + | > loader_time: 0.005 (0.004510052063885857) + + + --> TIME: 2025-04-18 23:03:56 -- STEP: 42/142 -- GLOBAL_STEP: 379750 + | > loss_disc: 2.4884250164031982 (2.4564403125217984) + | > loss_disc_real_0: 0.11079206317663193 (0.1296268761867569) + | > loss_disc_real_1: 0.19889435172080994 (0.205913279737745) + | > loss_disc_real_2: 0.22422349452972412 (0.22482848876998537) + | > loss_disc_real_3: 0.22817085683345795 (0.2261211701801845) + | > loss_disc_real_4: 0.19891856610774994 (0.23089287891274407) + | > loss_disc_real_5: 0.2093893140554428 (0.23203599559409277) + | > loss_0: 2.4884250164031982 (2.4564403125217984) + | > grad_norm_0: tensor(14.3259, device='cuda:0') (tensor(7.9167, device='cuda:0')) + | > loss_gen: 2.2581982612609863 (2.3345848378681007) + | > loss_kl: 2.4777588844299316 (2.4893604687282016) + | > loss_feat: 7.188551902770996 (7.071483294169108) + | > loss_mel: 19.327009201049805 (17.909736179170153) + | > loss_duration: 1.903883457183838 (1.9347484225318545) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.15540313720703 (31.73991366795131) + | > grad_norm_1: tensor(374.2373, device='cuda:0') (tensor(173.9387, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.3621 (0.34245561985742484) + | > loader_time: 0.006 (0.00508482683272589) + + + --> TIME: 2025-04-18 23:04:06 -- STEP: 67/142 -- GLOBAL_STEP: 379775 + | > loss_disc: 2.400991439819336 (2.4557308296659097) + | > loss_disc_real_0: 0.07142090797424316 (0.13019641789037784) + | > loss_disc_real_1: 0.16328027844429016 (0.20633666017162267) + | > loss_disc_real_2: 0.23719193041324615 (0.2234808288848222) + | > loss_disc_real_3: 0.24144423007965088 (0.2272577594910095) + | > loss_disc_real_4: 0.22437508404254913 (0.22970539531601009) + | > loss_disc_real_5: 0.23594848811626434 (0.22952578650481664) + | > loss_0: 2.400991439819336 (2.4557308296659097) + | > grad_norm_0: tensor(26.2248, device='cuda:0') (tensor(11.2934, device='cuda:0')) + | > loss_gen: 2.4417591094970703 (2.3530663483178444) + | > loss_kl: 2.4302425384521484 (2.492668105595147) + | > loss_feat: 8.039762496948242 (7.1134745612073305) + | > loss_mel: 18.796728134155273 (18.084593388571665) + | > loss_duration: 1.8109047412872314 (1.919963267312121) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.5193977355957 (31.9637659841509) + | > grad_norm_1: tensor(558.9454, device='cuda:0') (tensor(237.6290, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.3784 (0.35395756764198416) + | > loader_time: 0.0057 (0.005390750828073987) + + + --> TIME: 2025-04-18 23:04:16 -- STEP: 92/142 -- GLOBAL_STEP: 379800 + | > loss_disc: 2.4555180072784424 (2.4685408809910654) + | > loss_disc_real_0: 0.20155882835388184 (0.1314202978721131) + | > loss_disc_real_1: 0.2345178723335266 (0.20533752603375394) + | > loss_disc_real_2: 0.2528906464576721 (0.2259174013591331) + | > loss_disc_real_3: 0.229352205991745 (0.22849117381417233) + | > loss_disc_real_4: 0.21311621367931366 (0.2306194055987441) + | > loss_disc_real_5: 0.2502002418041229 (0.23161535467142644) + | > loss_0: 2.4555180072784424 (2.4685408809910654) + | > grad_norm_0: tensor(16.3249, device='cuda:0') (tensor(12.2831, device='cuda:0')) + | > loss_gen: 2.476456642150879 (2.3536960586257605) + | > loss_kl: 2.494642972946167 (2.4885054956311756) + | > loss_feat: 7.4767045974731445 (7.054518637449845) + | > loss_mel: 19.47395896911621 (18.082596799601685) + | > loss_duration: 1.7892100811004639 (1.90737730394239) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.710975646972656 (31.88669443130493) + | > grad_norm_1: tensor(119.7002, device='cuda:0') (tensor(250.4362, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.4118 (0.36531029058539344) + | > loader_time: 0.0073 (0.005696509195410686) + + + --> TIME: 2025-04-18 23:04:27 -- STEP: 117/142 -- GLOBAL_STEP: 379825 + | > loss_disc: 2.5316002368927 (2.4767022030985255) + | > loss_disc_real_0: 0.13845418393611908 (0.13133537094307757) + | > loss_disc_real_1: 0.22139869630336761 (0.20726219430947915) + | > loss_disc_real_2: 0.2558789551258087 (0.22642039246538764) + | > loss_disc_real_3: 0.2281724512577057 (0.22899438912033016) + | > loss_disc_real_4: 0.23954641819000244 (0.23124400061419886) + | > loss_disc_real_5: 0.24412111937999725 (0.23232258894504645) + | > loss_0: 2.5316002368927 (2.4767022030985255) + | > grad_norm_0: tensor(4.0532, device='cuda:0') (tensor(11.4062, device='cuda:0')) + | > loss_gen: 2.3306469917297363 (2.3402315062335415) + | > loss_kl: 2.3664071559906006 (2.4857597330696555) + | > loss_feat: 7.4452385902404785 (6.967759014194847) + | > loss_mel: 18.140125274658203 (17.975475963364314) + | > loss_duration: 1.8089667558670044 (1.893171920735612) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.09138488769531 (31.662398167145557) + | > grad_norm_1: tensor(130.6203, device='cuda:0') (tensor(232.6513, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.4354 (0.37640393493521923) + | > loader_time: 0.0072 (0.005969308380387786) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4191250801086426 (2.4191250801086426) + | > loss_disc_real_0: 0.14075718820095062 (0.14075718820095062) + | > loss_disc_real_1: 0.1943853348493576 (0.1943853348493576) + | > loss_disc_real_2: 0.20841412246227264 (0.20841412246227264) + | > loss_disc_real_3: 0.23584122955799103 (0.23584122955799103) + | > loss_disc_real_4: 0.24186699092388153 (0.24186699092388153) + | > loss_disc_real_5: 0.24937716126441956 (0.24937716126441956) + | > loss_0: 2.4191250801086426 (2.4191250801086426) + | > loss_gen: 2.3742103576660156 (2.3742103576660156) + | > loss_kl: 2.432142496109009 (2.432142496109009) + | > loss_feat: 7.954658031463623 (7.954658031463623) + | > loss_mel: 20.16853141784668 (20.16853141784668) + | > loss_duration: 1.8218705654144287 (1.8218705654144287) + | > loss_1: 34.75141143798828 (34.75141143798828) + + --> STEP: 1 + | > loss_disc: 2.535792350769043 (2.535792350769043) + | > loss_disc_real_0: 0.19484193623065948 (0.19484193623065948) + | > loss_disc_real_1: 0.20934262871742249 (0.20934262871742249) + | > loss_disc_real_2: 0.22988924384117126 (0.22988924384117126) + | > loss_disc_real_3: 0.2413693070411682 (0.2413693070411682) + | > loss_disc_real_4: 0.25907137989997864 (0.25907137989997864) + | > loss_disc_real_5: 0.302704393863678 (0.302704393863678) + | > loss_0: 2.535792350769043 (2.535792350769043) + | > loss_gen: 2.4805688858032227 (2.4805688858032227) + | > loss_kl: 2.55747389793396 (2.55747389793396) + | > loss_feat: 6.134952068328857 (6.134952068328857) + | > loss_mel: 18.87334632873535 (18.87334632873535) + | > loss_duration: 1.860304355621338 (1.860304355621338) + | > loss_1: 31.906644821166992 (31.906644821166992) + + --> STEP: 2 + | > loss_disc: 2.4768667221069336 (2.5063295364379883) + | > loss_disc_real_0: 0.20627298951148987 (0.20055746287107468) + | > loss_disc_real_1: 0.1956939697265625 (0.2025182992219925) + | > loss_disc_real_2: 0.23907329142093658 (0.23448126763105392) + | > loss_disc_real_3: 0.22687315940856934 (0.23412123322486877) + | > loss_disc_real_4: 0.23828989267349243 (0.24868063628673553) + | > loss_disc_real_5: 0.23584674298763275 (0.26927556842565536) + | > loss_0: 2.4768667221069336 (2.5063295364379883) + | > loss_gen: 2.446404218673706 (2.4634865522384644) + | > loss_kl: 2.3895816802978516 (2.4735277891159058) + | > loss_feat: 6.715182304382324 (6.425067186355591) + | > loss_mel: 17.879758834838867 (18.37655258178711) + | > loss_duration: 1.7738397121429443 (1.8170720338821411) + | > loss_1: 31.20476722717285 (31.555706024169922) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038641691207885742 (+0.0005134344100952148) + | > avg_loss_disc: 2.5063295364379883 (+0.026488184928894043) + | > avg_loss_disc_real_0: 0.20055746287107468 (+0.03583475947380066) + | > avg_loss_disc_real_1: 0.2025182992219925 (-0.013702832162380219) + | > avg_loss_disc_real_2: 0.23448126763105392 (+0.02067294716835022) + | > avg_loss_disc_real_3: 0.23412123322486877 (-0.03376922011375427) + | > avg_loss_disc_real_4: 0.24868063628673553 (+0.004113219678401947) + | > avg_loss_disc_real_5: 0.26927556842565536 (+0.013400934636592865) + | > avg_loss_0: 2.5063295364379883 (+0.026488184928894043) + | > avg_loss_gen: 2.4634865522384644 (+0.13957464694976807) + | > avg_loss_kl: 2.4735277891159058 (-0.07426226139068604) + | > avg_loss_feat: 6.425067186355591 (+0.6676502227783203) + | > avg_loss_mel: 18.37655258178711 (+1.4228930473327637) + | > avg_loss_duration: 1.8170720338821411 (-0.17958545684814453) + | > avg_loss_1: 31.555706024169922 (+1.9762706756591797) + + + > EPOCH: 4/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:04:39)  + + --> TIME: 2025-04-18 23:04:40 -- STEP: 0/142 -- GLOBAL_STEP: 379850 + | > loss_disc: 2.46537184715271 (2.46537184715271) + | > loss_disc_real_0: 0.16504886746406555 (0.16504886746406555) + | > loss_disc_real_1: 0.19821907579898834 (0.19821907579898834) + | > loss_disc_real_2: 0.22730621695518494 (0.22730621695518494) + | > loss_disc_real_3: 0.23381927609443665 (0.23381927609443665) + | > loss_disc_real_4: 0.25611305236816406 (0.25611305236816406) + | > loss_disc_real_5: 0.23424455523490906 (0.23424455523490906) + | > loss_0: 2.46537184715271 (2.46537184715271) + | > grad_norm_0: tensor(6.4431, device='cuda:0') (tensor(6.4431, device='cuda:0')) + | > loss_gen: 2.573780059814453 (2.573780059814453) + | > loss_kl: 2.4183902740478516 (2.4183902740478516) + | > loss_feat: 7.664832592010498 (7.664832592010498) + | > loss_mel: 19.716983795166016 (19.716983795166016) + | > loss_duration: 1.8497341871261597 (1.8497341871261597) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.22372055053711 (34.22372055053711) + | > grad_norm_1: tensor(139.1965, device='cuda:0') (tensor(139.1965, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.3345 (0.3344545364379883) + | > loader_time: 0.3086 (0.30863428115844727) + + + --> TIME: 2025-04-18 23:04:48 -- STEP: 25/142 -- GLOBAL_STEP: 379875 + | > loss_disc: 2.429353952407837 (2.4663631153106684) + | > loss_disc_real_0: 0.08815807104110718 (0.12344497323036194) + | > loss_disc_real_1: 0.18393956124782562 (0.20925274908542632) + | > loss_disc_real_2: 0.24087047576904297 (0.22648566722869873) + | > loss_disc_real_3: 0.23576445877552032 (0.2317729252576828) + | > loss_disc_real_4: 0.23703497648239136 (0.2317894995212555) + | > loss_disc_real_5: 0.2650662362575531 (0.23447175264358522) + | > loss_0: 2.429353952407837 (2.4663631153106684) + | > grad_norm_0: tensor(4.3742, device='cuda:0') (tensor(6.7650, device='cuda:0')) + | > loss_gen: 2.404764175415039 (2.3147775077819817) + | > loss_kl: 2.4739840030670166 (2.417619428634643) + | > loss_feat: 6.865951061248779 (6.932325172424316) + | > loss_mel: 16.82145118713379 (18.026942825317384) + | > loss_duration: 1.8085978031158447 (1.7978698873519898) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.37474822998047 (31.48953483581543) + | > grad_norm_1: tensor(74.6288, device='cuda:0') (tensor(114.2770, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.3425 (0.33067974090576174) + | > loader_time: 0.0051 (0.004892549514770508) + + + --> TIME: 2025-04-18 23:04:58 -- STEP: 50/142 -- GLOBAL_STEP: 379900 + | > loss_disc: 2.515613555908203 (2.473444418907165) + | > loss_disc_real_0: 0.09096547216176987 (0.12601814970374106) + | > loss_disc_real_1: 0.21063053607940674 (0.2065293762087822) + | > loss_disc_real_2: 0.22338928282260895 (0.22580602496862412) + | > loss_disc_real_3: 0.2502813935279846 (0.22946737080812454) + | > loss_disc_real_4: 0.26416072249412537 (0.23264027506113052) + | > loss_disc_real_5: 0.23402471840381622 (0.23394728660583497) + | > loss_0: 2.515613555908203 (2.473444418907165) + | > grad_norm_0: tensor(5.8625, device='cuda:0') (tensor(6.6887, device='cuda:0')) + | > loss_gen: 2.2871885299682617 (2.315496873855591) + | > loss_kl: 2.38070011138916 (2.412496395111083) + | > loss_feat: 7.061817646026611 (6.911861457824707) + | > loss_mel: 18.635498046875 (18.07376457214355) + | > loss_duration: 1.824074387550354 (1.7880784940719605) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.18927764892578 (31.501697769165037) + | > grad_norm_1: tensor(108.8282, device='cuda:0') (tensor(115.3661, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.3673 (0.3432823991775512) + | > loader_time: 0.0057 (0.005292506217956541) + + + --> TIME: 2025-04-18 23:05:07 -- STEP: 75/142 -- GLOBAL_STEP: 379925 + | > loss_disc: 2.373603105545044 (2.475948546727498) + | > loss_disc_real_0: 0.09039495885372162 (0.12468226859966913) + | > loss_disc_real_1: 0.1584213674068451 (0.2071187420686086) + | > loss_disc_real_2: 0.19700583815574646 (0.22558713813622794) + | > loss_disc_real_3: 0.18413977324962616 (0.2287579047679901) + | > loss_disc_real_4: 0.19793164730072021 (0.23418960591157278) + | > loss_disc_real_5: 0.18163953721523285 (0.23462736924489339) + | > loss_0: 2.373603105545044 (2.475948546727498) + | > grad_norm_0: tensor(8.1865, device='cuda:0') (tensor(6.6595, device='cuda:0')) + | > loss_gen: 2.37770676612854 (2.3199852752685546) + | > loss_kl: 2.284075975418091 (2.4013581943511957) + | > loss_feat: 7.428130626678467 (6.915280475616455) + | > loss_mel: 17.67123031616211 (18.121659520467123) + | > loss_duration: 1.7232506275177002 (1.7880657704671223) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.48439598083496 (31.546349334716798) + | > grad_norm_1: tensor(366.2978, device='cuda:0') (tensor(128.2836, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.4004 (0.355534652074178) + | > loader_time: 0.0065 (0.005589418411254881) + + + --> TIME: 2025-04-18 23:05:17 -- STEP: 100/142 -- GLOBAL_STEP: 379950 + | > loss_disc: 2.4355268478393555 (2.4776892948150633) + | > loss_disc_real_0: 0.13755597174167633 (0.1273721317201853) + | > loss_disc_real_1: 0.17692506313323975 (0.20750488683581353) + | > loss_disc_real_2: 0.22081205248832703 (0.2258893546462059) + | > loss_disc_real_3: 0.23663799464702606 (0.22910078033804893) + | > loss_disc_real_4: 0.210221067070961 (0.23265658110380172) + | > loss_disc_real_5: 0.24439869821071625 (0.23290905728936195) + | > loss_0: 2.4355268478393555 (2.4776892948150633) + | > grad_norm_0: tensor(7.7007, device='cuda:0') (tensor(7.9773, device='cuda:0')) + | > loss_gen: 2.5081968307495117 (2.320824732780457) + | > loss_kl: 2.436676263809204 (2.3954678845405573) + | > loss_feat: 8.146313667297363 (6.880082397460938) + | > loss_mel: 19.48375129699707 (18.113763103485113) + | > loss_duration: 1.755202293395996 (1.7861027467250823) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.330142974853516 (31.496240940093994) + | > grad_norm_1: tensor(301.1526, device='cuda:0') (tensor(154.9397, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.4197 (0.36700096130371096) + | > loader_time: 0.0072 (0.005866186618804932) + + + --> TIME: 2025-04-18 23:05:28 -- STEP: 125/142 -- GLOBAL_STEP: 379975 + | > loss_disc: 2.5795536041259766 (2.4850573387145998) + | > loss_disc_real_0: 0.1250026524066925 (0.12789405834674833) + | > loss_disc_real_1: 0.21146035194396973 (0.2076650458574295) + | > loss_disc_real_2: 0.3045467734336853 (0.22774069452285767) + | > loss_disc_real_3: 0.2783688008785248 (0.23025123286247254) + | > loss_disc_real_4: 0.23121017217636108 (0.23256574249267578) + | > loss_disc_real_5: 0.2796487808227539 (0.23431008875370027) + | > loss_0: 2.5795536041259766 (2.4850573387145998) + | > grad_norm_0: tensor(4.7204, device='cuda:0') (tensor(8.3091, device='cuda:0')) + | > loss_gen: 2.432375907897949 (2.3221840724945073) + | > loss_kl: 2.368927240371704 (2.3898585453033445) + | > loss_feat: 6.82604455947876 (6.866373432159424) + | > loss_mel: 18.855464935302734 (18.048398956298836) + | > loss_duration: 1.7801275253295898 (1.7818258094787598) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.262943267822266 (31.40864094543457) + | > grad_norm_1: tensor(224.2784, device='cuda:0') (tensor(169.1489, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.4372 (0.37922832107543947) + | > loader_time: 0.0077 (0.006152938842773437) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4646010398864746 (2.4646010398864746) + | > loss_disc_real_0: 0.1673482209444046 (0.1673482209444046) + | > loss_disc_real_1: 0.2489471137523651 (0.2489471137523651) + | > loss_disc_real_2: 0.2449670433998108 (0.2449670433998108) + | > loss_disc_real_3: 0.2644263207912445 (0.2644263207912445) + | > loss_disc_real_4: 0.23964111506938934 (0.23964111506938934) + | > loss_disc_real_5: 0.2809161841869354 (0.2809161841869354) + | > loss_0: 2.4646010398864746 (2.4646010398864746) + | > loss_gen: 2.624315023422241 (2.624315023422241) + | > loss_kl: 2.4088358879089355 (2.4088358879089355) + | > loss_feat: 6.7110137939453125 (6.7110137939453125) + | > loss_mel: 17.473590850830078 (17.473590850830078) + | > loss_duration: 1.7681891918182373 (1.7681891918182373) + | > loss_1: 30.985944747924805 (30.985944747924805) + + --> STEP: 1 + | > loss_disc: 2.5804944038391113 (2.5804944038391113) + | > loss_disc_real_0: 0.1772521585226059 (0.1772521585226059) + | > loss_disc_real_1: 0.26520928740501404 (0.26520928740501404) + | > loss_disc_real_2: 0.25053712725639343 (0.25053712725639343) + | > loss_disc_real_3: 0.24264003336429596 (0.24264003336429596) + | > loss_disc_real_4: 0.23893627524375916 (0.23893627524375916) + | > loss_disc_real_5: 0.2652305066585541 (0.2652305066585541) + | > loss_0: 2.5804944038391113 (2.5804944038391113) + | > loss_gen: 2.3145811557769775 (2.3145811557769775) + | > loss_kl: 2.410183906555176 (2.410183906555176) + | > loss_feat: 6.756065368652344 (6.756065368652344) + | > loss_mel: 18.435758590698242 (18.435758590698242) + | > loss_duration: 1.8082711696624756 (1.8082711696624756) + | > loss_1: 31.72486114501953 (31.72486114501953) + + --> STEP: 2 + | > loss_disc: 2.5069799423217773 (2.5437371730804443) + | > loss_disc_real_0: 0.18603254854679108 (0.1816423535346985) + | > loss_disc_real_1: 0.2483634054660797 (0.2567863464355469) + | > loss_disc_real_2: 0.26146847009658813 (0.2560027986764908) + | > loss_disc_real_3: 0.2500952184200287 (0.24636762589216232) + | > loss_disc_real_4: 0.2388889342546463 (0.23891260474920273) + | > loss_disc_real_5: 0.24530698359012604 (0.25526874512434006) + | > loss_0: 2.5069799423217773 (2.5437371730804443) + | > loss_gen: 2.5943002700805664 (2.454440712928772) + | > loss_kl: 2.295886278152466 (2.353035092353821) + | > loss_feat: 7.505889415740967 (7.130977392196655) + | > loss_mel: 18.82779884338379 (18.631778717041016) + | > loss_duration: 1.7423813343048096 (1.7753262519836426) + | > loss_1: 32.96625518798828 (32.345558166503906) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038008689880371094 (-6.330013275146484e-05) + | > avg_loss_disc: 2.5437371730804443 (+0.037407636642456055) + | > avg_loss_disc_real_0: 0.1816423535346985 (-0.01891510933637619) + | > avg_loss_disc_real_1: 0.2567863464355469 (+0.05426804721355438) + | > avg_loss_disc_real_2: 0.2560027986764908 (+0.02152153104543686) + | > avg_loss_disc_real_3: 0.24636762589216232 (+0.012246392667293549) + | > avg_loss_disc_real_4: 0.23891260474920273 (-0.009768031537532806) + | > avg_loss_disc_real_5: 0.25526874512434006 (-0.014006823301315308) + | > avg_loss_0: 2.5437371730804443 (+0.037407636642456055) + | > avg_loss_gen: 2.454440712928772 (-0.009045839309692383) + | > avg_loss_kl: 2.353035092353821 (-0.12049269676208496) + | > avg_loss_feat: 7.130977392196655 (+0.7059102058410645) + | > avg_loss_mel: 18.631778717041016 (+0.25522613525390625) + | > avg_loss_duration: 1.7753262519836426 (-0.041745781898498535) + | > avg_loss_1: 32.345558166503906 (+0.7898521423339844) + + + > EPOCH: 5/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:05:37)  + + --> TIME: 2025-04-18 23:05:41 -- STEP: 8/142 -- GLOBAL_STEP: 380000 + | > loss_disc: 2.5272042751312256 (2.5471290946006775) + | > loss_disc_real_0: 0.09505557268857956 (0.12779416190460324) + | > loss_disc_real_1: 0.19354689121246338 (0.21439522691071033) + | > loss_disc_real_2: 0.20367416739463806 (0.224288422614336) + | > loss_disc_real_3: 0.2221994400024414 (0.23825574107468128) + | > loss_disc_real_4: 0.19650930166244507 (0.2305346541106701) + | > loss_disc_real_5: 0.21146363019943237 (0.24999697506427765) + | > loss_0: 2.5272042751312256 (2.5471290946006775) + | > grad_norm_0: tensor(13.5166, device='cuda:0') (tensor(12.5391, device='cuda:0')) + | > loss_gen: 2.125366687774658 (2.2480037808418274) + | > loss_kl: 2.409226179122925 (2.3820774853229523) + | > loss_feat: 6.449163913726807 (6.731683433055878) + | > loss_mel: 17.192214965820312 (17.680230259895325) + | > loss_duration: 1.7446595430374146 (1.7554255723953247) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.920631408691406 (30.797420263290405) + | > grad_norm_1: tensor(170.5149, device='cuda:0') (tensor(276.9331, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.3227 (0.32169121503829956) + | > loader_time: 0.0046 (0.004227727651596069) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_380000.pth + + --> TIME: 2025-04-18 23:05:51 -- STEP: 33/142 -- GLOBAL_STEP: 380025 + | > loss_disc: 2.4290688037872314 (2.4650403947541206) + | > loss_disc_real_0: 0.15829269587993622 (0.12055715410546823) + | > loss_disc_real_1: 0.19963742792606354 (0.20911610397425565) + | > loss_disc_real_2: 0.21381445229053497 (0.2239617732438174) + | > loss_disc_real_3: 0.20248669385910034 (0.2283471425374349) + | > loss_disc_real_4: 0.24682727456092834 (0.2347420472087282) + | > loss_disc_real_5: 0.20883625745773315 (0.2283801847335064) + | > loss_0: 2.4290688037872314 (2.4650403947541206) + | > grad_norm_0: tensor(18.1862, device='cuda:0') (tensor(12.0011, device='cuda:0')) + | > loss_gen: 2.30594539642334 (2.326597300442783) + | > loss_kl: 2.269902467727661 (2.3561601783290054) + | > loss_feat: 6.906639099121094 (6.979025840759277) + | > loss_mel: 17.67806053161621 (18.010716842882562) + | > loss_duration: 1.7146655321121216 (1.736358480020003) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.875215530395508 (31.408858848340582) + | > grad_norm_1: tensor(304.3083, device='cuda:0') (tensor(294.7308, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.5284 (0.3432435989379883) + | > loader_time: 0.0052 (0.0049439488035259846) + + + --> TIME: 2025-04-18 23:06:00 -- STEP: 58/142 -- GLOBAL_STEP: 380050 + | > loss_disc: 2.4180104732513428 (2.4685505299732604) + | > loss_disc_real_0: 0.12266837805509567 (0.12320413581770041) + | > loss_disc_real_1: 0.19579780101776123 (0.2071214916891065) + | > loss_disc_real_2: 0.2250010073184967 (0.22707314147003765) + | > loss_disc_real_3: 0.23912380635738373 (0.22962590461147242) + | > loss_disc_real_4: 0.24796926975250244 (0.2314794585108757) + | > loss_disc_real_5: 0.25687289237976074 (0.23150138454190616) + | > loss_0: 2.4180104732513428 (2.4685505299732604) + | > grad_norm_0: tensor(17.2678, device='cuda:0') (tensor(14.8214, device='cuda:0')) + | > loss_gen: 2.487642765045166 (2.354575169497524) + | > loss_kl: 2.3129491806030273 (2.3759834067574856) + | > loss_feat: 6.823955059051514 (6.982290202173694) + | > loss_mel: 19.127958297729492 (18.062421815148717) + | > loss_duration: 1.761852741241455 (1.734748817723373) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.51435852050781 (31.510019631221375) + | > grad_norm_1: tensor(196.5940, device='cuda:0') (tensor(309.1023, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.3854 (0.3522375575427351) + | > loader_time: 0.0082 (0.005352423108857253) + + + --> TIME: 2025-04-18 23:06:10 -- STEP: 83/142 -- GLOBAL_STEP: 380075 + | > loss_disc: 2.417720079421997 (2.465383118893726) + | > loss_disc_real_0: 0.10517232865095139 (0.1245117305093501) + | > loss_disc_real_1: 0.19610418379306793 (0.20665257217654262) + | > loss_disc_real_2: 0.20834510028362274 (0.22584718250366578) + | > loss_disc_real_3: 0.20887596905231476 (0.22919244777007275) + | > loss_disc_real_4: 0.17314189672470093 (0.23009058061134385) + | > loss_disc_real_5: 0.23043292760849 (0.23114535032984723) + | > loss_0: 2.417720079421997 (2.465383118893726) + | > grad_norm_0: tensor(9.3044, device='cuda:0') (tensor(14.6411, device='cuda:0')) + | > loss_gen: 2.345536708831787 (2.3512200849602034) + | > loss_kl: 2.3982245922088623 (2.374106769102164) + | > loss_feat: 6.744957447052002 (6.987050648195198) + | > loss_mel: 18.261831283569336 (18.080866917070136) + | > loss_duration: 1.8151118755340576 (1.7358506185462677) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.565662384033203 (31.529095109686793) + | > grad_norm_1: tensor(105.9153, device='cuda:0') (tensor(304.2592, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.3945 (0.36313184485378036) + | > loader_time: 0.0067 (0.005613611405154309) + + + --> TIME: 2025-04-18 23:06:20 -- STEP: 108/142 -- GLOBAL_STEP: 380100 + | > loss_disc: 2.5026416778564453 (2.4727761767528667) + | > loss_disc_real_0: 0.16802869737148285 (0.12705114835666287) + | > loss_disc_real_1: 0.2615552842617035 (0.20704102792121745) + | > loss_disc_real_2: 0.23954427242279053 (0.22632593744330937) + | > loss_disc_real_3: 0.22521816194057465 (0.22875541931501142) + | > loss_disc_real_4: 0.23358578979969025 (0.23139488007183429) + | > loss_disc_real_5: 0.22486966848373413 (0.23237749603059557) + | > loss_0: 2.5026416778564453 (2.4727761767528667) + | > grad_norm_0: tensor(4.9182, device='cuda:0') (tensor(13.3655, device='cuda:0')) + | > loss_gen: 2.4858295917510986 (2.342721802216991) + | > loss_kl: 2.441298723220825 (2.382049101370352) + | > loss_feat: 6.242527961730957 (6.94773938037731) + | > loss_mel: 17.86997413635254 (18.019421789381244) + | > loss_duration: 1.7315113544464111 (1.7354075621675562) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.771142959594727 (31.4273397481) + | > grad_norm_1: tensor(77.8508, device='cuda:0') (tensor(281.6674, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.4179 (0.3726320951073258) + | > loader_time: 0.0075 (0.0058690640661451556) + + + --> TIME: 2025-04-18 23:06:31 -- STEP: 133/142 -- GLOBAL_STEP: 380125 + | > loss_disc: 2.6124746799468994 (2.478940861565725) + | > loss_disc_real_0: 0.13644786179065704 (0.12897768471025883) + | > loss_disc_real_1: 0.1917724460363388 (0.20776156151205077) + | > loss_disc_real_2: 0.20027047395706177 (0.22607621270462983) + | > loss_disc_real_3: 0.208419069647789 (0.22872049479108109) + | > loss_disc_real_4: 0.23768389225006104 (0.231724904219907) + | > loss_disc_real_5: 0.24036620557308197 (0.2331749317341281) + | > loss_0: 2.6124746799468994 (2.478940861565725) + | > grad_norm_0: tensor(9.6429, device='cuda:0') (tensor(12.7278, device='cuda:0')) + | > loss_gen: 2.1892943382263184 (2.3303148262482853) + | > loss_kl: 2.4046390056610107 (2.3766379463941525) + | > loss_feat: 5.977970600128174 (6.881876267884907) + | > loss_mel: 16.406536102294922 (17.93744177567332) + | > loss_duration: 1.7706000804901123 (1.732929141001594) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.749040603637695 (31.25919997423215) + | > grad_norm_1: tensor(38.3648, device='cuda:0') (tensor(263.7512, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.4324 (0.38344593514177133) + | > loader_time: 0.0075 (0.006128528064354918) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3516688346862793 (2.3516688346862793) + | > loss_disc_real_0: 0.09771163016557693 (0.09771163016557693) + | > loss_disc_real_1: 0.2442663311958313 (0.2442663311958313) + | > loss_disc_real_2: 0.24775412678718567 (0.24775412678718567) + | > loss_disc_real_3: 0.2446889728307724 (0.2446889728307724) + | > loss_disc_real_4: 0.1963229924440384 (0.1963229924440384) + | > loss_disc_real_5: 0.2094944715499878 (0.2094944715499878) + | > loss_0: 2.3516688346862793 (2.3516688346862793) + | > loss_gen: 2.3888895511627197 (2.3888895511627197) + | > loss_kl: 2.476658582687378 (2.476658582687378) + | > loss_feat: 6.7649688720703125 (6.7649688720703125) + | > loss_mel: 18.569551467895508 (18.569551467895508) + | > loss_duration: 1.7599483728408813 (1.7599483728408813) + | > loss_1: 31.960018157958984 (31.960018157958984) + + --> STEP: 1 + | > loss_disc: 2.388929843902588 (2.388929843902588) + | > loss_disc_real_0: 0.10548277199268341 (0.10548277199268341) + | > loss_disc_real_1: 0.22842390835285187 (0.22842390835285187) + | > loss_disc_real_2: 0.25010278820991516 (0.25010278820991516) + | > loss_disc_real_3: 0.22985170781612396 (0.22985170781612396) + | > loss_disc_real_4: 0.21823851764202118 (0.21823851764202118) + | > loss_disc_real_5: 0.2151496708393097 (0.2151496708393097) + | > loss_0: 2.388929843902588 (2.388929843902588) + | > loss_gen: 2.3717424869537354 (2.3717424869537354) + | > loss_kl: 2.617173433303833 (2.617173433303833) + | > loss_feat: 6.746603012084961 (6.746603012084961) + | > loss_mel: 18.246994018554688 (18.246994018554688) + | > loss_duration: 1.7719316482543945 (1.7719316482543945) + | > loss_1: 31.754444122314453 (31.754444122314453) + + --> STEP: 2 + | > loss_disc: 2.6334495544433594 (2.5111896991729736) + | > loss_disc_real_0: 0.16146276891231537 (0.1334727704524994) + | > loss_disc_real_1: 0.23929765820503235 (0.2338607832789421) + | > loss_disc_real_2: 0.22051425278186798 (0.23530852049589157) + | > loss_disc_real_3: 0.24529734253883362 (0.2375745251774788) + | > loss_disc_real_4: 0.25293001532554626 (0.23558426648378372) + | > loss_disc_real_5: 0.27768903970718384 (0.24641935527324677) + | > loss_0: 2.6334495544433594 (2.5111896991729736) + | > loss_gen: 2.132842540740967 (2.252292513847351) + | > loss_kl: 2.6873016357421875 (2.6522375345230103) + | > loss_feat: 6.146342754364014 (6.446472883224487) + | > loss_mel: 17.8791446685791 (18.063069343566895) + | > loss_duration: 1.7075371742248535 (1.739734411239624) + | > loss_1: 30.55316925048828 (31.153806686401367) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038406848907470703 (+3.981590270996094e-05) + | > avg_loss_disc: 2.5111896991729736 (-0.0325474739074707) + | > avg_loss_disc_real_0: 0.1334727704524994 (-0.0481695830821991) + | > avg_loss_disc_real_1: 0.2338607832789421 (-0.022925563156604767) + | > avg_loss_disc_real_2: 0.23530852049589157 (-0.020694278180599213) + | > avg_loss_disc_real_3: 0.2375745251774788 (-0.008793100714683533) + | > avg_loss_disc_real_4: 0.23558426648378372 (-0.0033283382654190063) + | > avg_loss_disc_real_5: 0.24641935527324677 (-0.008849389851093292) + | > avg_loss_0: 2.5111896991729736 (-0.0325474739074707) + | > avg_loss_gen: 2.252292513847351 (-0.2021481990814209) + | > avg_loss_kl: 2.6522375345230103 (+0.29920244216918945) + | > avg_loss_feat: 6.446472883224487 (-0.684504508972168) + | > avg_loss_mel: 18.063069343566895 (-0.5687093734741211) + | > avg_loss_duration: 1.739734411239624 (-0.035591840744018555) + | > avg_loss_1: 31.153806686401367 (-1.191751480102539) + + + > EPOCH: 6/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:06:37)  + + --> TIME: 2025-04-18 23:06:43 -- STEP: 16/142 -- GLOBAL_STEP: 380150 + | > loss_disc: 2.6240460872650146 (2.4387026131153107) + | > loss_disc_real_0: 0.13364578783512115 (0.11815980961546302) + | > loss_disc_real_1: 0.18562822043895721 (0.20849913638085127) + | > loss_disc_real_2: 0.2841067314147949 (0.22845870349556208) + | > loss_disc_real_3: 0.2898542582988739 (0.22684691566973925) + | > loss_disc_real_4: 0.2690795958042145 (0.22714193910360336) + | > loss_disc_real_5: 0.2531653046607971 (0.22504843678325415) + | > loss_0: 2.6240460872650146 (2.4387026131153107) + | > grad_norm_0: tensor(8.6019, device='cuda:0') (tensor(10.7177, device='cuda:0')) + | > loss_gen: 2.2976903915405273 (2.4037476629018784) + | > loss_kl: 2.356337070465088 (2.3268517553806305) + | > loss_feat: 6.3595991134643555 (7.2186959981918335) + | > loss_mel: 17.454736709594727 (18.13027930259705) + | > loss_duration: 1.697044014930725 (1.6997791528701782) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.165409088134766 (31.779353976249695) + | > grad_norm_1: tensor(317.4481, device='cuda:0') (tensor(273.6440, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.3361 (0.32599061727523804) + | > loader_time: 0.0052 (0.004623919725418091) + + + --> TIME: 2025-04-18 23:06:52 -- STEP: 41/142 -- GLOBAL_STEP: 380175 + | > loss_disc: 2.3996360301971436 (2.4514152712938264) + | > loss_disc_real_0: 0.11768009513616562 (0.12046787415335818) + | > loss_disc_real_1: 0.19468477368354797 (0.20448084466341065) + | > loss_disc_real_2: 0.19283051788806915 (0.22696395618159595) + | > loss_disc_real_3: 0.2176651507616043 (0.22762775384798284) + | > loss_disc_real_4: 0.22722700238227844 (0.2301590762487272) + | > loss_disc_real_5: 0.2225935459136963 (0.23189410276529265) + | > loss_0: 2.3996360301971436 (2.4514152712938264) + | > grad_norm_0: tensor(8.7602, device='cuda:0') (tensor(11.2493, device='cuda:0')) + | > loss_gen: 2.4053399562835693 (2.3910190593905565) + | > loss_kl: 2.354419708251953 (2.3315662523595297) + | > loss_feat: 7.321584224700928 (7.246379724363002) + | > loss_mel: 17.27142906188965 (18.42699181161276) + | > loss_duration: 1.6801912784576416 (1.7009079398178473) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.0329647064209 (32.096864886400176) + | > grad_norm_1: tensor(105.3995, device='cuda:0') (tensor(251.5261, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.3608 (0.3389776683435207) + | > loader_time: 0.006 (0.005143340040997761) + + + --> TIME: 2025-04-18 23:07:01 -- STEP: 66/142 -- GLOBAL_STEP: 380200 + | > loss_disc: 2.457697629928589 (2.450679970510078) + | > loss_disc_real_0: 0.11980104446411133 (0.12274075028571216) + | > loss_disc_real_1: 0.20915980637073517 (0.2047123915769837) + | > loss_disc_real_2: 0.23874351382255554 (0.22782889234297204) + | > loss_disc_real_3: 0.22283810377120972 (0.2283351890968554) + | > loss_disc_real_4: 0.2659040093421936 (0.23068757084282962) + | > loss_disc_real_5: 0.19546732306480408 (0.23040708764032883) + | > loss_0: 2.457697629928589 (2.450679970510078) + | > grad_norm_0: tensor(8.6265, device='cuda:0') (tensor(10.3109, device='cuda:0')) + | > loss_gen: 2.2953526973724365 (2.3781004349390673) + | > loss_kl: 2.374992609024048 (2.3290107828198057) + | > loss_feat: 6.659093856811523 (7.139740828311805) + | > loss_mel: 18.02239990234375 (18.327155864599984) + | > loss_duration: 1.6795618534088135 (1.7057193210630706) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.031400680541992 (31.87972724798954) + | > grad_norm_1: tensor(267.2233, device='cuda:0') (tensor(229.4170, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.3799 (0.3517676627997196) + | > loader_time: 0.0067 (0.005427190751740427) + + + --> TIME: 2025-04-18 23:07:11 -- STEP: 91/142 -- GLOBAL_STEP: 380225 + | > loss_disc: 2.427908182144165 (2.4659144852187604) + | > loss_disc_real_0: 0.1294441968202591 (0.12651989568065816) + | > loss_disc_real_1: 0.2268802970647812 (0.2076112252997828) + | > loss_disc_real_2: 0.22245945036411285 (0.22681031089562637) + | > loss_disc_real_3: 0.23867742717266083 (0.22875176075395648) + | > loss_disc_real_4: 0.2282482534646988 (0.22964208335666866) + | > loss_disc_real_5: 0.24055825173854828 (0.23092366018138089) + | > loss_0: 2.427908182144165 (2.4659144852187604) + | > grad_norm_0: tensor(13.9957, device='cuda:0') (tensor(10.2903, device='cuda:0')) + | > loss_gen: 2.4039359092712402 (2.364121489472442) + | > loss_kl: 2.1734726428985596 (2.3282858513213776) + | > loss_feat: 7.160719394683838 (7.098475173279479) + | > loss_mel: 17.42853355407715 (18.325033229785966) + | > loss_duration: 1.6899644136428833 (1.7079213045455597) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.856626510620117 (31.823836965875312) + | > grad_norm_1: tensor(320.6831, device='cuda:0') (tensor(219.3391, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.3983 (0.36298567384153935) + | > loader_time: 0.0062 (0.005694653961684679) + + + --> TIME: 2025-04-18 23:07:22 -- STEP: 116/142 -- GLOBAL_STEP: 380250 + | > loss_disc: 2.5323164463043213 (2.467450766727842) + | > loss_disc_real_0: 0.10916375368833542 (0.1274541146924784) + | > loss_disc_real_1: 0.2728443443775177 (0.20791635174175788) + | > loss_disc_real_2: 0.24389418959617615 (0.2259900787010275) + | > loss_disc_real_3: 0.23493482172489166 (0.22921130562136913) + | > loss_disc_real_4: 0.2317045032978058 (0.22935076006527605) + | > loss_disc_real_5: 0.23865480720996857 (0.23093404929185735) + | > loss_0: 2.5323164463043213 (2.467450766727842) + | > grad_norm_0: tensor(11.6815, device='cuda:0') (tensor(11.1632, device='cuda:0')) + | > loss_gen: 2.521139621734619 (2.3661609908630097) + | > loss_kl: 2.1642262935638428 (2.326694938643226) + | > loss_feat: 7.076613426208496 (7.083527445793152) + | > loss_mel: 18.573474884033203 (18.348984784093403) + | > loss_duration: 1.6709299087524414 (1.7074595484240302) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.006385803222656 (31.83282763382484) + | > grad_norm_1: tensor(460.3886, device='cuda:0') (tensor(244.0458, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.4258 (0.3737727753047285) + | > loader_time: 0.0067 (0.005934098671222556) + + + --> TIME: 2025-04-18 23:07:33 -- STEP: 141/142 -- GLOBAL_STEP: 380275 + | > loss_disc: 2.316760778427124 (2.4711734859655934) + | > loss_disc_real_0: 0.0726163238286972 (0.1282276511720732) + | > loss_disc_real_1: 0.20477506518363953 (0.20774420599142707) + | > loss_disc_real_2: 0.24628014862537384 (0.22695823478783275) + | > loss_disc_real_3: 0.2345011681318283 (0.22935812883343257) + | > loss_disc_real_4: 0.20817965269088745 (0.2304408278659726) + | > loss_disc_real_5: 0.24954329431056976 (0.23138549849919393) + | > loss_0: 2.316760778427124 (2.4711734859655934) + | > grad_norm_0: tensor(24.1018, device='cuda:0') (tensor(12.5348, device='cuda:0')) + | > loss_gen: 2.651538372039795 (2.3684292617419085) + | > loss_kl: 2.0146381855010986 (2.319819140941539) + | > loss_feat: 8.594587326049805 (7.050273841154491) + | > loss_mel: 18.332117080688477 (18.254619463115727) + | > loss_duration: 1.604046106338501 (1.706093969074547) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.19692611694336 (31.699235577955314) + | > grad_norm_1: tensor(750.9153, device='cuda:0') (tensor(257.8336, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.1968 (0.3831442805892187) + | > loader_time: 0.0026 (0.006133438002133202) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.59651780128479 (2.59651780128479) + | > loss_disc_real_0: 0.13307319581508636 (0.13307319581508636) + | > loss_disc_real_1: 0.2568834125995636 (0.2568834125995636) + | > loss_disc_real_2: 0.23290298879146576 (0.23290298879146576) + | > loss_disc_real_3: 0.2755237817764282 (0.2755237817764282) + | > loss_disc_real_4: 0.2639429569244385 (0.2639429569244385) + | > loss_disc_real_5: 0.205542653799057 (0.205542653799057) + | > loss_0: 2.59651780128479 (2.59651780128479) + | > loss_gen: 2.26900577545166 (2.26900577545166) + | > loss_kl: 2.2140402793884277 (2.2140402793884277) + | > loss_feat: 7.353967189788818 (7.353967189788818) + | > loss_mel: 19.41730499267578 (19.41730499267578) + | > loss_duration: 1.6839566230773926 (1.6839566230773926) + | > loss_1: 32.93827438354492 (32.93827438354492) + + --> STEP: 1 + | > loss_disc: 2.581003427505493 (2.581003427505493) + | > loss_disc_real_0: 0.13078922033309937 (0.13078922033309937) + | > loss_disc_real_1: 0.22629988193511963 (0.22629988193511963) + | > loss_disc_real_2: 0.19625483453273773 (0.19625483453273773) + | > loss_disc_real_3: 0.2327718883752823 (0.2327718883752823) + | > loss_disc_real_4: 0.21011067926883698 (0.21011067926883698) + | > loss_disc_real_5: 0.2107856720685959 (0.2107856720685959) + | > loss_0: 2.581003427505493 (2.581003427505493) + | > loss_gen: 2.0592453479766846 (2.0592453479766846) + | > loss_kl: 2.3713204860687256 (2.3713204860687256) + | > loss_feat: 6.798072814941406 (6.798072814941406) + | > loss_mel: 18.7423152923584 (18.7423152923584) + | > loss_duration: 1.7736819982528687 (1.7736819982528687) + | > loss_1: 31.74463653564453 (31.74463653564453) + + --> STEP: 2 + | > loss_disc: 2.585249900817871 (2.583126664161682) + | > loss_disc_real_0: 0.18934202194213867 (0.16006562113761902) + | > loss_disc_real_1: 0.28033196926116943 (0.25331592559814453) + | > loss_disc_real_2: 0.23203302919864655 (0.21414393186569214) + | > loss_disc_real_3: 0.20977355539798737 (0.22127272188663483) + | > loss_disc_real_4: 0.23223067820072174 (0.22117067873477936) + | > loss_disc_real_5: 0.19210565090179443 (0.20144566148519516) + | > loss_0: 2.585249900817871 (2.583126664161682) + | > loss_gen: 2.2014944553375244 (2.1303699016571045) + | > loss_kl: 2.2649760246276855 (2.3181482553482056) + | > loss_feat: 6.462553977966309 (6.630313396453857) + | > loss_mel: 17.886564254760742 (18.31443977355957) + | > loss_duration: 1.6819384098052979 (1.7278102040290833) + | > loss_1: 30.497526168823242 (31.121081352233887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037081241607666016 (-0.00013256072998046875) + | > avg_loss_disc: 2.583126664161682 (+0.0719369649887085) + | > avg_loss_disc_real_0: 0.16006562113761902 (+0.02659285068511963) + | > avg_loss_disc_real_1: 0.25331592559814453 (+0.019455142319202423) + | > avg_loss_disc_real_2: 0.21414393186569214 (-0.021164588630199432) + | > avg_loss_disc_real_3: 0.22127272188663483 (-0.016301803290843964) + | > avg_loss_disc_real_4: 0.22117067873477936 (-0.014413587749004364) + | > avg_loss_disc_real_5: 0.20144566148519516 (-0.044973693788051605) + | > avg_loss_0: 2.583126664161682 (+0.0719369649887085) + | > avg_loss_gen: 2.1303699016571045 (-0.12192261219024658) + | > avg_loss_kl: 2.3181482553482056 (-0.3340892791748047) + | > avg_loss_feat: 6.630313396453857 (+0.18384051322937012) + | > avg_loss_mel: 18.31443977355957 (+0.2513704299926758) + | > avg_loss_duration: 1.7278102040290833 (-0.011924207210540771) + | > avg_loss_1: 31.121081352233887 (-0.03272533416748047) + + + > EPOCH: 7/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:07:35)  + + --> TIME: 2025-04-18 23:07:44 -- STEP: 24/142 -- GLOBAL_STEP: 380300 + | > loss_disc: 2.5005319118499756 (2.43926078081131) + | > loss_disc_real_0: 0.11423599720001221 (0.11524257250130177) + | > loss_disc_real_1: 0.1993543803691864 (0.2087639756500721) + | > loss_disc_real_2: 0.24195590615272522 (0.2271166251351436) + | > loss_disc_real_3: 0.2542368769645691 (0.22752735577523708) + | > loss_disc_real_4: 0.2641579508781433 (0.22869225280980268) + | > loss_disc_real_5: 0.2620484530925751 (0.22956952390571436) + | > loss_0: 2.5005319118499756 (2.43926078081131) + | > grad_norm_0: tensor(13.9171, device='cuda:0') (tensor(14.0092, device='cuda:0')) + | > loss_gen: 2.510309934616089 (2.4053451220194497) + | > loss_kl: 2.361631393432617 (2.273084739844004) + | > loss_feat: 6.986241340637207 (7.339127759138743) + | > loss_mel: 18.745256423950195 (18.448958237965904) + | > loss_duration: 1.6934053897857666 (1.6787741382916768) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.296844482421875 (32.14529005686442) + | > grad_norm_1: tensor(45.3022, device='cuda:0') (tensor(310.8075, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.3417 (0.33183323343594867) + | > loader_time: 0.0053 (0.004870136578877767) + + + --> TIME: 2025-04-18 23:07:53 -- STEP: 49/142 -- GLOBAL_STEP: 380325 + | > loss_disc: 2.52408504486084 (2.4478973077268016) + | > loss_disc_real_0: 0.11081161350011826 (0.11723776374544416) + | > loss_disc_real_1: 0.21604478359222412 (0.2095669799921464) + | > loss_disc_real_2: 0.23397661745548248 (0.2269231956832263) + | > loss_disc_real_3: 0.24815332889556885 (0.22887229372044) + | > loss_disc_real_4: 0.24407772719860077 (0.22989713020470678) + | > loss_disc_real_5: 0.21476134657859802 (0.23074057242091822) + | > loss_0: 2.52408504486084 (2.4478973077268016) + | > grad_norm_0: tensor(7.9788, device='cuda:0') (tensor(13.2520, device='cuda:0')) + | > loss_gen: 2.1377511024475098 (2.387370537738411) + | > loss_kl: 2.2174253463745117 (2.2711356124099433) + | > loss_feat: 6.492288589477539 (7.176150068944814) + | > loss_mel: 17.217615127563477 (18.21030474682244) + | > loss_duration: 1.667236566543579 (1.6786124681939885) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.732316970825195 (31.723573684692383) + | > grad_norm_1: tensor(72.5774, device='cuda:0') (tensor(308.2632, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.3681 (0.3439037897148911) + | > loader_time: 0.0056 (0.005232859630973971) + + + --> TIME: 2025-04-18 23:08:02 -- STEP: 74/142 -- GLOBAL_STEP: 380350 + | > loss_disc: 2.613125801086426 (2.4618943607484978) + | > loss_disc_real_0: 0.19482333958148956 (0.12289297218258316) + | > loss_disc_real_1: 0.21712258458137512 (0.2088243856623366) + | > loss_disc_real_2: 0.23765307664871216 (0.22506865677801338) + | > loss_disc_real_3: 0.23349067568778992 (0.23063333533905647) + | > loss_disc_real_4: 0.24625591933727264 (0.22952644849145734) + | > loss_disc_real_5: 0.3202863931655884 (0.22986105164966067) + | > loss_0: 2.613125801086426 (2.4618943607484978) + | > grad_norm_0: tensor(27.9349, device='cuda:0') (tensor(15.8229, device='cuda:0')) + | > loss_gen: 2.4538323879241943 (2.3798377481666773) + | > loss_kl: 2.158463478088379 (2.267319759807072) + | > loss_feat: 6.740176200866699 (7.137028597496651) + | > loss_mel: 19.02286720275879 (18.198226142574008) + | > loss_duration: 1.6692728996276855 (1.6841489463239103) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.044612884521484 (31.666561307133854) + | > grad_norm_1: tensor(178.2895, device='cuda:0') (tensor(326.6345, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.3853 (0.3555057564297238) + | > loader_time: 0.0063 (0.005539359273137272) + + + --> TIME: 2025-04-18 23:08:13 -- STEP: 99/142 -- GLOBAL_STEP: 380375 + | > loss_disc: 2.57348895072937 (2.4683131858555956) + | > loss_disc_real_0: 0.12944811582565308 (0.12469518203476461) + | > loss_disc_real_1: 0.2340160459280014 (0.20808989246084234) + | > loss_disc_real_2: 0.25736138224601746 (0.2257960842113302) + | > loss_disc_real_3: 0.23334667086601257 (0.23046841961566847) + | > loss_disc_real_4: 0.25110217928886414 (0.23098697641281166) + | > loss_disc_real_5: 0.27777454257011414 (0.23181936370603967) + | > loss_0: 2.57348895072937 (2.4683131858555956) + | > grad_norm_0: tensor(11.8770, device='cuda:0') (tensor(16.9381, device='cuda:0')) + | > loss_gen: 2.3989381790161133 (2.3887123146442453) + | > loss_kl: 2.2169060707092285 (2.2835520084458167) + | > loss_feat: 6.588110446929932 (7.089937426827171) + | > loss_mel: 17.969411849975586 (18.17221798559633) + | > loss_duration: 1.701011300086975 (1.6881870922416147) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.874378204345703 (31.62260699031329) + | > grad_norm_1: tensor(212.0221, device='cuda:0') (tensor(337.5298, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.401 (0.3673878944281376) + | > loader_time: 0.0069 (0.005833110424003216) + + + --> TIME: 2025-04-18 23:08:23 -- STEP: 124/142 -- GLOBAL_STEP: 380400 + | > loss_disc: 2.531679391860962 (2.4704538910619673) + | > loss_disc_real_0: 0.1684676855802536 (0.12655273990164842) + | > loss_disc_real_1: 0.24358198046684265 (0.20765718697540222) + | > loss_disc_real_2: 0.2180856615304947 (0.22583396071868558) + | > loss_disc_real_3: 0.22158055007457733 (0.23102396089703806) + | > loss_disc_real_4: 0.2193751335144043 (0.2305673536273741) + | > loss_disc_real_5: 0.22928257286548615 (0.23158846218739787) + | > loss_0: 2.531679391860962 (2.4704538910619673) + | > grad_norm_0: tensor(12.0186, device='cuda:0') (tensor(16.0746, device='cuda:0')) + | > loss_gen: 2.2292566299438477 (2.371661889937616) + | > loss_kl: 2.2816431522369385 (2.277451240247296) + | > loss_feat: 6.081235408782959 (7.014630386906285) + | > loss_mel: 17.112417221069336 (18.09895466219995) + | > loss_duration: 1.7201693058013916 (1.688783453356835) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.424720764160156 (31.451481757625455) + | > grad_norm_1: tensor(185.3532, device='cuda:0') (tensor(321.9313, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.4333 (0.37842293131736016) + | > loader_time: 0.007 (0.006029457815231815) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3791773319244385 (2.3791773319244385) + | > loss_disc_real_0: 0.15305234491825104 (0.15305234491825104) + | > loss_disc_real_1: 0.20648115873336792 (0.20648115873336792) + | > loss_disc_real_2: 0.247804194688797 (0.247804194688797) + | > loss_disc_real_3: 0.23199549317359924 (0.23199549317359924) + | > loss_disc_real_4: 0.24056608974933624 (0.24056608974933624) + | > loss_disc_real_5: 0.19888746738433838 (0.19888746738433838) + | > loss_0: 2.3791773319244385 (2.3791773319244385) + | > loss_gen: 2.4692368507385254 (2.4692368507385254) + | > loss_kl: 2.1251535415649414 (2.1251535415649414) + | > loss_feat: 7.055164337158203 (7.055164337158203) + | > loss_mel: 19.131603240966797 (19.131603240966797) + | > loss_duration: 1.6892590522766113 (1.6892590522766113) + | > loss_1: 32.47041702270508 (32.47041702270508) + + --> STEP: 1 + | > loss_disc: 2.4882850646972656 (2.4882850646972656) + | > loss_disc_real_0: 0.17732007801532745 (0.17732007801532745) + | > loss_disc_real_1: 0.23461392521858215 (0.23461392521858215) + | > loss_disc_real_2: 0.2476959079504013 (0.2476959079504013) + | > loss_disc_real_3: 0.21969711780548096 (0.21969711780548096) + | > loss_disc_real_4: 0.20602144300937653 (0.20602144300937653) + | > loss_disc_real_5: 0.25455617904663086 (0.25455617904663086) + | > loss_0: 2.4882850646972656 (2.4882850646972656) + | > loss_gen: 2.3942463397979736 (2.3942463397979736) + | > loss_kl: 2.3045127391815186 (2.3045127391815186) + | > loss_feat: 5.944453716278076 (5.944453716278076) + | > loss_mel: 16.975160598754883 (16.975160598754883) + | > loss_duration: 1.7673193216323853 (1.7673193216323853) + | > loss_1: 29.385692596435547 (29.385692596435547) + + --> STEP: 2 + | > loss_disc: 2.4715375900268555 (2.4799113273620605) + | > loss_disc_real_0: 0.20132581889629364 (0.18932294845581055) + | > loss_disc_real_1: 0.2062867134809494 (0.22045031934976578) + | > loss_disc_real_2: 0.2686287760734558 (0.25816234201192856) + | > loss_disc_real_3: 0.2261834740638733 (0.22294029593467712) + | > loss_disc_real_4: 0.22360172867774963 (0.21481158584356308) + | > loss_disc_real_5: 0.24428565800189972 (0.2494209185242653) + | > loss_0: 2.4715375900268555 (2.4799113273620605) + | > loss_gen: 2.3898847103118896 (2.3920655250549316) + | > loss_kl: 2.2792603969573975 (2.291886568069458) + | > loss_feat: 6.552253246307373 (6.248353481292725) + | > loss_mel: 18.548980712890625 (17.762070655822754) + | > loss_duration: 1.672745943069458 (1.7200326323509216) + | > loss_1: 31.443124771118164 (30.414408683776855) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003744840621948242 (+3.6716461181640625e-05) + | > avg_loss_disc: 2.4799113273620605 (-0.10321533679962158) + | > avg_loss_disc_real_0: 0.18932294845581055 (+0.02925732731819153) + | > avg_loss_disc_real_1: 0.22045031934976578 (-0.032865606248378754) + | > avg_loss_disc_real_2: 0.25816234201192856 (+0.04401841014623642) + | > avg_loss_disc_real_3: 0.22294029593467712 (+0.0016675740480422974) + | > avg_loss_disc_real_4: 0.21481158584356308 (-0.006359092891216278) + | > avg_loss_disc_real_5: 0.2494209185242653 (+0.04797525703907013) + | > avg_loss_0: 2.4799113273620605 (-0.10321533679962158) + | > avg_loss_gen: 2.3920655250549316 (+0.26169562339782715) + | > avg_loss_kl: 2.291886568069458 (-0.02626168727874756) + | > avg_loss_feat: 6.248353481292725 (-0.3819599151611328) + | > avg_loss_mel: 17.762070655822754 (-0.5523691177368164) + | > avg_loss_duration: 1.7200326323509216 (-0.007777571678161621) + | > avg_loss_1: 30.414408683776855 (-0.7066726684570312) + + + > EPOCH: 8/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:08:33)  + + --> TIME: 2025-04-18 23:08:36 -- STEP: 7/142 -- GLOBAL_STEP: 380425 + | > loss_disc: 2.620464563369751 (2.4801387786865234) + | > loss_disc_real_0: 0.10118160396814346 (0.12813573011330195) + | > loss_disc_real_1: 0.2110891044139862 (0.20982173936707632) + | > loss_disc_real_2: 0.27377811074256897 (0.222445119704519) + | > loss_disc_real_3: 0.22414134442806244 (0.2342931181192398) + | > loss_disc_real_4: 0.2377537041902542 (0.22432005192552293) + | > loss_disc_real_5: 0.22451381385326385 (0.21578267003808702) + | > loss_0: 2.620464563369751 (2.4801387786865234) + | > grad_norm_0: tensor(14.0180, device='cuda:0') (tensor(8.7565, device='cuda:0')) + | > loss_gen: 2.2571539878845215 (2.340618406023298) + | > loss_kl: 2.0861971378326416 (2.2313924857548306) + | > loss_feat: 7.39729642868042 (7.0295345442635675) + | > loss_mel: 17.588970184326172 (17.72558375767299) + | > loss_duration: 1.6824216842651367 (1.6681763444628035) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.012039184570312 (30.99530601501465) + | > grad_norm_1: tensor(258.7914, device='cuda:0') (tensor(164.6628, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.3515 (0.32667698178972515) + | > loader_time: 0.0042 (0.004518202372959682) + + + --> TIME: 2025-04-18 23:08:44 -- STEP: 32/142 -- GLOBAL_STEP: 380450 + | > loss_disc: 2.28009033203125 (2.484880305826664) + | > loss_disc_real_0: 0.09525542706251144 (0.12697599129751325) + | > loss_disc_real_1: 0.18602335453033447 (0.21532377926632762) + | > loss_disc_real_2: 0.2336633801460266 (0.23026027157902718) + | > loss_disc_real_3: 0.21874578297138214 (0.23083044961094856) + | > loss_disc_real_4: 0.19868139922618866 (0.22922122292220592) + | > loss_disc_real_5: 0.16951945424079895 (0.22377571137622) + | > loss_0: 2.28009033203125 (2.484880305826664) + | > grad_norm_0: tensor(8.1242, device='cuda:0') (tensor(9.7487, device='cuda:0')) + | > loss_gen: 2.5095832347869873 (2.33652750402689) + | > loss_kl: 2.1334617137908936 (2.2355391532182693) + | > loss_feat: 7.688268661499023 (7.004488855600357) + | > loss_mel: 17.72628402709961 (18.049181938171387) + | > loss_duration: 1.6370329856872559 (1.661110993474722) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.694631576538086 (31.286848545074463) + | > grad_norm_1: tensor(264.2940, device='cuda:0') (tensor(246.3102, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.3607 (0.3369507119059563) + | > loader_time: 0.0056 (0.004941105842590332) + + + --> TIME: 2025-04-18 23:08:54 -- STEP: 57/142 -- GLOBAL_STEP: 380475 + | > loss_disc: 2.4833807945251465 (2.4762467794251033) + | > loss_disc_real_0: 0.15586774051189423 (0.1264976947976832) + | > loss_disc_real_1: 0.16090774536132812 (0.21252359789714478) + | > loss_disc_real_2: 0.22247286140918732 (0.2286949881858993) + | > loss_disc_real_3: 0.21178147196769714 (0.23126324357693656) + | > loss_disc_real_4: 0.2146110236644745 (0.23116184483494676) + | > loss_disc_real_5: 0.2731439173221588 (0.22814519415821946) + | > loss_0: 2.4833807945251465 (2.4762467794251033) + | > grad_norm_0: tensor(7.6522, device='cuda:0') (tensor(8.6826, device='cuda:0')) + | > loss_gen: 2.3568854331970215 (2.343886178836487) + | > loss_kl: 2.358264207839966 (2.2500483613265194) + | > loss_feat: 6.804070472717285 (6.983072598775228) + | > loss_mel: 18.104717254638672 (17.963686390926966) + | > loss_duration: 1.7154842615127563 (1.6636260141406143) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.33942222595215 (31.20431963602702) + | > grad_norm_1: tensor(461.7289, device='cuda:0') (tensor(225.5109, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.3736 (0.3479991210134406) + | > loader_time: 0.0059 (0.00534015371088396) + + + --> TIME: 2025-04-18 23:09:04 -- STEP: 82/142 -- GLOBAL_STEP: 380500 + | > loss_disc: 2.4607574939727783 (2.4707696030779585) + | > loss_disc_real_0: 0.08945693075656891 (0.1265529186987295) + | > loss_disc_real_1: 0.1759537309408188 (0.2113732899107584) + | > loss_disc_real_2: 0.20895156264305115 (0.22780439839130495) + | > loss_disc_real_3: 0.22587144374847412 (0.22897359365370215) + | > loss_disc_real_4: 0.20054925978183746 (0.2291460675073833) + | > loss_disc_real_5: 0.20716074109077454 (0.22869887693626126) + | > loss_0: 2.4607574939727783 (2.4707696030779585) + | > grad_norm_0: tensor(10.6729, device='cuda:0') (tensor(9.7474, device='cuda:0')) + | > loss_gen: 2.3850491046905518 (2.3464314181630206) + | > loss_kl: 2.3714797496795654 (2.26285601825249) + | > loss_feat: 7.4785027503967285 (6.965224958047634) + | > loss_mel: 18.922260284423828 (17.99319381248661) + | > loss_duration: 1.7416130304336548 (1.6685740584280433) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.89890670776367 (31.236280278461734) + | > grad_norm_1: tensor(302.5410, device='cuda:0') (tensor(233.7316, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.3925 (0.35991597757106886) + | > loader_time: 0.0062 (0.005630964186133408) + + + --> TIME: 2025-04-18 23:09:14 -- STEP: 107/142 -- GLOBAL_STEP: 380525 + | > loss_disc: 2.526669502258301 (2.475504913062693) + | > loss_disc_real_0: 0.12739841639995575 (0.1276499904065488) + | > loss_disc_real_1: 0.22994600236415863 (0.21167274042267667) + | > loss_disc_real_2: 0.25118014216423035 (0.22908264008637902) + | > loss_disc_real_3: 0.20191970467567444 (0.22897198657009088) + | > loss_disc_real_4: 0.22000336647033691 (0.22981265229042444) + | > loss_disc_real_5: 0.23767079412937164 (0.22914755038011855) + | > loss_0: 2.526669502258301 (2.475504913062693) + | > grad_norm_0: tensor(5.8389, device='cuda:0') (tensor(9.6407, device='cuda:0')) + | > loss_gen: 2.360849618911743 (2.340663263730914) + | > loss_kl: 2.2130932807922363 (2.2665060725167545) + | > loss_feat: 7.002750873565674 (6.921764453994894) + | > loss_mel: 17.023086547851562 (17.95000308919177) + | > loss_duration: 1.65045964717865 (1.6724619932263811) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.2502384185791 (31.151398935050608) + | > grad_norm_1: tensor(130.7116, device='cuda:0') (tensor(225.3519, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.4125 (0.37010723630958636) + | > loader_time: 0.0066 (0.005895480931362259) + + + --> TIME: 2025-04-18 23:09:25 -- STEP: 132/142 -- GLOBAL_STEP: 380550 + | > loss_disc: 2.563467025756836 (2.477893493392251) + | > loss_disc_real_0: 0.18540993332862854 (0.1295030573100754) + | > loss_disc_real_1: 0.21324029564857483 (0.21077703255595584) + | > loss_disc_real_2: 0.22476288676261902 (0.22880848090757022) + | > loss_disc_real_3: 0.23379597067832947 (0.22942609893101634) + | > loss_disc_real_4: 0.21143196523189545 (0.23047568186214476) + | > loss_disc_real_5: 0.21267977356910706 (0.2293513368702296) + | > loss_0: 2.563467025756836 (2.477893493392251) + | > grad_norm_0: tensor(5.2396, device='cuda:0') (tensor(9.3145, device='cuda:0')) + | > loss_gen: 2.133662700653076 (2.3352416789892954) + | > loss_kl: 2.2185842990875244 (2.2633348468578234) + | > loss_feat: 6.319526672363281 (6.882820942185142) + | > loss_mel: 16.63088035583496 (17.923067389112536) + | > loss_duration: 1.660627841949463 (1.6720965007940929) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.963281631469727 (31.076561465407863) + | > grad_norm_1: tensor(273.0078, device='cuda:0') (tensor(211.3822, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.4364 (0.3813599149386089) + | > loader_time: 0.0069 (0.006130816358508486) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6595306396484375 (2.6595306396484375) + | > loss_disc_real_0: 0.21179671585559845 (0.21179671585559845) + | > loss_disc_real_1: 0.2566603124141693 (0.2566603124141693) + | > loss_disc_real_2: 0.20515304803848267 (0.20515304803848267) + | > loss_disc_real_3: 0.3204939067363739 (0.3204939067363739) + | > loss_disc_real_4: 0.23955941200256348 (0.23955941200256348) + | > loss_disc_real_5: 0.26871466636657715 (0.26871466636657715) + | > loss_0: 2.6595306396484375 (2.6595306396484375) + | > loss_gen: 2.4264206886291504 (2.4264206886291504) + | > loss_kl: 2.162407875061035 (2.162407875061035) + | > loss_feat: 5.750835418701172 (5.750835418701172) + | > loss_mel: 17.090044021606445 (17.090044021606445) + | > loss_duration: 1.695668339729309 (1.695668339729309) + | > loss_1: 29.125377655029297 (29.125377655029297) + + --> STEP: 1 + | > loss_disc: 2.4531354904174805 (2.4531354904174805) + | > loss_disc_real_0: 0.16053979098796844 (0.16053979098796844) + | > loss_disc_real_1: 0.2092876434326172 (0.2092876434326172) + | > loss_disc_real_2: 0.1684921681880951 (0.1684921681880951) + | > loss_disc_real_3: 0.27402833104133606 (0.27402833104133606) + | > loss_disc_real_4: 0.2521991729736328 (0.2521991729736328) + | > loss_disc_real_5: 0.24314448237419128 (0.24314448237419128) + | > loss_0: 2.4531354904174805 (2.4531354904174805) + | > loss_gen: 2.472320556640625 (2.472320556640625) + | > loss_kl: 2.4794445037841797 (2.4794445037841797) + | > loss_feat: 6.729292392730713 (6.729292392730713) + | > loss_mel: 20.38669204711914 (20.38669204711914) + | > loss_duration: 1.7598075866699219 (1.7598075866699219) + | > loss_1: 33.82755661010742 (33.82755661010742) + + --> STEP: 2 + | > loss_disc: 2.50020694732666 (2.4766712188720703) + | > loss_disc_real_0: 0.24958693981170654 (0.2050633653998375) + | > loss_disc_real_1: 0.24172443151474 (0.2255060374736786) + | > loss_disc_real_2: 0.22176238894462585 (0.19512727856636047) + | > loss_disc_real_3: 0.2783164083957672 (0.27617236971855164) + | > loss_disc_real_4: 0.2117309421300888 (0.2319650575518608) + | > loss_disc_real_5: 0.27517205476760864 (0.25915826857089996) + | > loss_0: 2.50020694732666 (2.4766712188720703) + | > loss_gen: 2.69020676612854 (2.5812636613845825) + | > loss_kl: 2.3532116413116455 (2.4163280725479126) + | > loss_feat: 8.38529109954834 (7.557291746139526) + | > loss_mel: 19.168367385864258 (19.7775297164917) + | > loss_duration: 1.643718957901001 (1.7017632722854614) + | > loss_1: 34.24079513549805 (34.034175872802734) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037238597869873047 (-2.09808349609375e-05) + | > avg_loss_disc: 2.4766712188720703 (-0.0032401084899902344) + | > avg_loss_disc_real_0: 0.2050633653998375 (+0.015740416944026947) + | > avg_loss_disc_real_1: 0.2255060374736786 (+0.005055718123912811) + | > avg_loss_disc_real_2: 0.19512727856636047 (-0.06303506344556808) + | > avg_loss_disc_real_3: 0.27617236971855164 (+0.05323207378387451) + | > avg_loss_disc_real_4: 0.2319650575518608 (+0.01715347170829773) + | > avg_loss_disc_real_5: 0.25915826857089996 (+0.009737350046634674) + | > avg_loss_0: 2.4766712188720703 (-0.0032401084899902344) + | > avg_loss_gen: 2.5812636613845825 (+0.18919813632965088) + | > avg_loss_kl: 2.4163280725479126 (+0.12444150447845459) + | > avg_loss_feat: 7.557291746139526 (+1.3089382648468018) + | > avg_loss_mel: 19.7775297164917 (+2.0154590606689453) + | > avg_loss_duration: 1.7017632722854614 (-0.018269360065460205) + | > avg_loss_1: 34.034175872802734 (+3.619767189025879) + + + > EPOCH: 9/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:09:31)  + + --> TIME: 2025-04-18 23:09:36 -- STEP: 15/142 -- GLOBAL_STEP: 380575 + | > loss_disc: 2.4687423706054688 (2.46378870010376) + | > loss_disc_real_0: 0.09517128765583038 (0.12597834865252178) + | > loss_disc_real_1: 0.1917402446269989 (0.20008600254853567) + | > loss_disc_real_2: 0.22696150839328766 (0.22333355645338696) + | > loss_disc_real_3: 0.19358326494693756 (0.23058949708938598) + | > loss_disc_real_4: 0.22071948647499084 (0.22650887072086334) + | > loss_disc_real_5: 0.18188022077083588 (0.22763920525709788) + | > loss_0: 2.4687423706054688 (2.46378870010376) + | > grad_norm_0: tensor(8.1398, device='cuda:0') (tensor(7.5375, device='cuda:0')) + | > loss_gen: 2.44350004196167 (2.3169578393300374) + | > loss_kl: 2.1460227966308594 (2.1765574296315515) + | > loss_feat: 8.10033893585205 (6.9824701627095545) + | > loss_mel: 19.685710906982422 (17.760317611694337) + | > loss_duration: 1.6289182901382446 (1.6422766049702961) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.00448989868164 (30.878579584757485) + | > grad_norm_1: tensor(426.9055, device='cuda:0') (tensor(223.5719, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.3346 (0.32560598055521645) + | > loader_time: 0.0055 (0.004495954513549805) + + + --> TIME: 2025-04-18 23:09:45 -- STEP: 40/142 -- GLOBAL_STEP: 380600 + | > loss_disc: 2.421807289123535 (2.4790111660957335) + | > loss_disc_real_0: 0.11410083621740341 (0.1264407662674784) + | > loss_disc_real_1: 0.19794027507305145 (0.20544756166636943) + | > loss_disc_real_2: 0.22840425372123718 (0.2291170470416546) + | > loss_disc_real_3: 0.20426228642463684 (0.2306167632341385) + | > loss_disc_real_4: 0.2015390694141388 (0.23236434534192085) + | > loss_disc_real_5: 0.227864608168602 (0.23204147033393382) + | > loss_0: 2.421807289123535 (2.4790111660957335) + | > grad_norm_0: tensor(5.4292, device='cuda:0') (tensor(9.6422, device='cuda:0')) + | > loss_gen: 2.3401455879211426 (2.32836794257164) + | > loss_kl: 2.1789679527282715 (2.223303294181824) + | > loss_feat: 6.418951511383057 (6.980243551731109) + | > loss_mel: 16.891389846801758 (17.87823343276977) + | > loss_duration: 1.597022533416748 (1.6414718896150589) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.426477432250977 (31.051620054244996) + | > grad_norm_1: tensor(189.9707, device='cuda:0') (tensor(242.6152, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.3584 (0.3394194424152375) + | > loader_time: 0.0056 (0.005075287818908692) + + + --> TIME: 2025-04-18 23:09:55 -- STEP: 65/142 -- GLOBAL_STEP: 380625 + | > loss_disc: 2.4024505615234375 (2.4671374284304113) + | > loss_disc_real_0: 0.08714032918214798 (0.12591547954540974) + | > loss_disc_real_1: 0.16235116124153137 (0.20455434643305265) + | > loss_disc_real_2: 0.22175346314907074 (0.22794773349395164) + | > loss_disc_real_3: 0.22065988183021545 (0.23117385552479672) + | > loss_disc_real_4: 0.19084987044334412 (0.23034688348953541) + | > loss_disc_real_5: 0.23876714706420898 (0.2316098900941702) + | > loss_0: 2.4024505615234375 (2.4671374284304113) + | > grad_norm_0: tensor(8.4447, device='cuda:0') (tensor(9.9574, device='cuda:0')) + | > loss_gen: 2.450371742248535 (2.338458035542415) + | > loss_kl: 2.248049020767212 (2.2360127925872795) + | > loss_feat: 7.38054084777832 (7.02090311050415) + | > loss_mel: 17.8906192779541 (17.914625343909623) + | > loss_duration: 1.6509873867034912 (1.6521380717937764) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.620569229125977 (31.16213745704064) + | > grad_norm_1: tensor(198.4732, device='cuda:0') (tensor(249.1668, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.3881 (0.35138915868905884) + | > loader_time: 0.0064 (0.0054484954247107885) + + + --> TIME: 2025-04-18 23:10:05 -- STEP: 90/142 -- GLOBAL_STEP: 380650 + | > loss_disc: 2.483393907546997 (2.4728448311487843) + | > loss_disc_real_0: 0.13379518687725067 (0.1278201304376124) + | > loss_disc_real_1: 0.2450672835111618 (0.20696458303266102) + | > loss_disc_real_2: 0.2639457583427429 (0.227902386089166) + | > loss_disc_real_3: 0.21468621492385864 (0.22992629094256295) + | > loss_disc_real_4: 0.22862432897090912 (0.2303452303012212) + | > loss_disc_real_5: 0.2076505571603775 (0.23109796593586604) + | > loss_0: 2.483393907546997 (2.4728448311487843) + | > grad_norm_0: tensor(9.8693, device='cuda:0') (tensor(9.6897, device='cuda:0')) + | > loss_gen: 2.3433213233947754 (2.335050826602512) + | > loss_kl: 2.2215893268585205 (2.2506672011481377) + | > loss_feat: 6.485057353973389 (6.918528482649061) + | > loss_mel: 16.538606643676758 (17.88292098575167) + | > loss_duration: 1.672427773475647 (1.6581779532962375) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.261003494262695 (31.045345497131347) + | > grad_norm_1: tensor(57.7868, device='cuda:0') (tensor(235.8600, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.4013 (0.3625320699479845) + | > loader_time: 0.0065 (0.005742438634236656) + + + --> TIME: 2025-04-18 23:10:15 -- STEP: 115/142 -- GLOBAL_STEP: 380675 + | > loss_disc: 2.4422316551208496 (2.4687307378520145) + | > loss_disc_real_0: 0.15494488179683685 (0.127517377682354) + | > loss_disc_real_1: 0.17458882927894592 (0.20630979408388553) + | > loss_disc_real_2: 0.21493524312973022 (0.2278631329536438) + | > loss_disc_real_3: 0.24545511603355408 (0.2295840260775193) + | > loss_disc_real_4: 0.23915228247642517 (0.23040805199871892) + | > loss_disc_real_5: 0.19589896500110626 (0.22960642589175184) + | > loss_0: 2.4422316551208496 (2.4687307378520145) + | > grad_norm_0: tensor(11.7362, device='cuda:0') (tensor(10.8763, device='cuda:0')) + | > loss_gen: 2.4356868267059326 (2.3412449090377145) + | > loss_kl: 2.3451874256134033 (2.253735563029412) + | > loss_feat: 6.8888163566589355 (6.916766506692637) + | > loss_mel: 17.921798706054688 (17.89759776903234) + | > loss_duration: 1.6656765937805176 (1.6604665476342906) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.257165908813477 (31.0698113317075) + | > grad_norm_1: tensor(240.3923, device='cuda:0') (tensor(254.5258, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.4225 (0.37354556373927905) + | > loader_time: 0.0071 (0.0060626755590024225) + + + --> TIME: 2025-04-18 23:10:27 -- STEP: 140/142 -- GLOBAL_STEP: 380700 + | > loss_disc: 2.5806357860565186 (2.4720005682536543) + | > loss_disc_real_0: 0.10534673929214478 (0.1274310592561959) + | > loss_disc_real_1: 0.2042468637228012 (0.20706183974231993) + | > loss_disc_real_2: 0.21408818662166595 (0.22806825776185308) + | > loss_disc_real_3: 0.18911534547805786 (0.23012290565030916) + | > loss_disc_real_4: 0.23572419583797455 (0.2312960789671966) + | > loss_disc_real_5: 0.19678658246994019 (0.23032979081783975) + | > loss_0: 2.5806357860565186 (2.4720005682536543) + | > grad_norm_0: tensor(9.5263, device='cuda:0') (tensor(11.4957, device='cuda:0')) + | > loss_gen: 2.172375202178955 (2.3437825441360474) + | > loss_kl: 2.1183066368103027 (2.2561715415545858) + | > loss_feat: 6.589782238006592 (6.90578885419028) + | > loss_mel: 16.701171875 (17.863916206359857) + | > loss_duration: 1.6806774139404297 (1.660970379625048) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.262311935424805 (31.03062949861799) + | > grad_norm_1: tensor(249.4506, device='cuda:0') (tensor(262.4738, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.4638 (0.3851575681141445) + | > loader_time: 0.0078 (0.006294941902160646) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.372070550918579 (2.372070550918579) + | > loss_disc_real_0: 0.18780933320522308 (0.18780933320522308) + | > loss_disc_real_1: 0.20407450199127197 (0.20407450199127197) + | > loss_disc_real_2: 0.22163823246955872 (0.22163823246955872) + | > loss_disc_real_3: 0.22932207584381104 (0.22932207584381104) + | > loss_disc_real_4: 0.2415103018283844 (0.2415103018283844) + | > loss_disc_real_5: 0.24231015145778656 (0.24231015145778656) + | > loss_0: 2.372070550918579 (2.372070550918579) + | > loss_gen: 2.638213872909546 (2.638213872909546) + | > loss_kl: 2.59592342376709 (2.59592342376709) + | > loss_feat: 7.555443286895752 (7.555443286895752) + | > loss_mel: 19.640872955322266 (19.640872955322266) + | > loss_duration: 1.669318675994873 (1.669318675994873) + | > loss_1: 34.09977340698242 (34.09977340698242) + + --> STEP: 1 + | > loss_disc: 2.5224595069885254 (2.5224595069885254) + | > loss_disc_real_0: 0.18130198121070862 (0.18130198121070862) + | > loss_disc_real_1: 0.2396387755870819 (0.2396387755870819) + | > loss_disc_real_2: 0.2338055968284607 (0.2338055968284607) + | > loss_disc_real_3: 0.23893992602825165 (0.23893992602825165) + | > loss_disc_real_4: 0.23495566844940186 (0.23495566844940186) + | > loss_disc_real_5: 0.2212614268064499 (0.2212614268064499) + | > loss_0: 2.5224595069885254 (2.5224595069885254) + | > loss_gen: 2.4713613986968994 (2.4713613986968994) + | > loss_kl: 2.3565924167633057 (2.3565924167633057) + | > loss_feat: 6.796494483947754 (6.796494483947754) + | > loss_mel: 18.46392822265625 (18.46392822265625) + | > loss_duration: 1.7543187141418457 (1.7543187141418457) + | > loss_1: 31.842695236206055 (31.842695236206055) + + --> STEP: 2 + | > loss_disc: 2.759566307067871 (2.6410129070281982) + | > loss_disc_real_0: 0.24419814348220825 (0.21275006234645844) + | > loss_disc_real_1: 0.3268415629863739 (0.2832401692867279) + | > loss_disc_real_2: 0.2862459123134613 (0.260025754570961) + | > loss_disc_real_3: 0.24288156628608704 (0.24091074615716934) + | > loss_disc_real_4: 0.24057486653327942 (0.23776526749134064) + | > loss_disc_real_5: 0.26302069425582886 (0.24214106053113937) + | > loss_0: 2.759566307067871 (2.6410129070281982) + | > loss_gen: 2.4082281589508057 (2.4397947788238525) + | > loss_kl: 2.3582675457000732 (2.3574299812316895) + | > loss_feat: 5.999439716339111 (6.397967100143433) + | > loss_mel: 15.64355182647705 (17.05374002456665) + | > loss_duration: 1.6444876194000244 (1.699403166770935) + | > loss_1: 28.053974151611328 (29.94833469390869) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036782026290893555 (-4.565715789794922e-05) + | > avg_loss_disc: 2.6410129070281982 (+0.16434168815612793) + | > avg_loss_disc_real_0: 0.21275006234645844 (+0.007686696946620941) + | > avg_loss_disc_real_1: 0.2832401692867279 (+0.057734131813049316) + | > avg_loss_disc_real_2: 0.260025754570961 (+0.06489847600460052) + | > avg_loss_disc_real_3: 0.24091074615716934 (-0.035261623561382294) + | > avg_loss_disc_real_4: 0.23776526749134064 (+0.005800209939479828) + | > avg_loss_disc_real_5: 0.24214106053113937 (-0.01701720803976059) + | > avg_loss_0: 2.6410129070281982 (+0.16434168815612793) + | > avg_loss_gen: 2.4397947788238525 (-0.14146888256072998) + | > avg_loss_kl: 2.3574299812316895 (-0.058898091316223145) + | > avg_loss_feat: 6.397967100143433 (-1.1593246459960938) + | > avg_loss_mel: 17.05374002456665 (-2.723789691925049) + | > avg_loss_duration: 1.699403166770935 (-0.002360105514526367) + | > avg_loss_1: 29.94833469390869 (-4.085841178894043) + + + > EPOCH: 10/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:10:29)  + + --> TIME: 2025-04-18 23:10:37 -- STEP: 23/142 -- GLOBAL_STEP: 380725 + | > loss_disc: 2.4451992511749268 (2.4227730916894) + | > loss_disc_real_0: 0.12579558789730072 (0.12177895171486813) + | > loss_disc_real_1: 0.2160821408033371 (0.20435603222121362) + | > loss_disc_real_2: 0.26024165749549866 (0.22644368915454202) + | > loss_disc_real_3: 0.2342742383480072 (0.23121222529722296) + | > loss_disc_real_4: 0.2467556595802307 (0.231220411217731) + | > loss_disc_real_5: 0.2113286554813385 (0.22707661921563355) + | > loss_0: 2.4451992511749268 (2.4227730916894) + | > grad_norm_0: tensor(7.1539, device='cuda:0') (tensor(14.9861, device='cuda:0')) + | > loss_gen: 2.2826099395751953 (2.3859822853751806) + | > loss_kl: 2.230245590209961 (2.2813284915426504) + | > loss_feat: 5.738759994506836 (7.136945662291153) + | > loss_mel: 16.882278442382812 (18.187564683997113) + | > loss_duration: 1.6541897058486938 (1.627652593280958) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.788084030151367 (31.619474079297937) + | > grad_norm_1: tensor(69.4583, device='cuda:0') (tensor(291.7929, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.3479 (0.33236104509104863) + | > loader_time: 0.0053 (0.004699002141537873) + + + --> TIME: 2025-04-18 23:10:47 -- STEP: 48/142 -- GLOBAL_STEP: 380750 + | > loss_disc: 2.4888391494750977 (2.4488314737876253) + | > loss_disc_real_0: 0.12774457037448883 (0.12152209924533963) + | > loss_disc_real_1: 0.2518959939479828 (0.20701097883284092) + | > loss_disc_real_2: 0.2281031757593155 (0.22658319460848966) + | > loss_disc_real_3: 0.2269483357667923 (0.2311413288116455) + | > loss_disc_real_4: 0.19724003970623016 (0.2301416223247846) + | > loss_disc_real_5: 0.21216408908367157 (0.22863801351437965) + | > loss_0: 2.4888391494750977 (2.4488314737876253) + | > grad_norm_0: tensor(4.5342, device='cuda:0') (tensor(12.5847, device='cuda:0')) + | > loss_gen: 2.379638433456421 (2.362274537483852) + | > loss_kl: 2.044776201248169 (2.2425250013669333) + | > loss_feat: 7.304337978363037 (7.063294728597005) + | > loss_mel: 18.13161277770996 (17.981601754824318) + | > loss_duration: 1.6139190196990967 (1.6317548503478367) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.474285125732422 (31.281450986862183) + | > grad_norm_1: tensor(144.4419, device='cuda:0') (tensor(282.2213, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.369 (0.34927086035410554) + | > loader_time: 0.0056 (0.005178694923718768) + + + --> TIME: 2025-04-18 23:10:57 -- STEP: 73/142 -- GLOBAL_STEP: 380775 + | > loss_disc: 2.4505715370178223 (2.452386349847872) + | > loss_disc_real_0: 0.13905997574329376 (0.12423825355833523) + | > loss_disc_real_1: 0.18520042300224304 (0.20791716791995585) + | > loss_disc_real_2: 0.19506388902664185 (0.22513937603120934) + | > loss_disc_real_3: 0.24051061272621155 (0.22938985163218353) + | > loss_disc_real_4: 0.23031263053417206 (0.22954703887848005) + | > loss_disc_real_5: 0.2393677830696106 (0.23068920982210603) + | > loss_0: 2.4505715370178223 (2.452386349847872) + | > grad_norm_0: tensor(6.7795, device='cuda:0') (tensor(11.2289, device='cuda:0')) + | > loss_gen: 2.2363414764404297 (2.3622559455976093) + | > loss_kl: 2.120384454727173 (2.232372684021519) + | > loss_feat: 6.891925811767578 (7.027557947864271) + | > loss_mel: 17.671510696411133 (17.878163063362848) + | > loss_duration: 1.6900533437728882 (1.6388305164363286) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.61021614074707 (31.13918022260274) + | > grad_norm_1: tensor(231.6870, device='cuda:0') (tensor(264.1966, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.4036 (0.3630506829039691) + | > loader_time: 0.0064 (0.005600609191476481) + + + --> TIME: 2025-04-18 23:11:07 -- STEP: 98/142 -- GLOBAL_STEP: 380800 + | > loss_disc: 2.3851139545440674 (2.464687933727187) + | > loss_disc_real_0: 0.16602125763893127 (0.1275506348786305) + | > loss_disc_real_1: 0.19595666229724884 (0.20855532905885155) + | > loss_disc_real_2: 0.22699137032032013 (0.2259574970420526) + | > loss_disc_real_3: 0.22457917034626007 (0.22996516343282195) + | > loss_disc_real_4: 0.21093447506427765 (0.22917142662466788) + | > loss_disc_real_5: 0.2106684297323227 (0.22978008918616236) + | > loss_0: 2.3851139545440674 (2.464687933727187) + | > grad_norm_0: tensor(24.3447, device='cuda:0') (tensor(12.1132, device='cuda:0')) + | > loss_gen: 2.381582260131836 (2.3452190112094473) + | > loss_kl: 2.18766450881958 (2.232942645647087) + | > loss_feat: 6.938902378082275 (6.936689026501714) + | > loss_mel: 17.326946258544922 (17.84675135904428) + | > loss_duration: 1.5979701280593872 (1.6443215316655684) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.433067321777344 (31.00592366043402) + | > grad_norm_1: tensor(374.4696, device='cuda:0') (tensor(269.4463, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.4157 (0.37666227136339453) + | > loader_time: 0.0072 (0.005917619685737452) + + + --> TIME: 2025-04-18 23:11:18 -- STEP: 123/142 -- GLOBAL_STEP: 380825 + | > loss_disc: 2.470848321914673 (2.4722717060306207) + | > loss_disc_real_0: 0.10205080360174179 (0.13027170457975654) + | > loss_disc_real_1: 0.20304681360721588 (0.20809270029630123) + | > loss_disc_real_2: 0.19118493795394897 (0.2255524044356695) + | > loss_disc_real_3: 0.21019957959651947 (0.23057937064791115) + | > loss_disc_real_4: 0.2117053121328354 (0.2303769896185495) + | > loss_disc_real_5: 0.22309933602809906 (0.23109493471258055) + | > loss_0: 2.470848321914673 (2.4722717060306207) + | > grad_norm_0: tensor(18.9185, device='cuda:0') (tensor(12.0418, device='cuda:0')) + | > loss_gen: 2.4097073078155518 (2.3407577596059648) + | > loss_kl: 2.2221620082855225 (2.228447697996124) + | > loss_feat: 6.802265167236328 (6.899135492681488) + | > loss_mel: 17.10056495666504 (17.78987099872372) + | > loss_duration: 1.6218912601470947 (1.648003271924771) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.156591415405273 (30.906215342079722) + | > grad_norm_1: tensor(346.5238, device='cuda:0') (tensor(256.9171, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.4387 (0.387172580734501) + | > loader_time: 0.0074 (0.006181685905146403) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.609647274017334 (2.609647274017334) + | > loss_disc_real_0: 0.06225297600030899 (0.06225297600030899) + | > loss_disc_real_1: 0.17042334377765656 (0.17042334377765656) + | > loss_disc_real_2: 0.2223329246044159 (0.2223329246044159) + | > loss_disc_real_3: 0.242732971906662 (0.242732971906662) + | > loss_disc_real_4: 0.22071722149848938 (0.22071722149848938) + | > loss_disc_real_5: 0.2036609947681427 (0.2036609947681427) + | > loss_0: 2.609647274017334 (2.609647274017334) + | > loss_gen: 1.8362058401107788 (1.8362058401107788) + | > loss_kl: 2.4449238777160645 (2.4449238777160645) + | > loss_feat: 6.760971546173096 (6.760971546173096) + | > loss_mel: 19.556922912597656 (19.556922912597656) + | > loss_duration: 1.6858961582183838 (1.6858961582183838) + | > loss_1: 32.28491973876953 (32.28491973876953) + + --> STEP: 1 + | > loss_disc: 2.679607629776001 (2.679607629776001) + | > loss_disc_real_0: 0.06776441633701324 (0.06776441633701324) + | > loss_disc_real_1: 0.20239472389221191 (0.20239472389221191) + | > loss_disc_real_2: 0.24918405711650848 (0.24918405711650848) + | > loss_disc_real_3: 0.23722702264785767 (0.23722702264785767) + | > loss_disc_real_4: 0.25713545083999634 (0.25713545083999634) + | > loss_disc_real_5: 0.22485986351966858 (0.22485986351966858) + | > loss_0: 2.679607629776001 (2.679607629776001) + | > loss_gen: 1.8552169799804688 (1.8552169799804688) + | > loss_kl: 2.419215202331543 (2.419215202331543) + | > loss_feat: 5.567510604858398 (5.567510604858398) + | > loss_mel: 18.40614128112793 (18.40614128112793) + | > loss_duration: 1.7347807884216309 (1.7347807884216309) + | > loss_1: 29.982866287231445 (29.982866287231445) + + --> STEP: 2 + | > loss_disc: 2.580686569213867 (2.630147099494934) + | > loss_disc_real_0: 0.06758059561252594 (0.06767250597476959) + | > loss_disc_real_1: 0.1889493465423584 (0.19567203521728516) + | > loss_disc_real_2: 0.26994746923446655 (0.2595657631754875) + | > loss_disc_real_3: 0.2533237040042877 (0.2452753633260727) + | > loss_disc_real_4: 0.22798553109169006 (0.2425604909658432) + | > loss_disc_real_5: 0.21120570600032806 (0.21803278475999832) + | > loss_0: 2.580686569213867 (2.630147099494934) + | > loss_gen: 1.8431484699249268 (1.8491827249526978) + | > loss_kl: 2.252769708633423 (2.335992455482483) + | > loss_feat: 6.215933799743652 (5.891722202301025) + | > loss_mel: 17.1077823638916 (17.756961822509766) + | > loss_duration: 1.6420855522155762 (1.6884331703186035) + | > loss_1: 29.06171989440918 (29.522293090820312) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003965616226196289 (+0.0002874135971069336) + | > avg_loss_disc: 2.630147099494934 (-0.01086580753326416) + | > avg_loss_disc_real_0: 0.06767250597476959 (-0.14507755637168884) + | > avg_loss_disc_real_1: 0.19567203521728516 (-0.08756813406944275) + | > avg_loss_disc_real_2: 0.2595657631754875 (-0.0004599913954734802) + | > avg_loss_disc_real_3: 0.2452753633260727 (+0.004364617168903351) + | > avg_loss_disc_real_4: 0.2425604909658432 (+0.0047952234745025635) + | > avg_loss_disc_real_5: 0.21803278475999832 (-0.024108275771141052) + | > avg_loss_0: 2.630147099494934 (-0.01086580753326416) + | > avg_loss_gen: 1.8491827249526978 (-0.5906120538711548) + | > avg_loss_kl: 2.335992455482483 (-0.021437525749206543) + | > avg_loss_feat: 5.891722202301025 (-0.5062448978424072) + | > avg_loss_mel: 17.756961822509766 (+0.7032217979431152) + | > avg_loss_duration: 1.6884331703186035 (-0.010969996452331543) + | > avg_loss_1: 29.522293090820312 (-0.4260416030883789) + + + > EPOCH: 11/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:11:28)  + + --> TIME: 2025-04-18 23:11:31 -- STEP: 6/142 -- GLOBAL_STEP: 380850 + | > loss_disc: 2.6064095497131348 (2.5151642163594565) + | > loss_disc_real_0: 0.08614011853933334 (0.13748661801218987) + | > loss_disc_real_1: 0.18937626481056213 (0.21313500901063284) + | > loss_disc_real_2: 0.2035812884569168 (0.23604539036750793) + | > loss_disc_real_3: 0.19052650034427643 (0.22428376227617264) + | > loss_disc_real_4: 0.1936570107936859 (0.23773828148841858) + | > loss_disc_real_5: 0.2179044634103775 (0.23052944242954254) + | > loss_0: 2.6064095497131348 (2.5151642163594565) + | > grad_norm_0: tensor(10.5243, device='cuda:0') (tensor(12.4733, device='cuda:0')) + | > loss_gen: 2.0649847984313965 (2.3126277923583984) + | > loss_kl: 2.1635782718658447 (2.1992061932881675) + | > loss_feat: 6.28004264831543 (6.716936190923055) + | > loss_mel: 16.412446975708008 (17.581715901692707) + | > loss_duration: 1.616847038269043 (1.6320934891700745) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.537899017333984 (30.44257990519206) + | > grad_norm_1: tensor(237.3669, device='cuda:0') (tensor(218.9364, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.3275 (0.32312385241190594) + | > loader_time: 0.0043 (0.004312594731648763) + + + --> TIME: 2025-04-18 23:11:39 -- STEP: 31/142 -- GLOBAL_STEP: 380875 + | > loss_disc: 2.6257874965667725 (2.46770429611206) + | > loss_disc_real_0: 0.07239638268947601 (0.12708006414674944) + | > loss_disc_real_1: 0.2308952808380127 (0.20855907182539662) + | > loss_disc_real_2: 0.25370875000953674 (0.22721997095692542) + | > loss_disc_real_3: 0.22525791823863983 (0.22900135334460966) + | > loss_disc_real_4: 0.2585306167602539 (0.23101648832521132) + | > loss_disc_real_5: 0.23843319714069366 (0.23016228647001327) + | > loss_0: 2.6257874965667725 (2.46770429611206) + | > grad_norm_0: tensor(13.3534, device='cuda:0') (tensor(10.1804, device='cuda:0')) + | > loss_gen: 2.168224334716797 (2.346583443303261) + | > loss_kl: 2.1803011894226074 (2.1849865144298923) + | > loss_feat: 6.838766098022461 (6.834944740418465) + | > loss_mel: 17.23979949951172 (17.964047801110056) + | > loss_duration: 1.6162925958633423 (1.6278025411790418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.043384552001953 (30.958365101968088) + | > grad_norm_1: tensor(325.4153, device='cuda:0') (tensor(259.6373, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.3519 (0.3347338015033353) + | > loader_time: 0.0055 (0.004786276048229587) + + + --> TIME: 2025-04-18 23:11:49 -- STEP: 56/142 -- GLOBAL_STEP: 380900 + | > loss_disc: 2.450430393218994 (2.453395507165363) + | > loss_disc_real_0: 0.138361856341362 (0.12639922941369672) + | > loss_disc_real_1: 0.1473742425441742 (0.2052710953035525) + | > loss_disc_real_2: 0.27213090658187866 (0.22640470282307693) + | > loss_disc_real_3: 0.24592150747776031 (0.22835771606436797) + | > loss_disc_real_4: 0.26191356778144836 (0.2303723855210202) + | > loss_disc_real_5: 0.22236397862434387 (0.22979447591517652) + | > loss_0: 2.450430393218994 (2.453395507165363) + | > grad_norm_0: tensor(18.3711, device='cuda:0') (tensor(10.5338, device='cuda:0')) + | > loss_gen: 2.4843227863311768 (2.3652193418570917) + | > loss_kl: 2.118905782699585 (2.1828932336398537) + | > loss_feat: 6.817146301269531 (6.9482507194791525) + | > loss_mel: 17.86767578125 (18.008512207439967) + | > loss_duration: 1.6835532188415527 (1.6305412671395711) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.97160530090332 (31.135416780199325) + | > grad_norm_1: tensor(439.9450, device='cuda:0') (tensor(265.7733, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.3726 (0.34713072010449003) + | > loader_time: 0.0061 (0.005182279007775443) + + + --> TIME: 2025-04-18 23:11:59 -- STEP: 81/142 -- GLOBAL_STEP: 380925 + | > loss_disc: 2.529494524002075 (2.4694512685139975) + | > loss_disc_real_0: 0.118113212287426 (0.12719120434772824) + | > loss_disc_real_1: 0.21002429723739624 (0.20777774280236086) + | > loss_disc_real_2: 0.2192959487438202 (0.22799600348060514) + | > loss_disc_real_3: 0.22156666219234467 (0.22988473044501412) + | > loss_disc_real_4: 0.22600208222866058 (0.23082918646158995) + | > loss_disc_real_5: 0.24549713730812073 (0.23176229460003936) + | > loss_0: 2.529494524002075 (2.4694512685139975) + | > grad_norm_0: tensor(11.5926, device='cuda:0') (tensor(10.9241, device='cuda:0')) + | > loss_gen: 2.3048095703125 (2.3578078158107796) + | > loss_kl: 2.3113749027252197 (2.1970408198274214) + | > loss_feat: 5.638503551483154 (6.946443810875033) + | > loss_mel: 17.252178192138672 (18.021944740672172) + | > loss_duration: 1.6190156936645508 (1.6352551027580544) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.12588119506836 (31.15849228847174) + | > grad_norm_1: tensor(255.3278, device='cuda:0') (tensor(271.2956, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.3968 (0.36063900111634056) + | > loader_time: 0.0065 (0.005559223669546622) + + + --> TIME: 2025-04-18 23:12:09 -- STEP: 106/142 -- GLOBAL_STEP: 380950 + | > loss_disc: 2.4567832946777344 (2.473786016680161) + | > loss_disc_real_0: 0.15132884681224823 (0.12870518821028043) + | > loss_disc_real_1: 0.19471167027950287 (0.20791954139493543) + | > loss_disc_real_2: 0.21715298295021057 (0.2268998513244233) + | > loss_disc_real_3: 0.22465024888515472 (0.2308751752354064) + | > loss_disc_real_4: 0.22332534193992615 (0.22946614251946504) + | > loss_disc_real_5: 0.22885315120220184 (0.23235985960038202) + | > loss_0: 2.4567832946777344 (2.473786016680161) + | > grad_norm_0: tensor(12.4269, device='cuda:0') (tensor(11.2286, device='cuda:0')) + | > loss_gen: 2.331331968307495 (2.3433872146426498) + | > loss_kl: 2.268019676208496 (2.213799501365086) + | > loss_feat: 6.910897731781006 (6.879898611104713) + | > loss_mel: 17.81134033203125 (17.964268531439448) + | > loss_duration: 1.6773688793182373 (1.641405754494217) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.99895668029785 (31.042759571435315) + | > grad_norm_1: tensor(197.7431, device='cuda:0') (tensor(261.5273, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.4158 (0.37233693644685567) + | > loader_time: 0.0064 (0.00581162155799146) + + + --> TIME: 2025-04-18 23:12:20 -- STEP: 131/142 -- GLOBAL_STEP: 380975 + | > loss_disc: 2.4564857482910156 (2.480945265020124) + | > loss_disc_real_0: 0.14068081974983215 (0.12973514915877632) + | > loss_disc_real_1: 0.2287437915802002 (0.20908728537668703) + | > loss_disc_real_2: 0.22726024687290192 (0.22735535734482393) + | > loss_disc_real_3: 0.2321176826953888 (0.23096119948015867) + | > loss_disc_real_4: 0.24061967432498932 (0.23080094124524647) + | > loss_disc_real_5: 0.20676125586032867 (0.2315745764333783) + | > loss_0: 2.4564857482910156 (2.480945265020124) + | > grad_norm_0: tensor(7.6949, device='cuda:0') (tensor(11.2172, device='cuda:0')) + | > loss_gen: 2.2369251251220703 (2.335925955808799) + | > loss_kl: 2.140275239944458 (2.2081703393513923) + | > loss_feat: 6.34499454498291 (6.847022846454882) + | > loss_mel: 16.44826316833496 (17.91238227873359) + | > loss_duration: 1.6163561344146729 (1.6413225426928688) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.786813735961914 (30.94482389843191) + | > grad_norm_1: tensor(146.0838, device='cuda:0') (tensor(258.0094, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.4454 (0.3848715065089801) + | > loader_time: 0.0074 (0.0060786254533374585) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4482297897338867 (2.4482297897338867) + | > loss_disc_real_0: 0.13340790569782257 (0.13340790569782257) + | > loss_disc_real_1: 0.295689195394516 (0.295689195394516) + | > loss_disc_real_2: 0.24470503628253937 (0.24470503628253937) + | > loss_disc_real_3: 0.2366933673620224 (0.2366933673620224) + | > loss_disc_real_4: 0.23149242997169495 (0.23149242997169495) + | > loss_disc_real_5: 0.25439563393592834 (0.25439563393592834) + | > loss_0: 2.4482297897338867 (2.4482297897338867) + | > loss_gen: 2.401370048522949 (2.401370048522949) + | > loss_kl: 2.3659563064575195 (2.3659563064575195) + | > loss_feat: 6.031509876251221 (6.031509876251221) + | > loss_mel: 16.9562931060791 (16.9562931060791) + | > loss_duration: 1.6589080095291138 (1.6589080095291138) + | > loss_1: 29.414039611816406 (29.414039611816406) + + --> STEP: 1 + | > loss_disc: 2.5357115268707275 (2.5357115268707275) + | > loss_disc_real_0: 0.11448869854211807 (0.11448869854211807) + | > loss_disc_real_1: 0.28376448154449463 (0.28376448154449463) + | > loss_disc_real_2: 0.25691697001457214 (0.25691697001457214) + | > loss_disc_real_3: 0.2967616319656372 (0.2967616319656372) + | > loss_disc_real_4: 0.2522147595882416 (0.2522147595882416) + | > loss_disc_real_5: 0.26106083393096924 (0.26106083393096924) + | > loss_0: 2.5357115268707275 (2.5357115268707275) + | > loss_gen: 2.467785120010376 (2.467785120010376) + | > loss_kl: 2.4433417320251465 (2.4433417320251465) + | > loss_feat: 7.40929651260376 (7.40929651260376) + | > loss_mel: 20.058517456054688 (20.058517456054688) + | > loss_duration: 1.7348214387893677 (1.7348214387893677) + | > loss_1: 34.11376190185547 (34.11376190185547) + + --> STEP: 2 + | > loss_disc: 2.42022705078125 (2.4779692888259888) + | > loss_disc_real_0: 0.07845879346132278 (0.09647374600172043) + | > loss_disc_real_1: 0.26948437094688416 (0.2766244262456894) + | > loss_disc_real_2: 0.24756121635437012 (0.25223909318447113) + | > loss_disc_real_3: 0.2678762972354889 (0.28231896460056305) + | > loss_disc_real_4: 0.22179271280765533 (0.23700373619794846) + | > loss_disc_real_5: 0.2420949637889862 (0.2515778988599777) + | > loss_0: 2.42022705078125 (2.4779692888259888) + | > loss_gen: 2.4249069690704346 (2.4463460445404053) + | > loss_kl: 2.2373416423797607 (2.3403416872024536) + | > loss_feat: 8.22403335571289 (7.816664934158325) + | > loss_mel: 18.08180046081543 (19.07015895843506) + | > loss_duration: 1.6123452186584473 (1.6735833287239075) + | > loss_1: 32.58042907714844 (33.34709548950195) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003396153450012207 (-0.000569462776184082) + | > avg_loss_disc: 2.4779692888259888 (-0.1521778106689453) + | > avg_loss_disc_real_0: 0.09647374600172043 (+0.028801240026950836) + | > avg_loss_disc_real_1: 0.2766244262456894 (+0.08095239102840424) + | > avg_loss_disc_real_2: 0.25223909318447113 (-0.007326669991016388) + | > avg_loss_disc_real_3: 0.28231896460056305 (+0.037043601274490356) + | > avg_loss_disc_real_4: 0.23700373619794846 (-0.005556754767894745) + | > avg_loss_disc_real_5: 0.2515778988599777 (+0.0335451140999794) + | > avg_loss_0: 2.4779692888259888 (-0.1521778106689453) + | > avg_loss_gen: 2.4463460445404053 (+0.5971633195877075) + | > avg_loss_kl: 2.3403416872024536 (+0.004349231719970703) + | > avg_loss_feat: 7.816664934158325 (+1.9249427318572998) + | > avg_loss_mel: 19.07015895843506 (+1.313197135925293) + | > avg_loss_duration: 1.6735833287239075 (-0.014849841594696045) + | > avg_loss_1: 33.34709548950195 (+3.8248023986816406) + + + > EPOCH: 12/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:12:27)  + + --> TIME: 2025-04-18 23:12:32 -- STEP: 14/142 -- GLOBAL_STEP: 381000 + | > loss_disc: 2.4883570671081543 (2.4606597934450423) + | > loss_disc_real_0: 0.12440039217472076 (0.12119516038468905) + | > loss_disc_real_1: 0.18174421787261963 (0.2098608762025833) + | > loss_disc_real_2: 0.22225289046764374 (0.22647661396435328) + | > loss_disc_real_3: 0.2343144416809082 (0.23320053198507854) + | > loss_disc_real_4: 0.2535881996154785 (0.22947123646736145) + | > loss_disc_real_5: 0.23095622658729553 (0.24210217701537268) + | > loss_0: 2.4883570671081543 (2.4606597934450423) + | > grad_norm_0: tensor(8.2739, device='cuda:0') (tensor(7.7950, device='cuda:0')) + | > loss_gen: 2.2670745849609375 (2.3442153930664062) + | > loss_kl: 2.0778472423553467 (2.198984163148063) + | > loss_feat: 7.183913230895996 (6.930000066757202) + | > loss_mel: 18.111675262451172 (17.792728969029017) + | > loss_duration: 1.6228680610656738 (1.6090105516569955) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.263378143310547 (30.874939373561315) + | > grad_norm_1: tensor(240.6027, device='cuda:0') (tensor(193.2000, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.3281 (0.3271718706403459) + | > loader_time: 0.0047 (0.004505310739789691) + + + --> TIME: 2025-04-18 23:12:41 -- STEP: 39/142 -- GLOBAL_STEP: 381025 + | > loss_disc: 2.480924129486084 (2.467659534552159) + | > loss_disc_real_0: 0.17980992794036865 (0.12166375800585136) + | > loss_disc_real_1: 0.17670270800590515 (0.20974068076182634) + | > loss_disc_real_2: 0.24399396777153015 (0.23033181788065496) + | > loss_disc_real_3: 0.2125767469406128 (0.22751951523316213) + | > loss_disc_real_4: 0.21350426971912384 (0.23031304738460442) + | > loss_disc_real_5: 0.2565062940120697 (0.2362081928130908) + | > loss_0: 2.480924129486084 (2.467659534552159) + | > grad_norm_0: tensor(8.1817, device='cuda:0') (tensor(7.3392, device='cuda:0')) + | > loss_gen: 2.156062602996826 (2.343739931400006) + | > loss_kl: 2.118086338043213 (2.1881656921826877) + | > loss_feat: 6.0114827156066895 (7.0379258669339695) + | > loss_mel: 15.844154357910156 (18.046811519524983) + | > loss_duration: 1.6321637630462646 (1.6136598862134492) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.761951446533203 (31.23030320191995) + | > grad_norm_1: tensor(68.0704, device='cuda:0') (tensor(156.9057, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.36 (0.34027638802161586) + | > loader_time: 0.0062 (0.00502621821868114) + + + --> TIME: 2025-04-18 23:12:51 -- STEP: 64/142 -- GLOBAL_STEP: 381050 + | > loss_disc: 2.4130396842956543 (2.465618990361691) + | > loss_disc_real_0: 0.07111173123121262 (0.12459426501300186) + | > loss_disc_real_1: 0.25028878450393677 (0.20910104969516397) + | > loss_disc_real_2: 0.2314474731683731 (0.22839603596366942) + | > loss_disc_real_3: 0.22827528417110443 (0.22788045089691877) + | > loss_disc_real_4: 0.22862057387828827 (0.23067851550877094) + | > loss_disc_real_5: 0.21949784457683563 (0.2352477228268981) + | > loss_0: 2.4130396842956543 (2.465618990361691) + | > grad_norm_0: tensor(6.5223, device='cuda:0') (tensor(7.0192, device='cuda:0')) + | > loss_gen: 2.3686630725860596 (2.351245399564505) + | > loss_kl: 2.2663984298706055 (2.186900107190013) + | > loss_feat: 7.548359394073486 (7.015818752348423) + | > loss_mel: 19.627004623413086 (18.023003995418545) + | > loss_duration: 1.6575154066085815 (1.6205268874764442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.46794128417969 (31.197495251893997) + | > grad_norm_1: tensor(158.6596, device='cuda:0') (tensor(147.6491, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.3803 (0.352555014193058) + | > loader_time: 0.0061 (0.005392227321863174) + + + --> TIME: 2025-04-18 23:13:01 -- STEP: 89/142 -- GLOBAL_STEP: 381075 + | > loss_disc: 2.3771533966064453 (2.4711383460612786) + | > loss_disc_real_0: 0.1087585911154747 (0.12599615602011094) + | > loss_disc_real_1: 0.19118937849998474 (0.20974819211477644) + | > loss_disc_real_2: 0.2002055048942566 (0.2270530624670929) + | > loss_disc_real_3: 0.22514201700687408 (0.22826636457041408) + | > loss_disc_real_4: 0.23799586296081543 (0.23091211630387254) + | > loss_disc_real_5: 0.22766776382923126 (0.23373971712053493) + | > loss_0: 2.3771533966064453 (2.4711383460612786) + | > grad_norm_0: tensor(16.1007, device='cuda:0') (tensor(8.1729, device='cuda:0')) + | > loss_gen: 2.4710192680358887 (2.352722178684192) + | > loss_kl: 2.2068214416503906 (2.195894398046343) + | > loss_feat: 7.543357849121094 (7.000847714670589) + | > loss_mel: 18.770265579223633 (18.082944869995117) + | > loss_duration: 1.6035540103912354 (1.6265955212410919) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.59502029418945 (31.259004892927877) + | > grad_norm_1: tensor(548.1625, device='cuda:0') (tensor(183.6602, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.4074 (0.3651623350850652) + | > loader_time: 0.0064 (0.005669773294684592) + + + --> TIME: 2025-04-18 23:13:11 -- STEP: 114/142 -- GLOBAL_STEP: 381100 + | > loss_disc: 2.732745885848999 (2.4736908381445364) + | > loss_disc_real_0: 0.2413802444934845 (0.12763566451898797) + | > loss_disc_real_1: 0.2240753322839737 (0.20924721122310871) + | > loss_disc_real_2: 0.23970778286457062 (0.2280756376291576) + | > loss_disc_real_3: 0.252167671918869 (0.22889425056545357) + | > loss_disc_real_4: 0.21172595024108887 (0.2307477495387981) + | > loss_disc_real_5: 0.2328028529882431 (0.23351636437470452) + | > loss_0: 2.732745885848999 (2.4736908381445364) + | > grad_norm_0: tensor(31.9334, device='cuda:0') (tensor(9.6694, device='cuda:0')) + | > loss_gen: 2.1196444034576416 (2.3545350338283346) + | > loss_kl: 2.1526126861572266 (2.1890023622596457) + | > loss_feat: 6.461606025695801 (6.967842390662746) + | > loss_mel: 17.289321899414062 (18.0577550687288) + | > loss_duration: 1.6414110660552979 (1.628972135092083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.664596557617188 (31.198107150563022) + | > grad_norm_1: tensor(421.6412, device='cuda:0') (tensor(211.5227, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.4274 (0.37715832810652883) + | > loader_time: 0.0068 (0.005916407233790348) + + + --> TIME: 2025-04-18 23:13:23 -- STEP: 139/142 -- GLOBAL_STEP: 381125 + | > loss_disc: 2.4986934661865234 (2.4812959355416058) + | > loss_disc_real_0: 0.07363609224557877 (0.1300612762838388) + | > loss_disc_real_1: 0.1944137066602707 (0.21084269859807955) + | > loss_disc_real_2: 0.20711803436279297 (0.22814254527040523) + | > loss_disc_real_3: 0.22278264164924622 (0.228735709040285) + | > loss_disc_real_4: 0.2302512675523758 (0.23103039738514441) + | > loss_disc_real_5: 0.19150911271572113 (0.23282296419572487) + | > loss_0: 2.4986934661865234 (2.4812959355416058) + | > grad_norm_0: tensor(16.4082, device='cuda:0') (tensor(11.5266, device='cuda:0')) + | > loss_gen: 2.2872776985168457 (2.3553723362709986) + | > loss_kl: 2.2418861389160156 (2.1871538222264886) + | > loss_feat: 6.409308910369873 (6.909890754617376) + | > loss_mel: 15.79871654510498 (17.972351527042527) + | > loss_duration: 1.638349175453186 (1.6306755645669626) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.375539779663086 (31.055444141086056) + | > grad_norm_1: tensor(469.0993, device='cuda:0') (tensor(233.1580, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.4529 (0.3888822442336048) + | > loader_time: 0.0071 (0.006161020814086036) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.585599184036255 (2.585599184036255) + | > loss_disc_real_0: 0.24154844880104065 (0.24154844880104065) + | > loss_disc_real_1: 0.24128258228302002 (0.24128258228302002) + | > loss_disc_real_2: 0.2822970747947693 (0.2822970747947693) + | > loss_disc_real_3: 0.2739998996257782 (0.2739998996257782) + | > loss_disc_real_4: 0.24244579672813416 (0.24244579672813416) + | > loss_disc_real_5: 0.2264128029346466 (0.2264128029346466) + | > loss_0: 2.585599184036255 (2.585599184036255) + | > loss_gen: 2.63200044631958 (2.63200044631958) + | > loss_kl: 2.136664628982544 (2.136664628982544) + | > loss_feat: 6.661512851715088 (6.661512851715088) + | > loss_mel: 18.11277961730957 (18.11277961730957) + | > loss_duration: 1.6768946647644043 (1.6768946647644043) + | > loss_1: 31.219852447509766 (31.219852447509766) + + --> STEP: 1 + | > loss_disc: 2.5315799713134766 (2.5315799713134766) + | > loss_disc_real_0: 0.2537163197994232 (0.2537163197994232) + | > loss_disc_real_1: 0.20588229596614838 (0.20588229596614838) + | > loss_disc_real_2: 0.2525756061077118 (0.2525756061077118) + | > loss_disc_real_3: 0.23770911991596222 (0.23770911991596222) + | > loss_disc_real_4: 0.19884084165096283 (0.19884084165096283) + | > loss_disc_real_5: 0.2165815681219101 (0.2165815681219101) + | > loss_0: 2.5315799713134766 (2.5315799713134766) + | > loss_gen: 2.427049160003662 (2.427049160003662) + | > loss_kl: 2.4228901863098145 (2.4228901863098145) + | > loss_feat: 6.966449737548828 (6.966449737548828) + | > loss_mel: 19.60963249206543 (19.60963249206543) + | > loss_duration: 1.7188982963562012 (1.7188982963562012) + | > loss_1: 33.144920349121094 (33.144920349121094) + + --> STEP: 2 + | > loss_disc: 2.4952638149261475 (2.513421893119812) + | > loss_disc_real_0: 0.25896376371383667 (0.25634004175662994) + | > loss_disc_real_1: 0.24620912969112396 (0.22604571282863617) + | > loss_disc_real_2: 0.23907798528671265 (0.24582679569721222) + | > loss_disc_real_3: 0.22665497660636902 (0.23218204826116562) + | > loss_disc_real_4: 0.20237261056900024 (0.20060672610998154) + | > loss_disc_real_5: 0.2103261649608612 (0.21345386654138565) + | > loss_0: 2.4952638149261475 (2.513421893119812) + | > loss_gen: 2.4507086277008057 (2.438878893852234) + | > loss_kl: 2.113840103149414 (2.2683651447296143) + | > loss_feat: 6.1688551902771 (6.567652463912964) + | > loss_mel: 17.311582565307617 (18.460607528686523) + | > loss_duration: 1.6214494705200195 (1.6701738834381104) + | > loss_1: 29.66643524169922 (31.405677795410156) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036104917526245117 (+0.0002143383026123047) + | > avg_loss_disc: 2.513421893119812 (+0.03545260429382324) + | > avg_loss_disc_real_0: 0.25634004175662994 (+0.15986629575490952) + | > avg_loss_disc_real_1: 0.22604571282863617 (-0.05057871341705322) + | > avg_loss_disc_real_2: 0.24582679569721222 (-0.006412297487258911) + | > avg_loss_disc_real_3: 0.23218204826116562 (-0.05013691633939743) + | > avg_loss_disc_real_4: 0.20060672610998154 (-0.03639701008796692) + | > avg_loss_disc_real_5: 0.21345386654138565 (-0.03812403231859207) + | > avg_loss_0: 2.513421893119812 (+0.03545260429382324) + | > avg_loss_gen: 2.438878893852234 (-0.007467150688171387) + | > avg_loss_kl: 2.2683651447296143 (-0.07197654247283936) + | > avg_loss_feat: 6.567652463912964 (-1.2490124702453613) + | > avg_loss_mel: 18.460607528686523 (-0.6095514297485352) + | > avg_loss_duration: 1.6701738834381104 (-0.003409445285797119) + | > avg_loss_1: 31.405677795410156 (-1.9414176940917969) + + + > EPOCH: 13/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:13:25)  + + --> TIME: 2025-04-18 23:13:33 -- STEP: 22/142 -- GLOBAL_STEP: 381150 + | > loss_disc: 2.4194626808166504 (2.432363759387623) + | > loss_disc_real_0: 0.11611366271972656 (0.1213810376145623) + | > loss_disc_real_1: 0.2355889230966568 (0.20637855949726971) + | > loss_disc_real_2: 0.22144447267055511 (0.22163786061785437) + | > loss_disc_real_3: 0.24247273802757263 (0.2253283140334216) + | > loss_disc_real_4: 0.2514578104019165 (0.23516898060386832) + | > loss_disc_real_5: 0.23203608393669128 (0.22303167866034943) + | > loss_0: 2.4194626808166504 (2.432363759387623) + | > grad_norm_0: tensor(12.0538, device='cuda:0') (tensor(16.9968, device='cuda:0')) + | > loss_gen: 2.5332858562469482 (2.3961625424298374) + | > loss_kl: 2.121326208114624 (2.092692944136533) + | > loss_feat: 7.123753547668457 (7.335006952285767) + | > loss_mel: 18.150590896606445 (18.42084789276123) + | > loss_duration: 1.55050528049469 (1.6009001677686518) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.479461669921875 (31.845610705288973) + | > grad_norm_1: tensor(286.3130, device='cuda:0') (tensor(329.0980, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.342 (0.3309761502526023) + | > loader_time: 0.0051 (0.004650885408574882) + + + --> TIME: 2025-04-18 23:13:43 -- STEP: 47/142 -- GLOBAL_STEP: 381175 + | > loss_disc: 2.295689821243286 (2.4469205369340616) + | > loss_disc_real_0: 0.1304665058851242 (0.12300844934392482) + | > loss_disc_real_1: 0.22620399296283722 (0.20433172266534033) + | > loss_disc_real_2: 0.20415794849395752 (0.22272443993294494) + | > loss_disc_real_3: 0.193955197930336 (0.22770457223374793) + | > loss_disc_real_4: 0.1879606395959854 (0.23471298845524483) + | > loss_disc_real_5: 0.19376035034656525 (0.23159809188639863) + | > loss_0: 2.295689821243286 (2.4469205369340616) + | > grad_norm_0: tensor(6.4732, device='cuda:0') (tensor(14.3531, device='cuda:0')) + | > loss_gen: 2.5021629333496094 (2.3735362823973305) + | > loss_kl: 2.201162576675415 (2.11368474808145) + | > loss_feat: 7.691657066345215 (7.170306002840083) + | > loss_mel: 17.741031646728516 (18.143185940194638) + | > loss_duration: 1.5885367393493652 (1.60726638550454) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.724550247192383 (31.407979437645444) + | > grad_norm_1: tensor(187.6236, device='cuda:0') (tensor(294.2034, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.3701 (0.34341636109859386) + | > loader_time: 0.0057 (0.005165120388599149) + + + --> TIME: 2025-04-18 23:13:52 -- STEP: 72/142 -- GLOBAL_STEP: 381200 + | > loss_disc: 2.4622035026550293 (2.455531828933292) + | > loss_disc_real_0: 0.11942937225103378 (0.12369351616750161) + | > loss_disc_real_1: 0.19531068205833435 (0.20513125446935496) + | > loss_disc_real_2: 0.22696565091609955 (0.2264986712899473) + | > loss_disc_real_3: 0.19964012503623962 (0.22731833884285557) + | > loss_disc_real_4: 0.24405784904956818 (0.23250555619597435) + | > loss_disc_real_5: 0.20402103662490845 (0.2308106134749121) + | > loss_0: 2.4622035026550293 (2.455531828933292) + | > grad_norm_0: tensor(6.4484, device='cuda:0') (tensor(11.7762, device='cuda:0')) + | > loss_gen: 2.278531789779663 (2.3618462880452475) + | > loss_kl: 2.0519204139709473 (2.136754751205444) + | > loss_feat: 7.408456325531006 (7.124013430542416) + | > loss_mel: 18.765729904174805 (18.18145766523149) + | > loss_duration: 1.5971629619598389 (1.614248823788431) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.101802825927734 (31.418321000205147) + | > grad_norm_1: tensor(136.4220, device='cuda:0') (tensor(244.5302, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.3947 (0.356213026576572) + | > loader_time: 0.0063 (0.005515969461864893) + + + --> TIME: 2025-04-18 23:14:03 -- STEP: 97/142 -- GLOBAL_STEP: 381225 + | > loss_disc: 2.545787811279297 (2.473462353047637) + | > loss_disc_real_0: 0.12399561703205109 (0.12567861554856163) + | > loss_disc_real_1: 0.23863251507282257 (0.2086689143451219) + | > loss_disc_real_2: 0.229932963848114 (0.22695652986924672) + | > loss_disc_real_3: 0.22911851108074188 (0.22982961768956528) + | > loss_disc_real_4: 0.20587986707687378 (0.23330155076439849) + | > loss_disc_real_5: 0.24215514957904816 (0.23366173256918327) + | > loss_0: 2.545787811279297 (2.473462353047637) + | > grad_norm_0: tensor(7.9864, device='cuda:0') (tensor(10.6880, device='cuda:0')) + | > loss_gen: 2.2587790489196777 (2.3436356991836713) + | > loss_kl: 2.149240255355835 (2.151131782335105) + | > loss_feat: 6.981389999389648 (7.038725587510571) + | > loss_mel: 18.000755310058594 (18.150049219426418) + | > loss_duration: 1.6253397464752197 (1.6199405672623939) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.015504837036133 (31.30348292085313) + | > grad_norm_1: tensor(166.3208, device='cuda:0') (tensor(213.7386, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.4175 (0.3684474020889124) + | > loader_time: 0.0068 (0.0058479063289681625) + + + --> TIME: 2025-04-18 23:14:13 -- STEP: 122/142 -- GLOBAL_STEP: 381250 + | > loss_disc: 2.549896001815796 (2.476081254052334) + | > loss_disc_real_0: 0.16965900361537933 (0.12824014061298528) + | > loss_disc_real_1: 0.2254146933555603 (0.20899198387489945) + | > loss_disc_real_2: 0.230948805809021 (0.226164632889091) + | > loss_disc_real_3: 0.24754440784454346 (0.22956613170318915) + | > loss_disc_real_4: 0.24144084751605988 (0.23264499594930743) + | > loss_disc_real_5: 0.2700217366218567 (0.23309896312287595) + | > loss_0: 2.549896001815796 (2.476081254052334) + | > grad_norm_0: tensor(4.6840, device='cuda:0') (tensor(9.7287, device='cuda:0')) + | > loss_gen: 2.2785139083862305 (2.336673077989798) + | > loss_kl: 1.9984970092773438 (2.149113979495941) + | > loss_feat: 6.267755508422852 (6.981352442600688) + | > loss_mel: 16.932334899902344 (18.099968542818157) + | > loss_duration: 1.6099927425384521 (1.622744044319528) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.087093353271484 (31.18985210481237) + | > grad_norm_1: tensor(41.2260, device='cuda:0') (tensor(189.2951, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.45 (0.3806980828769871) + | > loader_time: 0.0066 (0.006132075043975327) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.352630615234375 (2.352630615234375) + | > loss_disc_real_0: 0.11885368824005127 (0.11885368824005127) + | > loss_disc_real_1: 0.21822085976600647 (0.21822085976600647) + | > loss_disc_real_2: 0.20245139300823212 (0.20245139300823212) + | > loss_disc_real_3: 0.2548540234565735 (0.2548540234565735) + | > loss_disc_real_4: 0.18895699083805084 (0.18895699083805084) + | > loss_disc_real_5: 0.20285369455814362 (0.20285369455814362) + | > loss_0: 2.352630615234375 (2.352630615234375) + | > loss_gen: 2.431523323059082 (2.431523323059082) + | > loss_kl: 2.1776022911071777 (2.1776022911071777) + | > loss_feat: 7.1585187911987305 (7.1585187911987305) + | > loss_mel: 19.53498077392578 (19.53498077392578) + | > loss_duration: 1.6458035707473755 (1.6458035707473755) + | > loss_1: 32.948429107666016 (32.948429107666016) + + --> STEP: 1 + | > loss_disc: 2.357654571533203 (2.357654571533203) + | > loss_disc_real_0: 0.12736456096172333 (0.12736456096172333) + | > loss_disc_real_1: 0.22847773134708405 (0.22847773134708405) + | > loss_disc_real_2: 0.22678908705711365 (0.22678908705711365) + | > loss_disc_real_3: 0.24717116355895996 (0.24717116355895996) + | > loss_disc_real_4: 0.19115352630615234 (0.19115352630615234) + | > loss_disc_real_5: 0.2166408896446228 (0.2166408896446228) + | > loss_0: 2.357654571533203 (2.357654571533203) + | > loss_gen: 2.3970067501068115 (2.3970067501068115) + | > loss_kl: 2.4101979732513428 (2.4101979732513428) + | > loss_feat: 6.666438102722168 (6.666438102722168) + | > loss_mel: 20.36817169189453 (20.36817169189453) + | > loss_duration: 1.7205302715301514 (1.7205302715301514) + | > loss_1: 33.562347412109375 (33.562347412109375) + + --> STEP: 2 + | > loss_disc: 2.3924384117126465 (2.375046491622925) + | > loss_disc_real_0: 0.19750294089317322 (0.16243375092744827) + | > loss_disc_real_1: 0.22680509090423584 (0.22764141112565994) + | > loss_disc_real_2: 0.2034512460231781 (0.21512016654014587) + | > loss_disc_real_3: 0.20597080886363983 (0.2265709862112999) + | > loss_disc_real_4: 0.18574495613574982 (0.18844924122095108) + | > loss_disc_real_5: 0.18780416250228882 (0.2022225260734558) + | > loss_0: 2.3924384117126465 (2.375046491622925) + | > loss_gen: 2.37734317779541 (2.387174963951111) + | > loss_kl: 2.2308380603790283 (2.3205180168151855) + | > loss_feat: 7.037114143371582 (6.851776123046875) + | > loss_mel: 16.77205467224121 (18.57011318206787) + | > loss_duration: 1.638098955154419 (1.6793146133422852) + | > loss_1: 30.055450439453125 (31.80889892578125) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003924727439880371 (+0.0003142356872558594) + | > avg_loss_disc: 2.375046491622925 (-0.1383754014968872) + | > avg_loss_disc_real_0: 0.16243375092744827 (-0.09390629082918167) + | > avg_loss_disc_real_1: 0.22764141112565994 (+0.0015956982970237732) + | > avg_loss_disc_real_2: 0.21512016654014587 (-0.030706629157066345) + | > avg_loss_disc_real_3: 0.2265709862112999 (-0.005611062049865723) + | > avg_loss_disc_real_4: 0.18844924122095108 (-0.012157484889030457) + | > avg_loss_disc_real_5: 0.2022225260734558 (-0.01123134046792984) + | > avg_loss_0: 2.375046491622925 (-0.1383754014968872) + | > avg_loss_gen: 2.387174963951111 (-0.05170392990112305) + | > avg_loss_kl: 2.3205180168151855 (+0.05215287208557129) + | > avg_loss_feat: 6.851776123046875 (+0.28412365913391113) + | > avg_loss_mel: 18.57011318206787 (+0.10950565338134766) + | > avg_loss_duration: 1.6793146133422852 (+0.009140729904174805) + | > avg_loss_1: 31.80889892578125 (+0.40322113037109375) + + + > EPOCH: 14/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:14:24)  + + --> TIME: 2025-04-18 23:14:26 -- STEP: 5/142 -- GLOBAL_STEP: 381275 + | > loss_disc: 2.3325793743133545 (2.366227912902832) + | > loss_disc_real_0: 0.15111671388149261 (0.12608536928892136) + | > loss_disc_real_1: 0.22134597599506378 (0.2099319130182266) + | > loss_disc_real_2: 0.20993369817733765 (0.2292884409427643) + | > loss_disc_real_3: 0.24476125836372375 (0.22627781927585602) + | > loss_disc_real_4: 0.20598815381526947 (0.21575047075748444) + | > loss_disc_real_5: 0.2506990134716034 (0.22299950420856476) + | > loss_0: 2.3325793743133545 (2.366227912902832) + | > grad_norm_0: tensor(12.2248, device='cuda:0') (tensor(16.3691, device='cuda:0')) + | > loss_gen: 2.602529525756836 (2.460096025466919) + | > loss_kl: 2.1475555896759033 (2.106451225280762) + | > loss_feat: 7.840147018432617 (7.489865016937256) + | > loss_mel: 18.207914352416992 (18.232378005981445) + | > loss_duration: 1.5798838138580322 (1.6087207555770875) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.37803268432617 (31.897510528564453) + | > grad_norm_1: tensor(402.3383, device='cuda:0') (tensor(340.5916, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.3277 (0.32338433265686034) + | > loader_time: 0.0042 (0.00428476333618164) + + + --> TIME: 2025-04-18 23:14:35 -- STEP: 30/142 -- GLOBAL_STEP: 381300 + | > loss_disc: 2.4216818809509277 (2.4311492760976154) + | > loss_disc_real_0: 0.07831303030252457 (0.12192802305022875) + | > loss_disc_real_1: 0.1866264045238495 (0.20511541565259298) + | > loss_disc_real_2: 0.22837653756141663 (0.22912098467350006) + | > loss_disc_real_3: 0.23776546120643616 (0.22601508498191833) + | > loss_disc_real_4: 0.23466628789901733 (0.2291736548145612) + | > loss_disc_real_5: 0.20927096903324127 (0.2259951079885165) + | > loss_0: 2.4216818809509277 (2.4311492760976154) + | > grad_norm_0: tensor(13.1753, device='cuda:0') (tensor(13.1159, device='cuda:0')) + | > loss_gen: 2.345918655395508 (2.384050464630127) + | > loss_kl: 2.051830291748047 (2.137211040655771) + | > loss_feat: 7.606610298156738 (7.205535729726155) + | > loss_mel: 18.29924964904785 (18.13784844080607) + | > loss_duration: 1.5924079418182373 (1.5943341970443725) + | > amp_scaler: 512.0 (358.4) + | > loss_1: 31.89601707458496 (31.45897954305013) + | > grad_norm_1: tensor(341.7877, device='cuda:0') (tensor(292.3168, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.349 (0.33531163533528635) + | > loader_time: 0.0055 (0.004958128929138185) + + + --> TIME: 2025-04-18 23:14:44 -- STEP: 55/142 -- GLOBAL_STEP: 381325 + | > loss_disc: 2.446309804916382 (2.438611177964644) + | > loss_disc_real_0: 0.1252169907093048 (0.1240857801654122) + | > loss_disc_real_1: 0.20666959881782532 (0.2072680327025327) + | > loss_disc_real_2: 0.22689160704612732 (0.22608236101540652) + | > loss_disc_real_3: 0.2468850463628769 (0.22612703415480526) + | > loss_disc_real_4: 0.2532764673233032 (0.22962080348621716) + | > loss_disc_real_5: 0.26774367690086365 (0.22968523664907975) + | > loss_0: 2.446309804916382 (2.438611177964644) + | > grad_norm_0: tensor(10.7283, device='cuda:0') (tensor(12.1811, device='cuda:0')) + | > loss_gen: 2.4211320877075195 (2.375304798646406) + | > loss_kl: 2.1369516849517822 (2.1258258017626677) + | > loss_feat: 7.385334491729736 (7.069237795743075) + | > loss_mel: 19.019691467285156 (18.054095545682042) + | > loss_duration: 1.6479403972625732 (1.5967040061950684) + | > amp_scaler: 512.0 (428.2181818181818) + | > loss_1: 32.61104965209961 (31.22116778980602) + | > grad_norm_1: tensor(249.1909, device='cuda:0') (tensor(272.1937, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.3724 (0.34716589667580344) + | > loader_time: 0.006 (0.005267620086669923) + + + --> TIME: 2025-04-18 23:14:54 -- STEP: 80/142 -- GLOBAL_STEP: 381350 + | > loss_disc: 2.480229377746582 (2.454410356283188) + | > loss_disc_real_0: 0.11303066462278366 (0.12512797433882947) + | > loss_disc_real_1: 0.17691105604171753 (0.2069094879552722) + | > loss_disc_real_2: 0.22777196764945984 (0.22659372631460428) + | > loss_disc_real_3: 0.2086690217256546 (0.22763568870723247) + | > loss_disc_real_4: 0.22482721507549286 (0.23049030676484109) + | > loss_disc_real_5: 0.19607198238372803 (0.22974828239530326) + | > loss_0: 2.480229377746582 (2.454410356283188) + | > grad_norm_0: tensor(12.7741, device='cuda:0') (tensor(11.7741, device='cuda:0')) + | > loss_gen: 2.21665358543396 (2.359362971782684) + | > loss_kl: 2.1343002319335938 (2.13373337239027) + | > loss_feat: 7.077871322631836 (7.003877401351929) + | > loss_mel: 17.824926376342773 (17.969403302669527) + | > loss_duration: 1.6304562091827393 (1.6029957234859467) + | > amp_scaler: 256.0 (409.59999999999997) + | > loss_1: 30.88420867919922 (31.069372749328615) + | > grad_norm_1: tensor(250.4046, device='cuda:0') (tensor(263.8070, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.4149 (0.36148526370525363) + | > loader_time: 0.0065 (0.0056199520826339736) + + + --> TIME: 2025-04-18 23:15:05 -- STEP: 105/142 -- GLOBAL_STEP: 381375 + | > loss_disc: 2.573122501373291 (2.4689041160401843) + | > loss_disc_real_0: 0.212171271443367 (0.12754522441398525) + | > loss_disc_real_1: 0.23924945294857025 (0.20810243175143286) + | > loss_disc_real_2: 0.24111853539943695 (0.22728030383586884) + | > loss_disc_real_3: 0.20410078763961792 (0.22908902423722402) + | > loss_disc_real_4: 0.22312648594379425 (0.23247426336719876) + | > loss_disc_real_5: 0.23717942833900452 (0.23064030891373044) + | > loss_0: 2.573122501373291 (2.4689041160401843) + | > grad_norm_0: tensor(16.1971, device='cuda:0') (tensor(12.2482, device='cuda:0')) + | > loss_gen: 2.134831190109253 (2.3523045403616774) + | > loss_kl: 2.2191920280456543 (2.150868278458004) + | > loss_feat: 6.004179000854492 (6.965862841833205) + | > loss_mel: 16.309654235839844 (17.966538656325564) + | > loss_duration: 1.635080337524414 (1.6104916845049178) + | > amp_scaler: 256.0 (373.0285714285714) + | > loss_1: 28.302936553955078 (31.04606612069266) + | > grad_norm_1: tensor(252.0052, device='cuda:0') (tensor(269.7915, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.4303 (0.37483111336117697) + | > loader_time: 0.007 (0.005887111028035484) + + + --> TIME: 2025-04-18 23:15:16 -- STEP: 130/142 -- GLOBAL_STEP: 381400 + | > loss_disc: 2.3949553966522217 (2.469494738945594) + | > loss_disc_real_0: 0.13600292801856995 (0.12772155020099416) + | > loss_disc_real_1: 0.20829474925994873 (0.20818168922112537) + | > loss_disc_real_2: 0.23688654601573944 (0.22657413219030087) + | > loss_disc_real_3: 0.22382812201976776 (0.22900453404738352) + | > loss_disc_real_4: 0.22290939092636108 (0.23224808126688004) + | > loss_disc_real_5: 0.24302296340465546 (0.2305619614628645) + | > loss_0: 2.3949553966522217 (2.469494738945594) + | > grad_norm_0: tensor(18.4315, device='cuda:0') (tensor(12.8593, device='cuda:0')) + | > loss_gen: 2.455162763595581 (2.3488714713316705) + | > loss_kl: 2.122096300125122 (2.151783089454357) + | > loss_feat: 7.576201915740967 (6.929697392537044) + | > loss_mel: 19.650291442871094 (17.932835798997143) + | > loss_duration: 1.6014879941940308 (1.613294072334583) + | > amp_scaler: 256.0 (350.5230769230769) + | > loss_1: 33.40523910522461 (30.97648193652813) + | > grad_norm_1: tensor(515.9943, device='cuda:0') (tensor(287.3843, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.4618 (0.3884600364244901) + | > loader_time: 0.0078 (0.006139469146728517) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5048441886901855 (2.5048441886901855) + | > loss_disc_real_0: 0.13177092373371124 (0.13177092373371124) + | > loss_disc_real_1: 0.20517295598983765 (0.20517295598983765) + | > loss_disc_real_2: 0.23694007098674774 (0.23694007098674774) + | > loss_disc_real_3: 0.2695218026638031 (0.2695218026638031) + | > loss_disc_real_4: 0.2082996964454651 (0.2082996964454651) + | > loss_disc_real_5: 0.2726004719734192 (0.2726004719734192) + | > loss_0: 2.5048441886901855 (2.5048441886901855) + | > loss_gen: 2.296522617340088 (2.296522617340088) + | > loss_kl: 2.44805908203125 (2.44805908203125) + | > loss_feat: 6.816963195800781 (6.816963195800781) + | > loss_mel: 19.205041885375977 (19.205041885375977) + | > loss_duration: 1.6209834814071655 (1.6209834814071655) + | > loss_1: 32.387569427490234 (32.387569427490234) + + --> STEP: 1 + | > loss_disc: 2.4539334774017334 (2.4539334774017334) + | > loss_disc_real_0: 0.15626566112041473 (0.15626566112041473) + | > loss_disc_real_1: 0.16285592317581177 (0.16285592317581177) + | > loss_disc_real_2: 0.2269824743270874 (0.2269824743270874) + | > loss_disc_real_3: 0.20863458514213562 (0.20863458514213562) + | > loss_disc_real_4: 0.20824123919010162 (0.20824123919010162) + | > loss_disc_real_5: 0.22599302232265472 (0.22599302232265472) + | > loss_0: 2.4539334774017334 (2.4539334774017334) + | > loss_gen: 2.14898943901062 (2.14898943901062) + | > loss_kl: 2.3156955242156982 (2.3156955242156982) + | > loss_feat: 5.849241256713867 (5.849241256713867) + | > loss_mel: 16.497499465942383 (16.497499465942383) + | > loss_duration: 1.7003417015075684 (1.7003417015075684) + | > loss_1: 28.511768341064453 (28.511768341064453) + + --> STEP: 2 + | > loss_disc: 2.5574777126312256 (2.5057055950164795) + | > loss_disc_real_0: 0.16315427422523499 (0.15970996767282486) + | > loss_disc_real_1: 0.24612495303153992 (0.20449043810367584) + | > loss_disc_real_2: 0.2326565831899643 (0.22981952875852585) + | > loss_disc_real_3: 0.23089300096035004 (0.21976379305124283) + | > loss_disc_real_4: 0.22458688914775848 (0.21641406416893005) + | > loss_disc_real_5: 0.23079486191272736 (0.22839394211769104) + | > loss_0: 2.5574777126312256 (2.5057055950164795) + | > loss_gen: 2.3167734146118164 (2.2328814268112183) + | > loss_kl: 2.1086699962615967 (2.2121827602386475) + | > loss_feat: 6.352975845336914 (6.101108551025391) + | > loss_mel: 17.997365951538086 (17.247432708740234) + | > loss_duration: 1.5889933109283447 (1.6446675062179565) + | > loss_1: 30.364776611328125 (29.43827247619629) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0030938386917114258 (-0.0008308887481689453) + | > avg_loss_disc: 2.5057055950164795 (+0.1306591033935547) + | > avg_loss_disc_real_0: 0.15970996767282486 (-0.002723783254623413) + | > avg_loss_disc_real_1: 0.20449043810367584 (-0.0231509730219841) + | > avg_loss_disc_real_2: 0.22981952875852585 (+0.014699362218379974) + | > avg_loss_disc_real_3: 0.21976379305124283 (-0.006807193160057068) + | > avg_loss_disc_real_4: 0.21641406416893005 (+0.027964822947978973) + | > avg_loss_disc_real_5: 0.22839394211769104 (+0.02617141604423523) + | > avg_loss_0: 2.5057055950164795 (+0.1306591033935547) + | > avg_loss_gen: 2.2328814268112183 (-0.15429353713989258) + | > avg_loss_kl: 2.2121827602386475 (-0.10833525657653809) + | > avg_loss_feat: 6.101108551025391 (-0.7506675720214844) + | > avg_loss_mel: 17.247432708740234 (-1.3226804733276367) + | > avg_loss_duration: 1.6446675062179565 (-0.03464710712432861) + | > avg_loss_1: 29.43827247619629 (-2.370626449584961) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_381412.pth + + > EPOCH: 15/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:15:24)  + + --> TIME: 2025-04-18 23:15:29 -- STEP: 13/142 -- GLOBAL_STEP: 381425 + | > loss_disc: 2.416860580444336 (2.412879650409405) + | > loss_disc_real_0: 0.09276717901229858 (0.1165535942866252) + | > loss_disc_real_1: 0.19400979578495026 (0.20107165093605334) + | > loss_disc_real_2: 0.2229911834001541 (0.22704265438593352) + | > loss_disc_real_3: 0.24788108468055725 (0.22636264218733862) + | > loss_disc_real_4: 0.2373269498348236 (0.22318818935981163) + | > loss_disc_real_5: 0.24531593918800354 (0.22442983778623435) + | > loss_0: 2.416860580444336 (2.412879650409405) + | > grad_norm_0: tensor(5.3661, device='cuda:0') (tensor(9.8194, device='cuda:0')) + | > loss_gen: 2.614318609237671 (2.407375665811392) + | > loss_kl: 2.112860918045044 (2.153327776835515) + | > loss_feat: 7.351232051849365 (7.333786817697378) + | > loss_mel: 17.290782928466797 (17.97143980172964) + | > loss_duration: 1.608925700187683 (1.5804181282336895) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.978118896484375 (31.446348043588493) + | > grad_norm_1: tensor(232.5416, device='cuda:0') (tensor(212.7884, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.3296 (0.3250566996060885) + | > loader_time: 0.0052 (0.004376117999737079) + + + --> TIME: 2025-04-18 23:15:38 -- STEP: 38/142 -- GLOBAL_STEP: 381450 + | > loss_disc: 2.428213119506836 (2.444464928225467) + | > loss_disc_real_0: 0.13404172658920288 (0.12046817964629124) + | > loss_disc_real_1: 0.22360727190971375 (0.20671343685765015) + | > loss_disc_real_2: 0.24398452043533325 (0.2272728791362361) + | > loss_disc_real_3: 0.25736477971076965 (0.22707587362904297) + | > loss_disc_real_4: 0.23217929899692535 (0.22685421375851883) + | > loss_disc_real_5: 0.21200332045555115 (0.23154660275107936) + | > loss_0: 2.428213119506836 (2.444464928225467) + | > grad_norm_0: tensor(10.2607, device='cuda:0') (tensor(9.5754, device='cuda:0')) + | > loss_gen: 2.3834099769592285 (2.368911542390522) + | > loss_kl: 2.045253038406372 (2.12993765191028) + | > loss_feat: 7.172939777374268 (7.085056217093217) + | > loss_mel: 18.2657527923584 (17.91951380277935) + | > loss_duration: 1.5880627632141113 (1.5847255650319552) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.45541763305664 (31.088144904688786) + | > grad_norm_1: tensor(95.4735, device='cuda:0') (tensor(200.2440, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.3578 (0.34231455075113393) + | > loader_time: 0.0057 (0.004976856081109298) + + + --> TIME: 2025-04-18 23:15:48 -- STEP: 63/142 -- GLOBAL_STEP: 381475 + | > loss_disc: 2.51399302482605 (2.480832058285909) + | > loss_disc_real_0: 0.15157511830329895 (0.12775706275114943) + | > loss_disc_real_1: 0.1954452246427536 (0.20814102036612375) + | > loss_disc_real_2: 0.24121685326099396 (0.2294599593631805) + | > loss_disc_real_3: 0.23843802511692047 (0.22974459614072526) + | > loss_disc_real_4: 0.24315166473388672 (0.23067550384809102) + | > loss_disc_real_5: 0.2108592391014099 (0.23200228029773348) + | > loss_0: 2.51399302482605 (2.480832058285909) + | > grad_norm_0: tensor(3.8229, device='cuda:0') (tensor(8.5521, device='cuda:0')) + | > loss_gen: 2.313845634460449 (2.336903189855908) + | > loss_kl: 2.3527090549468994 (2.131042558049398) + | > loss_feat: 6.450446605682373 (6.925285460456969) + | > loss_mel: 18.071247100830078 (17.890416024223207) + | > loss_duration: 1.65120267868042 (1.5929243943047902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.83945083618164 (30.876571776374938) + | > grad_norm_1: tensor(55.5916, device='cuda:0') (tensor(162.0647, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.3969 (0.3563210888514443) + | > loader_time: 0.0059 (0.005320344652448382) + + + --> TIME: 2025-04-18 23:15:58 -- STEP: 88/142 -- GLOBAL_STEP: 381500 + | > loss_disc: 2.5893266201019287 (2.4767997996373596) + | > loss_disc_real_0: 0.1282954066991806 (0.13169791016050364) + | > loss_disc_real_1: 0.2033437043428421 (0.20659602670506996) + | > loss_disc_real_2: 0.23920708894729614 (0.22721545516767286) + | > loss_disc_real_3: 0.2130521535873413 (0.22830564410171725) + | > loss_disc_real_4: 0.2729036509990692 (0.23037934184751727) + | > loss_disc_real_5: 0.23705622553825378 (0.23212784867395053) + | > loss_0: 2.5893266201019287 (2.4767997996373596) + | > grad_norm_0: tensor(4.0483, device='cuda:0') (tensor(8.1003, device='cuda:0')) + | > loss_gen: 2.2224302291870117 (2.326366803862832) + | > loss_kl: 2.3813440799713135 (2.143287375569343) + | > loss_feat: 6.481783390045166 (6.8384312770583415) + | > loss_mel: 17.446964263916016 (17.906335819851265) + | > loss_duration: 1.6937917470932007 (1.6009810458530083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.226316452026367 (30.8154023994099) + | > grad_norm_1: tensor(57.4686, device='cuda:0') (tensor(141.8028, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.4046 (0.3706676391038027) + | > loader_time: 0.0067 (0.0056081089106473055) + + + --> TIME: 2025-04-18 23:16:09 -- STEP: 113/142 -- GLOBAL_STEP: 381525 + | > loss_disc: 2.5211024284362793 (2.484652458039004) + | > loss_disc_real_0: 0.12960878014564514 (0.13309039313972532) + | > loss_disc_real_1: 0.20259249210357666 (0.20761094829150006) + | > loss_disc_real_2: 0.24850928783416748 (0.22752365865538607) + | > loss_disc_real_3: 0.23255608975887299 (0.23086613152934388) + | > loss_disc_real_4: 0.23353227972984314 (0.23145574937879512) + | > loss_disc_real_5: 0.2123095840215683 (0.23238233325228227) + | > loss_0: 2.5211024284362793 (2.484652458039004) + | > grad_norm_0: tensor(11.4098, device='cuda:0') (tensor(8.1817, device='cuda:0')) + | > loss_gen: 2.1934103965759277 (2.3171446787572547) + | > loss_kl: 2.1341123580932617 (2.1425966826160394) + | > loss_feat: 6.268827438354492 (6.828763759241695) + | > loss_mel: 16.544919967651367 (17.91082958626536) + | > loss_duration: 1.6253529787063599 (1.6040193791938042) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.766624450683594 (30.8033541957889) + | > grad_norm_1: tensor(143.0110, device='cuda:0') (tensor(142.4665, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.4375 (0.38267674488303927) + | > loader_time: 0.007 (0.005913637380684371) + + + --> TIME: 2025-04-18 23:16:21 -- STEP: 138/142 -- GLOBAL_STEP: 381550 + | > loss_disc: 2.5307400226593018 (2.4876134896623907) + | > loss_disc_real_0: 0.1727602183818817 (0.13489031354370332) + | > loss_disc_real_1: 0.22272123396396637 (0.2085489941679913) + | > loss_disc_real_2: 0.2621287703514099 (0.22785414917313535) + | > loss_disc_real_3: 0.250914990901947 (0.23012246137511902) + | > loss_disc_real_4: 0.26412323117256165 (0.23093170510685962) + | > loss_disc_real_5: 0.2287272810935974 (0.23225027247183566) + | > loss_0: 2.5307400226593018 (2.4876134896623907) + | > grad_norm_0: tensor(20.0338, device='cuda:0') (tensor(8.6856, device='cuda:0')) + | > loss_gen: 2.27457332611084 (2.3108497253362685) + | > loss_kl: 2.1587867736816406 (2.1371950939081708) + | > loss_feat: 5.652552127838135 (6.773052664770597) + | > loss_mel: 16.184194564819336 (17.81833651445914) + | > loss_duration: 1.6391034126281738 (1.6066819023394934) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.909210205078125 (30.646116007929262) + | > grad_norm_1: tensor(147.5546, device='cuda:0') (tensor(155.8955, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.4618 (0.3954755326975946) + | > loader_time: 0.0071 (0.006112563437309819) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.387092113494873 (2.387092113494873) + | > loss_disc_real_0: 0.07273832708597183 (0.07273832708597183) + | > loss_disc_real_1: 0.19472748041152954 (0.19472748041152954) + | > loss_disc_real_2: 0.22007840871810913 (0.22007840871810913) + | > loss_disc_real_3: 0.21944573521614075 (0.21944573521614075) + | > loss_disc_real_4: 0.23950859904289246 (0.23950859904289246) + | > loss_disc_real_5: 0.2380724549293518 (0.2380724549293518) + | > loss_0: 2.387092113494873 (2.387092113494873) + | > loss_gen: 2.187831401824951 (2.187831401824951) + | > loss_kl: 2.1574130058288574 (2.1574130058288574) + | > loss_feat: 7.923992156982422 (7.923992156982422) + | > loss_mel: 19.38640022277832 (19.38640022277832) + | > loss_duration: 1.667787790298462 (1.667787790298462) + | > loss_1: 33.32342529296875 (33.32342529296875) + + --> STEP: 1 + | > loss_disc: 2.5087103843688965 (2.5087103843688965) + | > loss_disc_real_0: 0.0926048681139946 (0.0926048681139946) + | > loss_disc_real_1: 0.24907955527305603 (0.24907955527305603) + | > loss_disc_real_2: 0.23614156246185303 (0.23614156246185303) + | > loss_disc_real_3: 0.25286737084388733 (0.25286737084388733) + | > loss_disc_real_4: 0.2416493147611618 (0.2416493147611618) + | > loss_disc_real_5: 0.2569243311882019 (0.2569243311882019) + | > loss_0: 2.5087103843688965 (2.5087103843688965) + | > loss_gen: 2.162179470062256 (2.162179470062256) + | > loss_kl: 2.4804675579071045 (2.4804675579071045) + | > loss_feat: 6.426706314086914 (6.426706314086914) + | > loss_mel: 18.212039947509766 (18.212039947509766) + | > loss_duration: 1.6909908056259155 (1.6909908056259155) + | > loss_1: 30.972383499145508 (30.972383499145508) + + --> STEP: 2 + | > loss_disc: 2.584017276763916 (2.5463638305664062) + | > loss_disc_real_0: 0.11950919032096863 (0.10605702921748161) + | > loss_disc_real_1: 0.2342856079339981 (0.24168258160352707) + | > loss_disc_real_2: 0.2207765281200409 (0.22845904529094696) + | > loss_disc_real_3: 0.22195123136043549 (0.2374093011021614) + | > loss_disc_real_4: 0.21261966228485107 (0.22713448852300644) + | > loss_disc_real_5: 0.2127113938331604 (0.23481786251068115) + | > loss_0: 2.584017276763916 (2.5463638305664062) + | > loss_gen: 1.9494960308074951 (2.0558377504348755) + | > loss_kl: 2.078925609588623 (2.2796965837478638) + | > loss_feat: 4.833077907562256 (5.629892110824585) + | > loss_mel: 15.920903205871582 (17.066471576690674) + | > loss_duration: 1.5985383987426758 (1.6447646021842957) + | > loss_1: 26.380939483642578 (28.676661491394043) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003106236457824707 (+1.239776611328125e-05) + | > avg_loss_disc: 2.5463638305664062 (+0.04065823554992676) + | > avg_loss_disc_real_0: 0.10605702921748161 (-0.053652938455343246) + | > avg_loss_disc_real_1: 0.24168258160352707 (+0.03719214349985123) + | > avg_loss_disc_real_2: 0.22845904529094696 (-0.001360483467578888) + | > avg_loss_disc_real_3: 0.2374093011021614 (+0.01764550805091858) + | > avg_loss_disc_real_4: 0.22713448852300644 (+0.010720424354076385) + | > avg_loss_disc_real_5: 0.23481786251068115 (+0.006423920392990112) + | > avg_loss_0: 2.5463638305664062 (+0.04065823554992676) + | > avg_loss_gen: 2.0558377504348755 (-0.17704367637634277) + | > avg_loss_kl: 2.2796965837478638 (+0.06751382350921631) + | > avg_loss_feat: 5.629892110824585 (-0.47121644020080566) + | > avg_loss_mel: 17.066471576690674 (-0.18096113204956055) + | > avg_loss_duration: 1.6447646021842957 (+9.709596633911133e-05) + | > avg_loss_1: 28.676661491394043 (-0.7616109848022461) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_381554.pth + + > EPOCH: 16/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:16:25)  + + --> TIME: 2025-04-18 23:16:33 -- STEP: 21/142 -- GLOBAL_STEP: 381575 + | > loss_disc: 2.4050967693328857 (2.4317336763654436) + | > loss_disc_real_0: 0.14610973000526428 (0.1205413369905381) + | > loss_disc_real_1: 0.21184422075748444 (0.20045993441627138) + | > loss_disc_real_2: 0.21693673729896545 (0.22677252548081533) + | > loss_disc_real_3: 0.24442540109157562 (0.23021758738018216) + | > loss_disc_real_4: 0.23833134770393372 (0.22981189971878416) + | > loss_disc_real_5: 0.22472891211509705 (0.2256044518379938) + | > loss_0: 2.4050967693328857 (2.4317336763654436) + | > grad_norm_0: tensor(19.3666, device='cuda:0') (tensor(15.6369, device='cuda:0')) + | > loss_gen: 2.3411457538604736 (2.3977463585989818) + | > loss_kl: 2.0106186866760254 (2.090348692167373) + | > loss_feat: 7.548243999481201 (7.191373211996896) + | > loss_mel: 18.81413459777832 (18.314411072503955) + | > loss_duration: 1.5685100555419922 (1.5698932976949782) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.28265380859375 (31.563772655668714) + | > grad_norm_1: tensor(335.6598, device='cuda:0') (tensor(293.2428, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.3388 (0.328509217216855) + | > loader_time: 0.0052 (0.004682416007632301) + + + --> TIME: 2025-04-18 23:16:42 -- STEP: 46/142 -- GLOBAL_STEP: 381600 + | > loss_disc: 2.434760093688965 (2.442356923352117) + | > loss_disc_real_0: 0.13518556952476501 (0.12333588941913584) + | > loss_disc_real_1: 0.2031884342432022 (0.2040882042568663) + | > loss_disc_real_2: 0.24868889153003693 (0.22689172690329346) + | > loss_disc_real_3: 0.20931637287139893 (0.22906434050072794) + | > loss_disc_real_4: 0.23399704694747925 (0.22981906941403513) + | > loss_disc_real_5: 0.22309748828411102 (0.22871640648530878) + | > loss_0: 2.434760093688965 (2.442356923352117) + | > grad_norm_0: tensor(8.2926, device='cuda:0') (tensor(17.6608, device='cuda:0')) + | > loss_gen: 2.4964053630828857 (2.4042295113853784) + | > loss_kl: 2.055250644683838 (2.107965741468512) + | > loss_feat: 6.7133636474609375 (7.057905031287151) + | > loss_mel: 17.50480079650879 (18.04260450860728) + | > loss_duration: 1.5739895105361938 (1.576063713301783) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.343809127807617 (31.18876851123312) + | > grad_norm_1: tensor(434.1380, device='cuda:0') (tensor(314.6833, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.3669 (0.34143090248107905) + | > loader_time: 0.0056 (0.005076973334602689) + + + --> TIME: 2025-04-18 23:16:51 -- STEP: 71/142 -- GLOBAL_STEP: 381625 + | > loss_disc: 2.4344735145568848 (2.4444058915259133) + | > loss_disc_real_0: 0.14334024488925934 (0.12463319180926806) + | > loss_disc_real_1: 0.20828290283679962 (0.20435054243450435) + | > loss_disc_real_2: 0.21220703423023224 (0.2249437068969431) + | > loss_disc_real_3: 0.23269261419773102 (0.22921801767718625) + | > loss_disc_real_4: 0.22740894556045532 (0.2307387672259774) + | > loss_disc_real_5: 0.22806048393249512 (0.22978178401228408) + | > loss_0: 2.4344735145568848 (2.4444058915259133) + | > grad_norm_0: tensor(21.1026, device='cuda:0') (tensor(17.4918, device='cuda:0')) + | > loss_gen: 2.3140969276428223 (2.403094792030227) + | > loss_kl: 2.10852313041687 (2.1091413397184553) + | > loss_feat: 7.01778507232666 (7.071787928191709) + | > loss_mel: 19.125686645507812 (17.968465657301344) + | > loss_duration: 1.5947740077972412 (1.5845114563552427) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.160865783691406 (31.137000983869527) + | > grad_norm_1: tensor(536.7271, device='cuda:0') (tensor(320.4447, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.3919 (0.35385148290177465) + | > loader_time: 0.0064 (0.005434053044923595) + + + --> TIME: 2025-04-18 23:17:01 -- STEP: 96/142 -- GLOBAL_STEP: 381650 + | > loss_disc: 2.4765238761901855 (2.4482956106464067) + | > loss_disc_real_0: 0.1069490909576416 (0.12416883655047666) + | > loss_disc_real_1: 0.1763353943824768 (0.2050698489571611) + | > loss_disc_real_2: 0.20589271187782288 (0.22543845310186347) + | > loss_disc_real_3: 0.22734159231185913 (0.2299672633719941) + | > loss_disc_real_4: 0.18554292619228363 (0.22881029018511376) + | > loss_disc_real_5: 0.22059501707553864 (0.22968733931581178) + | > loss_0: 2.4765238761901855 (2.4482956106464067) + | > grad_norm_0: tensor(11.4485, device='cuda:0') (tensor(15.7991, device='cuda:0')) + | > loss_gen: 2.204200267791748 (2.3831213042140016) + | > loss_kl: 2.117140769958496 (2.1170703868071237) + | > loss_feat: 7.439720153808594 (7.012912486990293) + | > loss_mel: 18.666244506835938 (17.98067248860995) + | > loss_duration: 1.60823392868042 (1.59359955539306) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.03554153442383 (31.087376077969868) + | > grad_norm_1: tensor(389.1456, device='cuda:0') (tensor(312.2103, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.4052 (0.3656825721263884) + | > loader_time: 0.0068 (0.00573169191678365) + + + --> TIME: 2025-04-18 23:17:12 -- STEP: 121/142 -- GLOBAL_STEP: 381675 + | > loss_disc: 2.402947425842285 (2.450695087101834) + | > loss_disc_real_0: 0.12577477097511292 (0.12502868008638215) + | > loss_disc_real_1: 0.15511742234230042 (0.20468644243626555) + | > loss_disc_real_2: 0.2173861712217331 (0.22565642799720292) + | > loss_disc_real_3: 0.2656871974468231 (0.2301559621883818) + | > loss_disc_real_4: 0.23630931973457336 (0.23011163849968555) + | > loss_disc_real_5: 0.25275763869285583 (0.22947114102603974) + | > loss_0: 2.402947425842285 (2.450695087101834) + | > grad_norm_0: tensor(6.5598, device='cuda:0') (tensor(14.8362, device='cuda:0')) + | > loss_gen: 2.437216281890869 (2.3702135598363965) + | > loss_kl: 2.1936206817626953 (2.1216860861817675) + | > loss_feat: 7.349118232727051 (6.952959383814788) + | > loss_mel: 19.059226989746094 (17.911061862283507) + | > loss_duration: 1.6593317985534668 (1.5981207563857402) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.69851303100586 (30.95404155195252) + | > grad_norm_1: tensor(261.9933, device='cuda:0') (tensor(302.4389, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.441 (0.3775076215917413) + | > loader_time: 0.0066 (0.0060002074753942565) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.568617105484009 (2.568617105484009) + | > loss_disc_real_0: 0.13258658349514008 (0.13258658349514008) + | > loss_disc_real_1: 0.19891557097434998 (0.19891557097434998) + | > loss_disc_real_2: 0.2571254074573517 (0.2571254074573517) + | > loss_disc_real_3: 0.20324259996414185 (0.20324259996414185) + | > loss_disc_real_4: 0.24764898419380188 (0.24764898419380188) + | > loss_disc_real_5: 0.2917240262031555 (0.2917240262031555) + | > loss_0: 2.568617105484009 (2.568617105484009) + | > loss_gen: 2.183558225631714 (2.183558225631714) + | > loss_kl: 2.3047280311584473 (2.3047280311584473) + | > loss_feat: 6.871758460998535 (6.871758460998535) + | > loss_mel: 18.741201400756836 (18.741201400756836) + | > loss_duration: 1.6433370113372803 (1.6433370113372803) + | > loss_1: 31.744583129882812 (31.744583129882812) + + --> STEP: 1 + | > loss_disc: 2.4663796424865723 (2.4663796424865723) + | > loss_disc_real_0: 0.11304085701704025 (0.11304085701704025) + | > loss_disc_real_1: 0.19587469100952148 (0.19587469100952148) + | > loss_disc_real_2: 0.2358698695898056 (0.2358698695898056) + | > loss_disc_real_3: 0.2275996059179306 (0.2275996059179306) + | > loss_disc_real_4: 0.2499876171350479 (0.2499876171350479) + | > loss_disc_real_5: 0.25584352016448975 (0.25584352016448975) + | > loss_0: 2.4663796424865723 (2.4663796424865723) + | > loss_gen: 2.270745277404785 (2.270745277404785) + | > loss_kl: 2.3964860439300537 (2.3964860439300537) + | > loss_feat: 6.968695163726807 (6.968695163726807) + | > loss_mel: 18.495372772216797 (18.495372772216797) + | > loss_duration: 1.6849663257598877 (1.6849663257598877) + | > loss_1: 31.816265106201172 (31.816265106201172) + + --> STEP: 2 + | > loss_disc: 2.521911144256592 (2.494145393371582) + | > loss_disc_real_0: 0.14617717266082764 (0.12960901483893394) + | > loss_disc_real_1: 0.23166923224925995 (0.21377196162939072) + | > loss_disc_real_2: 0.2106555998325348 (0.2232627347111702) + | > loss_disc_real_3: 0.20763751864433289 (0.21761856228113174) + | > loss_disc_real_4: 0.21087756752967834 (0.23043259233236313) + | > loss_disc_real_5: 0.2663304805755615 (0.26108700037002563) + | > loss_0: 2.521911144256592 (2.494145393371582) + | > loss_gen: 2.3705878257751465 (2.320666551589966) + | > loss_kl: 2.0948336124420166 (2.245659828186035) + | > loss_feat: 6.720241546630859 (6.844468355178833) + | > loss_mel: 19.813831329345703 (19.15460205078125) + | > loss_duration: 1.6140143871307373 (1.6494903564453125) + | > loss_1: 32.61351013183594 (32.214887619018555) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004033803939819336 (+0.0009275674819946289) + | > avg_loss_disc: 2.494145393371582 (-0.05221843719482422) + | > avg_loss_disc_real_0: 0.12960901483893394 (+0.02355198562145233) + | > avg_loss_disc_real_1: 0.21377196162939072 (-0.027910619974136353) + | > avg_loss_disc_real_2: 0.2232627347111702 (-0.005196310579776764) + | > avg_loss_disc_real_3: 0.21761856228113174 (-0.019790738821029663) + | > avg_loss_disc_real_4: 0.23043259233236313 (+0.0032981038093566895) + | > avg_loss_disc_real_5: 0.26108700037002563 (+0.026269137859344482) + | > avg_loss_0: 2.494145393371582 (-0.05221843719482422) + | > avg_loss_gen: 2.320666551589966 (+0.26482880115509033) + | > avg_loss_kl: 2.245659828186035 (-0.03403675556182861) + | > avg_loss_feat: 6.844468355178833 (+1.214576244354248) + | > avg_loss_mel: 19.15460205078125 (+2.088130474090576) + | > avg_loss_duration: 1.6494903564453125 (+0.004725754261016846) + | > avg_loss_1: 32.214887619018555 (+3.5382261276245117) + + + > EPOCH: 17/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:17:23)  + + --> TIME: 2025-04-18 23:17:25 -- STEP: 4/142 -- GLOBAL_STEP: 381700 + | > loss_disc: 2.48018479347229 (2.47796493768692) + | > loss_disc_real_0: 0.0998026430606842 (0.11963940598070621) + | > loss_disc_real_1: 0.17033657431602478 (0.21274074539542198) + | > loss_disc_real_2: 0.2454186975955963 (0.256456945091486) + | > loss_disc_real_3: 0.2701144516468048 (0.24493931978940964) + | > loss_disc_real_4: 0.25940659642219543 (0.24274617806077003) + | > loss_disc_real_5: 0.2157820612192154 (0.24014639481902122) + | > loss_0: 2.48018479347229 (2.47796493768692) + | > grad_norm_0: tensor(4.1173, device='cuda:0') (tensor(6.3013, device='cuda:0')) + | > loss_gen: 2.3151066303253174 (2.4166675806045532) + | > loss_kl: 2.2190282344818115 (2.156327247619629) + | > loss_feat: 7.180691242218018 (6.987395405769348) + | > loss_mel: 16.638463973999023 (17.555708408355713) + | > loss_duration: 1.5904265642166138 (1.5940449833869934) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.94371795654297 (30.710143566131592) + | > grad_norm_1: tensor(128.9498, device='cuda:0') (tensor(151.9882, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.3183 (0.3223809003829956) + | > loader_time: 0.0052 (0.004291117191314697) + + + --> TIME: 2025-04-18 23:17:34 -- STEP: 29/142 -- GLOBAL_STEP: 381725 + | > loss_disc: 2.610269069671631 (2.4617337687262184) + | > loss_disc_real_0: 0.156365305185318 (0.12359864984093043) + | > loss_disc_real_1: 0.23257200419902802 (0.21141131068098135) + | > loss_disc_real_2: 0.21664020419120789 (0.22821149743836502) + | > loss_disc_real_3: 0.2602485418319702 (0.23083704504473457) + | > loss_disc_real_4: 0.2515278458595276 (0.23025316513817887) + | > loss_disc_real_5: 0.2512693703174591 (0.23150635382224774) + | > loss_0: 2.610269069671631 (2.4617337687262184) + | > grad_norm_0: tensor(7.4023, device='cuda:0') (tensor(8.3344, device='cuda:0')) + | > loss_gen: 2.223773956298828 (2.358164589980553) + | > loss_kl: 2.089329957962036 (2.0899809804455995) + | > loss_feat: 6.659463882446289 (6.995254681028169) + | > loss_mel: 16.66762924194336 (17.840006203487) + | > loss_duration: 1.5558297634124756 (1.5730596410817113) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.196027755737305 (30.856466359105603) + | > grad_norm_1: tensor(186.0925, device='cuda:0') (tensor(199.3106, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.3459 (0.3346064419581972) + | > loader_time: 0.0052 (0.004895687103271484) + + + --> TIME: 2025-04-18 23:17:43 -- STEP: 54/142 -- GLOBAL_STEP: 381750 + | > loss_disc: 2.4884958267211914 (2.463111400604249) + | > loss_disc_real_0: 0.1136375293135643 (0.1268560425550849) + | > loss_disc_real_1: 0.24568815529346466 (0.20962410420179367) + | > loss_disc_real_2: 0.20423433184623718 (0.22607447372542488) + | > loss_disc_real_3: 0.23058870434761047 (0.2292457068407977) + | > loss_disc_real_4: 0.22261711955070496 (0.22771131881961115) + | > loss_disc_real_5: 0.22826901078224182 (0.2305900079232675) + | > loss_0: 2.4884958267211914 (2.463111400604249) + | > grad_norm_0: tensor(13.1036, device='cuda:0') (tensor(9.3532, device='cuda:0')) + | > loss_gen: 2.396061897277832 (2.3493209017647634) + | > loss_kl: 2.289682149887085 (2.0865433392701327) + | > loss_feat: 7.421959400177002 (7.024031029807197) + | > loss_mel: 18.432147979736328 (17.928970177968342) + | > loss_duration: 1.558635950088501 (1.5721060699886746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.098487854003906 (30.960971761632848) + | > grad_norm_1: tensor(170.1071, device='cuda:0') (tensor(208.7477, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.3673 (0.34677295331601754) + | > loader_time: 0.0058 (0.005258083343505859) + + + --> TIME: 2025-04-18 23:17:53 -- STEP: 79/142 -- GLOBAL_STEP: 381775 + | > loss_disc: 2.5342907905578613 (2.469623677338227) + | > loss_disc_real_0: 0.15624327957630157 (0.12785865507925612) + | > loss_disc_real_1: 0.21615339815616608 (0.2112330369934251) + | > loss_disc_real_2: 0.2507278025150299 (0.22717818240576151) + | > loss_disc_real_3: 0.23393504321575165 (0.22947629380829726) + | > loss_disc_real_4: 0.3019908368587494 (0.22957246590264235) + | > loss_disc_real_5: 0.23873130977153778 (0.23124562505679794) + | > loss_0: 2.5342907905578613 (2.469623677338227) + | > grad_norm_0: tensor(9.0188, device='cuda:0') (tensor(9.9962, device='cuda:0')) + | > loss_gen: 2.136186122894287 (2.3445200950284546) + | > loss_kl: 2.1863017082214355 (2.0988383504408814) + | > loss_feat: 5.9540696144104 (7.00976496708544) + | > loss_mel: 17.034923553466797 (17.948417687717875) + | > loss_duration: 1.6549339294433594 (1.5817212907573845) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.966415405273438 (30.983262653592266) + | > grad_norm_1: tensor(96.0288, device='cuda:0') (tensor(235.8263, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.3857 (0.35908362231677105) + | > loader_time: 0.0067 (0.00557675240915033) + + + --> TIME: 2025-04-18 23:18:03 -- STEP: 104/142 -- GLOBAL_STEP: 381800 + | > loss_disc: 2.5035030841827393 (2.468117803335192) + | > loss_disc_real_0: 0.13098318874835968 (0.12913706970329464) + | > loss_disc_real_1: 0.18947763741016388 (0.20893163377275834) + | > loss_disc_real_2: 0.19308525323867798 (0.22677572186176592) + | > loss_disc_real_3: 0.2334422618150711 (0.229242007749585) + | > loss_disc_real_4: 0.21413873136043549 (0.22894974969900572) + | > loss_disc_real_5: 0.2611251473426819 (0.2316882163286209) + | > loss_0: 2.5035030841827393 (2.468117803335192) + | > grad_norm_0: tensor(17.2847, device='cuda:0') (tensor(10.7807, device='cuda:0')) + | > loss_gen: 2.2736988067626953 (2.346021085977554) + | > loss_kl: 2.1322221755981445 (2.101964174554898) + | > loss_feat: 6.915462493896484 (6.97541100703753) + | > loss_mel: 17.998563766479492 (17.904790612367485) + | > loss_duration: 1.6352684497833252 (1.5909948750184133) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.955217361450195 (30.919182025469265) + | > grad_norm_1: tensor(347.2268, device='cuda:0') (tensor(254.1977, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.4125 (0.37051792557422947) + | > loader_time: 0.0072 (0.005848983159432044) + + + --> TIME: 2025-04-18 23:18:14 -- STEP: 129/142 -- GLOBAL_STEP: 381825 + | > loss_disc: 2.4775948524475098 (2.468011693436972) + | > loss_disc_real_0: 0.15413494408130646 (0.12970556902099944) + | > loss_disc_real_1: 0.21327394247055054 (0.20916719318822372) + | > loss_disc_real_2: 0.19049714505672455 (0.22755238540874895) + | > loss_disc_real_3: 0.2089218646287918 (0.22847635258537854) + | > loss_disc_real_4: 0.1930961161851883 (0.22906511456005332) + | > loss_disc_real_5: 0.2305700182914734 (0.23139125385949777) + | > loss_0: 2.4775948524475098 (2.468011693436972) + | > grad_norm_0: tensor(18.3002, device='cuda:0') (tensor(11.5023, device='cuda:0')) + | > loss_gen: 2.2112295627593994 (2.3502843915954106) + | > loss_kl: 2.0822947025299072 (2.0991160481475117) + | > loss_feat: 6.564685821533203 (6.978721112243889) + | > loss_mel: 17.252918243408203 (17.96703083570614) + | > loss_duration: 1.5901784896850586 (1.5950585049252177) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.701305389404297 (30.990211072818255) + | > grad_norm_1: tensor(255.4033, device='cuda:0') (tensor(260.3666, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.4431 (0.38256821891134113) + | > loader_time: 0.0076 (0.006134086801100147) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.614264965057373 (2.614264965057373) + | > loss_disc_real_0: 0.21502447128295898 (0.21502447128295898) + | > loss_disc_real_1: 0.22995023429393768 (0.22995023429393768) + | > loss_disc_real_2: 0.24005120992660522 (0.24005120992660522) + | > loss_disc_real_3: 0.23643411695957184 (0.23643411695957184) + | > loss_disc_real_4: 0.23781326413154602 (0.23781326413154602) + | > loss_disc_real_5: 0.2392592877149582 (0.2392592877149582) + | > loss_0: 2.614264965057373 (2.614264965057373) + | > loss_gen: 2.2295544147491455 (2.2295544147491455) + | > loss_kl: 2.228827476501465 (2.228827476501465) + | > loss_feat: 5.975221157073975 (5.975221157073975) + | > loss_mel: 16.93091583251953 (16.93091583251953) + | > loss_duration: 1.660414695739746 (1.660414695739746) + | > loss_1: 29.024932861328125 (29.024932861328125) + + --> STEP: 1 + | > loss_disc: 2.5136778354644775 (2.5136778354644775) + | > loss_disc_real_0: 0.17074079811573029 (0.17074079811573029) + | > loss_disc_real_1: 0.21659508347511292 (0.21659508347511292) + | > loss_disc_real_2: 0.22216950356960297 (0.22216950356960297) + | > loss_disc_real_3: 0.23568013310432434 (0.23568013310432434) + | > loss_disc_real_4: 0.22318878769874573 (0.22318878769874573) + | > loss_disc_real_5: 0.22794963419437408 (0.22794963419437408) + | > loss_0: 2.5136778354644775 (2.5136778354644775) + | > loss_gen: 2.262965440750122 (2.262965440750122) + | > loss_kl: 2.4665791988372803 (2.4665791988372803) + | > loss_feat: 6.8378400802612305 (6.8378400802612305) + | > loss_mel: 18.491878509521484 (18.491878509521484) + | > loss_duration: 1.682420253753662 (1.682420253753662) + | > loss_1: 31.74168586730957 (31.74168586730957) + + --> STEP: 2 + | > loss_disc: 2.731003761291504 (2.6223407983779907) + | > loss_disc_real_0: 0.18306167423725128 (0.17690123617649078) + | > loss_disc_real_1: 0.23299431800842285 (0.22479470074176788) + | > loss_disc_real_2: 0.2552245855331421 (0.23869704455137253) + | > loss_disc_real_3: 0.24755994975566864 (0.2416200414299965) + | > loss_disc_real_4: 0.2300235778093338 (0.22660618275403976) + | > loss_disc_real_5: 0.22079195082187653 (0.2243707925081253) + | > loss_0: 2.731003761291504 (2.6223407983779907) + | > loss_gen: 1.9550060033798218 (2.108985722064972) + | > loss_kl: 2.1834607124328613 (2.325019955635071) + | > loss_feat: 5.683252811431885 (6.260546445846558) + | > loss_mel: 15.878591537475586 (17.185235023498535) + | > loss_duration: 1.5980061292648315 (1.6402131915092468) + | > loss_1: 27.298315048217773 (29.520000457763672) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035247802734375 (-0.0005090236663818359) + | > avg_loss_disc: 2.6223407983779907 (+0.1281954050064087) + | > avg_loss_disc_real_0: 0.17690123617649078 (+0.04729222133755684) + | > avg_loss_disc_real_1: 0.22479470074176788 (+0.011022739112377167) + | > avg_loss_disc_real_2: 0.23869704455137253 (+0.015434309840202332) + | > avg_loss_disc_real_3: 0.2416200414299965 (+0.024001479148864746) + | > avg_loss_disc_real_4: 0.22660618275403976 (-0.0038264095783233643) + | > avg_loss_disc_real_5: 0.2243707925081253 (-0.03671620786190033) + | > avg_loss_0: 2.6223407983779907 (+0.1281954050064087) + | > avg_loss_gen: 2.108985722064972 (-0.2116808295249939) + | > avg_loss_kl: 2.325019955635071 (+0.07936012744903564) + | > avg_loss_feat: 6.260546445846558 (-0.5839219093322754) + | > avg_loss_mel: 17.185235023498535 (-1.9693670272827148) + | > avg_loss_duration: 1.6402131915092468 (-0.009277164936065674) + | > avg_loss_1: 29.520000457763672 (-2.694887161254883) + + + > EPOCH: 18/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:18:22)  + + --> TIME: 2025-04-18 23:18:27 -- STEP: 12/142 -- GLOBAL_STEP: 381850 + | > loss_disc: 2.4265382289886475 (2.478985329469045) + | > loss_disc_real_0: 0.14194026589393616 (0.1307845680663983) + | > loss_disc_real_1: 0.22707505524158478 (0.21419486527641615) + | > loss_disc_real_2: 0.2461608350276947 (0.23771755894025168) + | > loss_disc_real_3: 0.21213030815124512 (0.23123476281762123) + | > loss_disc_real_4: 0.22764182090759277 (0.22409757475058237) + | > loss_disc_real_5: 0.22662916779518127 (0.24038558329145113) + | > loss_0: 2.4265382289886475 (2.478985329469045) + | > grad_norm_0: tensor(21.5488, device='cuda:0') (tensor(17.9313, device='cuda:0')) + | > loss_gen: 2.497213125228882 (2.42233153184255) + | > loss_kl: 2.0312283039093018 (2.0693142910798388) + | > loss_feat: 7.537107467651367 (7.015711943308513) + | > loss_mel: 19.09113311767578 (17.67273457845052) + | > loss_duration: 1.5813078880310059 (1.5670480628808339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.73799133300781 (30.74714008967082) + | > grad_norm_1: tensor(185.7905, device='cuda:0') (tensor(376.0843, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.3298 (0.32754333813985187) + | > loader_time: 0.0045 (0.0045196811358133955) + + + --> TIME: 2025-04-18 23:18:35 -- STEP: 37/142 -- GLOBAL_STEP: 381875 + | > loss_disc: 2.510295867919922 (2.458734718528954) + | > loss_disc_real_0: 0.1327393800020218 (0.1201815758202527) + | > loss_disc_real_1: 0.21997374296188354 (0.2109197394267933) + | > loss_disc_real_2: 0.25233539938926697 (0.2312368645861342) + | > loss_disc_real_3: 0.2543017566204071 (0.2275728063808905) + | > loss_disc_real_4: 0.23194478452205658 (0.22833239710008776) + | > loss_disc_real_5: 0.23163004219532013 (0.2350567402066411) + | > loss_0: 2.510295867919922 (2.458734718528954) + | > grad_norm_0: tensor(11.0821, device='cuda:0') (tensor(17.2796, device='cuda:0')) + | > loss_gen: 2.3074452877044678 (2.399281031376607) + | > loss_kl: 2.136120557785034 (2.0704773374505945) + | > loss_feat: 6.1904191970825195 (7.043130732871391) + | > loss_mel: 16.8294677734375 (17.836791579787796) + | > loss_duration: 1.589536428451538 (1.5679075170207668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.052989959716797 (30.91758815662281) + | > grad_norm_1: tensor(349.5805, device='cuda:0') (tensor(364.0114, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.3606 (0.34305026724531845) + | > loader_time: 0.0055 (0.00509459907944138) + + + --> TIME: 2025-04-18 23:18:45 -- STEP: 62/142 -- GLOBAL_STEP: 381900 + | > loss_disc: 2.46756911277771 (2.4598970528571846) + | > loss_disc_real_0: 0.18445247411727905 (0.12392235030570338) + | > loss_disc_real_1: 0.188552126288414 (0.20827840388782562) + | > loss_disc_real_2: 0.22396650910377502 (0.22869567020285514) + | > loss_disc_real_3: 0.2307642698287964 (0.22843553846882236) + | > loss_disc_real_4: 0.23497506976127625 (0.22791545934254123) + | > loss_disc_real_5: 0.255511075258255 (0.23519983575228723) + | > loss_0: 2.46756911277771 (2.4598970528571846) + | > grad_norm_0: tensor(29.9358, device='cuda:0') (tensor(16.7734, device='cuda:0')) + | > loss_gen: 2.4106862545013428 (2.380637976431078) + | > loss_kl: 2.232503890991211 (2.076127834858433) + | > loss_feat: 7.087152481079102 (6.983675326070478) + | > loss_mel: 18.895931243896484 (17.80493543994042) + | > loss_duration: 1.6326045989990234 (1.5739457491905458) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.258880615234375 (30.819322401477443) + | > grad_norm_1: tensor(206.5000, device='cuda:0') (tensor(337.6313, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.3917 (0.35725353994677145) + | > loader_time: 0.0065 (0.005465715162215694) + + + --> TIME: 2025-04-18 23:18:55 -- STEP: 87/142 -- GLOBAL_STEP: 381925 + | > loss_disc: 2.5206527709960938 (2.4681099940990587) + | > loss_disc_real_0: 0.17050889134407043 (0.1259410475519882) + | > loss_disc_real_1: 0.19156025350093842 (0.20728809874633264) + | > loss_disc_real_2: 0.2442302405834198 (0.22842923508978438) + | > loss_disc_real_3: 0.27227431535720825 (0.22772289624159364) + | > loss_disc_real_4: 0.22633224725723267 (0.228347672299407) + | > loss_disc_real_5: 0.21584169566631317 (0.23667597616540975) + | > loss_0: 2.5206527709960938 (2.4681099940990587) + | > grad_norm_0: tensor(17.3995, device='cuda:0') (tensor(17.0829, device='cuda:0')) + | > loss_gen: 2.2988030910491943 (2.3733044492787334) + | > loss_kl: 2.22538685798645 (2.0944061457425707) + | > loss_feat: 6.230353355407715 (6.9535151404896) + | > loss_mel: 17.320043563842773 (17.851246724183532) + | > loss_duration: 1.6519653797149658 (1.581913309535761) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.726552963256836 (30.85438581444751) + | > grad_norm_1: tensor(357.7988, device='cuda:0') (tensor(331.9118, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.4138 (0.3704280990293658) + | > loader_time: 0.0068 (0.005781187408271877) + + + --> TIME: 2025-04-18 23:19:06 -- STEP: 112/142 -- GLOBAL_STEP: 381950 + | > loss_disc: 2.4344871044158936 (2.469285439167705) + | > loss_disc_real_0: 0.12105318158864975 (0.12495713108884433) + | > loss_disc_real_1: 0.17437219619750977 (0.2075827880097287) + | > loss_disc_real_2: 0.24388004839420319 (0.22889371415866272) + | > loss_disc_real_3: 0.23704056441783905 (0.22782643192580768) + | > loss_disc_real_4: 0.2287082076072693 (0.22968095554304974) + | > loss_disc_real_5: 0.23629944026470184 (0.23572882291461741) + | > loss_0: 2.4344871044158936 (2.469285439167705) + | > grad_norm_0: tensor(8.1532, device='cuda:0') (tensor(15.5696, device='cuda:0')) + | > loss_gen: 2.260841131210327 (2.358225045459612) + | > loss_kl: 2.044865846633911 (2.09153162688017) + | > loss_feat: 6.316978931427002 (6.936192252806255) + | > loss_mel: 15.597820281982422 (17.90465473277227) + | > loss_duration: 1.5788874626159668 (1.5859493879335267) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.799394607543945 (30.87655314377376) + | > grad_norm_1: tensor(259.0294, device='cuda:0') (tensor(311.3717, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.4406 (0.383143112063408) + | > loader_time: 0.0075 (0.006058365106582642) + + + --> TIME: 2025-04-18 23:19:18 -- STEP: 137/142 -- GLOBAL_STEP: 381975 + | > loss_disc: 2.586355686187744 (2.473448702888768) + | > loss_disc_real_0: 0.14344076812267303 (0.1264292309536551) + | > loss_disc_real_1: 0.2009061574935913 (0.20715789866708492) + | > loss_disc_real_2: 0.21080800890922546 (0.22823536221998453) + | > loss_disc_real_3: 0.2298722267150879 (0.22883802597975209) + | > loss_disc_real_4: 0.2066815048456192 (0.22964974519980214) + | > loss_disc_real_5: 0.1923235058784485 (0.23596711250117225) + | > loss_0: 2.586355686187744 (2.473448702888768) + | > grad_norm_0: tensor(6.8729, device='cuda:0') (tensor(14.5874, device='cuda:0')) + | > loss_gen: 2.16042423248291 (2.351261128474327) + | > loss_kl: 1.9729558229446411 (2.093476132754863) + | > loss_feat: 6.8047075271606445 (6.919607496609653) + | > loss_mel: 18.203838348388672 (17.90165141725191) + | > loss_duration: 1.6026465892791748 (1.5898304159623866) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.744571685791016 (30.855826663274836) + | > grad_norm_1: tensor(215.3380, device='cuda:0') (tensor(291.6391, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.4534 (0.3955121423206191) + | > loader_time: 0.008 (0.006330484891459889) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3737123012542725 (2.3737123012542725) + | > loss_disc_real_0: 0.09204518049955368 (0.09204518049955368) + | > loss_disc_real_1: 0.2177344411611557 (0.2177344411611557) + | > loss_disc_real_2: 0.2361791878938675 (0.2361791878938675) + | > loss_disc_real_3: 0.23286612331867218 (0.23286612331867218) + | > loss_disc_real_4: 0.19075308740139008 (0.19075308740139008) + | > loss_disc_real_5: 0.24757984280586243 (0.24757984280586243) + | > loss_0: 2.3737123012542725 (2.3737123012542725) + | > loss_gen: 2.331367254257202 (2.331367254257202) + | > loss_kl: 2.1619420051574707 (2.1619420051574707) + | > loss_feat: 7.126014709472656 (7.126014709472656) + | > loss_mel: 20.42668914794922 (20.42668914794922) + | > loss_duration: 1.6367716789245605 (1.6367716789245605) + | > loss_1: 33.68278503417969 (33.68278503417969) + + --> STEP: 1 + | > loss_disc: 2.3957953453063965 (2.3957953453063965) + | > loss_disc_real_0: 0.1002127081155777 (0.1002127081155777) + | > loss_disc_real_1: 0.2412026822566986 (0.2412026822566986) + | > loss_disc_real_2: 0.24091869592666626 (0.24091869592666626) + | > loss_disc_real_3: 0.2180052548646927 (0.2180052548646927) + | > loss_disc_real_4: 0.23837457597255707 (0.23837457597255707) + | > loss_disc_real_5: 0.22868502140045166 (0.22868502140045166) + | > loss_0: 2.3957953453063965 (2.3957953453063965) + | > loss_gen: 2.342724561691284 (2.342724561691284) + | > loss_kl: 2.3927929401397705 (2.3927929401397705) + | > loss_feat: 6.757347583770752 (6.757347583770752) + | > loss_mel: 19.33513832092285 (19.33513832092285) + | > loss_duration: 1.6796307563781738 (1.6796307563781738) + | > loss_1: 32.507633209228516 (32.507633209228516) + + --> STEP: 2 + | > loss_disc: 2.561234951019287 (2.478515148162842) + | > loss_disc_real_0: 0.08967263996601105 (0.09494267404079437) + | > loss_disc_real_1: 0.22336992621421814 (0.23228630423545837) + | > loss_disc_real_2: 0.23422253131866455 (0.2375706136226654) + | > loss_disc_real_3: 0.2383517622947693 (0.228178508579731) + | > loss_disc_real_4: 0.2506037950515747 (0.2444891855120659) + | > loss_disc_real_5: 0.25833895802497864 (0.24351198971271515) + | > loss_0: 2.561234951019287 (2.478515148162842) + | > loss_gen: 2.0734400749206543 (2.2080823183059692) + | > loss_kl: 2.074307441711426 (2.233550190925598) + | > loss_feat: 6.624918460845947 (6.69113302230835) + | > loss_mel: 17.66195297241211 (18.49854564666748) + | > loss_duration: 1.5785802602767944 (1.6291055083274841) + | > loss_1: 30.013200759887695 (31.260416984558105) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034564733505249023 (-6.830692291259766e-05) + | > avg_loss_disc: 2.478515148162842 (-0.14382565021514893) + | > avg_loss_disc_real_0: 0.09494267404079437 (-0.08195856213569641) + | > avg_loss_disc_real_1: 0.23228630423545837 (+0.007491603493690491) + | > avg_loss_disc_real_2: 0.2375706136226654 (-0.0011264309287071228) + | > avg_loss_disc_real_3: 0.228178508579731 (-0.013441532850265503) + | > avg_loss_disc_real_4: 0.2444891855120659 (+0.017883002758026123) + | > avg_loss_disc_real_5: 0.24351198971271515 (+0.019141197204589844) + | > avg_loss_0: 2.478515148162842 (-0.14382565021514893) + | > avg_loss_gen: 2.2080823183059692 (+0.09909659624099731) + | > avg_loss_kl: 2.233550190925598 (-0.09146976470947266) + | > avg_loss_feat: 6.69113302230835 (+0.430586576461792) + | > avg_loss_mel: 18.49854564666748 (+1.3133106231689453) + | > avg_loss_duration: 1.6291055083274841 (-0.011107683181762695) + | > avg_loss_1: 31.260416984558105 (+1.7404165267944336) + + + > EPOCH: 19/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:19:21)  + + --> TIME: 2025-04-18 23:19:29 -- STEP: 20/142 -- GLOBAL_STEP: 382000 + | > loss_disc: 2.447171449661255 (2.4938660264015198) + | > loss_disc_real_0: 0.10761623829603195 (0.13140744119882583) + | > loss_disc_real_1: 0.21765559911727905 (0.21332074627280234) + | > loss_disc_real_2: 0.24147070944309235 (0.22899567037820817) + | > loss_disc_real_3: 0.22617347538471222 (0.23117417171597482) + | > loss_disc_real_4: 0.24949799478054047 (0.22832807078957557) + | > loss_disc_real_5: 0.20561735332012177 (0.23066331595182418) + | > loss_0: 2.447171449661255 (2.4938660264015198) + | > grad_norm_0: tensor(14.4413, device='cuda:0') (tensor(9.1734, device='cuda:0')) + | > loss_gen: 2.3932254314422607 (2.3146883845329285) + | > loss_kl: 2.193106174468994 (2.0333984375) + | > loss_feat: 7.117249965667725 (6.760003590583802) + | > loss_mel: 18.554767608642578 (17.54261965751648) + | > loss_duration: 1.6196658611297607 (1.5557893812656403) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.878013610839844 (30.20649919509888) + | > grad_norm_1: tensor(138.3708, device='cuda:0') (tensor(148.9848, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.3402 (0.32860277891159057) + | > loader_time: 0.005 (0.004762709140777588) + + + --> TIME: 2025-04-18 23:19:38 -- STEP: 45/142 -- GLOBAL_STEP: 382025 + | > loss_disc: 2.4740371704101562 (2.4876266585456004) + | > loss_disc_real_0: 0.11880695819854736 (0.12685544027222523) + | > loss_disc_real_1: 0.20081892609596252 (0.21243418223328062) + | > loss_disc_real_2: 0.20417937636375427 (0.22596386141247218) + | > loss_disc_real_3: 0.20342378318309784 (0.22901299463378058) + | > loss_disc_real_4: 0.24256858229637146 (0.23307392431630028) + | > loss_disc_real_5: 0.2379937767982483 (0.2326232310798433) + | > loss_0: 2.4740371704101562 (2.4876266585456004) + | > grad_norm_0: tensor(13.7927, device='cuda:0') (tensor(10.0333, device='cuda:0')) + | > loss_gen: 2.4324097633361816 (2.317640113830567) + | > loss_kl: 2.104405641555786 (2.053324863645765) + | > loss_feat: 7.102173805236816 (6.787167210049099) + | > loss_mel: 18.839996337890625 (17.52934559716118) + | > loss_duration: 1.5894761085510254 (1.5624642610549926) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.06846237182617 (30.249942186143663) + | > grad_norm_1: tensor(345.2690, device='cuda:0') (tensor(213.1224, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.3655 (0.341557937198215) + | > loader_time: 0.006 (0.0051847139994303385) + + + --> TIME: 2025-04-18 23:19:47 -- STEP: 70/142 -- GLOBAL_STEP: 382050 + | > loss_disc: 2.438462734222412 (2.4877658707754953) + | > loss_disc_real_0: 0.10330207645893097 (0.1303256130644253) + | > loss_disc_real_1: 0.22154109179973602 (0.21093473242861885) + | > loss_disc_real_2: 0.2449425905942917 (0.22617576782192503) + | > loss_disc_real_3: 0.2647012174129486 (0.23080572187900544) + | > loss_disc_real_4: 0.24061328172683716 (0.23097862558705465) + | > loss_disc_real_5: 0.2309465855360031 (0.2316189555185182) + | > loss_0: 2.438462734222412 (2.4877658707754953) + | > grad_norm_0: tensor(6.0271, device='cuda:0') (tensor(11.3474, device='cuda:0')) + | > loss_gen: 2.453422784805298 (2.324393044199262) + | > loss_kl: 2.0982236862182617 (2.0672959344727646) + | > loss_feat: 7.091944694519043 (6.851394217354911) + | > loss_mel: 18.74989891052246 (17.728643076760427) + | > loss_duration: 1.6044032573699951 (1.570507938521249) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.997892379760742 (30.542234202793665) + | > grad_norm_1: tensor(193.4909, device='cuda:0') (tensor(232.1627, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.3926 (0.3545714310237339) + | > loader_time: 0.0061 (0.005528436388288226) + + + --> TIME: 2025-04-18 23:19:58 -- STEP: 95/142 -- GLOBAL_STEP: 382075 + | > loss_disc: 2.546766757965088 (2.4865518494656205) + | > loss_disc_real_0: 0.1954597681760788 (0.13298713078624325) + | > loss_disc_real_1: 0.22110027074813843 (0.21072914067067597) + | > loss_disc_real_2: 0.20059196650981903 (0.22596920524772846) + | > loss_disc_real_3: 0.2049158364534378 (0.22906700560921117) + | > loss_disc_real_4: 0.20790134370326996 (0.23289245570960798) + | > loss_disc_real_5: 0.22069287300109863 (0.23116515341557956) + | > loss_0: 2.546766757965088 (2.4865518494656205) + | > grad_norm_0: tensor(15.5220, device='cuda:0') (tensor(12.9696, device='cuda:0')) + | > loss_gen: 2.354769468307495 (2.3565037175228727) + | > loss_kl: 2.083160400390625 (2.0757144413496302) + | > loss_feat: 6.551905155181885 (6.897571423179225) + | > loss_mel: 18.58963394165039 (17.845279201708344) + | > loss_duration: 1.6323471069335938 (1.5792455899088007) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.211814880371094 (30.754314282065945) + | > grad_norm_1: tensor(247.1707, device='cuda:0') (tensor(258.7047, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.4049 (0.36618016142594184) + | > loader_time: 0.0066 (0.00577438003138492) + + + --> TIME: 2025-04-18 23:20:08 -- STEP: 120/142 -- GLOBAL_STEP: 382100 + | > loss_disc: 2.552335739135742 (2.489124991496403) + | > loss_disc_real_0: 0.21893095970153809 (0.13433862229188284) + | > loss_disc_real_1: 0.18414311110973358 (0.21051571952799955) + | > loss_disc_real_2: 0.2311393767595291 (0.22623026954631012) + | > loss_disc_real_3: 0.23682498931884766 (0.2295697153856357) + | > loss_disc_real_4: 0.24623778462409973 (0.23265024572610854) + | > loss_disc_real_5: 0.292675644159317 (0.232274883116285) + | > loss_0: 2.552335739135742 (2.489124991496403) + | > grad_norm_0: tensor(35.9060, device='cuda:0') (tensor(15.3543, device='cuda:0')) + | > loss_gen: 2.3659701347351074 (2.3583898762861897) + | > loss_kl: 2.0864946842193604 (2.073667981227238) + | > loss_feat: 6.4349517822265625 (6.893294823169708) + | > loss_mel: 17.490436553955078 (17.90519646803539) + | > loss_duration: 1.601781964302063 (1.5832369297742843) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.97963523864746 (30.813785950342815) + | > grad_norm_1: tensor(343.8032, device='cuda:0') (tensor(278.8100, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.4382 (0.3778536876042684) + | > loader_time: 0.0082 (0.006047791242599488) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5717668533325195 (2.5717668533325195) + | > loss_disc_real_0: 0.15767474472522736 (0.15767474472522736) + | > loss_disc_real_1: 0.21734333038330078 (0.21734333038330078) + | > loss_disc_real_2: 0.28475454449653625 (0.28475454449653625) + | > loss_disc_real_3: 0.2833404242992401 (0.2833404242992401) + | > loss_disc_real_4: 0.24790866672992706 (0.24790866672992706) + | > loss_disc_real_5: 0.2636956572532654 (0.2636956572532654) + | > loss_0: 2.5717668533325195 (2.5717668533325195) + | > loss_gen: 2.262423038482666 (2.262423038482666) + | > loss_kl: 2.619460344314575 (2.619460344314575) + | > loss_feat: 6.107544422149658 (6.107544422149658) + | > loss_mel: 17.29281234741211 (17.29281234741211) + | > loss_duration: 1.6466386318206787 (1.6466386318206787) + | > loss_1: 29.928878784179688 (29.928878784179688) + + --> STEP: 1 + | > loss_disc: 2.396636724472046 (2.396636724472046) + | > loss_disc_real_0: 0.09804876893758774 (0.09804876893758774) + | > loss_disc_real_1: 0.2135300636291504 (0.2135300636291504) + | > loss_disc_real_2: 0.2392471730709076 (0.2392471730709076) + | > loss_disc_real_3: 0.2219269871711731 (0.2219269871711731) + | > loss_disc_real_4: 0.22529728710651398 (0.22529728710651398) + | > loss_disc_real_5: 0.1941293179988861 (0.1941293179988861) + | > loss_0: 2.396636724472046 (2.396636724472046) + | > loss_gen: 2.166567325592041 (2.166567325592041) + | > loss_kl: 2.331125259399414 (2.331125259399414) + | > loss_feat: 7.427112102508545 (7.427112102508545) + | > loss_mel: 19.474931716918945 (19.474931716918945) + | > loss_duration: 1.68158757686615 (1.68158757686615) + | > loss_1: 33.08132553100586 (33.08132553100586) + + --> STEP: 2 + | > loss_disc: 2.59059739112854 (2.493617057800293) + | > loss_disc_real_0: 0.11899063736200333 (0.10851970314979553) + | > loss_disc_real_1: 0.2112904042005539 (0.21241023391485214) + | > loss_disc_real_2: 0.3106995224952698 (0.2749733477830887) + | > loss_disc_real_3: 0.2574203908443451 (0.2396736890077591) + | > loss_disc_real_4: 0.28827014565467834 (0.25678371638059616) + | > loss_disc_real_5: 0.2481788843870163 (0.2211541011929512) + | > loss_0: 2.59059739112854 (2.493617057800293) + | > loss_gen: 2.249040365219116 (2.2078038454055786) + | > loss_kl: 2.0358972549438477 (2.183511257171631) + | > loss_feat: 6.408198833465576 (6.9176554679870605) + | > loss_mel: 18.96088218688965 (19.217906951904297) + | > loss_duration: 1.5757648944854736 (1.6286762356758118) + | > loss_1: 31.22978401184082 (32.15555477142334) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003486156463623047 (+2.968311309814453e-05) + | > avg_loss_disc: 2.493617057800293 (+0.015101909637451172) + | > avg_loss_disc_real_0: 0.10851970314979553 (+0.01357702910900116) + | > avg_loss_disc_real_1: 0.21241023391485214 (-0.01987607032060623) + | > avg_loss_disc_real_2: 0.2749733477830887 (+0.03740273416042328) + | > avg_loss_disc_real_3: 0.2396736890077591 (+0.011495180428028107) + | > avg_loss_disc_real_4: 0.25678371638059616 (+0.012294530868530273) + | > avg_loss_disc_real_5: 0.2211541011929512 (-0.022357888519763947) + | > avg_loss_0: 2.493617057800293 (+0.015101909637451172) + | > avg_loss_gen: 2.2078038454055786 (-0.000278472900390625) + | > avg_loss_kl: 2.183511257171631 (-0.050038933753967285) + | > avg_loss_feat: 6.9176554679870605 (+0.22652244567871094) + | > avg_loss_mel: 19.217906951904297 (+0.7193613052368164) + | > avg_loss_duration: 1.6286762356758118 (-0.0004292726516723633) + | > avg_loss_1: 32.15555477142334 (+0.8951377868652344) + + + > EPOCH: 20/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:20:20)  + + --> TIME: 2025-04-18 23:20:22 -- STEP: 3/142 -- GLOBAL_STEP: 382125 + | > loss_disc: 2.5234670639038086 (2.510450998942057) + | > loss_disc_real_0: 0.12897270917892456 (0.14499995112419128) + | > loss_disc_real_1: 0.20612651109695435 (0.2141966074705124) + | > loss_disc_real_2: 0.21277198195457458 (0.2269552250703176) + | > loss_disc_real_3: 0.2671041488647461 (0.26016901930173236) + | > loss_disc_real_4: 0.2651728093624115 (0.26212214430173236) + | > loss_disc_real_5: 0.2702743709087372 (0.2510486990213394) + | > loss_0: 2.5234670639038086 (2.510450998942057) + | > grad_norm_0: tensor(16.3885, device='cuda:0') (tensor(10.9356, device='cuda:0')) + | > loss_gen: 2.2669448852539062 (2.3472413221995034) + | > loss_kl: 2.1088767051696777 (2.0548632939656577) + | > loss_feat: 6.856079578399658 (6.76027234395345) + | > loss_mel: 18.22115707397461 (17.278210322062176) + | > loss_duration: 1.5732300281524658 (1.5559242169062297) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.026288986206055 (29.996512095133465) + | > grad_norm_1: tensor(251.9439, device='cuda:0') (tensor(216.8701, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.3216 (0.3216230869293213) + | > loader_time: 0.0036 (0.00388487180074056) + + + --> TIME: 2025-04-18 23:20:30 -- STEP: 28/142 -- GLOBAL_STEP: 382150 + | > loss_disc: 2.63533616065979 (2.4477087344442094) + | > loss_disc_real_0: 0.10360615700483322 (0.11655426291482789) + | > loss_disc_real_1: 0.2271641045808792 (0.2084605379828385) + | > loss_disc_real_2: 0.26697614789009094 (0.22497902757355145) + | > loss_disc_real_3: 0.2858678102493286 (0.23251261029924666) + | > loss_disc_real_4: 0.271075963973999 (0.23557176760264806) + | > loss_disc_real_5: 0.28573229908943176 (0.22841356375387736) + | > loss_0: 2.63533616065979 (2.4477087344442094) + | > grad_norm_0: tensor(8.8084, device='cuda:0') (tensor(11.4631, device='cuda:0')) + | > loss_gen: 2.313115358352661 (2.3403910228184293) + | > loss_kl: 1.9731532335281372 (2.009280932801111) + | > loss_feat: 6.520702838897705 (7.027861339705331) + | > loss_mel: 17.590585708618164 (17.889475890568324) + | > loss_duration: 1.597920298576355 (1.5591309411185128) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9954776763916 (30.826140335627965) + | > grad_norm_1: tensor(156.2109, device='cuda:0') (tensor(268.1749, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.3448 (0.3341902749879019) + | > loader_time: 0.0055 (0.004733383655548095) + + + --> TIME: 2025-04-18 23:20:39 -- STEP: 53/142 -- GLOBAL_STEP: 382175 + | > loss_disc: 2.5141615867614746 (2.4417062030648284) + | > loss_disc_real_0: 0.13112816214561462 (0.12069055002253011) + | > loss_disc_real_1: 0.22530125081539154 (0.20491665011306978) + | > loss_disc_real_2: 0.2178293615579605 (0.22430183752527777) + | > loss_disc_real_3: 0.2166890799999237 (0.23143577069606422) + | > loss_disc_real_4: 0.23195292055606842 (0.23231629015139812) + | > loss_disc_real_5: 0.24483682215213776 (0.227904331009343) + | > loss_0: 2.5141615867614746 (2.4417062030648284) + | > grad_norm_0: tensor(3.5486, device='cuda:0') (tensor(9.5942, device='cuda:0')) + | > loss_gen: 2.240190267562866 (2.3449744278529914) + | > loss_kl: 1.9084582328796387 (2.0241423800306504) + | > loss_feat: 7.324389457702637 (7.074643027107671) + | > loss_mel: 17.933395385742188 (18.002204391191597) + | > loss_duration: 1.5176887512207031 (1.5582835449362702) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.924121856689453 (31.004247917319244) + | > grad_norm_1: tensor(41.0101, device='cuda:0') (tensor(226.7723, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.3664 (0.3459113948750046) + | > loader_time: 0.0062 (0.00506368673072671) + + + --> TIME: 2025-04-18 23:20:49 -- STEP: 78/142 -- GLOBAL_STEP: 382200 + | > loss_disc: 2.4282636642456055 (2.448623427977929) + | > loss_disc_real_0: 0.11283339560031891 (0.12285189215953533) + | > loss_disc_real_1: 0.2081281989812851 (0.20673459653670972) + | > loss_disc_real_2: 0.21792344748973846 (0.22509307643541923) + | > loss_disc_real_3: 0.25830575823783875 (0.23037642183212134) + | > loss_disc_real_4: 0.22526587545871735 (0.23209273757842871) + | > loss_disc_real_5: 0.2235550433397293 (0.22913587207977587) + | > loss_0: 2.4282636642456055 (2.448623427977929) + | > grad_norm_0: tensor(7.9639, device='cuda:0') (tensor(9.2119, device='cuda:0')) + | > loss_gen: 2.3780086040496826 (2.337520360946656) + | > loss_kl: 2.161320686340332 (2.0452241118137655) + | > loss_feat: 6.279636859893799 (6.983036554776705) + | > loss_mel: 17.71718978881836 (17.981814506726394) + | > loss_duration: 1.6005594730377197 (1.5666259221541576) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.136714935302734 (30.914221519078964) + | > grad_norm_1: tensor(137.0747, device='cuda:0') (tensor(201.7132, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.3915 (0.35860887246254153) + | > loader_time: 0.0064 (0.0053998017922425885) + + + --> TIME: 2025-04-18 23:21:00 -- STEP: 103/142 -- GLOBAL_STEP: 382225 + | > loss_disc: 2.675869941711426 (2.462670409563676) + | > loss_disc_real_0: 0.11357853561639786 (0.12580527632849886) + | > loss_disc_real_1: 0.194423645734787 (0.20941166261446129) + | > loss_disc_real_2: 0.22452926635742188 (0.2260915702118457) + | > loss_disc_real_3: 0.24066612124443054 (0.2297141806593219) + | > loss_disc_real_4: 0.24800914525985718 (0.23067649866192086) + | > loss_disc_real_5: 0.21680419147014618 (0.22910366286930528) + | > loss_0: 2.675869941711426 (2.462670409563676) + | > grad_norm_0: tensor(8.3549, device='cuda:0') (tensor(9.2359, device='cuda:0')) + | > loss_gen: 2.070157766342163 (2.320718392585089) + | > loss_kl: 2.0854990482330322 (2.060329581927328) + | > loss_feat: 5.620511531829834 (6.908866479558852) + | > loss_mel: 17.06351089477539 (17.964265036351467) + | > loss_duration: 1.6293127536773682 (1.575617438381158) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.468992233276367 (30.82979700403306) + | > grad_norm_1: tensor(46.4400, device='cuda:0') (tensor(196.3858, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.4141 (0.36988272018802965) + | > loader_time: 0.0068 (0.005683401255931669) + + + --> TIME: 2025-04-18 23:21:11 -- STEP: 128/142 -- GLOBAL_STEP: 382250 + | > loss_disc: 2.4344091415405273 (2.4634443819522858) + | > loss_disc_real_0: 0.11123519390821457 (0.127889653434977) + | > loss_disc_real_1: 0.21860763430595398 (0.20860064285807312) + | > loss_disc_real_2: 0.22843141853809357 (0.2257774262689054) + | > loss_disc_real_3: 0.20769359171390533 (0.23010807298123837) + | > loss_disc_real_4: 0.23767557740211487 (0.2300199921010062) + | > loss_disc_real_5: 0.23097963631153107 (0.22964218142442405) + | > loss_0: 2.4344091415405273 (2.4634443819522858) + | > grad_norm_0: tensor(10.6400, device='cuda:0') (tensor(8.9805, device='cuda:0')) + | > loss_gen: 2.2953641414642334 (2.316828878596426) + | > loss_kl: 2.034259796142578 (2.0655593769624834) + | > loss_feat: 7.101314544677734 (6.892650060355663) + | > loss_mel: 19.2353515625 (17.933884821832173) + | > loss_duration: 1.6083141565322876 (1.580673431046307) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.27460479736328 (30.78959660232067) + | > grad_norm_1: tensor(286.2791, device='cuda:0') (tensor(187.6999, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.4355 (0.3820816818624734) + | > loader_time: 0.0068 (0.005999751389026642) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5676345825195312 (2.5676345825195312) + | > loss_disc_real_0: 0.19671714305877686 (0.19671714305877686) + | > loss_disc_real_1: 0.24885466694831848 (0.24885466694831848) + | > loss_disc_real_2: 0.21721728146076202 (0.21721728146076202) + | > loss_disc_real_3: 0.2925945818424225 (0.2925945818424225) + | > loss_disc_real_4: 0.2707478702068329 (0.2707478702068329) + | > loss_disc_real_5: 0.29404786229133606 (0.29404786229133606) + | > loss_0: 2.5676345825195312 (2.5676345825195312) + | > loss_gen: 2.6776788234710693 (2.6776788234710693) + | > loss_kl: 2.1759815216064453 (2.1759815216064453) + | > loss_feat: 7.120119571685791 (7.120119571685791) + | > loss_mel: 19.559480667114258 (19.559480667114258) + | > loss_duration: 1.6469875574111938 (1.6469875574111938) + | > loss_1: 33.18024826049805 (33.18024826049805) + + --> STEP: 1 + | > loss_disc: 2.371455669403076 (2.371455669403076) + | > loss_disc_real_0: 0.16999982297420502 (0.16999982297420502) + | > loss_disc_real_1: 0.24496017396450043 (0.24496017396450043) + | > loss_disc_real_2: 0.2068472057580948 (0.2068472057580948) + | > loss_disc_real_3: 0.2571190893650055 (0.2571190893650055) + | > loss_disc_real_4: 0.2569006681442261 (0.2569006681442261) + | > loss_disc_real_5: 0.25674039125442505 (0.25674039125442505) + | > loss_0: 2.371455669403076 (2.371455669403076) + | > loss_gen: 2.838582754135132 (2.838582754135132) + | > loss_kl: 2.4265778064727783 (2.4265778064727783) + | > loss_feat: 6.931934833526611 (6.931934833526611) + | > loss_mel: 19.300857543945312 (19.300857543945312) + | > loss_duration: 1.66043221950531 (1.66043221950531) + | > loss_1: 33.15838623046875 (33.15838623046875) + + --> STEP: 2 + | > loss_disc: 2.3945791721343994 (2.383017420768738) + | > loss_disc_real_0: 0.21163581311702728 (0.19081781804561615) + | > loss_disc_real_1: 0.20321302115917206 (0.22408659756183624) + | > loss_disc_real_2: 0.21031102538108826 (0.20857911556959152) + | > loss_disc_real_3: 0.21092252433300018 (0.23402080684900284) + | > loss_disc_real_4: 0.21392372250556946 (0.23541219532489777) + | > loss_disc_real_5: 0.2961178123950958 (0.27642910182476044) + | > loss_0: 2.3945791721343994 (2.383017420768738) + | > loss_gen: 2.699288845062256 (2.768935799598694) + | > loss_kl: 2.1314401626586914 (2.279008984565735) + | > loss_feat: 7.71946382522583 (7.325699329376221) + | > loss_mel: 17.817346572875977 (18.559102058410645) + | > loss_duration: 1.585526466369629 (1.6229793429374695) + | > loss_1: 31.953067779541016 (32.55572700500488) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037050247192382812 (+0.00021886825561523438) + | > avg_loss_disc: 2.383017420768738 (-0.11059963703155518) + | > avg_loss_disc_real_0: 0.19081781804561615 (+0.08229811489582062) + | > avg_loss_disc_real_1: 0.22408659756183624 (+0.0116763636469841) + | > avg_loss_disc_real_2: 0.20857911556959152 (-0.06639423221349716) + | > avg_loss_disc_real_3: 0.23402080684900284 (-0.005652882158756256) + | > avg_loss_disc_real_4: 0.23541219532489777 (-0.021371521055698395) + | > avg_loss_disc_real_5: 0.27642910182476044 (+0.055275000631809235) + | > avg_loss_0: 2.383017420768738 (-0.11059963703155518) + | > avg_loss_gen: 2.768935799598694 (+0.5611319541931152) + | > avg_loss_kl: 2.279008984565735 (+0.095497727394104) + | > avg_loss_feat: 7.325699329376221 (+0.40804386138916016) + | > avg_loss_mel: 18.559102058410645 (-0.6588048934936523) + | > avg_loss_duration: 1.6229793429374695 (-0.005696892738342285) + | > avg_loss_1: 32.55572700500488 (+0.40017223358154297) + + + > EPOCH: 21/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:21:18)  + + --> TIME: 2025-04-18 23:21:23 -- STEP: 11/142 -- GLOBAL_STEP: 382275 + | > loss_disc: 2.5432887077331543 (2.4435732581398706) + | > loss_disc_real_0: 0.1482391506433487 (0.1264503300189972) + | > loss_disc_real_1: 0.24485449492931366 (0.2090171751650897) + | > loss_disc_real_2: 0.2838265895843506 (0.2197637910192663) + | > loss_disc_real_3: 0.2769577205181122 (0.23138566315174103) + | > loss_disc_real_4: 0.26667505502700806 (0.22869034111499786) + | > loss_disc_real_5: 0.2415037751197815 (0.23384308815002441) + | > loss_0: 2.5432887077331543 (2.4435732581398706) + | > grad_norm_0: tensor(8.2831, device='cuda:0') (tensor(6.8960, device='cuda:0')) + | > loss_gen: 2.4520933628082275 (2.405169226906516) + | > loss_kl: 1.9836325645446777 (2.015559955076738) + | > loss_feat: 6.580628871917725 (7.471084854819558) + | > loss_mel: 17.53371238708496 (18.58631723577326) + | > loss_duration: 1.5213336944580078 (1.5399077263745395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.071399688720703 (32.0180383162065) + | > grad_norm_1: tensor(176.8774, device='cuda:0') (tensor(130.8309, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.3295 (0.3239947665821422) + | > loader_time: 0.0048 (0.004362258044156161) + + + --> TIME: 2025-04-18 23:21:31 -- STEP: 36/142 -- GLOBAL_STEP: 382300 + | > loss_disc: 2.307419538497925 (2.4767507844501075) + | > loss_disc_real_0: 0.10693865269422531 (0.1302588687588771) + | > loss_disc_real_1: 0.19756916165351868 (0.20826302758521503) + | > loss_disc_real_2: 0.20362524688243866 (0.22318056681089932) + | > loss_disc_real_3: 0.21345174312591553 (0.2275147313872973) + | > loss_disc_real_4: 0.20763851702213287 (0.23084288090467453) + | > loss_disc_real_5: 0.2100047767162323 (0.2328097824421194) + | > loss_0: 2.307419538497925 (2.4767507844501075) + | > grad_norm_0: tensor(7.0990, device='cuda:0') (tensor(9.0488, device='cuda:0')) + | > loss_gen: 2.5914723873138428 (2.3358073896831937) + | > loss_kl: 2.1331002712249756 (2.025565869278378) + | > loss_feat: 7.455002307891846 (7.003058340814379) + | > loss_mel: 17.81614112854004 (18.034029404322307) + | > loss_duration: 1.5384376049041748 (1.5501962337228987) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.53415298461914 (30.94865714179145) + | > grad_norm_1: tensor(325.2202, device='cuda:0') (tensor(179.3920, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.3509 (0.3373018370734321) + | > loader_time: 0.0053 (0.004897461997138129) + + + --> TIME: 2025-04-18 23:21:41 -- STEP: 61/142 -- GLOBAL_STEP: 382325 + | > loss_disc: 2.3629720211029053 (2.494474411010743) + | > loss_disc_real_0: 0.0995127484202385 (0.13157257816342058) + | > loss_disc_real_1: 0.24390652775764465 (0.20977815497116964) + | > loss_disc_real_2: 0.2260836511850357 (0.22740289007054001) + | > loss_disc_real_3: 0.2628462314605713 (0.22919774617328018) + | > loss_disc_real_4: 0.23363010585308075 (0.23297710692296264) + | > loss_disc_real_5: 0.20115315914154053 (0.2329093081052186) + | > loss_0: 2.3629720211029053 (2.494474411010743) + | > grad_norm_0: tensor(17.1177, device='cuda:0') (tensor(10.5180, device='cuda:0')) + | > loss_gen: 2.5949923992156982 (2.326254090324778) + | > loss_kl: 1.9657071828842163 (2.0465263538673284) + | > loss_feat: 7.486708164215088 (6.90081991914843) + | > loss_mel: 18.89120101928711 (17.939045140000648) + | > loss_duration: 1.559138298034668 (1.5574045181274414) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.497745513916016 (30.770049923756083) + | > grad_norm_1: tensor(474.0622, device='cuda:0') (tensor(214.4492, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.381 (0.3509555957356437) + | > loader_time: 0.0057 (0.005249070339515561) + + + --> TIME: 2025-04-18 23:21:51 -- STEP: 86/142 -- GLOBAL_STEP: 382350 + | > loss_disc: 2.3806095123291016 (2.4863147707872613) + | > loss_disc_real_0: 0.09281224012374878 (0.1321264417885348) + | > loss_disc_real_1: 0.1803780496120453 (0.20874261266963426) + | > loss_disc_real_2: 0.2262856364250183 (0.2281852950190389) + | > loss_disc_real_3: 0.2359606921672821 (0.2283658518694168) + | > loss_disc_real_4: 0.22832797467708588 (0.23183984326761822) + | > loss_disc_real_5: 0.2441108375787735 (0.23251488198374592) + | > loss_0: 2.3806095123291016 (2.4863147707872613) + | > grad_norm_0: tensor(7.3094, device='cuda:0') (tensor(10.2992, device='cuda:0')) + | > loss_gen: 2.4160964488983154 (2.3316336537516396) + | > loss_kl: 2.1931307315826416 (2.067380937032922) + | > loss_feat: 6.76956033706665 (6.93243652720784) + | > loss_mel: 16.631933212280273 (18.041003493375566) + | > loss_duration: 1.603743553161621 (1.567545341890912) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.614463806152344 (30.939999913060387) + | > grad_norm_1: tensor(251.1274, device='cuda:0') (tensor(218.3653, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.4009 (0.36255800724029535) + | > loader_time: 0.0069 (0.005569283352341764) + + + --> TIME: 2025-04-18 23:22:02 -- STEP: 111/142 -- GLOBAL_STEP: 382375 + | > loss_disc: 2.4232711791992188 (2.488989920229525) + | > loss_disc_real_0: 0.12273599952459335 (0.1324011502904935) + | > loss_disc_real_1: 0.2151046246290207 (0.20937831323963027) + | > loss_disc_real_2: 0.23436930775642395 (0.22826996472504762) + | > loss_disc_real_3: 0.24429452419281006 (0.22976875238053435) + | > loss_disc_real_4: 0.2229660451412201 (0.23222080436912743) + | > loss_disc_real_5: 0.2246253788471222 (0.2312854615686176) + | > loss_0: 2.4232711791992188 (2.488989920229525) + | > grad_norm_0: tensor(11.7182, device='cuda:0') (tensor(11.0844, device='cuda:0')) + | > loss_gen: 2.4382917881011963 (2.339956081665313) + | > loss_kl: 2.0490641593933105 (2.0789815359287434) + | > loss_feat: 6.183497428894043 (6.9682486980885) + | > loss_mel: 16.999265670776367 (18.08402439495466) + | > loss_duration: 1.6148943901062012 (1.574051008568154) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.28501319885254 (31.045261675173098) + | > grad_norm_1: tensor(240.0140, device='cuda:0') (tensor(238.5742, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.4272 (0.3756075399415987) + | > loader_time: 0.0068 (0.00581758516328829) + + + --> TIME: 2025-04-18 23:22:13 -- STEP: 136/142 -- GLOBAL_STEP: 382400 + | > loss_disc: 2.434795379638672 (2.488677915404825) + | > loss_disc_real_0: 0.08715925365686417 (0.1320983464007869) + | > loss_disc_real_1: 0.1623287796974182 (0.20952974730993018) + | > loss_disc_real_2: 0.21984094381332397 (0.228364336578285) + | > loss_disc_real_3: 0.2597302198410034 (0.2303088900140103) + | > loss_disc_real_4: 0.22513197362422943 (0.23168943384114435) + | > loss_disc_real_5: 0.2263043224811554 (0.2319953988361008) + | > loss_0: 2.434795379638672 (2.488677915404825) + | > grad_norm_0: tensor(16.9920, device='cuda:0') (tensor(11.4892, device='cuda:0')) + | > loss_gen: 2.2678210735321045 (2.3389889054438644) + | > loss_kl: 2.0137221813201904 (2.0809107089743897) + | > loss_feat: 7.255373001098633 (6.9238552731626175) + | > loss_mel: 18.84212303161621 (18.000336794292238) + | > loss_duration: 1.598016381263733 (1.5777573147240809) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.977054595947266 (30.9218489001779) + | > grad_norm_1: tensor(390.2118, device='cuda:0') (tensor(245.9322, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.4632 (0.38755866359261903) + | > loader_time: 0.0078 (0.006089219275642845) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5578453540802 (2.5578453540802) + | > loss_disc_real_0: 0.15554754436016083 (0.15554754436016083) + | > loss_disc_real_1: 0.262773722410202 (0.262773722410202) + | > loss_disc_real_2: 0.24904146790504456 (0.24904146790504456) + | > loss_disc_real_3: 0.2620633840560913 (0.2620633840560913) + | > loss_disc_real_4: 0.2077324539422989 (0.2077324539422989) + | > loss_disc_real_5: 0.24439525604248047 (0.24439525604248047) + | > loss_0: 2.5578453540802 (2.5578453540802) + | > loss_gen: 2.182302951812744 (2.182302951812744) + | > loss_kl: 2.506279945373535 (2.506279945373535) + | > loss_feat: 6.073001861572266 (6.073001861572266) + | > loss_mel: 17.33549690246582 (17.33549690246582) + | > loss_duration: 1.6099674701690674 (1.6099674701690674) + | > loss_1: 29.707050323486328 (29.707050323486328) + + --> STEP: 1 + | > loss_disc: 2.500936985015869 (2.500936985015869) + | > loss_disc_real_0: 0.12554901838302612 (0.12554901838302612) + | > loss_disc_real_1: 0.29106780886650085 (0.29106780886650085) + | > loss_disc_real_2: 0.24254989624023438 (0.24254989624023438) + | > loss_disc_real_3: 0.2487202286720276 (0.2487202286720276) + | > loss_disc_real_4: 0.23423629999160767 (0.23423629999160767) + | > loss_disc_real_5: 0.2563675045967102 (0.2563675045967102) + | > loss_0: 2.500936985015869 (2.500936985015869) + | > loss_gen: 2.39685320854187 (2.39685320854187) + | > loss_kl: 2.4363906383514404 (2.4363906383514404) + | > loss_feat: 7.09883451461792 (7.09883451461792) + | > loss_mel: 19.80945587158203 (19.80945587158203) + | > loss_duration: 1.6787925958633423 (1.6787925958633423) + | > loss_1: 33.420326232910156 (33.420326232910156) + + --> STEP: 2 + | > loss_disc: 2.481147050857544 (2.4910420179367065) + | > loss_disc_real_0: 0.1279575526714325 (0.1267532855272293) + | > loss_disc_real_1: 0.26819196343421936 (0.2796298861503601) + | > loss_disc_real_2: 0.26323384046554565 (0.25289186835289) + | > loss_disc_real_3: 0.24936890602111816 (0.24904456734657288) + | > loss_disc_real_4: 0.2155415564775467 (0.22488892823457718) + | > loss_disc_real_5: 0.1987493932247162 (0.2275584489107132) + | > loss_0: 2.481147050857544 (2.4910420179367065) + | > loss_gen: 2.273200035095215 (2.3350266218185425) + | > loss_kl: 2.305586576461792 (2.370988607406616) + | > loss_feat: 6.676048755645752 (6.887441635131836) + | > loss_mel: 17.62682342529297 (18.7181396484375) + | > loss_duration: 1.5929973125457764 (1.6358949542045593) + | > loss_1: 30.47465705871582 (31.94749164581299) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003681778907775879 (-2.3245811462402344e-05) + | > avg_loss_disc: 2.4910420179367065 (+0.10802459716796875) + | > avg_loss_disc_real_0: 0.1267532855272293 (-0.06406453251838684) + | > avg_loss_disc_real_1: 0.2796298861503601 (+0.055543288588523865) + | > avg_loss_disc_real_2: 0.25289186835289 (+0.04431275278329849) + | > avg_loss_disc_real_3: 0.24904456734657288 (+0.015023760497570038) + | > avg_loss_disc_real_4: 0.22488892823457718 (-0.010523267090320587) + | > avg_loss_disc_real_5: 0.2275584489107132 (-0.04887065291404724) + | > avg_loss_0: 2.4910420179367065 (+0.10802459716796875) + | > avg_loss_gen: 2.3350266218185425 (-0.43390917778015137) + | > avg_loss_kl: 2.370988607406616 (+0.09197962284088135) + | > avg_loss_feat: 6.887441635131836 (-0.43825769424438477) + | > avg_loss_mel: 18.7181396484375 (+0.15903759002685547) + | > avg_loss_duration: 1.6358949542045593 (+0.012915611267089844) + | > avg_loss_1: 31.94749164581299 (-0.6082353591918945) + + + > EPOCH: 22/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:22:17)  + + --> TIME: 2025-04-18 23:22:24 -- STEP: 19/142 -- GLOBAL_STEP: 382425 + | > loss_disc: 2.4791030883789062 (2.482372948997899) + | > loss_disc_real_0: 0.07998346537351608 (0.11859822312467977) + | > loss_disc_real_1: 0.19660678505897522 (0.2116129037580992) + | > loss_disc_real_2: 0.22399987280368805 (0.23154443345571818) + | > loss_disc_real_3: 0.23971863090991974 (0.23440246597716682) + | > loss_disc_real_4: 0.2249133586883545 (0.23345854407862612) + | > loss_disc_real_5: 0.2926253080368042 (0.23309436597322164) + | > loss_0: 2.4791030883789062 (2.482372948997899) + | > grad_norm_0: tensor(11.3135, device='cuda:0') (tensor(15.8927, device='cuda:0')) + | > loss_gen: 2.409696578979492 (2.384662577980443) + | > loss_kl: 1.9027693271636963 (1.9981574259306256) + | > loss_feat: 7.372391223907471 (7.253442965055767) + | > loss_mel: 18.40669822692871 (18.1539955641094) + | > loss_duration: 1.5484364032745361 (1.5396335752386796) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.639991760253906 (31.329891606381064) + | > grad_norm_1: tensor(36.0508, device='cuda:0') (tensor(313.8175, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.3355 (0.32789451197574015) + | > loader_time: 0.0051 (0.0047122051841334296) + + + --> TIME: 2025-04-18 23:22:33 -- STEP: 44/142 -- GLOBAL_STEP: 382450 + | > loss_disc: 2.5086731910705566 (2.471432301131162) + | > loss_disc_real_0: 0.15684174001216888 (0.12187677926637909) + | > loss_disc_real_1: 0.210340678691864 (0.2123067406090823) + | > loss_disc_real_2: 0.2741958498954773 (0.23053718053481795) + | > loss_disc_real_3: 0.2517905533313751 (0.23391331833871928) + | > loss_disc_real_4: 0.2156858891248703 (0.23224447261203418) + | > loss_disc_real_5: 0.21307867765426636 (0.22998335314067928) + | > loss_0: 2.5086731910705566 (2.471432301131162) + | > grad_norm_0: tensor(19.2017, device='cuda:0') (tensor(14.3579, device='cuda:0')) + | > loss_gen: 2.206022024154663 (2.3570353876460683) + | > loss_kl: 2.0037219524383545 (2.0316457694227044) + | > loss_feat: 6.810155391693115 (7.0353346629576246) + | > loss_mel: 18.787893295288086 (17.873360850594253) + | > loss_duration: 1.598755121231079 (1.551219417290254) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.40654754638672 (30.848595792596992) + | > grad_norm_1: tensor(216.3326, device='cuda:0') (tensor(304.5575, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.3575 (0.3426117951219732) + | > loader_time: 0.0052 (0.0050556226210160685) + + + --> TIME: 2025-04-18 23:22:43 -- STEP: 69/142 -- GLOBAL_STEP: 382475 + | > loss_disc: 2.5376200675964355 (2.483537466629692) + | > loss_disc_real_0: 0.19362138211727142 (0.12802658748367554) + | > loss_disc_real_1: 0.24596063792705536 (0.20919384558995566) + | > loss_disc_real_2: 0.25101083517074585 (0.22929000616937445) + | > loss_disc_real_3: 0.25392311811447144 (0.23344925326713617) + | > loss_disc_real_4: 0.2651222348213196 (0.2340178744516511) + | > loss_disc_real_5: 0.24626539647579193 (0.2303753851548485) + | > loss_0: 2.5376200675964355 (2.483537466629692) + | > grad_norm_0: tensor(14.0841, device='cuda:0') (tensor(16.4622, device='cuda:0')) + | > loss_gen: 2.4943854808807373 (2.3645042440165644) + | > loss_kl: 2.0984108448028564 (2.0484292196190883) + | > loss_feat: 6.403019905090332 (6.970514905625495) + | > loss_mel: 18.777923583984375 (17.886538021806338) + | > loss_duration: 1.6331006288528442 (1.5588184906088787) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.40683937072754 (30.82880473482436) + | > grad_norm_1: tensor(309.4185, device='cuda:0') (tensor(307.3922, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.3936 (0.35413456999737286) + | > loader_time: 0.0063 (0.0053783534229665565) + + + --> TIME: 2025-04-18 23:22:53 -- STEP: 94/142 -- GLOBAL_STEP: 382500 + | > loss_disc: 2.5734689235687256 (2.4780838464168795) + | > loss_disc_real_0: 0.11447741836309433 (0.12881727714804891) + | > loss_disc_real_1: 0.19207558035850525 (0.209326427033607) + | > loss_disc_real_2: 0.24296382069587708 (0.2295161910513614) + | > loss_disc_real_3: 0.24486595392227173 (0.23252858522724598) + | > loss_disc_real_4: 0.28211602568626404 (0.23175151883921724) + | > loss_disc_real_5: 0.3275834023952484 (0.23091879154139378) + | > loss_0: 2.5734689235687256 (2.4780838464168795) + | > grad_norm_0: tensor(4.8738, device='cuda:0') (tensor(17.6209, device='cuda:0')) + | > loss_gen: 2.5334949493408203 (2.3754246488530586) + | > loss_kl: 2.089531660079956 (2.053191443707081) + | > loss_feat: 6.098980903625488 (6.949415359091251) + | > loss_mel: 17.98420524597168 (17.90323223966232) + | > loss_duration: 1.58943772315979 (1.567909878619174) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.295650482177734 (30.849173424091745) + | > grad_norm_1: tensor(290.3526, device='cuda:0') (tensor(319.3536, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.4005 (0.36572483752636215) + | > loader_time: 0.0067 (0.005752175412279494) + + + --> TIME: 2025-04-18 23:23:04 -- STEP: 119/142 -- GLOBAL_STEP: 382525 + | > loss_disc: 2.5555620193481445 (2.47893922669547) + | > loss_disc_real_0: 0.16596777737140656 (0.1302795585464029) + | > loss_disc_real_1: 0.18340353667736053 (0.20880169991184683) + | > loss_disc_real_2: 0.25982755422592163 (0.22958833404949733) + | > loss_disc_real_3: 0.22232398390769958 (0.23113980426006958) + | > loss_disc_real_4: 0.25481775403022766 (0.23158983197532781) + | > loss_disc_real_5: 0.24802443385124207 (0.22918688950418425) + | > loss_0: 2.5555620193481445 (2.47893922669547) + | > grad_norm_0: tensor(14.8667, device='cuda:0') (tensor(17.6621, device='cuda:0')) + | > loss_gen: 2.322277784347534 (2.363724177624999) + | > loss_kl: 2.041853427886963 (2.06027733979105) + | > loss_feat: 6.285732269287109 (6.931867339030034) + | > loss_mel: 18.596359252929688 (17.946415740902665) + | > loss_duration: 1.6455817222595215 (1.5721591330375995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.891803741455078 (30.87444369532481) + | > grad_norm_1: tensor(306.8940, device='cuda:0') (tensor(322.3012, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.4262 (0.37731417888352864) + | > loader_time: 0.0071 (0.006008088087835232) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.512538433074951 (2.512538433074951) + | > loss_disc_real_0: 0.1633380502462387 (0.1633380502462387) + | > loss_disc_real_1: 0.22986137866973877 (0.22986137866973877) + | > loss_disc_real_2: 0.2543509900569916 (0.2543509900569916) + | > loss_disc_real_3: 0.22560547292232513 (0.22560547292232513) + | > loss_disc_real_4: 0.20197506248950958 (0.20197506248950958) + | > loss_disc_real_5: 0.24003925919532776 (0.24003925919532776) + | > loss_0: 2.512538433074951 (2.512538433074951) + | > loss_gen: 2.386673927307129 (2.386673927307129) + | > loss_kl: 2.1962850093841553 (2.1962850093841553) + | > loss_feat: 5.985623836517334 (5.985623836517334) + | > loss_mel: 17.241458892822266 (17.241458892822266) + | > loss_duration: 1.650542974472046 (1.650542974472046) + | > loss_1: 29.460582733154297 (29.460582733154297) + + --> STEP: 1 + | > loss_disc: 2.416116952896118 (2.416116952896118) + | > loss_disc_real_0: 0.11806713789701462 (0.11806713789701462) + | > loss_disc_real_1: 0.2196972817182541 (0.2196972817182541) + | > loss_disc_real_2: 0.21449263393878937 (0.21449263393878937) + | > loss_disc_real_3: 0.18830159306526184 (0.18830159306526184) + | > loss_disc_real_4: 0.20939099788665771 (0.20939099788665771) + | > loss_disc_real_5: 0.19434909522533417 (0.19434909522533417) + | > loss_0: 2.416116952896118 (2.416116952896118) + | > loss_gen: 2.262686014175415 (2.262686014175415) + | > loss_kl: 2.2803943157196045 (2.2803943157196045) + | > loss_feat: 6.721050262451172 (6.721050262451172) + | > loss_mel: 18.497777938842773 (18.497777938842773) + | > loss_duration: 1.65485417842865 (1.65485417842865) + | > loss_1: 31.41676139831543 (31.41676139831543) + + --> STEP: 2 + | > loss_disc: 2.3475515842437744 (2.3818342685699463) + | > loss_disc_real_0: 0.13306748867034912 (0.12556731328368187) + | > loss_disc_real_1: 0.20835252106189728 (0.21402490139007568) + | > loss_disc_real_2: 0.20729245245456696 (0.21089254319667816) + | > loss_disc_real_3: 0.17515169084072113 (0.18172664195299149) + | > loss_disc_real_4: 0.19309356808662415 (0.20124228298664093) + | > loss_disc_real_5: 0.17242293059825897 (0.18338601291179657) + | > loss_0: 2.3475515842437744 (2.3818342685699463) + | > loss_gen: 2.1873393058776855 (2.2250126600265503) + | > loss_kl: 1.9663764238357544 (2.1233853697776794) + | > loss_feat: 6.4511895179748535 (6.586119890213013) + | > loss_mel: 16.258316040039062 (17.378046989440918) + | > loss_duration: 1.62577486038208 (1.640314519405365) + | > loss_1: 28.488996505737305 (29.952878952026367) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003934741020202637 (+0.0002529621124267578) + | > avg_loss_disc: 2.3818342685699463 (-0.10920774936676025) + | > avg_loss_disc_real_0: 0.12556731328368187 (-0.0011859722435474396) + | > avg_loss_disc_real_1: 0.21402490139007568 (-0.06560498476028442) + | > avg_loss_disc_real_2: 0.21089254319667816 (-0.04199932515621185) + | > avg_loss_disc_real_3: 0.18172664195299149 (-0.06731792539358139) + | > avg_loss_disc_real_4: 0.20124228298664093 (-0.02364664524793625) + | > avg_loss_disc_real_5: 0.18338601291179657 (-0.044172435998916626) + | > avg_loss_0: 2.3818342685699463 (-0.10920774936676025) + | > avg_loss_gen: 2.2250126600265503 (-0.11001396179199219) + | > avg_loss_kl: 2.1233853697776794 (-0.24760323762893677) + | > avg_loss_feat: 6.586119890213013 (-0.30132174491882324) + | > avg_loss_mel: 17.378046989440918 (-1.340092658996582) + | > avg_loss_duration: 1.640314519405365 (+0.004419565200805664) + | > avg_loss_1: 29.952878952026367 (-1.994612693786621) + + + > EPOCH: 23/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:23:15)  + + --> TIME: 2025-04-18 23:23:17 -- STEP: 2/142 -- GLOBAL_STEP: 382550 + | > loss_disc: 2.4452548027038574 (2.4424495697021484) + | > loss_disc_real_0: 0.11564777046442032 (0.12481312826275826) + | > loss_disc_real_1: 0.21090447902679443 (0.2158714458346367) + | > loss_disc_real_2: 0.2417144924402237 (0.24138066917657852) + | > loss_disc_real_3: 0.20423047244548798 (0.20690830051898956) + | > loss_disc_real_4: 0.2141244113445282 (0.20964156091213226) + | > loss_disc_real_5: 0.1774633377790451 (0.19074026495218277) + | > loss_0: 2.4452548027038574 (2.4424495697021484) + | > grad_norm_0: tensor(3.6137, device='cuda:0') (tensor(6.5079, device='cuda:0')) + | > loss_gen: 2.193988800048828 (2.1931211948394775) + | > loss_kl: 2.012454032897949 (1.9816243648529053) + | > loss_feat: 6.958076477050781 (6.83663272857666) + | > loss_mel: 17.017820358276367 (16.749332427978516) + | > loss_duration: 1.572510004043579 (1.536159634590149) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.75484848022461 (29.2968692779541) + | > grad_norm_1: tensor(96.7145, device='cuda:0') (tensor(174.6546, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.3201 (0.32380354404449463) + | > loader_time: 0.0038 (0.003820061683654785) + + + --> TIME: 2025-04-18 23:23:25 -- STEP: 27/142 -- GLOBAL_STEP: 382575 + | > loss_disc: 2.380899429321289 (2.4586386150783963) + | > loss_disc_real_0: 0.10717063397169113 (0.12109681577594192) + | > loss_disc_real_1: 0.22166557610034943 (0.20848555697335136) + | > loss_disc_real_2: 0.19733692705631256 (0.2291045669052336) + | > loss_disc_real_3: 0.2081974595785141 (0.22971748257124866) + | > loss_disc_real_4: 0.20512445271015167 (0.23261327544848123) + | > loss_disc_real_5: 0.1921006590127945 (0.22774453130033281) + | > loss_0: 2.380899429321289 (2.4586386150783963) + | > grad_norm_0: tensor(7.5373, device='cuda:0') (tensor(8.4846, device='cuda:0')) + | > loss_gen: 2.251007080078125 (2.3552254394248684) + | > loss_kl: 1.917521357536316 (1.9811278802377206) + | > loss_feat: 6.628770351409912 (7.183156984823722) + | > loss_mel: 16.483295440673828 (18.09543757968478) + | > loss_duration: 1.5718591213226318 (1.5438130564159818) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.852453231811523 (31.15876105979637) + | > grad_norm_1: tensor(95.8279, device='cuda:0') (tensor(237.0208, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.355 (0.3353328263318097) + | > loader_time: 0.0048 (0.004804655357643412) + + + --> TIME: 2025-04-18 23:23:35 -- STEP: 52/142 -- GLOBAL_STEP: 382600 + | > loss_disc: 2.4713406562805176 (2.4474788170594435) + | > loss_disc_real_0: 0.16434255242347717 (0.12671478551167706) + | > loss_disc_real_1: 0.2209187000989914 (0.20788448275281832) + | > loss_disc_real_2: 0.22706370055675507 (0.2266836596223024) + | > loss_disc_real_3: 0.21460439264774323 (0.22772930849056977) + | > loss_disc_real_4: 0.2750617265701294 (0.22963781540210432) + | > loss_disc_real_5: 0.20250211656093597 (0.22488340907371962) + | > loss_0: 2.4713406562805176 (2.4474788170594435) + | > grad_norm_0: tensor(10.0978, device='cuda:0') (tensor(9.8334, device='cuda:0')) + | > loss_gen: 2.3882546424865723 (2.363925067278056) + | > loss_kl: 2.036612033843994 (2.0067951564605417) + | > loss_feat: 6.460069179534912 (7.128015087201045) + | > loss_mel: 19.060068130493164 (18.06577297357412) + | > loss_duration: 1.5988547801971436 (1.546211458169497) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.54385757446289 (31.1107199375446) + | > grad_norm_1: tensor(150.8081, device='cuda:0') (tensor(243.7501, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.3756 (0.3516693482032189) + | > loader_time: 0.0059 (0.005232393741607665) + + + --> TIME: 2025-04-18 23:23:45 -- STEP: 77/142 -- GLOBAL_STEP: 382625 + | > loss_disc: 2.4545934200286865 (2.459523417732932) + | > loss_disc_real_0: 0.18422077596187592 (0.131024168199533) + | > loss_disc_real_1: 0.18859218060970306 (0.20730333514027782) + | > loss_disc_real_2: 0.23041513562202454 (0.22664738204572107) + | > loss_disc_real_3: 0.2128840535879135 (0.22779707030042426) + | > loss_disc_real_4: 0.19458574056625366 (0.23031449066354082) + | > loss_disc_real_5: 0.22460125386714935 (0.22659686904448967) + | > loss_0: 2.4545934200286865 (2.459523417732932) + | > grad_norm_0: tensor(21.0108, device='cuda:0') (tensor(11.0481, device='cuda:0')) + | > loss_gen: 2.3819282054901123 (2.355870736109747) + | > loss_kl: 2.070584774017334 (2.0176696421264046) + | > loss_feat: 6.371973991394043 (7.085958610881459) + | > loss_mel: 16.421184539794922 (18.044978525731466) + | > loss_duration: 1.5476492643356323 (1.555178081834471) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.793319702148438 (31.05965572208553) + | > grad_norm_1: tensor(453.6026, device='cuda:0') (tensor(250.9176, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.3931 (0.36603082310069696) + | > loader_time: 0.0062 (0.005533193613027597) + + + --> TIME: 2025-04-18 23:23:55 -- STEP: 102/142 -- GLOBAL_STEP: 382650 + | > loss_disc: 2.47912859916687 (2.4609856371786076) + | > loss_disc_real_0: 0.10398224741220474 (0.13063010142422188) + | > loss_disc_real_1: 0.20181138813495636 (0.20790241439552867) + | > loss_disc_real_2: 0.22625479102134705 (0.22593559441613217) + | > loss_disc_real_3: 0.2202872484922409 (0.22926133973341362) + | > loss_disc_real_4: 0.24161586165428162 (0.23202918542950762) + | > loss_disc_real_5: 0.21364104747772217 (0.22710379329966565) + | > loss_0: 2.47912859916687 (2.4609856371786076) + | > grad_norm_0: tensor(14.8741, device='cuda:0') (tensor(11.5661, device='cuda:0')) + | > loss_gen: 2.3249948024749756 (2.3502820823706823) + | > loss_kl: 2.106104612350464 (2.0305761741656885) + | > loss_feat: 6.863896369934082 (7.02573405527601) + | > loss_mel: 18.587188720703125 (18.04605912227256) + | > loss_duration: 1.5579231977462769 (1.5638960786894254) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.440107345581055 (31.016547670551375) + | > grad_norm_1: tensor(277.4598, device='cuda:0') (tensor(263.2874, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.4225 (0.37565517893024514) + | > loader_time: 0.0072 (0.005772403642243031) + + + --> TIME: 2025-04-18 23:24:06 -- STEP: 127/142 -- GLOBAL_STEP: 382675 + | > loss_disc: 2.4208710193634033 (2.4672708154663323) + | > loss_disc_real_0: 0.18139488995075226 (0.13116405135768605) + | > loss_disc_real_1: 0.2303716093301773 (0.20902395271879481) + | > loss_disc_real_2: 0.22324499487876892 (0.22733185798164426) + | > loss_disc_real_3: 0.26428136229515076 (0.22972361708250572) + | > loss_disc_real_4: 0.22103489935398102 (0.23100941141289988) + | > loss_disc_real_5: 0.23275648057460785 (0.22711285819688182) + | > loss_0: 2.4208710193634033 (2.4672708154663323) + | > grad_norm_0: tensor(5.3962, device='cuda:0') (tensor(11.3657, device='cuda:0')) + | > loss_gen: 2.33874249458313 (2.3404560389481177) + | > loss_kl: 2.1777024269104004 (2.0373064748884184) + | > loss_feat: 6.772961139678955 (6.981529010562446) + | > loss_mel: 18.176855087280273 (18.06292189200093) + | > loss_duration: 1.5998605489730835 (1.5690294325821041) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.06612205505371 (30.991243001982923) + | > grad_norm_1: tensor(133.5792, device='cuda:0') (tensor(263.7346, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.4394 (0.38665087955204513) + | > loader_time: 0.0072 (0.006055835663802981) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4871551990509033 (2.4871551990509033) + | > loss_disc_real_0: 0.1369558423757553 (0.1369558423757553) + | > loss_disc_real_1: 0.21199631690979004 (0.21199631690979004) + | > loss_disc_real_2: 0.23737961053848267 (0.23737961053848267) + | > loss_disc_real_3: 0.21511004865169525 (0.21511004865169525) + | > loss_disc_real_4: 0.22081393003463745 (0.22081393003463745) + | > loss_disc_real_5: 0.2200394868850708 (0.2200394868850708) + | > loss_0: 2.4871551990509033 (2.4871551990509033) + | > loss_gen: 2.1531355381011963 (2.1531355381011963) + | > loss_kl: 2.29935359954834 (2.29935359954834) + | > loss_feat: 6.306576251983643 (6.306576251983643) + | > loss_mel: 16.914087295532227 (16.914087295532227) + | > loss_duration: 1.6453266143798828 (1.6453266143798828) + | > loss_1: 29.318479537963867 (29.318479537963867) + + --> STEP: 1 + | > loss_disc: 2.493260622024536 (2.493260622024536) + | > loss_disc_real_0: 0.13916373252868652 (0.13916373252868652) + | > loss_disc_real_1: 0.20632466673851013 (0.20632466673851013) + | > loss_disc_real_2: 0.21507951617240906 (0.21507951617240906) + | > loss_disc_real_3: 0.20959939062595367 (0.20959939062595367) + | > loss_disc_real_4: 0.21230414509773254 (0.21230414509773254) + | > loss_disc_real_5: 0.2022397369146347 (0.2022397369146347) + | > loss_0: 2.493260622024536 (2.493260622024536) + | > loss_gen: 1.975827932357788 (1.975827932357788) + | > loss_kl: 2.449138641357422 (2.449138641357422) + | > loss_feat: 5.647488594055176 (5.647488594055176) + | > loss_mel: 16.37480354309082 (16.37480354309082) + | > loss_duration: 1.6761279106140137 (1.6761279106140137) + | > loss_1: 28.123388290405273 (28.123388290405273) + + --> STEP: 2 + | > loss_disc: 2.539228677749634 (2.516244649887085) + | > loss_disc_real_0: 0.19849729537963867 (0.1688305139541626) + | > loss_disc_real_1: 0.19505508244037628 (0.2006898745894432) + | > loss_disc_real_2: 0.22188858687877655 (0.2184840515255928) + | > loss_disc_real_3: 0.21732324361801147 (0.21346131712198257) + | > loss_disc_real_4: 0.22087253630161285 (0.2165883406996727) + | > loss_disc_real_5: 0.20275714993476868 (0.2024984434247017) + | > loss_0: 2.539228677749634 (2.516244649887085) + | > loss_gen: 1.9846404790878296 (1.9802342057228088) + | > loss_kl: 2.300785779953003 (2.3749622106552124) + | > loss_feat: 5.243520736694336 (5.445504665374756) + | > loss_mel: 14.383842468261719 (15.37932300567627) + | > loss_duration: 1.6171910762786865 (1.64665949344635) + | > loss_1: 25.52998161315918 (26.826684951782227) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00371551513671875 (-0.00021922588348388672) + | > avg_loss_disc: 2.516244649887085 (+0.13441038131713867) + | > avg_loss_disc_real_0: 0.1688305139541626 (+0.04326320067048073) + | > avg_loss_disc_real_1: 0.2006898745894432 (-0.013335026800632477) + | > avg_loss_disc_real_2: 0.2184840515255928 (+0.007591508328914642) + | > avg_loss_disc_real_3: 0.21346131712198257 (+0.03173467516899109) + | > avg_loss_disc_real_4: 0.2165883406996727 (+0.015346057713031769) + | > avg_loss_disc_real_5: 0.2024984434247017 (+0.01911243051290512) + | > avg_loss_0: 2.516244649887085 (+0.13441038131713867) + | > avg_loss_gen: 1.9802342057228088 (-0.24477845430374146) + | > avg_loss_kl: 2.3749622106552124 (+0.25157684087753296) + | > avg_loss_feat: 5.445504665374756 (-1.1406152248382568) + | > avg_loss_mel: 15.37932300567627 (-1.9987239837646484) + | > avg_loss_duration: 1.64665949344635 (+0.006344974040985107) + | > avg_loss_1: 26.826684951782227 (-3.1261940002441406) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_382690.pth + + > EPOCH: 24/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:24:15)  + + --> TIME: 2025-04-18 23:24:19 -- STEP: 10/142 -- GLOBAL_STEP: 382700 + | > loss_disc: 2.440488815307617 (2.4162962436676025) + | > loss_disc_real_0: 0.13200382888317108 (0.12383725196123123) + | > loss_disc_real_1: 0.1796012818813324 (0.19742777198553085) + | > loss_disc_real_2: 0.23713551461696625 (0.22837065458297728) + | > loss_disc_real_3: 0.21539396047592163 (0.23209158778190614) + | > loss_disc_real_4: 0.22327902913093567 (0.22827300131320954) + | > loss_disc_real_5: 0.22625011205673218 (0.23914653360843657) + | > loss_0: 2.440488815307617 (2.4162962436676025) + | > grad_norm_0: tensor(4.0865, device='cuda:0') (tensor(7.9093, device='cuda:0')) + | > loss_gen: 2.3879234790802 (2.4431642293930054) + | > loss_kl: 2.0256526470184326 (2.02414208650589) + | > loss_feat: 7.703553199768066 (7.439204359054566) + | > loss_mel: 18.717180252075195 (18.395129013061524) + | > loss_duration: 1.4960771799087524 (1.5363897323608398) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.330387115478516 (31.83802947998047) + | > grad_norm_1: tensor(133.1048, device='cuda:0') (tensor(183.3543, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.3543 (0.32756402492523196) + | > loader_time: 0.0051 (0.004327797889709472) + + + --> TIME: 2025-04-18 23:24:28 -- STEP: 35/142 -- GLOBAL_STEP: 382725 + | > loss_disc: 2.480612277984619 (2.476074368613107) + | > loss_disc_real_0: 0.1521177589893341 (0.12850903081042425) + | > loss_disc_real_1: 0.19163987040519714 (0.20780823699065618) + | > loss_disc_real_2: 0.21262380480766296 (0.22815355488232203) + | > loss_disc_real_3: 0.22558414936065674 (0.23031345563275474) + | > loss_disc_real_4: 0.22905831038951874 (0.23319746255874635) + | > loss_disc_real_5: 0.22985637187957764 (0.23441432671887533) + | > loss_0: 2.480612277984619 (2.476074368613107) + | > grad_norm_0: tensor(6.2709, device='cuda:0') (tensor(11.3828, device='cuda:0')) + | > loss_gen: 2.3364341259002686 (2.3567509855542865) + | > loss_kl: 2.064330816268921 (2.006564470699855) + | > loss_feat: 6.431780815124512 (7.022484629494803) + | > loss_mel: 16.774883270263672 (18.052781432015557) + | > loss_duration: 1.5579681396484375 (1.5386755909238543) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.16539764404297 (30.977257210867744) + | > grad_norm_1: tensor(99.6071, device='cuda:0') (tensor(272.5122, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.3554 (0.3382921014513288) + | > loader_time: 0.0063 (0.004935353142874581) + + + --> TIME: 2025-04-18 23:24:38 -- STEP: 60/142 -- GLOBAL_STEP: 382750 + | > loss_disc: 2.6412389278411865 (2.4672983407974245) + | > loss_disc_real_0: 0.16927137970924377 (0.12870850202937925) + | > loss_disc_real_1: 0.192457914352417 (0.20698945547143618) + | > loss_disc_real_2: 0.20067867636680603 (0.22643520111838977) + | > loss_disc_real_3: 0.23999163508415222 (0.23019458651542662) + | > loss_disc_real_4: 0.21918655931949615 (0.2313606063524882) + | > loss_disc_real_5: 0.24590346217155457 (0.23137441004316012) + | > loss_0: 2.6412389278411865 (2.4672983407974245) + | > grad_norm_0: tensor(7.5855, device='cuda:0') (tensor(10.8625, device='cuda:0')) + | > loss_gen: 2.176682949066162 (2.353674709796906) + | > loss_kl: 1.8791970014572144 (2.0107506314913435) + | > loss_feat: 6.26915979385376 (7.020806304613749) + | > loss_mel: 16.796405792236328 (17.930316909154257) + | > loss_duration: 1.5865974426269531 (1.548133271932602) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.708044052124023 (30.863681856791178) + | > grad_norm_1: tensor(107.4245, device='cuda:0') (tensor(259.0310, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.3752 (0.34953122138977055) + | > loader_time: 0.0063 (0.005325937271118164) + + + --> TIME: 2025-04-18 23:24:48 -- STEP: 85/142 -- GLOBAL_STEP: 382775 + | > loss_disc: 2.590031623840332 (2.4785798689898315) + | > loss_disc_real_0: 0.16100583970546722 (0.13124293746317142) + | > loss_disc_real_1: 0.19825898110866547 (0.20783250647432663) + | > loss_disc_real_2: 0.2185770571231842 (0.2271952338078443) + | > loss_disc_real_3: 0.2379521280527115 (0.23058494048960068) + | > loss_disc_real_4: 0.20911484956741333 (0.23107514188570136) + | > loss_disc_real_5: 0.2481154054403305 (0.2325546375092338) + | > loss_0: 2.590031623840332 (2.4785798689898315) + | > grad_norm_0: tensor(28.7692, device='cuda:0') (tensor(10.3200, device='cuda:0')) + | > loss_gen: 2.203688144683838 (2.3476740276112285) + | > loss_kl: 2.1151742935180664 (2.030042154648725) + | > loss_feat: 6.938024044036865 (6.954135053298053) + | > loss_mel: 18.609355926513672 (17.934152928520653) + | > loss_duration: 1.5745216608047485 (1.5584942691466384) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.440763473510742 (30.824498412188362) + | > grad_norm_1: tensor(617.7122, device='cuda:0') (tensor(244.5570, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.3955 (0.3617966623867261) + | > loader_time: 0.0062 (0.005617525998283835) + + + --> TIME: 2025-04-18 23:24:58 -- STEP: 110/142 -- GLOBAL_STEP: 382800 + | > loss_disc: 2.5917410850524902 (2.4842072660272763) + | > loss_disc_real_0: 0.17788811028003693 (0.1337576345286587) + | > loss_disc_real_1: 0.21228057146072388 (0.20852027156136252) + | > loss_disc_real_2: 0.26752135157585144 (0.22802471315318887) + | > loss_disc_real_3: 0.2523113191127777 (0.23034842488440602) + | > loss_disc_real_4: 0.27546289563179016 (0.23168983486565678) + | > loss_disc_real_5: 0.21249806880950928 (0.23178291727196088) + | > loss_0: 2.5917410850524902 (2.4842072660272763) + | > grad_norm_0: tensor(16.2479, device='cuda:0') (tensor(10.9436, device='cuda:0')) + | > loss_gen: 2.2250773906707764 (2.338572348247876) + | > loss_kl: 2.1482319831848145 (2.0355277386578643) + | > loss_feat: 6.863514423370361 (6.905136268789118) + | > loss_mel: 18.829326629638672 (17.931175570054496) + | > loss_duration: 1.593266487121582 (1.5656679099256339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.65941619873047 (30.7760798887773) + | > grad_norm_1: tensor(48.6765, device='cuda:0') (tensor(247.9535, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.4384 (0.37354588725350135) + | > loader_time: 0.0074 (0.005917861244895241) + + + --> TIME: 2025-04-18 23:25:09 -- STEP: 135/142 -- GLOBAL_STEP: 382825 + | > loss_disc: 2.520843982696533 (2.483816085038361) + | > loss_disc_real_0: 0.2467021942138672 (0.1351424717240864) + | > loss_disc_real_1: 0.22668032348155975 (0.20764348242017958) + | > loss_disc_real_2: 0.24674391746520996 (0.22890637340369047) + | > loss_disc_real_3: 0.25986772775650024 (0.2301106255363535) + | > loss_disc_real_4: 0.2384859025478363 (0.23084635889088667) + | > loss_disc_real_5: 0.2309952676296234 (0.23103195671682006) + | > loss_0: 2.520843982696533 (2.483816085038361) + | > grad_norm_0: tensor(10.6654, device='cuda:0') (tensor(12.0577, device='cuda:0')) + | > loss_gen: 2.557219982147217 (2.34227802488539) + | > loss_kl: 2.0044920444488525 (2.034929832705745) + | > loss_feat: 6.802884578704834 (6.886238762184425) + | > loss_mel: 16.24733543395996 (17.83946461147733) + | > loss_duration: 1.5484440326690674 (1.5697030252880517) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.160377502441406 (30.67261427420157) + | > grad_norm_1: tensor(327.9932, device='cuda:0') (tensor(258.8307, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.4545 (0.3858357235237405) + | > loader_time: 0.0081 (0.006194902349401403) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4756217002868652 (2.4756217002868652) + | > loss_disc_real_0: 0.11926038563251495 (0.11926038563251495) + | > loss_disc_real_1: 0.2606799304485321 (0.2606799304485321) + | > loss_disc_real_2: 0.2290324866771698 (0.2290324866771698) + | > loss_disc_real_3: 0.23503164947032928 (0.23503164947032928) + | > loss_disc_real_4: 0.28918856382369995 (0.28918856382369995) + | > loss_disc_real_5: 0.2528266906738281 (0.2528266906738281) + | > loss_0: 2.4756217002868652 (2.4756217002868652) + | > loss_gen: 2.4424896240234375 (2.4424896240234375) + | > loss_kl: 2.309075355529785 (2.309075355529785) + | > loss_feat: 7.0738935470581055 (7.0738935470581055) + | > loss_mel: 18.940622329711914 (18.940622329711914) + | > loss_duration: 1.6217308044433594 (1.6217308044433594) + | > loss_1: 32.38780975341797 (32.38780975341797) + + --> STEP: 1 + | > loss_disc: 2.5027740001678467 (2.5027740001678467) + | > loss_disc_real_0: 0.13896183669567108 (0.13896183669567108) + | > loss_disc_real_1: 0.22679570317268372 (0.22679570317268372) + | > loss_disc_real_2: 0.2315780520439148 (0.2315780520439148) + | > loss_disc_real_3: 0.21557345986366272 (0.21557345986366272) + | > loss_disc_real_4: 0.26013603806495667 (0.26013603806495667) + | > loss_disc_real_5: 0.22825780510902405 (0.22825780510902405) + | > loss_0: 2.5027740001678467 (2.5027740001678467) + | > loss_gen: 2.1336193084716797 (2.1336193084716797) + | > loss_kl: 2.2253549098968506 (2.2253549098968506) + | > loss_feat: 5.938492774963379 (5.938492774963379) + | > loss_mel: 15.72778034210205 (15.72778034210205) + | > loss_duration: 1.667912483215332 (1.667912483215332) + | > loss_1: 27.693161010742188 (27.693161010742188) + + --> STEP: 2 + | > loss_disc: 2.560204029083252 (2.5314890146255493) + | > loss_disc_real_0: 0.09930424392223358 (0.11913304030895233) + | > loss_disc_real_1: 0.3262085020542145 (0.2765021026134491) + | > loss_disc_real_2: 0.23892104625701904 (0.23524954915046692) + | > loss_disc_real_3: 0.24236589670181274 (0.22896967828273773) + | > loss_disc_real_4: 0.2958608865737915 (0.2779984623193741) + | > loss_disc_real_5: 0.25653499364852905 (0.24239639937877655) + | > loss_0: 2.560204029083252 (2.5314890146255493) + | > loss_gen: 2.4018361568450928 (2.2677277326583862) + | > loss_kl: 2.2030205726623535 (2.214187741279602) + | > loss_feat: 7.152604579925537 (6.545548677444458) + | > loss_mel: 19.198366165161133 (17.463073253631592) + | > loss_duration: 1.5747973918914795 (1.6213549375534058) + | > loss_1: 32.53062438964844 (30.111892700195312) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036841630935668945 (-3.135204315185547e-05) + | > avg_loss_disc: 2.5314890146255493 (+0.015244364738464355) + | > avg_loss_disc_real_0: 0.11913304030895233 (-0.049697473645210266) + | > avg_loss_disc_real_1: 0.2765021026134491 (+0.07581222802400589) + | > avg_loss_disc_real_2: 0.23524954915046692 (+0.016765497624874115) + | > avg_loss_disc_real_3: 0.22896967828273773 (+0.015508361160755157) + | > avg_loss_disc_real_4: 0.2779984623193741 (+0.061410121619701385) + | > avg_loss_disc_real_5: 0.24239639937877655 (+0.03989795595407486) + | > avg_loss_0: 2.5314890146255493 (+0.015244364738464355) + | > avg_loss_gen: 2.2677277326583862 (+0.2874935269355774) + | > avg_loss_kl: 2.214187741279602 (-0.16077446937561035) + | > avg_loss_feat: 6.545548677444458 (+1.1000440120697021) + | > avg_loss_mel: 17.463073253631592 (+2.0837502479553223) + | > avg_loss_duration: 1.6213549375534058 (-0.025304555892944336) + | > avg_loss_1: 30.111892700195312 (+3.285207748413086) + + + > EPOCH: 25/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:25:14)  + + --> TIME: 2025-04-18 23:25:21 -- STEP: 18/142 -- GLOBAL_STEP: 382850 + | > loss_disc: 2.469383955001831 (2.4734345409605236) + | > loss_disc_real_0: 0.10815978050231934 (0.1250867885020044) + | > loss_disc_real_1: 0.1982145607471466 (0.2120005347662502) + | > loss_disc_real_2: 0.21978642046451569 (0.22983458638191223) + | > loss_disc_real_3: 0.2273937314748764 (0.22825669745604196) + | > loss_disc_real_4: 0.2208613157272339 (0.23318682445420158) + | > loss_disc_real_5: 0.2161020189523697 (0.22827019376887214) + | > loss_0: 2.469383955001831 (2.4734345409605236) + | > grad_norm_0: tensor(11.4193, device='cuda:0') (tensor(20.8539, device='cuda:0')) + | > loss_gen: 2.3331356048583984 (2.3866973055733576) + | > loss_kl: 2.0437381267547607 (2.0088335474332175) + | > loss_feat: 6.554663181304932 (7.292689032024807) + | > loss_mel: 17.741119384765625 (18.34073495864868) + | > loss_duration: 1.555005431175232 (1.5286971860461764) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.2276611328125 (31.55765151977539) + | > grad_norm_1: tensor(184.9855, device='cuda:0') (tensor(343.5033, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.3396 (0.33216585053337944) + | > loader_time: 0.0051 (0.004754106203715008) + + + --> TIME: 2025-04-18 23:25:30 -- STEP: 43/142 -- GLOBAL_STEP: 382875 + | > loss_disc: 2.500816822052002 (2.4735457120939746) + | > loss_disc_real_0: 0.11030843108892441 (0.12450972355382386) + | > loss_disc_real_1: 0.22927714884281158 (0.20980728816154393) + | > loss_disc_real_2: 0.23281562328338623 (0.22893878124481024) + | > loss_disc_real_3: 0.2328898012638092 (0.22922516874102658) + | > loss_disc_real_4: 0.22812528908252716 (0.23240929079610248) + | > loss_disc_real_5: 0.22412645816802979 (0.2333948442409205) + | > loss_0: 2.500816822052002 (2.4735457120939746) + | > grad_norm_0: tensor(12.1226, device='cuda:0') (tensor(14.3285, device='cuda:0')) + | > loss_gen: 2.3038244247436523 (2.3419772026150723) + | > loss_kl: 1.9647507667541504 (2.0054211810577756) + | > loss_feat: 6.351977825164795 (7.069972625998563) + | > loss_mel: 17.476106643676758 (17.976026645926538) + | > loss_duration: 1.5477608442306519 (1.5366067498229274) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.644420623779297 (30.930003986802213) + | > grad_norm_1: tensor(199.0625, device='cuda:0') (tensor(277.0176, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.3648 (0.34294314717137536) + | > loader_time: 0.0062 (0.005178662233574446) + + + --> TIME: 2025-04-18 23:25:39 -- STEP: 68/142 -- GLOBAL_STEP: 382900 + | > loss_disc: 2.3594512939453125 (2.4571337068782135) + | > loss_disc_real_0: 0.12544000148773193 (0.12721354897846193) + | > loss_disc_real_1: 0.17622442543506622 (0.20659054881509611) + | > loss_disc_real_2: 0.21582607924938202 (0.22598594514762654) + | > loss_disc_real_3: 0.17445605993270874 (0.2270605726277127) + | > loss_disc_real_4: 0.2184733897447586 (0.22954782514887698) + | > loss_disc_real_5: 0.18731528520584106 (0.22995074663092105) + | > loss_0: 2.3594512939453125 (2.4571337068782135) + | > grad_norm_0: tensor(7.6888, device='cuda:0') (tensor(12.1548, device='cuda:0')) + | > loss_gen: 2.3800835609436035 (2.3444315791130057) + | > loss_kl: 2.0909104347229004 (2.0134187270613273) + | > loss_feat: 7.787159442901611 (7.009002208709717) + | > loss_mel: 17.618011474609375 (17.910394388086655) + | > loss_duration: 1.5557186603546143 (1.5452986114165363) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.431884765625 (30.822545275968665) + | > grad_norm_1: tensor(183.4422, device='cuda:0') (tensor(232.4340, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.3862 (0.35430189791847677) + | > loader_time: 0.0062 (0.005509713116814109) + + + --> TIME: 2025-04-18 23:25:50 -- STEP: 93/142 -- GLOBAL_STEP: 382925 + | > loss_disc: 2.55153226852417 (2.4713882220688688) + | > loss_disc_real_0: 0.150660440325737 (0.1287958360006732) + | > loss_disc_real_1: 0.17854435741901398 (0.20647658407688144) + | > loss_disc_real_2: 0.24189263582229614 (0.22610566692967568) + | > loss_disc_real_3: 0.2340787649154663 (0.22935918582383022) + | > loss_disc_real_4: 0.22828219830989838 (0.23142267523273344) + | > loss_disc_real_5: 0.2214764505624771 (0.23428911932053104) + | > loss_0: 2.55153226852417 (2.4713882220688688) + | > grad_norm_0: tensor(4.9666, device='cuda:0') (tensor(11.5513, device='cuda:0')) + | > loss_gen: 2.24994158744812 (2.3393376335020983) + | > loss_kl: 2.105560064315796 (2.029737296924796) + | > loss_feat: 6.452354431152344 (6.976679099503384) + | > loss_mel: 18.719013214111328 (17.94010006484165) + | > loss_duration: 1.5733888149261475 (1.5567923681710356) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.100257873535156 (30.84264623990623) + | > grad_norm_1: tensor(224.9356, device='cuda:0') (tensor(229.2302, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.4041 (0.3666692190272833) + | > loader_time: 0.0064 (0.0057852319491806855) + + + --> TIME: 2025-04-18 23:26:01 -- STEP: 118/142 -- GLOBAL_STEP: 382950 + | > loss_disc: 2.520784616470337 (2.47305302700754) + | > loss_disc_real_0: 0.10899458080530167 (0.12963499450835136) + | > loss_disc_real_1: 0.17662566900253296 (0.20666171585099175) + | > loss_disc_real_2: 0.25542694330215454 (0.22710006065287833) + | > loss_disc_real_3: 0.23071032762527466 (0.229434124241441) + | > loss_disc_real_4: 0.2555433213710785 (0.23009955327389603) + | > loss_disc_real_5: 0.2736892104148865 (0.2333337482759508) + | > loss_0: 2.520784616470337 (2.47305302700754) + | > grad_norm_0: tensor(11.3525, device='cuda:0') (tensor(11.7977, device='cuda:0')) + | > loss_gen: 2.272725820541382 (2.331128611402997) + | > loss_kl: 2.021583318710327 (2.0326376062328535) + | > loss_feat: 6.882183074951172 (6.923998396275407) + | > loss_mel: 18.287370681762695 (17.87582155809564) + | > loss_duration: 1.5626336336135864 (1.5624522677922652) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.02649688720703 (30.726038286241433) + | > grad_norm_1: tensor(267.5891, device='cuda:0') (tensor(233.4002, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.454 (0.3796371059902643) + | > loader_time: 0.0072 (0.006083894584138515) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6062371730804443 (2.6062371730804443) + | > loss_disc_real_0: 0.1487339586019516 (0.1487339586019516) + | > loss_disc_real_1: 0.21761897206306458 (0.21761897206306458) + | > loss_disc_real_2: 0.272348016500473 (0.272348016500473) + | > loss_disc_real_3: 0.24209663271903992 (0.24209663271903992) + | > loss_disc_real_4: 0.2213384509086609 (0.2213384509086609) + | > loss_disc_real_5: 0.2305266261100769 (0.2305266261100769) + | > loss_0: 2.6062371730804443 (2.6062371730804443) + | > loss_gen: 2.1720712184906006 (2.1720712184906006) + | > loss_kl: 2.282716989517212 (2.282716989517212) + | > loss_feat: 5.537275314331055 (5.537275314331055) + | > loss_mel: 16.08202362060547 (16.08202362060547) + | > loss_duration: 1.6440215110778809 (1.6440215110778809) + | > loss_1: 27.718109130859375 (27.718109130859375) + + --> STEP: 1 + | > loss_disc: 2.49302077293396 (2.49302077293396) + | > loss_disc_real_0: 0.13461200892925262 (0.13461200892925262) + | > loss_disc_real_1: 0.23402659595012665 (0.23402659595012665) + | > loss_disc_real_2: 0.24523988366127014 (0.24523988366127014) + | > loss_disc_real_3: 0.23511850833892822 (0.23511850833892822) + | > loss_disc_real_4: 0.23290807008743286 (0.23290807008743286) + | > loss_disc_real_5: 0.18939656019210815 (0.18939656019210815) + | > loss_0: 2.49302077293396 (2.49302077293396) + | > loss_gen: 2.2144289016723633 (2.2144289016723633) + | > loss_kl: 2.2809112071990967 (2.2809112071990967) + | > loss_feat: 6.812273025512695 (6.812273025512695) + | > loss_mel: 18.362777709960938 (18.362777709960938) + | > loss_duration: 1.6498992443084717 (1.6498992443084717) + | > loss_1: 31.320289611816406 (31.320289611816406) + + --> STEP: 2 + | > loss_disc: 2.4213554859161377 (2.457188129425049) + | > loss_disc_real_0: 0.144977867603302 (0.1397949382662773) + | > loss_disc_real_1: 0.2226836085319519 (0.22835510224103928) + | > loss_disc_real_2: 0.22797246277332306 (0.2366061732172966) + | > loss_disc_real_3: 0.2339303195476532 (0.2345244139432907) + | > loss_disc_real_4: 0.2690262198448181 (0.2509671449661255) + | > loss_disc_real_5: 0.2223861813545227 (0.20589137077331543) + | > loss_0: 2.4213554859161377 (2.457188129425049) + | > loss_gen: 2.436398506164551 (2.325413703918457) + | > loss_kl: 2.094909906387329 (2.187910556793213) + | > loss_feat: 6.843807220458984 (6.82804012298584) + | > loss_mel: 18.05447006225586 (18.2086238861084) + | > loss_duration: 1.604777455329895 (1.6273383498191833) + | > loss_1: 31.03436279296875 (31.177326202392578) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003643512725830078 (-4.0650367736816406e-05) + | > avg_loss_disc: 2.457188129425049 (-0.07430088520050049) + | > avg_loss_disc_real_0: 0.1397949382662773 (+0.02066189795732498) + | > avg_loss_disc_real_1: 0.22835510224103928 (-0.04814700037240982) + | > avg_loss_disc_real_2: 0.2366061732172966 (+0.0013566240668296814) + | > avg_loss_disc_real_3: 0.2345244139432907 (+0.0055547356605529785) + | > avg_loss_disc_real_4: 0.2509671449661255 (-0.027031317353248596) + | > avg_loss_disc_real_5: 0.20589137077331543 (-0.03650502860546112) + | > avg_loss_0: 2.457188129425049 (-0.07430088520050049) + | > avg_loss_gen: 2.325413703918457 (+0.0576859712600708) + | > avg_loss_kl: 2.187910556793213 (-0.02627718448638916) + | > avg_loss_feat: 6.82804012298584 (+0.28249144554138184) + | > avg_loss_mel: 18.2086238861084 (+0.7455506324768066) + | > avg_loss_duration: 1.6273383498191833 (+0.005983412265777588) + | > avg_loss_1: 31.177326202392578 (+1.0654335021972656) + + + > EPOCH: 26/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:26:13)  + + --> TIME: 2025-04-18 23:26:14 -- STEP: 1/142 -- GLOBAL_STEP: 382975 + | > loss_disc: 2.382765769958496 (2.382765769958496) + | > loss_disc_real_0: 0.1525520533323288 (0.1525520533323288) + | > loss_disc_real_1: 0.21742989122867584 (0.21742989122867584) + | > loss_disc_real_2: 0.2656013071537018 (0.2656013071537018) + | > loss_disc_real_3: 0.23875883221626282 (0.23875883221626282) + | > loss_disc_real_4: 0.24158146977424622 (0.24158146977424622) + | > loss_disc_real_5: 0.22431005537509918 (0.22431005537509918) + | > loss_0: 2.382765769958496 (2.382765769958496) + | > grad_norm_0: tensor(11.8385, device='cuda:0') (tensor(11.8385, device='cuda:0')) + | > loss_gen: 2.579421043395996 (2.579421043395996) + | > loss_kl: 1.9428460597991943 (1.9428460597991943) + | > loss_feat: 7.945273399353027 (7.945273399353027) + | > loss_mel: 18.40932846069336 (18.40932846069336) + | > loss_duration: 1.485724687576294 (1.485724687576294) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.36259460449219 (32.36259460449219) + | > grad_norm_1: tensor(397.0233, device='cuda:0') (tensor(397.0233, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.3357 (0.33571648597717285) + | > loader_time: 0.005 (0.004998207092285156) + + + --> TIME: 2025-04-18 23:26:23 -- STEP: 26/142 -- GLOBAL_STEP: 383000 + | > loss_disc: 2.4860191345214844 (2.4743921298247122) + | > loss_disc_real_0: 0.11072983592748642 (0.12563034089711994) + | > loss_disc_real_1: 0.22696369886398315 (0.2149467422411992) + | > loss_disc_real_2: 0.2141619622707367 (0.225369488963714) + | > loss_disc_real_3: 0.2444932907819748 (0.23013703295817742) + | > loss_disc_real_4: 0.21272920072078705 (0.22408701594059283) + | > loss_disc_real_5: 0.25533387064933777 (0.23076483034170592) + | > loss_0: 2.4860191345214844 (2.4743921298247122) + | > grad_norm_0: tensor(9.0581, device='cuda:0') (tensor(10.4449, device='cuda:0')) + | > loss_gen: 2.382129669189453 (2.344993627988375) + | > loss_kl: 1.964455485343933 (1.9812493782777052) + | > loss_feat: 6.9515767097473145 (7.06128426698538) + | > loss_mel: 18.706409454345703 (17.967088259183445) + | > loss_duration: 1.535049557685852 (1.5305508054219759) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.53961944580078 (30.885166021493767) + | > grad_norm_1: tensor(323.4215, device='cuda:0') (tensor(253.5205, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.3493 (0.3328965718929584) + | > loader_time: 0.0054 (0.0050340707485492416) + + + --> TIME: 2025-04-18 23:26:32 -- STEP: 51/142 -- GLOBAL_STEP: 383025 + | > loss_disc: 2.465369701385498 (2.4799621993420176) + | > loss_disc_real_0: 0.1102585718035698 (0.12689400288988562) + | > loss_disc_real_1: 0.21488572657108307 (0.21392768270829143) + | > loss_disc_real_2: 0.21282930672168732 (0.22623812889351563) + | > loss_disc_real_3: 0.19766834378242493 (0.22991037865479788) + | > loss_disc_real_4: 0.22755126655101776 (0.22746504551055385) + | > loss_disc_real_5: 0.18919585645198822 (0.2329858556097629) + | > loss_0: 2.465369701385498 (2.4799621993420176) + | > grad_norm_0: tensor(12.7664, device='cuda:0') (tensor(11.2473, device='cuda:0')) + | > loss_gen: 2.4126956462860107 (2.337027764787861) + | > loss_kl: 1.9650300741195679 (1.9854312714408426) + | > loss_feat: 7.464449882507324 (6.995010871513217) + | > loss_mel: 17.921852111816406 (17.973749646953515) + | > loss_duration: 1.5186445713043213 (1.5353008625554103) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.282670974731445 (30.826520171819947) + | > grad_norm_1: tensor(378.9840, device='cuda:0') (tensor(268.3358, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.3694 (0.34500888282177494) + | > loader_time: 0.0064 (0.005386791977227903) + + + --> TIME: 2025-04-18 23:26:42 -- STEP: 76/142 -- GLOBAL_STEP: 383050 + | > loss_disc: 2.61205792427063 (2.492193946712896) + | > loss_disc_real_0: 0.08926008641719818 (0.13088305475876524) + | > loss_disc_real_1: 0.22667977213859558 (0.2125140412857658) + | > loss_disc_real_2: 0.25099313259124756 (0.22784147215516945) + | > loss_disc_real_3: 0.23332419991493225 (0.2300711936856571) + | > loss_disc_real_4: 0.24473263323307037 (0.22909888233009137) + | > loss_disc_real_5: 0.2719629406929016 (0.2323314333824735) + | > loss_0: 2.61205792427063 (2.492193946712896) + | > grad_norm_0: tensor(15.1487, device='cuda:0') (tensor(13.6573, device='cuda:0')) + | > loss_gen: 2.342755079269409 (2.332873162470366) + | > loss_kl: 1.8363395929336548 (1.998947211002049) + | > loss_feat: 5.7343525886535645 (6.946660675500569) + | > loss_mel: 16.6624755859375 (18.009694802133666) + | > loss_duration: 1.577561616897583 (1.5460741316017352) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.153484344482422 (30.834249847813656) + | > grad_norm_1: tensor(185.1969, device='cuda:0') (tensor(282.1884, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.3929 (0.3572414046839664) + | > loader_time: 0.0063 (0.005645344131871274) + + + --> TIME: 2025-04-18 23:26:52 -- STEP: 101/142 -- GLOBAL_STEP: 383075 + | > loss_disc: 2.5083224773406982 (2.4945733877691896) + | > loss_disc_real_0: 0.23221583664417267 (0.13526042178273204) + | > loss_disc_real_1: 0.20622166991233826 (0.2116369786533979) + | > loss_disc_real_2: 0.2284194827079773 (0.2269300793362136) + | > loss_disc_real_3: 0.22349081933498383 (0.23018410669104888) + | > loss_disc_real_4: 0.22937631607055664 (0.22862877541839485) + | > loss_disc_real_5: 0.2726351022720337 (0.23231218622462585) + | > loss_0: 2.5083224773406982 (2.4945733877691896) + | > grad_norm_0: tensor(40.1764, device='cuda:0') (tensor(15.0354, device='cuda:0')) + | > loss_gen: 2.334617853164673 (2.3303871178390954) + | > loss_kl: 2.220825433731079 (2.0181994119493094) + | > loss_feat: 6.469169616699219 (6.876687224548642) + | > loss_mel: 17.319347381591797 (17.930502287232056) + | > loss_duration: 1.610172986984253 (1.555991812507705) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.954132080078125 (30.711767668771273) + | > grad_norm_1: tensor(393.3305, device='cuda:0') (tensor(291.9579, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.4299 (0.3687122363855343) + | > loader_time: 0.0068 (0.00592444674803479) + + + --> TIME: 2025-04-18 23:27:03 -- STEP: 126/142 -- GLOBAL_STEP: 383100 + | > loss_disc: 2.5040740966796875 (2.4913737017010886) + | > loss_disc_real_0: 0.24445834755897522 (0.13362305629111476) + | > loss_disc_real_1: 0.1692245900630951 (0.21068599228821103) + | > loss_disc_real_2: 0.2614336311817169 (0.22709457044090545) + | > loss_disc_real_3: 0.25258439779281616 (0.23071889768517206) + | > loss_disc_real_4: 0.2415614277124405 (0.2288994657851401) + | > loss_disc_real_5: 0.2354673147201538 (0.23322592096196282) + | > loss_0: 2.5040740966796875 (2.4913737017010886) + | > grad_norm_0: tensor(15.2120, device='cuda:0') (tensor(16.1084, device='cuda:0')) + | > loss_gen: 2.445122241973877 (2.341176782335554) + | > loss_kl: 1.9073597192764282 (2.0196721705179366) + | > loss_feat: 6.695074081420898 (6.861669555542961) + | > loss_mel: 17.24126625061035 (17.866462124718566) + | > loss_duration: 1.550271987915039 (1.560984666385348) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.839094161987305 (30.649965150015696) + | > grad_norm_1: tensor(332.0166, device='cuda:0') (tensor(299.8083, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.4559 (0.38281307333991643) + | > loader_time: 0.0081 (0.006207761310395741) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5106756687164307 (2.5106756687164307) + | > loss_disc_real_0: 0.1038251668214798 (0.1038251668214798) + | > loss_disc_real_1: 0.2870110869407654 (0.2870110869407654) + | > loss_disc_real_2: 0.25971028208732605 (0.25971028208732605) + | > loss_disc_real_3: 0.25639626383781433 (0.25639626383781433) + | > loss_disc_real_4: 0.2738102674484253 (0.2738102674484253) + | > loss_disc_real_5: 0.25447747111320496 (0.25447747111320496) + | > loss_0: 2.5106756687164307 (2.5106756687164307) + | > loss_gen: 2.4617722034454346 (2.4617722034454346) + | > loss_kl: 2.037402629852295 (2.037402629852295) + | > loss_feat: 6.329528331756592 (6.329528331756592) + | > loss_mel: 18.072525024414062 (18.072525024414062) + | > loss_duration: 1.6211698055267334 (1.6211698055267334) + | > loss_1: 30.52239990234375 (30.52239990234375) + + --> STEP: 1 + | > loss_disc: 2.505237579345703 (2.505237579345703) + | > loss_disc_real_0: 0.10442692041397095 (0.10442692041397095) + | > loss_disc_real_1: 0.2821822464466095 (0.2821822464466095) + | > loss_disc_real_2: 0.27915236353874207 (0.27915236353874207) + | > loss_disc_real_3: 0.21329480409622192 (0.21329480409622192) + | > loss_disc_real_4: 0.22391200065612793 (0.22391200065612793) + | > loss_disc_real_5: 0.22621029615402222 (0.22621029615402222) + | > loss_0: 2.505237579345703 (2.505237579345703) + | > loss_gen: 2.2764618396759033 (2.2764618396759033) + | > loss_kl: 2.378892660140991 (2.378892660140991) + | > loss_feat: 7.147660255432129 (7.147660255432129) + | > loss_mel: 19.82496452331543 (19.82496452331543) + | > loss_duration: 1.6732051372528076 (1.6732051372528076) + | > loss_1: 33.301185607910156 (33.301185607910156) + + --> STEP: 2 + | > loss_disc: 2.5932462215423584 (2.5492419004440308) + | > loss_disc_real_0: 0.09791966527700424 (0.1011732928454876) + | > loss_disc_real_1: 0.31991350650787354 (0.3010478764772415) + | > loss_disc_real_2: 0.30365249514579773 (0.2914024293422699) + | > loss_disc_real_3: 0.2550475001335144 (0.23417115211486816) + | > loss_disc_real_4: 0.2619113326072693 (0.2429116666316986) + | > loss_disc_real_5: 0.28288397192955017 (0.2545471340417862) + | > loss_0: 2.5932462215423584 (2.5492419004440308) + | > loss_gen: 2.5130674839019775 (2.3947646617889404) + | > loss_kl: 2.2336630821228027 (2.306277871131897) + | > loss_feat: 8.392261505126953 (7.769960880279541) + | > loss_mel: 18.595399856567383 (19.210182189941406) + | > loss_duration: 1.5856510400772095 (1.6294280886650085) + | > loss_1: 33.32004165649414 (33.31061363220215) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034967660903930664 (-0.00014674663543701172) + | > avg_loss_disc: 2.5492419004440308 (+0.09205377101898193) + | > avg_loss_disc_real_0: 0.1011732928454876 (-0.03862164542078972) + | > avg_loss_disc_real_1: 0.3010478764772415 (+0.07269277423620224) + | > avg_loss_disc_real_2: 0.2914024293422699 (+0.0547962561249733) + | > avg_loss_disc_real_3: 0.23417115211486816 (-0.0003532618284225464) + | > avg_loss_disc_real_4: 0.2429116666316986 (-0.00805547833442688) + | > avg_loss_disc_real_5: 0.2545471340417862 (+0.048655763268470764) + | > avg_loss_0: 2.5492419004440308 (+0.09205377101898193) + | > avg_loss_gen: 2.3947646617889404 (+0.0693509578704834) + | > avg_loss_kl: 2.306277871131897 (+0.11836731433868408) + | > avg_loss_feat: 7.769960880279541 (+0.9419207572937012) + | > avg_loss_mel: 19.210182189941406 (+1.0015583038330078) + | > avg_loss_duration: 1.6294280886650085 (+0.0020897388458251953) + | > avg_loss_1: 33.31061363220215 (+2.1332874298095703) + + + > EPOCH: 27/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:27:12)  + + --> TIME: 2025-04-18 23:27:16 -- STEP: 9/142 -- GLOBAL_STEP: 383125 + | > loss_disc: 2.457090377807617 (2.509312550226847) + | > loss_disc_real_0: 0.13575458526611328 (0.13170015066862106) + | > loss_disc_real_1: 0.20341962575912476 (0.21433389517996046) + | > loss_disc_real_2: 0.2267504632472992 (0.22986058559682634) + | > loss_disc_real_3: 0.1924867480993271 (0.22866387996408674) + | > loss_disc_real_4: 0.22374778985977173 (0.2421028713385264) + | > loss_disc_real_5: 0.24196211993694305 (0.23027306463983324) + | > loss_0: 2.457090377807617 (2.509312550226847) + | > grad_norm_0: tensor(5.9332, device='cuda:0') (tensor(7.2782, device='cuda:0')) + | > loss_gen: 2.3026044368743896 (2.309496800104777) + | > loss_kl: 2.009122848510742 (2.001690321498447) + | > loss_feat: 7.019842624664307 (7.004520999060737) + | > loss_mel: 18.59195327758789 (18.058691660563152) + | > loss_duration: 1.4969937801361084 (1.5261419084337022) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.420516967773438 (30.900541729397244) + | > grad_norm_1: tensor(90.7083, device='cuda:0') (tensor(109.7395, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.3235 (0.3241678343878852) + | > loader_time: 0.0045 (0.004226393169826931) + + + --> TIME: 2025-04-18 23:27:25 -- STEP: 34/142 -- GLOBAL_STEP: 383150 + | > loss_disc: 2.462845802307129 (2.4596206580891327) + | > loss_disc_real_0: 0.1629943698644638 (0.12749280780553818) + | > loss_disc_real_1: 0.24411852657794952 (0.2076737271512256) + | > loss_disc_real_2: 0.1859041154384613 (0.22525906738112955) + | > loss_disc_real_3: 0.21685221791267395 (0.22841603878666372) + | > loss_disc_real_4: 0.21309098601341248 (0.23379875588066437) + | > loss_disc_real_5: 0.23405247926712036 (0.22541605199084563) + | > loss_0: 2.462845802307129 (2.4596206580891327) + | > grad_norm_0: tensor(11.0899, device='cuda:0') (tensor(6.8527, device='cuda:0')) + | > loss_gen: 2.336244583129883 (2.3276985182481655) + | > loss_kl: 2.034740447998047 (1.9675465471604292) + | > loss_feat: 6.662554740905762 (7.101014501908246) + | > loss_mel: 18.111177444458008 (18.169123200809253) + | > loss_duration: 1.5567107200622559 (1.527707906330333) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.701427459716797 (31.093090898850384) + | > grad_norm_1: tensor(219.7223, device='cuda:0') (tensor(136.5862, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.3646 (0.3412281765657313) + | > loader_time: 0.0056 (0.005127303740557502) + + + --> TIME: 2025-04-18 23:27:34 -- STEP: 59/142 -- GLOBAL_STEP: 383175 + | > loss_disc: 2.3988418579101562 (2.4658401254880227) + | > loss_disc_real_0: 0.11702896654605865 (0.12895733155941555) + | > loss_disc_real_1: 0.1909860223531723 (0.2072253979868808) + | > loss_disc_real_2: 0.23141543567180634 (0.22634966747235444) + | > loss_disc_real_3: 0.22306609153747559 (0.2299988039974439) + | > loss_disc_real_4: 0.20154006779193878 (0.23209621845665623) + | > loss_disc_real_5: 0.20796222984790802 (0.22926085328651688) + | > loss_0: 2.3988418579101562 (2.4658401254880227) + | > grad_norm_0: tensor(6.3105, device='cuda:0') (tensor(7.9201, device='cuda:0')) + | > loss_gen: 2.363978624343872 (2.3267586312051547) + | > loss_kl: 1.9989100694656372 (1.9807587417505554) + | > loss_feat: 7.2052507400512695 (7.064561173067254) + | > loss_mel: 17.224992752075195 (18.023451497999293) + | > loss_duration: 1.5473350286483765 (1.5363019664408797) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.34046745300293 (30.931832071078027) + | > grad_norm_1: tensor(230.8413, device='cuda:0') (tensor(172.3990, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.3854 (0.355769003851939) + | > loader_time: 0.006 (0.00549959328215001) + + + --> TIME: 2025-04-18 23:27:44 -- STEP: 84/142 -- GLOBAL_STEP: 383200 + | > loss_disc: 2.4498496055603027 (2.4701479134105497) + | > loss_disc_real_0: 0.13653980195522308 (0.13077066093683243) + | > loss_disc_real_1: 0.16924764215946198 (0.2057112395053818) + | > loss_disc_real_2: 0.21409863233566284 (0.22676141418161846) + | > loss_disc_real_3: 0.21029284596443176 (0.2302818608780702) + | > loss_disc_real_4: 0.2251613438129425 (0.23273685876102673) + | > loss_disc_real_5: 0.24396176636219025 (0.23035735530512674) + | > loss_0: 2.4498496055603027 (2.4701479134105497) + | > grad_norm_0: tensor(5.3169, device='cuda:0') (tensor(7.6340, device='cuda:0')) + | > loss_gen: 2.335756540298462 (2.3287326892217) + | > loss_kl: 2.014617443084717 (1.9832266115006947) + | > loss_feat: 7.068881511688232 (7.014325096493676) + | > loss_mel: 18.830322265625 (18.001526083265038) + | > loss_duration: 1.630378007888794 (1.547491148823784) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.87995719909668 (30.875301701681956) + | > grad_norm_1: tensor(96.1481, device='cuda:0') (tensor(166.7335, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.4084 (0.3691292405128479) + | > loader_time: 0.0065 (0.005782848312741233) + + + --> TIME: 2025-04-18 23:27:55 -- STEP: 109/142 -- GLOBAL_STEP: 383225 + | > loss_disc: 2.4967689514160156 (2.4771712220043205) + | > loss_disc_real_0: 0.13122540712356567 (0.13224460287105053) + | > loss_disc_real_1: 0.1821839064359665 (0.20743549539955383) + | > loss_disc_real_2: 0.22013850510120392 (0.2267291733157744) + | > loss_disc_real_3: 0.21566741168498993 (0.2302623180621261) + | > loss_disc_real_4: 0.23207619786262512 (0.23251440669965306) + | > loss_disc_real_5: 0.2513573169708252 (0.2313784024584184) + | > loss_0: 2.4967689514160156 (2.4771712220043205) + | > grad_norm_0: tensor(8.0407, device='cuda:0') (tensor(7.3957, device='cuda:0')) + | > loss_gen: 2.266038417816162 (2.319580331854864) + | > loss_kl: 1.9991226196289062 (1.9911997646366784) + | > loss_feat: 6.565087795257568 (6.966777880257423) + | > loss_mel: 17.62808609008789 (18.02324282794917) + | > loss_duration: 1.5801074504852295 (1.5546082772246197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.038442611694336 (30.85540916722849) + | > grad_norm_1: tensor(232.2050, device='cuda:0') (tensor(149.4594, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.426 (0.3806834723971306) + | > loader_time: 0.0068 (0.006030594537017543) + + + --> TIME: 2025-04-18 23:28:07 -- STEP: 134/142 -- GLOBAL_STEP: 383250 + | > loss_disc: 2.533534288406372 (2.4808837460048156) + | > loss_disc_real_0: 0.16571824252605438 (0.13262059306030838) + | > loss_disc_real_1: 0.240194171667099 (0.2075643834115854) + | > loss_disc_real_2: 0.2380923181772232 (0.22698086522408387) + | > loss_disc_real_3: 0.2355903834104538 (0.23022683198327448) + | > loss_disc_real_4: 0.18492993712425232 (0.23141696459766645) + | > loss_disc_real_5: 0.2684935927391052 (0.23287274421595816) + | > loss_0: 2.533534288406372 (2.4808837460048156) + | > grad_norm_0: tensor(13.6755, device='cuda:0') (tensor(7.6699, device='cuda:0')) + | > loss_gen: 2.278101921081543 (2.3121860756802914) + | > loss_kl: 2.1428463459014893 (1.9986883873370156) + | > loss_feat: 6.373961925506592 (6.90875125287184) + | > loss_mel: 18.39724349975586 (17.950232406160723) + | > loss_duration: 1.597575068473816 (1.5590528498834642) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.78972816467285 (30.728911001290847) + | > grad_norm_1: tensor(201.0462, device='cuda:0') (tensor(162.7571, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.4483 (0.39322435322092547) + | > loader_time: 0.008 (0.006300714478563906) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4488229751586914 (2.4488229751586914) + | > loss_disc_real_0: 0.19723102450370789 (0.19723102450370789) + | > loss_disc_real_1: 0.19542500376701355 (0.19542500376701355) + | > loss_disc_real_2: 0.23874397575855255 (0.23874397575855255) + | > loss_disc_real_3: 0.23039625585079193 (0.23039625585079193) + | > loss_disc_real_4: 0.2571730613708496 (0.2571730613708496) + | > loss_disc_real_5: 0.23642805218696594 (0.23642805218696594) + | > loss_0: 2.4488229751586914 (2.4488229751586914) + | > loss_gen: 2.5958306789398193 (2.5958306789398193) + | > loss_kl: 1.9996752738952637 (1.9996752738952637) + | > loss_feat: 7.125336647033691 (7.125336647033691) + | > loss_mel: 19.547672271728516 (19.547672271728516) + | > loss_duration: 1.627697229385376 (1.627697229385376) + | > loss_1: 32.89621353149414 (32.89621353149414) + + --> STEP: 1 + | > loss_disc: 2.426086664199829 (2.426086664199829) + | > loss_disc_real_0: 0.173597514629364 (0.173597514629364) + | > loss_disc_real_1: 0.20940068364143372 (0.20940068364143372) + | > loss_disc_real_2: 0.26203733682632446 (0.26203733682632446) + | > loss_disc_real_3: 0.22190134227275848 (0.22190134227275848) + | > loss_disc_real_4: 0.25139155983924866 (0.25139155983924866) + | > loss_disc_real_5: 0.22009341418743134 (0.22009341418743134) + | > loss_0: 2.426086664199829 (2.426086664199829) + | > loss_gen: 2.5759098529815674 (2.5759098529815674) + | > loss_kl: 2.2677736282348633 (2.2677736282348633) + | > loss_feat: 7.048893928527832 (7.048893928527832) + | > loss_mel: 19.006898880004883 (19.006898880004883) + | > loss_duration: 1.6585137844085693 (1.6585137844085693) + | > loss_1: 32.55799102783203 (32.55799102783203) + + --> STEP: 2 + | > loss_disc: 2.4662327766418457 (2.4461597204208374) + | > loss_disc_real_0: 0.18085050582885742 (0.17722401022911072) + | > loss_disc_real_1: 0.23375260829925537 (0.22157664597034454) + | > loss_disc_real_2: 0.2271309792995453 (0.24458415806293488) + | > loss_disc_real_3: 0.22086381912231445 (0.22138258069753647) + | > loss_disc_real_4: 0.23466885089874268 (0.24303020536899567) + | > loss_disc_real_5: 0.24557331204414368 (0.2328333631157875) + | > loss_0: 2.4662327766418457 (2.4461597204208374) + | > loss_gen: 2.5021491050720215 (2.5390294790267944) + | > loss_kl: 2.271970272064209 (2.269871950149536) + | > loss_feat: 6.011932849884033 (6.530413389205933) + | > loss_mel: 15.803031921386719 (17.4049654006958) + | > loss_duration: 1.5708621740341187 (1.614687979221344) + | > loss_1: 28.15994644165039 (30.35896873474121) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032843351364135742 (-0.0002124309539794922) + | > avg_loss_disc: 2.4461597204208374 (-0.10308218002319336) + | > avg_loss_disc_real_0: 0.17722401022911072 (+0.07605071738362312) + | > avg_loss_disc_real_1: 0.22157664597034454 (-0.07947123050689697) + | > avg_loss_disc_real_2: 0.24458415806293488 (-0.04681827127933502) + | > avg_loss_disc_real_3: 0.22138258069753647 (-0.012788571417331696) + | > avg_loss_disc_real_4: 0.24303020536899567 (+0.0001185387372970581) + | > avg_loss_disc_real_5: 0.2328333631157875 (-0.021713770925998688) + | > avg_loss_0: 2.4461597204208374 (-0.10308218002319336) + | > avg_loss_gen: 2.5390294790267944 (+0.144264817237854) + | > avg_loss_kl: 2.269871950149536 (-0.03640592098236084) + | > avg_loss_feat: 6.530413389205933 (-1.2395474910736084) + | > avg_loss_mel: 17.4049654006958 (-1.8052167892456055) + | > avg_loss_duration: 1.614687979221344 (-0.01474010944366455) + | > avg_loss_1: 30.35896873474121 (-2.9516448974609375) + + + > EPOCH: 28/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:28:12)  + + --> TIME: 2025-04-18 23:28:18 -- STEP: 17/142 -- GLOBAL_STEP: 383275 + | > loss_disc: 2.370771646499634 (2.4053241645588592) + | > loss_disc_real_0: 0.09049791097640991 (0.11461271345615387) + | > loss_disc_real_1: 0.2341858595609665 (0.20216033476240494) + | > loss_disc_real_2: 0.230988547205925 (0.22426944795776815) + | > loss_disc_real_3: 0.24439676105976105 (0.22748286320882685) + | > loss_disc_real_4: 0.2028561532497406 (0.22375081917818854) + | > loss_disc_real_5: 0.2363055944442749 (0.22720475319553823) + | > loss_0: 2.370771646499634 (2.4053241645588592) + | > grad_norm_0: tensor(13.6354, device='cuda:0') (tensor(14.1461, device='cuda:0')) + | > loss_gen: 2.522282600402832 (2.4033625406377457) + | > loss_kl: 1.9675251245498657 (1.9364709363264196) + | > loss_feat: 7.600464344024658 (7.40632085239186) + | > loss_mel: 17.893648147583008 (18.05742701362161) + | > loss_duration: 1.4964830875396729 (1.5129308209699743) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.480403900146484 (31.3165127249325) + | > grad_norm_1: tensor(348.6438, device='cuda:0') (tensor(346.7288, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.3422 (0.3278006525600658) + | > loader_time: 0.0052 (0.0047374472898595475) + + + --> TIME: 2025-04-18 23:28:27 -- STEP: 42/142 -- GLOBAL_STEP: 383300 + | > loss_disc: 2.549229145050049 (2.4315268198649087) + | > loss_disc_real_0: 0.09670912474393845 (0.11723494582942554) + | > loss_disc_real_1: 0.2317628264427185 (0.20612276203575589) + | > loss_disc_real_2: 0.24456901848316193 (0.2263429853178206) + | > loss_disc_real_3: 0.23636825382709503 (0.22793918493248166) + | > loss_disc_real_4: 0.24719348549842834 (0.2264890560791606) + | > loss_disc_real_5: 0.23998890817165375 (0.22784832297336488) + | > loss_0: 2.549229145050049 (2.4315268198649087) + | > grad_norm_0: tensor(11.2453, device='cuda:0') (tensor(16.0030, device='cuda:0')) + | > loss_gen: 2.328176259994507 (2.4022521518525624) + | > loss_kl: 2.0184438228607178 (1.95319911695662) + | > loss_feat: 7.189820289611816 (7.332337288629441) + | > loss_mel: 17.926904678344727 (18.176887603033162) + | > loss_duration: 1.5537216663360596 (1.5252822949772789) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.01706886291504 (31.389958790370397) + | > grad_norm_1: tensor(106.4460, device='cuda:0') (tensor(370.9211, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.371 (0.3464070217950003) + | > loader_time: 0.0059 (0.005339736030215309) + + + --> TIME: 2025-04-18 23:28:37 -- STEP: 67/142 -- GLOBAL_STEP: 383325 + | > loss_disc: 2.560317277908325 (2.441693021290337) + | > loss_disc_real_0: 0.11264095455408096 (0.12048504179093375) + | > loss_disc_real_1: 0.20671771466732025 (0.20628188906320885) + | > loss_disc_real_2: 0.24667246639728546 (0.22519821751473554) + | > loss_disc_real_3: 0.22441141307353973 (0.22774503635826396) + | > loss_disc_real_4: 0.25302210450172424 (0.22658024972944119) + | > loss_disc_real_5: 0.23057584464550018 (0.2328110830997353) + | > loss_0: 2.560317277908325 (2.441693021290337) + | > grad_norm_0: tensor(8.1297, device='cuda:0') (tensor(13.5028, device='cuda:0')) + | > loss_gen: 2.2432734966278076 (2.3768291544558395) + | > loss_kl: 2.0310537815093994 (1.9638802738332037) + | > loss_feat: 7.186522006988525 (7.181386876462111) + | > loss_mel: 17.489734649658203 (17.98998203562267) + | > loss_duration: 1.486182689666748 (1.5349056489432036) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.436765670776367 (31.04698406048675) + | > grad_norm_1: tensor(230.5995, device='cuda:0') (tensor(300.0420, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.3911 (0.3601708945943348) + | > loader_time: 0.0066 (0.005709580521085369) + + + --> TIME: 2025-04-18 23:28:47 -- STEP: 92/142 -- GLOBAL_STEP: 383350 + | > loss_disc: 2.5236053466796875 (2.456197432849717) + | > loss_disc_real_0: 0.09152505546808243 (0.12456025839175867) + | > loss_disc_real_1: 0.16057245433330536 (0.20561740466434023) + | > loss_disc_real_2: 0.25894129276275635 (0.22621488538773163) + | > loss_disc_real_3: 0.21964363753795624 (0.2276173496375913) + | > loss_disc_real_4: 0.22320890426635742 (0.22861786769784015) + | > loss_disc_real_5: 0.22700123488903046 (0.23307136511025223) + | > loss_0: 2.5236053466796875 (2.456197432849717) + | > grad_norm_0: tensor(7.7328, device='cuda:0') (tensor(12.9781, device='cuda:0')) + | > loss_gen: 2.3617680072784424 (2.357356004093005) + | > loss_kl: 1.887492299079895 (1.9815014639626378) + | > loss_feat: 7.609620094299316 (7.045664751011392) + | > loss_mel: 19.216167449951172 (17.985406585361652) + | > loss_duration: 1.5288909673690796 (1.5466321719729381) + | > amp_scaler: 256.0 (272.69565217391306) + | > loss_1: 32.603939056396484 (30.916561002316683) + | > grad_norm_1: tensor(291.0623, device='cuda:0') (tensor(286.5265, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.4245 (0.37344425398370495) + | > loader_time: 0.007 (0.0059476157893305235) + + + --> TIME: 2025-04-18 23:28:58 -- STEP: 117/142 -- GLOBAL_STEP: 383375 + | > loss_disc: 2.401686668395996 (2.4639063635443) + | > loss_disc_real_0: 0.12641124427318573 (0.12738607231623084) + | > loss_disc_real_1: 0.20449277758598328 (0.20713704735295385) + | > loss_disc_real_2: 0.1980748027563095 (0.22630431815090343) + | > loss_disc_real_3: 0.2232380211353302 (0.22857733274626935) + | > loss_disc_real_4: 0.22846665978431702 (0.22929065910160032) + | > loss_disc_real_5: 0.20190948247909546 (0.23243191175990635) + | > loss_0: 2.401686668395996 (2.4639063635443) + | > grad_norm_0: tensor(26.0962, device='cuda:0') (tensor(14.8790, device='cuda:0')) + | > loss_gen: 2.2610392570495605 (2.3592224141471423) + | > loss_kl: 1.8889676332473755 (1.9923154696440086) + | > loss_feat: 7.339395999908447 (7.024101530384813) + | > loss_mel: 17.695087432861328 (17.99091544518105) + | > loss_duration: 1.5544865131378174 (1.5530850948431554) + | > amp_scaler: 256.0 (269.1282051282051) + | > loss_1: 30.738977432250977 (30.91963999495547) + | > grad_norm_1: tensor(348.0917, device='cuda:0') (tensor(295.6799, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.4478 (0.3855996396806505) + | > loader_time: 0.0069 (0.006206113049107739) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4491097927093506 (2.4491097927093506) + | > loss_disc_real_0: 0.1029118075966835 (0.1029118075966835) + | > loss_disc_real_1: 0.24069160223007202 (0.24069160223007202) + | > loss_disc_real_2: 0.2588791251182556 (0.2588791251182556) + | > loss_disc_real_3: 0.2644505500793457 (0.2644505500793457) + | > loss_disc_real_4: 0.2661064863204956 (0.2661064863204956) + | > loss_disc_real_5: 0.2482345849275589 (0.2482345849275589) + | > loss_0: 2.4491097927093506 (2.4491097927093506) + | > loss_gen: 2.478687286376953 (2.478687286376953) + | > loss_kl: 2.1311216354370117 (2.1311216354370117) + | > loss_feat: 7.762266159057617 (7.762266159057617) + | > loss_mel: 19.89778709411621 (19.89778709411621) + | > loss_duration: 1.6336716413497925 (1.6336716413497925) + | > loss_1: 33.903533935546875 (33.903533935546875) + + --> STEP: 1 + | > loss_disc: 2.6815338134765625 (2.6815338134765625) + | > loss_disc_real_0: 0.14313964545726776 (0.14313964545726776) + | > loss_disc_real_1: 0.2761104702949524 (0.2761104702949524) + | > loss_disc_real_2: 0.24868661165237427 (0.24868661165237427) + | > loss_disc_real_3: 0.2752211391925812 (0.2752211391925812) + | > loss_disc_real_4: 0.23767872154712677 (0.23767872154712677) + | > loss_disc_real_5: 0.2854818105697632 (0.2854818105697632) + | > loss_0: 2.6815338134765625 (2.6815338134765625) + | > loss_gen: 2.2069642543792725 (2.2069642543792725) + | > loss_kl: 2.389965534210205 (2.389965534210205) + | > loss_feat: 5.968186855316162 (5.968186855316162) + | > loss_mel: 18.378562927246094 (18.378562927246094) + | > loss_duration: 1.649080753326416 (1.649080753326416) + | > loss_1: 30.59276008605957 (30.59276008605957) + + --> STEP: 2 + | > loss_disc: 2.3681368827819824 (2.5248353481292725) + | > loss_disc_real_0: 0.12445452809333801 (0.1337970867753029) + | > loss_disc_real_1: 0.2065495252609253 (0.24132999777793884) + | > loss_disc_real_2: 0.18545274436473846 (0.21706967800855637) + | > loss_disc_real_3: 0.20629462599754333 (0.24075788259506226) + | > loss_disc_real_4: 0.2460031658411026 (0.24184094369411469) + | > loss_disc_real_5: 0.26767751574516296 (0.2765796631574631) + | > loss_0: 2.3681368827819824 (2.5248353481292725) + | > loss_gen: 2.4768645763397217 (2.341914415359497) + | > loss_kl: 1.8601449728012085 (2.125055253505707) + | > loss_feat: 7.743948459625244 (6.856067657470703) + | > loss_mel: 18.4883975982666 (18.433480262756348) + | > loss_duration: 1.5882748365402222 (1.618677794933319) + | > loss_1: 32.157630920410156 (31.375195503234863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00404810905456543 (+0.0007637739181518555) + | > avg_loss_disc: 2.5248353481292725 (+0.07867562770843506) + | > avg_loss_disc_real_0: 0.1337970867753029 (-0.04342692345380783) + | > avg_loss_disc_real_1: 0.24132999777793884 (+0.0197533518075943) + | > avg_loss_disc_real_2: 0.21706967800855637 (-0.02751448005437851) + | > avg_loss_disc_real_3: 0.24075788259506226 (+0.019375301897525787) + | > avg_loss_disc_real_4: 0.24184094369411469 (-0.0011892616748809814) + | > avg_loss_disc_real_5: 0.2765796631574631 (+0.04374630004167557) + | > avg_loss_0: 2.5248353481292725 (+0.07867562770843506) + | > avg_loss_gen: 2.341914415359497 (-0.19711506366729736) + | > avg_loss_kl: 2.125055253505707 (-0.14481669664382935) + | > avg_loss_feat: 6.856067657470703 (+0.3256542682647705) + | > avg_loss_mel: 18.433480262756348 (+1.0285148620605469) + | > avg_loss_duration: 1.618677794933319 (+0.003989815711975098) + | > avg_loss_1: 31.375195503234863 (+1.0162267684936523) + + + > EPOCH: 29/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:29:11)  + + --> TIME: 2025-04-18 23:29:12 -- STEP: 0/142 -- GLOBAL_STEP: 383400 + | > loss_disc: 2.404280662536621 (2.404280662536621) + | > loss_disc_real_0: 0.08392801135778427 (0.08392801135778427) + | > loss_disc_real_1: 0.21434809267520905 (0.21434809267520905) + | > loss_disc_real_2: 0.2390623390674591 (0.2390623390674591) + | > loss_disc_real_3: 0.2267339676618576 (0.2267339676618576) + | > loss_disc_real_4: 0.23585684597492218 (0.23585684597492218) + | > loss_disc_real_5: 0.23680411279201508 (0.23680411279201508) + | > loss_0: 2.404280662536621 (2.404280662536621) + | > grad_norm_0: tensor(27.7884, device='cuda:0') (tensor(27.7884, device='cuda:0')) + | > loss_gen: 2.442535638809204 (2.442535638809204) + | > loss_kl: 1.8640081882476807 (1.8640081882476807) + | > loss_feat: 7.105715274810791 (7.105715274810791) + | > loss_mel: 17.40068817138672 (17.40068817138672) + | > loss_duration: 1.5464138984680176 (1.5464138984680176) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.359359741210938 (30.359359741210938) + | > grad_norm_1: tensor(373.3683, device='cuda:0') (tensor(373.3683, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.342 (0.3420412540435791) + | > loader_time: 0.3009 (0.3009188175201416) + + + --> TIME: 2025-04-18 23:29:21 -- STEP: 25/142 -- GLOBAL_STEP: 383425 + | > loss_disc: 2.363417863845825 (2.4455786418914793) + | > loss_disc_real_0: 0.10231368988752365 (0.11537162974476814) + | > loss_disc_real_1: 0.17352911829948425 (0.20767646849155427) + | > loss_disc_real_2: 0.20761385560035706 (0.22616271197795867) + | > loss_disc_real_3: 0.2108563780784607 (0.23028251826763152) + | > loss_disc_real_4: 0.18467360734939575 (0.23096556067466736) + | > loss_disc_real_5: 0.21416662633419037 (0.23398668348789214) + | > loss_0: 2.363417863845825 (2.4455786418914793) + | > grad_norm_0: tensor(5.1356, device='cuda:0') (tensor(13.7780, device='cuda:0')) + | > loss_gen: 2.3443410396575928 (2.375251979827881) + | > loss_kl: 2.0535013675689697 (1.9790607261657716) + | > loss_feat: 7.858930587768555 (7.276490650177002) + | > loss_mel: 18.544328689575195 (18.232717819213867) + | > loss_duration: 1.55311918258667 (1.5207075881958005) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.35422134399414 (31.384228744506835) + | > grad_norm_1: tensor(89.8411, device='cuda:0') (tensor(305.8525, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.3435 (0.3329285144805908) + | > loader_time: 0.0054 (0.004998331069946289) + + + --> TIME: 2025-04-18 23:29:30 -- STEP: 50/142 -- GLOBAL_STEP: 383450 + | > loss_disc: 2.4946045875549316 (2.4456851434707643) + | > loss_disc_real_0: 0.10993602126836777 (0.12049401231110096) + | > loss_disc_real_1: 0.18499568104743958 (0.20651873469352722) + | > loss_disc_real_2: 0.2292858064174652 (0.22406730443239212) + | > loss_disc_real_3: 0.2416708618402481 (0.2272950753569603) + | > loss_disc_real_4: 0.23240333795547485 (0.23335049510002137) + | > loss_disc_real_5: 0.24910114705562592 (0.22943356543779372) + | > loss_0: 2.4946045875549316 (2.4456851434707643) + | > grad_norm_0: tensor(12.0848, device='cuda:0') (tensor(11.2710, device='cuda:0')) + | > loss_gen: 2.329770565032959 (2.3592570161819455) + | > loss_kl: 2.065666913986206 (1.9849910712242127) + | > loss_feat: 6.489446640014648 (7.135518264770508) + | > loss_mel: 16.843158721923828 (18.051150512695322) + | > loss_duration: 1.5737264156341553 (1.5250535273551937) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.30177116394043 (31.0559704208374) + | > grad_norm_1: tensor(139.1275, device='cuda:0') (tensor(252.6590, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.3677 (0.3453053712844848) + | > loader_time: 0.0055 (0.00524813175201416) + + + --> TIME: 2025-04-18 23:29:40 -- STEP: 75/142 -- GLOBAL_STEP: 383475 + | > loss_disc: 2.327810049057007 (2.4588637065887453) + | > loss_disc_real_0: 0.11175432056188583 (0.12439209764202436) + | > loss_disc_real_1: 0.20421583950519562 (0.20654072503248852) + | > loss_disc_real_2: 0.19184604287147522 (0.22465102632840475) + | > loss_disc_real_3: 0.20379996299743652 (0.2283001341422399) + | > loss_disc_real_4: 0.22543522715568542 (0.23328085283438366) + | > loss_disc_real_5: 0.2304001897573471 (0.23049875219662985) + | > loss_0: 2.327810049057007 (2.4588637065887453) + | > grad_norm_0: tensor(4.2387, device='cuda:0') (tensor(9.9663, device='cuda:0')) + | > loss_gen: 2.3640964031219482 (2.3477523454030353) + | > loss_kl: 1.9640367031097412 (2.0009305731455487) + | > loss_feat: 7.136388778686523 (7.061067924499512) + | > loss_mel: 17.556371688842773 (17.98343846638998) + | > loss_duration: 1.5127856731414795 (1.5348951133092243) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.53367805480957 (30.928084462483724) + | > grad_norm_1: tensor(99.2581, device='cuda:0') (tensor(204.2328, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.4004 (0.3578869724273681) + | > loader_time: 0.0062 (0.0055600865681966136) + + + --> TIME: 2025-04-18 23:29:50 -- STEP: 100/142 -- GLOBAL_STEP: 383500 + | > loss_disc: 2.513988494873047 (2.4673754000663752) + | > loss_disc_real_0: 0.16135044395923615 (0.12669001389294862) + | > loss_disc_real_1: 0.19319234788417816 (0.20595974907279013) + | > loss_disc_real_2: 0.19616906344890594 (0.22654279708862304) + | > loss_disc_real_3: 0.21315769851207733 (0.22821330934762954) + | > loss_disc_real_4: 0.22784879803657532 (0.23285003378987312) + | > loss_disc_real_5: 0.20202548801898956 (0.22968848645687104) + | > loss_0: 2.513988494873047 (2.4673754000663752) + | > grad_norm_0: tensor(14.4471, device='cuda:0') (tensor(9.3802, device='cuda:0')) + | > loss_gen: 2.2571561336517334 (2.3315077304840086) + | > loss_kl: 2.152404308319092 (2.02042433142662) + | > loss_feat: 5.576135158538818 (6.974513902664184) + | > loss_mel: 15.51065444946289 (17.9982041168213) + | > loss_duration: 1.5501452684402466 (1.5457453072071072) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.04649543762207 (30.870395450592042) + | > grad_norm_1: tensor(150.9736, device='cuda:0') (tensor(189.1979, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.4187 (0.36891801834106436) + | > loader_time: 0.007 (0.005775380134582517) + + + --> TIME: 2025-04-18 23:30:01 -- STEP: 125/142 -- GLOBAL_STEP: 383525 + | > loss_disc: 2.427428960800171 (2.472247690200805) + | > loss_disc_real_0: 0.1452222615480423 (0.12806998643279077) + | > loss_disc_real_1: 0.18785713613033295 (0.2067267383337021) + | > loss_disc_real_2: 0.23862114548683167 (0.22705481827259064) + | > loss_disc_real_3: 0.24072721600532532 (0.22918941485881805) + | > loss_disc_real_4: 0.2106057107448578 (0.2327011753320694) + | > loss_disc_real_5: 0.22787360846996307 (0.2324488068819046) + | > loss_0: 2.427428960800171 (2.472247690200805) + | > grad_norm_0: tensor(8.4473, device='cuda:0') (tensor(9.3617, device='cuda:0')) + | > loss_gen: 2.304835557937622 (2.325556406021118) + | > loss_kl: 2.0654914379119873 (2.026210783958434) + | > loss_feat: 7.114658832550049 (6.918250736236573) + | > loss_mel: 18.766834259033203 (17.934117057800297) + | > loss_duration: 1.6010037660598755 (1.5521223840713498) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.85282325744629 (30.756257446289062) + | > grad_norm_1: tensor(202.6368, device='cuda:0') (tensor(190.7481, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.44 (0.3809008827209472) + | > loader_time: 0.0076 (0.006031274795532225) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4979312419891357 (2.4979312419891357) + | > loss_disc_real_0: 0.19405491650104523 (0.19405491650104523) + | > loss_disc_real_1: 0.2458133101463318 (0.2458133101463318) + | > loss_disc_real_2: 0.27129513025283813 (0.27129513025283813) + | > loss_disc_real_3: 0.228906512260437 (0.228906512260437) + | > loss_disc_real_4: 0.202589750289917 (0.202589750289917) + | > loss_disc_real_5: 0.25964266061782837 (0.25964266061782837) + | > loss_0: 2.4979312419891357 (2.4979312419891357) + | > loss_gen: 2.4846880435943604 (2.4846880435943604) + | > loss_kl: 2.0136303901672363 (2.0136303901672363) + | > loss_feat: 6.703514575958252 (6.703514575958252) + | > loss_mel: 17.557737350463867 (17.557737350463867) + | > loss_duration: 1.6531656980514526 (1.6531656980514526) + | > loss_1: 30.412736892700195 (30.412736892700195) + + --> STEP: 1 + | > loss_disc: 2.560427665710449 (2.560427665710449) + | > loss_disc_real_0: 0.16651032865047455 (0.16651032865047455) + | > loss_disc_real_1: 0.20868507027626038 (0.20868507027626038) + | > loss_disc_real_2: 0.2565035820007324 (0.2565035820007324) + | > loss_disc_real_3: 0.2291392683982849 (0.2291392683982849) + | > loss_disc_real_4: 0.2362229973077774 (0.2362229973077774) + | > loss_disc_real_5: 0.24150918424129486 (0.24150918424129486) + | > loss_0: 2.560427665710449 (2.560427665710449) + | > loss_gen: 2.2155466079711914 (2.2155466079711914) + | > loss_kl: 2.4656319618225098 (2.4656319618225098) + | > loss_feat: 6.495840072631836 (6.495840072631836) + | > loss_mel: 16.87444496154785 (16.87444496154785) + | > loss_duration: 1.6662945747375488 (1.6662945747375488) + | > loss_1: 29.717758178710938 (29.717758178710938) + + --> STEP: 2 + | > loss_disc: 2.630457878112793 (2.595442771911621) + | > loss_disc_real_0: 0.22205764055252075 (0.19428398460149765) + | > loss_disc_real_1: 0.2123233675956726 (0.2105042189359665) + | > loss_disc_real_2: 0.22389812767505646 (0.24020085483789444) + | > loss_disc_real_3: 0.21669958531856537 (0.22291942685842514) + | > loss_disc_real_4: 0.21428601443767548 (0.22525450587272644) + | > loss_disc_real_5: 0.24558164179325104 (0.24354541301727295) + | > loss_0: 2.630457878112793 (2.595442771911621) + | > loss_gen: 2.2195072174072266 (2.217526912689209) + | > loss_kl: 2.150160312652588 (2.307896137237549) + | > loss_feat: 6.970705509185791 (6.7332727909088135) + | > loss_mel: 18.40681266784668 (17.640628814697266) + | > loss_duration: 1.5962532758712769 (1.6312739253044128) + | > loss_1: 31.34343910217285 (30.530598640441895) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003306150436401367 (-0.0007419586181640625) + | > avg_loss_disc: 2.595442771911621 (+0.07060742378234863) + | > avg_loss_disc_real_0: 0.19428398460149765 (+0.06048689782619476) + | > avg_loss_disc_real_1: 0.2105042189359665 (-0.03082577884197235) + | > avg_loss_disc_real_2: 0.24020085483789444 (+0.023131176829338074) + | > avg_loss_disc_real_3: 0.22291942685842514 (-0.017838455736637115) + | > avg_loss_disc_real_4: 0.22525450587272644 (-0.016586437821388245) + | > avg_loss_disc_real_5: 0.24354541301727295 (-0.033034250140190125) + | > avg_loss_0: 2.595442771911621 (+0.07060742378234863) + | > avg_loss_gen: 2.217526912689209 (-0.12438750267028809) + | > avg_loss_kl: 2.307896137237549 (+0.18284088373184204) + | > avg_loss_feat: 6.7332727909088135 (-0.12279486656188965) + | > avg_loss_mel: 17.640628814697266 (-0.792851448059082) + | > avg_loss_duration: 1.6312739253044128 (+0.01259613037109375) + | > avg_loss_1: 30.530598640441895 (-0.8445968627929688) + + + > EPOCH: 30/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:30:10)  + + --> TIME: 2025-04-18 23:30:13 -- STEP: 8/142 -- GLOBAL_STEP: 383550 + | > loss_disc: 2.368405818939209 (2.439185231924057) + | > loss_disc_real_0: 0.09026654064655304 (0.11845065746456385) + | > loss_disc_real_1: 0.18265900015830994 (0.2042416613548994) + | > loss_disc_real_2: 0.21595293283462524 (0.21734696812927723) + | > loss_disc_real_3: 0.1823497861623764 (0.22281592339277267) + | > loss_disc_real_4: 0.17345203459262848 (0.23062223754823208) + | > loss_disc_real_5: 0.20463070273399353 (0.2302214428782463) + | > loss_0: 2.368405818939209 (2.439185231924057) + | > grad_norm_0: tensor(5.6309, device='cuda:0') (tensor(7.5220, device='cuda:0')) + | > loss_gen: 2.3227157592773438 (2.3136261105537415) + | > loss_kl: 1.8826501369476318 (1.9096326231956482) + | > loss_feat: 7.426687240600586 (7.295899033546448) + | > loss_mel: 18.7481689453125 (18.355578660964966) + | > loss_duration: 1.5218631029129028 (1.5240519642829895) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.90208625793457 (31.39878797531128) + | > grad_norm_1: tensor(108.3560, device='cuda:0') (tensor(131.1193, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.3242 (0.3227314054965973) + | > loader_time: 0.0047 (0.004288136959075928) + + + --> TIME: 2025-04-18 23:30:22 -- STEP: 33/142 -- GLOBAL_STEP: 383575 + | > loss_disc: 2.4530134201049805 (2.4481194525054004) + | > loss_disc_real_0: 0.12478481978178024 (0.12225985978588913) + | > loss_disc_real_1: 0.21719051897525787 (0.2037685779911099) + | > loss_disc_real_2: 0.21858733892440796 (0.22433012678767694) + | > loss_disc_real_3: 0.2219369113445282 (0.23106537985079217) + | > loss_disc_real_4: 0.19669313728809357 (0.22944704014243503) + | > loss_disc_real_5: 0.22196413576602936 (0.2293753127257029) + | > loss_0: 2.4530134201049805 (2.4481194525054004) + | > grad_norm_0: tensor(4.5526, device='cuda:0') (tensor(7.6778, device='cuda:0')) + | > loss_gen: 2.335338830947876 (2.3296769965778696) + | > loss_kl: 1.9045910835266113 (1.9825269489577322) + | > loss_feat: 6.235131740570068 (7.105742324482311) + | > loss_mel: 16.42359161376953 (17.890697594844934) + | > loss_duration: 1.5109320878982544 (1.5224714640415076) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.40958595275879 (30.831115318067145) + | > grad_norm_1: tensor(74.1501, device='cuda:0') (tensor(109.2848, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.366 (0.33927757812268805) + | > loader_time: 0.0051 (0.0048693960363214665) + + + --> TIME: 2025-04-18 23:30:32 -- STEP: 58/142 -- GLOBAL_STEP: 383600 + | > loss_disc: 2.4759247303009033 (2.450467220668134) + | > loss_disc_real_0: 0.1624877005815506 (0.12420652742529738) + | > loss_disc_real_1: 0.19112563133239746 (0.20240262818747554) + | > loss_disc_real_2: 0.21206001937389374 (0.22374749928712845) + | > loss_disc_real_3: 0.22545284032821655 (0.22907689881735835) + | > loss_disc_real_4: 0.22838328778743744 (0.23159755869158383) + | > loss_disc_real_5: 0.20422981679439545 (0.23000379369176668) + | > loss_0: 2.4759247303009033 (2.450467220668134) + | > grad_norm_0: tensor(7.5036, device='cuda:0') (tensor(7.6338, device='cuda:0')) + | > loss_gen: 2.37597918510437 (2.3444760955613235) + | > loss_kl: 2.04940128326416 (1.9903929459637608) + | > loss_feat: 6.208436012268066 (7.154680868674969) + | > loss_mel: 18.453794479370117 (18.117065265260894) + | > loss_duration: 1.569197416305542 (1.5314048487564613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.65680694580078 (31.13801992350611) + | > grad_norm_1: tensor(135.3482, device='cuda:0') (tensor(126.9672, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.3814 (0.3535702927359219) + | > loader_time: 0.006 (0.0053029553643588355) + + + --> TIME: 2025-04-18 23:30:42 -- STEP: 83/142 -- GLOBAL_STEP: 383625 + | > loss_disc: 2.4649274349212646 (2.464495087244426) + | > loss_disc_real_0: 0.14565129578113556 (0.12898897182151495) + | > loss_disc_real_1: 0.21703854203224182 (0.20553563642932707) + | > loss_disc_real_2: 0.23560954630374908 (0.225191696401102) + | > loss_disc_real_3: 0.21446678042411804 (0.22942730856229024) + | > loss_disc_real_4: 0.2263854593038559 (0.2318931054997157) + | > loss_disc_real_5: 0.19887609779834747 (0.22975856137562947) + | > loss_0: 2.4649274349212646 (2.464495087244426) + | > grad_norm_0: tensor(5.9855, device='cuda:0') (tensor(7.6034, device='cuda:0')) + | > loss_gen: 2.2900261878967285 (2.336373153939304) + | > loss_kl: 2.0872983932495117 (1.9961238797888698) + | > loss_feat: 6.4782609939575195 (7.093110911817436) + | > loss_mel: 18.700937271118164 (18.046820192451936) + | > loss_duration: 1.6431328058242798 (1.5418588569365352) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.199655532836914 (31.014287006424134) + | > grad_norm_1: tensor(105.7597, device='cuda:0') (tensor(128.4691, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.4113 (0.37051775082048155) + | > loader_time: 0.0063 (0.005609632974647614) + + + --> TIME: 2025-04-18 23:30:53 -- STEP: 108/142 -- GLOBAL_STEP: 383650 + | > loss_disc: 2.5149528980255127 (2.4739574436788225) + | > loss_disc_real_0: 0.1327272206544876 (0.13023992821022315) + | > loss_disc_real_1: 0.23723378777503967 (0.2060011645985974) + | > loss_disc_real_2: 0.2363218069076538 (0.22685286140552274) + | > loss_disc_real_3: 0.22156241536140442 (0.229865953464199) + | > loss_disc_real_4: 0.23522129654884338 (0.23219213179416126) + | > loss_disc_real_5: 0.27654996514320374 (0.23067028599756736) + | > loss_0: 2.5149528980255127 (2.4739574436788225) + | > grad_norm_0: tensor(10.9892, device='cuda:0') (tensor(7.5549, device='cuda:0')) + | > loss_gen: 2.3194546699523926 (2.3253380612090773) + | > loss_kl: 1.9570260047912598 (2.0014722082349987) + | > loss_feat: 6.762699604034424 (7.008367458979289) + | > loss_mel: 18.93926429748535 (18.045201919696947) + | > loss_duration: 1.5700690746307373 (1.5498879838872839) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.548511505126953 (30.930267651875813) + | > grad_norm_1: tensor(336.2861, device='cuda:0') (tensor(130.8270, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.4344 (0.38186882381085996) + | > loader_time: 0.0075 (0.00588473567256221) + + + --> TIME: 2025-04-18 23:31:04 -- STEP: 133/142 -- GLOBAL_STEP: 383675 + | > loss_disc: 2.5178427696228027 (2.474870504293228) + | > loss_disc_real_0: 0.10685193538665771 (0.13076497239053694) + | > loss_disc_real_1: 0.2617417871952057 (0.20741609220666096) + | > loss_disc_real_2: 0.21450062096118927 (0.22702217292516752) + | > loss_disc_real_3: 0.2254822552204132 (0.2296086080316314) + | > loss_disc_real_4: 0.211561918258667 (0.23120785151657305) + | > loss_disc_real_5: 0.2161913514137268 (0.2301935025847944) + | > loss_0: 2.5178427696228027 (2.474870504293228) + | > grad_norm_0: tensor(9.1337, device='cuda:0') (tensor(8.2568, device='cuda:0')) + | > loss_gen: 2.294318914413452 (2.3278648638187485) + | > loss_kl: 2.0703346729278564 (2.0046322829741285) + | > loss_feat: 6.407938480377197 (6.957620868109222) + | > loss_mel: 17.54314613342285 (17.96282708734498) + | > loss_duration: 1.6573529243469238 (1.5548312448917474) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.97309112548828 (30.807776286189718) + | > grad_norm_1: tensor(247.9222, device='cuda:0') (tensor(151.7096, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.4469 (0.39447447411099773) + | > loader_time: 0.0078 (0.006161238017835115) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5647737979888916 (2.5647737979888916) + | > loss_disc_real_0: 0.12300018221139908 (0.12300018221139908) + | > loss_disc_real_1: 0.22059603035449982 (0.22059603035449982) + | > loss_disc_real_2: 0.25197502970695496 (0.25197502970695496) + | > loss_disc_real_3: 0.2951783537864685 (0.2951783537864685) + | > loss_disc_real_4: 0.2708972096443176 (0.2708972096443176) + | > loss_disc_real_5: 0.26974350214004517 (0.26974350214004517) + | > loss_0: 2.5647737979888916 (2.5647737979888916) + | > loss_gen: 2.2875022888183594 (2.2875022888183594) + | > loss_kl: 2.2232024669647217 (2.2232024669647217) + | > loss_feat: 6.830366134643555 (6.830366134643555) + | > loss_mel: 20.343095779418945 (20.343095779418945) + | > loss_duration: 1.6023824214935303 (1.6023824214935303) + | > loss_1: 33.28654861450195 (33.28654861450195) + + --> STEP: 1 + | > loss_disc: 2.515942335128784 (2.515942335128784) + | > loss_disc_real_0: 0.1207849308848381 (0.1207849308848381) + | > loss_disc_real_1: 0.23853692412376404 (0.23853692412376404) + | > loss_disc_real_2: 0.2230132818222046 (0.2230132818222046) + | > loss_disc_real_3: 0.2612968385219574 (0.2612968385219574) + | > loss_disc_real_4: 0.21292205154895782 (0.21292205154895782) + | > loss_disc_real_5: 0.23944352567195892 (0.23944352567195892) + | > loss_0: 2.515942335128784 (2.515942335128784) + | > loss_gen: 2.1692395210266113 (2.1692395210266113) + | > loss_kl: 2.210345983505249 (2.210345983505249) + | > loss_feat: 6.090103626251221 (6.090103626251221) + | > loss_mel: 17.33662223815918 (17.33662223815918) + | > loss_duration: 1.6609939336776733 (1.6609939336776733) + | > loss_1: 29.467304229736328 (29.467304229736328) + + --> STEP: 2 + | > loss_disc: 2.5619912147521973 (2.5389667749404907) + | > loss_disc_real_0: 0.13568158447742462 (0.12823325768113136) + | > loss_disc_real_1: 0.22952914237976074 (0.2340330332517624) + | > loss_disc_real_2: 0.20615935325622559 (0.2145863175392151) + | > loss_disc_real_3: 0.22032707929611206 (0.24081195890903473) + | > loss_disc_real_4: 0.2382732629776001 (0.22559765726327896) + | > loss_disc_real_5: 0.2565271258354187 (0.2479853257536888) + | > loss_0: 2.5619912147521973 (2.5389667749404907) + | > loss_gen: 2.117922782897949 (2.1435811519622803) + | > loss_kl: 2.21222186088562 (2.2112839221954346) + | > loss_feat: 6.889522075653076 (6.489812850952148) + | > loss_mel: 16.77916145324707 (17.057891845703125) + | > loss_duration: 1.5894112586975098 (1.6252025961875916) + | > loss_1: 29.588239669799805 (29.527771949768066) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003744363784790039 (+0.0004382133483886719) + | > avg_loss_disc: 2.5389667749404907 (-0.05647599697113037) + | > avg_loss_disc_real_0: 0.12823325768113136 (-0.06605072692036629) + | > avg_loss_disc_real_1: 0.2340330332517624 (+0.0235288143157959) + | > avg_loss_disc_real_2: 0.2145863175392151 (-0.025614537298679352) + | > avg_loss_disc_real_3: 0.24081195890903473 (+0.01789253205060959) + | > avg_loss_disc_real_4: 0.22559765726327896 (+0.00034315139055252075) + | > avg_loss_disc_real_5: 0.2479853257536888 (+0.004439912736415863) + | > avg_loss_0: 2.5389667749404907 (-0.05647599697113037) + | > avg_loss_gen: 2.1435811519622803 (-0.07394576072692871) + | > avg_loss_kl: 2.2112839221954346 (-0.09661221504211426) + | > avg_loss_feat: 6.489812850952148 (-0.24345993995666504) + | > avg_loss_mel: 17.057891845703125 (-0.5827369689941406) + | > avg_loss_duration: 1.6252025961875916 (-0.006071329116821289) + | > avg_loss_1: 29.527771949768066 (-1.0028266906738281) + + + > EPOCH: 31/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:31:10)  + + --> TIME: 2025-04-18 23:31:16 -- STEP: 16/142 -- GLOBAL_STEP: 383700 + | > loss_disc: 2.5508275032043457 (2.4331215620040894) + | > loss_disc_real_0: 0.11710266023874283 (0.12112339725717902) + | > loss_disc_real_1: 0.21565742790699005 (0.204303833656013) + | > loss_disc_real_2: 0.221009761095047 (0.2186245657503605) + | > loss_disc_real_3: 0.22848699986934662 (0.22292810678482056) + | > loss_disc_real_4: 0.21167023479938507 (0.22683893330395222) + | > loss_disc_real_5: 0.2507149875164032 (0.2333567412570119) + | > loss_0: 2.5508275032043457 (2.4331215620040894) + | > grad_norm_0: tensor(10.3402, device='cuda:0') (tensor(12.2657, device='cuda:0')) + | > loss_gen: 2.363752841949463 (2.3980691879987717) + | > loss_kl: 1.9924530982971191 (1.9681362882256508) + | > loss_feat: 7.012413024902344 (7.386266261339188) + | > loss_mel: 18.1226863861084 (18.23672270774841) + | > loss_duration: 1.4959850311279297 (1.5135766416788101) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.987289428710938 (31.50277078151703) + | > grad_norm_1: tensor(308.9324, device='cuda:0') (tensor(279.4922, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.3382 (0.3275030106306076) + | > loader_time: 0.0053 (0.004738286137580872) + + + --> TIME: 2025-04-18 23:31:25 -- STEP: 41/142 -- GLOBAL_STEP: 383725 + | > loss_disc: 2.409599781036377 (2.449230176646535) + | > loss_disc_real_0: 0.08745042979717255 (0.12051873526922087) + | > loss_disc_real_1: 0.16098661720752716 (0.20858353848864392) + | > loss_disc_real_2: 0.17577676475048065 (0.2230230144611219) + | > loss_disc_real_3: 0.21656213700771332 (0.226607042841795) + | > loss_disc_real_4: 0.2561107873916626 (0.23298291280502226) + | > loss_disc_real_5: 0.2211921513080597 (0.23202099451204625) + | > loss_0: 2.409599781036377 (2.449230176646535) + | > grad_norm_0: tensor(11.4339, device='cuda:0') (tensor(10.9843, device='cuda:0')) + | > loss_gen: 2.4410457611083984 (2.387712519343307) + | > loss_kl: 2.1243560314178467 (1.9858785140805129) + | > loss_feat: 7.416348457336426 (7.221019047062572) + | > loss_mel: 17.720487594604492 (18.13865615100395) + | > loss_duration: 1.500260353088379 (1.522174457224404) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.202499389648438 (31.255440781756146) + | > grad_norm_1: tensor(341.2433, device='cuda:0') (tensor(264.8208, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.3717 (0.34343094942046376) + | > loader_time: 0.0061 (0.00516489656960092) + + + --> TIME: 2025-04-18 23:31:35 -- STEP: 66/142 -- GLOBAL_STEP: 383750 + | > loss_disc: 2.501361608505249 (2.461326805028049) + | > loss_disc_real_0: 0.14373241364955902 (0.12463614130110452) + | > loss_disc_real_1: 0.24194373190402985 (0.21003959982684164) + | > loss_disc_real_2: 0.2061663269996643 (0.22587605046503473) + | > loss_disc_real_3: 0.21593327820301056 (0.22754289458195368) + | > loss_disc_real_4: 0.2358655035495758 (0.23123951956178201) + | > loss_disc_real_5: 0.25345170497894287 (0.2324775199998509) + | > loss_0: 2.501361608505249 (2.461326805028049) + | > grad_norm_0: tensor(9.3189, device='cuda:0') (tensor(11.6834, device='cuda:0')) + | > loss_gen: 2.0700290203094482 (2.3698668371547362) + | > loss_kl: 2.0842480659484863 (1.989208297296004) + | > loss_feat: 6.054381370544434 (7.0955049124631016) + | > loss_mel: 17.559539794921875 (18.075928254561) + | > loss_duration: 1.4996135234832764 (1.5330440149162754) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.267810821533203 (31.063552422956988) + | > grad_norm_1: tensor(51.1131, device='cuda:0') (tensor(265.5823, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.3919 (0.35803138848507043) + | > loader_time: 0.006 (0.005494403116630786) + + + --> TIME: 2025-04-18 23:31:45 -- STEP: 91/142 -- GLOBAL_STEP: 383775 + | > loss_disc: 2.415177822113037 (2.4660253970177624) + | > loss_disc_real_0: 0.14828145503997803 (0.12720883711353764) + | > loss_disc_real_1: 0.17076802253723145 (0.20964916525306282) + | > loss_disc_real_2: 0.21511833369731903 (0.22513088567571327) + | > loss_disc_real_3: 0.2160564512014389 (0.22890452762226482) + | > loss_disc_real_4: 0.18902988731861115 (0.23054224154451392) + | > loss_disc_real_5: 0.21958045661449432 (0.23092022931182776) + | > loss_0: 2.415177822113037 (2.4660253970177624) + | > grad_norm_0: tensor(20.7180, device='cuda:0') (tensor(12.1932, device='cuda:0')) + | > loss_gen: 2.416213274002075 (2.3550256687206237) + | > loss_kl: 1.9644980430603027 (2.0003197952941227) + | > loss_feat: 7.925628662109375 (6.993395763439136) + | > loss_mel: 18.073442459106445 (18.004809714935647) + | > loss_duration: 1.5605554580688477 (1.5432674400099031) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.940338134765625 (30.896818538288493) + | > grad_norm_1: tensor(516.7943, device='cuda:0') (tensor(275.1528, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.4132 (0.37197507344759445) + | > loader_time: 0.0062 (0.005757151069221917) + + + --> TIME: 2025-04-18 23:31:56 -- STEP: 116/142 -- GLOBAL_STEP: 383800 + | > loss_disc: 2.4847192764282227 (2.4698859535414597) + | > loss_disc_real_0: 0.18647322058677673 (0.12705586176237152) + | > loss_disc_real_1: 0.2124451994895935 (0.21039068339199857) + | > loss_disc_real_2: 0.22297869622707367 (0.2252869647124718) + | > loss_disc_real_3: 0.22421598434448242 (0.22910301515768314) + | > loss_disc_real_4: 0.22032098472118378 (0.23033373443217114) + | > loss_disc_real_5: 0.25584515929222107 (0.23174143685349102) + | > loss_0: 2.4847192764282227 (2.4698859535414597) + | > grad_norm_0: tensor(7.4981, device='cuda:0') (tensor(12.3947, device='cuda:0')) + | > loss_gen: 2.3863017559051514 (2.3577390847534967) + | > loss_kl: 1.9637601375579834 (2.0018321029071147) + | > loss_feat: 6.384407997131348 (6.932423410744502) + | > loss_mel: 17.788217544555664 (17.915676240263316) + | > loss_duration: 1.5344935655593872 (1.5484445598618737) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05718231201172 (30.756115502324597) + | > grad_norm_1: tensor(323.1922, device='cuda:0') (tensor(283.1314, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.4492 (0.38421966700718335) + | > loader_time: 0.0073 (0.006003036581236742) + + + --> TIME: 2025-04-18 23:32:07 -- STEP: 141/142 -- GLOBAL_STEP: 383825 + | > loss_disc: 2.5471301078796387 (2.471417968154799) + | > loss_disc_real_0: 0.27837514877319336 (0.12780434786216588) + | > loss_disc_real_1: 0.16874127089977264 (0.20945136079974208) + | > loss_disc_real_2: 0.15529483556747437 (0.2251445604554305) + | > loss_disc_real_3: 0.20544545352458954 (0.22917281640759596) + | > loss_disc_real_4: 0.17215777933597565 (0.2298641822016831) + | > loss_disc_real_5: 0.19289825856685638 (0.23084631604505768) + | > loss_0: 2.5471301078796387 (2.471417968154799) + | > grad_norm_0: tensor(65.4469, device='cuda:0') (tensor(13.7355, device='cuda:0')) + | > loss_gen: 2.30688214302063 (2.354010429788143) + | > loss_kl: 1.809685468673706 (2.0021795989773805) + | > loss_feat: 8.184358596801758 (6.927403893031127) + | > loss_mel: 18.56117820739746 (17.937728604526384) + | > loss_duration: 1.4337127208709717 (1.5529400972609821) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.29581832885742 (30.77426271912054) + | > grad_norm_1: tensor(359.1359, device='cuda:0') (tensor(298.2779, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.21 (0.39500521429887064) + | > loader_time: 0.0026 (0.00624912512217853) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5248184204101562 (2.5248184204101562) + | > loss_disc_real_0: 0.10685554891824722 (0.10685554891824722) + | > loss_disc_real_1: 0.2363021969795227 (0.2363021969795227) + | > loss_disc_real_2: 0.27544480562210083 (0.27544480562210083) + | > loss_disc_real_3: 0.25846341252326965 (0.25846341252326965) + | > loss_disc_real_4: 0.24972917139530182 (0.24972917139530182) + | > loss_disc_real_5: 0.2752837538719177 (0.2752837538719177) + | > loss_0: 2.5248184204101562 (2.5248184204101562) + | > loss_gen: 2.434105634689331 (2.434105634689331) + | > loss_kl: 2.022369146347046 (2.022369146347046) + | > loss_feat: 7.175604820251465 (7.175604820251465) + | > loss_mel: 18.88205337524414 (18.88205337524414) + | > loss_duration: 1.626199722290039 (1.626199722290039) + | > loss_1: 32.14033508300781 (32.14033508300781) + + --> STEP: 1 + | > loss_disc: 2.3780438899993896 (2.3780438899993896) + | > loss_disc_real_0: 0.11266728490591049 (0.11266728490591049) + | > loss_disc_real_1: 0.2679097652435303 (0.2679097652435303) + | > loss_disc_real_2: 0.25660520792007446 (0.25660520792007446) + | > loss_disc_real_3: 0.22642666101455688 (0.22642666101455688) + | > loss_disc_real_4: 0.26965752243995667 (0.26965752243995667) + | > loss_disc_real_5: 0.26336514949798584 (0.26336514949798584) + | > loss_0: 2.3780438899993896 (2.3780438899993896) + | > loss_gen: 2.7132365703582764 (2.7132365703582764) + | > loss_kl: 2.3117175102233887 (2.3117175102233887) + | > loss_feat: 7.638877868652344 (7.638877868652344) + | > loss_mel: 19.171619415283203 (19.171619415283203) + | > loss_duration: 1.6573445796966553 (1.6573445796966553) + | > loss_1: 33.492794036865234 (33.492794036865234) + + --> STEP: 2 + | > loss_disc: 2.556213855743408 (2.467128872871399) + | > loss_disc_real_0: 0.12496328353881836 (0.11881528422236443) + | > loss_disc_real_1: 0.25964120030403137 (0.2637754827737808) + | > loss_disc_real_2: 0.2246675044298172 (0.24063635617494583) + | > loss_disc_real_3: 0.24611830711364746 (0.23627248406410217) + | > loss_disc_real_4: 0.25140756368637085 (0.26053254306316376) + | > loss_disc_real_5: 0.23469959199428558 (0.2490323707461357) + | > loss_0: 2.556213855743408 (2.467128872871399) + | > loss_gen: 2.286452531814575 (2.499844551086426) + | > loss_kl: 2.1246957778930664 (2.2182066440582275) + | > loss_feat: 7.5030083656311035 (7.570943117141724) + | > loss_mel: 19.35621452331543 (19.263916969299316) + | > loss_duration: 1.6010260581970215 (1.6291853189468384) + | > loss_1: 32.87139892578125 (33.18209648132324) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035094022750854492 (-0.00023496150970458984) + | > avg_loss_disc: 2.467128872871399 (-0.0718379020690918) + | > avg_loss_disc_real_0: 0.11881528422236443 (-0.009417973458766937) + | > avg_loss_disc_real_1: 0.2637754827737808 (+0.029742449522018433) + | > avg_loss_disc_real_2: 0.24063635617494583 (+0.026050038635730743) + | > avg_loss_disc_real_3: 0.23627248406410217 (-0.004539474844932556) + | > avg_loss_disc_real_4: 0.26053254306316376 (+0.034934885799884796) + | > avg_loss_disc_real_5: 0.2490323707461357 (+0.0010470449924468994) + | > avg_loss_0: 2.467128872871399 (-0.0718379020690918) + | > avg_loss_gen: 2.499844551086426 (+0.3562633991241455) + | > avg_loss_kl: 2.2182066440582275 (+0.006922721862792969) + | > avg_loss_feat: 7.570943117141724 (+1.0811302661895752) + | > avg_loss_mel: 19.263916969299316 (+2.2060251235961914) + | > avg_loss_duration: 1.6291853189468384 (+0.003982722759246826) + | > avg_loss_1: 33.18209648132324 (+3.654324531555176) + + + > EPOCH: 32/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:32:09)  + + --> TIME: 2025-04-18 23:32:18 -- STEP: 24/142 -- GLOBAL_STEP: 383850 + | > loss_disc: 2.4367194175720215 (2.4193084736665087) + | > loss_disc_real_0: 0.15568770468235016 (0.11742602164546649) + | > loss_disc_real_1: 0.1875513643026352 (0.20672322685519853) + | > loss_disc_real_2: 0.18009911477565765 (0.22467997297644615) + | > loss_disc_real_3: 0.22333072125911713 (0.22493932830790678) + | > loss_disc_real_4: 0.2343386709690094 (0.2285482349495093) + | > loss_disc_real_5: 0.21796710789203644 (0.23297272560497126) + | > loss_0: 2.4367194175720215 (2.4193084736665087) + | > grad_norm_0: tensor(7.0367, device='cuda:0') (tensor(14.9095, device='cuda:0')) + | > loss_gen: 2.300065517425537 (2.465068111817042) + | > loss_kl: 1.8710490465164185 (1.95479354262352) + | > loss_feat: 6.5466413497924805 (7.270894328753154) + | > loss_mel: 17.412050247192383 (18.08772492408752) + | > loss_duration: 1.5795948505401611 (1.5164983918269475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.709402084350586 (31.294979651769) + | > grad_norm_1: tensor(58.0866, device='cuda:0') (tensor(297.0177, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.3489 (0.33283355832099915) + | > loader_time: 0.0056 (0.00505681832631429) + + + --> TIME: 2025-04-18 23:32:28 -- STEP: 49/142 -- GLOBAL_STEP: 383875 + | > loss_disc: 2.498324155807495 (2.430356118143821) + | > loss_disc_real_0: 0.12013354152441025 (0.11821441732499065) + | > loss_disc_real_1: 0.23374724388122559 (0.2086150618840237) + | > loss_disc_real_2: 0.24814322590827942 (0.225542814451821) + | > loss_disc_real_3: 0.23541603982448578 (0.22440408535149634) + | > loss_disc_real_4: 0.2626541554927826 (0.2276458390513245) + | > loss_disc_real_5: 0.23604531586170197 (0.23126360165829563) + | > loss_0: 2.498324155807495 (2.430356118143821) + | > grad_norm_0: tensor(3.5127, device='cuda:0') (tensor(12.3120, device='cuda:0')) + | > loss_gen: 2.3481321334838867 (2.4114011453122512) + | > loss_kl: 2.0538675785064697 (1.9500521226805083) + | > loss_feat: 6.7368693351745605 (7.14036564924279) + | > loss_mel: 18.86672592163086 (18.0292983930938) + | > loss_duration: 1.5324673652648926 (1.5221980883150685) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.538063049316406 (31.053315454599808) + | > grad_norm_1: tensor(44.3697, device='cuda:0') (tensor(270.2929, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.3689 (0.3521815660048504) + | > loader_time: 0.0058 (0.005415678024291992) + + + --> TIME: 2025-04-18 23:32:37 -- STEP: 74/142 -- GLOBAL_STEP: 383900 + | > loss_disc: 2.456779718399048 (2.4507822249386764) + | > loss_disc_real_0: 0.15406790375709534 (0.12234201443356436) + | > loss_disc_real_1: 0.1738933026790619 (0.20657933523526062) + | > loss_disc_real_2: 0.19823633134365082 (0.2260452575377516) + | > loss_disc_real_3: 0.206184983253479 (0.22786888762100324) + | > loss_disc_real_4: 0.2210949957370758 (0.23047890292631612) + | > loss_disc_real_5: 0.283810555934906 (0.23382288499458417) + | > loss_0: 2.456779718399048 (2.4507822249386764) + | > grad_norm_0: tensor(8.2003, device='cuda:0') (tensor(10.7045, device='cuda:0')) + | > loss_gen: 2.3157331943511963 (2.377461165995212) + | > loss_kl: 1.8509221076965332 (1.9672670267723702) + | > loss_feat: 7.7180256843566895 (7.048253291362041) + | > loss_mel: 17.6513729095459 (17.931813781325868) + | > loss_duration: 1.5478267669677734 (1.532147848928297) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.083881378173828 (30.85694323359309) + | > grad_norm_1: tensor(201.4869, device='cuda:0') (tensor(231.7022, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.3885 (0.3621593488229288) + | > loader_time: 0.0068 (0.005675799137837178) + + + --> TIME: 2025-04-18 23:32:48 -- STEP: 99/142 -- GLOBAL_STEP: 383925 + | > loss_disc: 2.4935848712921143 (2.4622107684010213) + | > loss_disc_real_0: 0.12394415587186813 (0.1253975872891118) + | > loss_disc_real_1: 0.2399076670408249 (0.20798042853071233) + | > loss_disc_real_2: 0.21420058608055115 (0.22608716048375524) + | > loss_disc_real_3: 0.22791549563407898 (0.2287525189946396) + | > loss_disc_real_4: 0.23451916873455048 (0.23073805688005505) + | > loss_disc_real_5: 0.2421998232603073 (0.23337349611701388) + | > loss_0: 2.4935848712921143 (2.4622107684010213) + | > grad_norm_0: tensor(9.7738, device='cuda:0') (tensor(9.8597, device='cuda:0')) + | > loss_gen: 2.20885968208313 (2.3557030865640356) + | > loss_kl: 2.076737403869629 (1.9938412844532667) + | > loss_feat: 6.375186920166016 (6.933603460138494) + | > loss_mel: 16.994415283203125 (17.82622206331503) + | > loss_duration: 1.5678460597991943 (1.5416085250449902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.223045349121094 (30.650978570032603) + | > grad_norm_1: tensor(249.8551, device='cuda:0') (tensor(213.9372, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.4083 (0.3723163941893915) + | > loader_time: 0.0065 (0.00591816083349363) + + + --> TIME: 2025-04-18 23:32:59 -- STEP: 124/142 -- GLOBAL_STEP: 383950 + | > loss_disc: 2.5181055068969727 (2.4627690603656163) + | > loss_disc_real_0: 0.09332702308893204 (0.12732182575329662) + | > loss_disc_real_1: 0.21623702347278595 (0.20790763199329376) + | > loss_disc_real_2: 0.20877809822559357 (0.22528799779472813) + | > loss_disc_real_3: 0.2425941675901413 (0.2281149826463192) + | > loss_disc_real_4: 0.23472559452056885 (0.22966789778682492) + | > loss_disc_real_5: 0.22445179522037506 (0.23124123232499247) + | > loss_0: 2.5181055068969727 (2.4627690603656163) + | > grad_norm_0: tensor(8.0538, device='cuda:0') (tensor(9.4367, device='cuda:0')) + | > loss_gen: 2.472337007522583 (2.341094003569695) + | > loss_kl: 2.1509923934936523 (2.0046395640219417) + | > loss_feat: 7.291296005249023 (6.8792312991234565) + | > loss_mel: 18.56740951538086 (17.730169365482947) + | > loss_duration: 1.6104252338409424 (1.5488096302555454) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.09246063232422 (30.50394401242656) + | > grad_norm_1: tensor(116.2786, device='cuda:0') (tensor(198.2672, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.4393 (0.38378143310546875) + | > loader_time: 0.0073 (0.006187158246194165) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5816493034362793 (2.5816493034362793) + | > loss_disc_real_0: 0.17531858384609222 (0.17531858384609222) + | > loss_disc_real_1: 0.22758682072162628 (0.22758682072162628) + | > loss_disc_real_2: 0.2776549756526947 (0.2776549756526947) + | > loss_disc_real_3: 0.26957598328590393 (0.26957598328590393) + | > loss_disc_real_4: 0.24427013099193573 (0.24427013099193573) + | > loss_disc_real_5: 0.2385791540145874 (0.2385791540145874) + | > loss_0: 2.5816493034362793 (2.5816493034362793) + | > loss_gen: 2.2921714782714844 (2.2921714782714844) + | > loss_kl: 2.0222084522247314 (2.0222084522247314) + | > loss_feat: 5.434564113616943 (5.434564113616943) + | > loss_mel: 16.829307556152344 (16.829307556152344) + | > loss_duration: 1.6366231441497803 (1.6366231441497803) + | > loss_1: 28.214874267578125 (28.214874267578125) + + --> STEP: 1 + | > loss_disc: 2.5271193981170654 (2.5271193981170654) + | > loss_disc_real_0: 0.15426091849803925 (0.15426091849803925) + | > loss_disc_real_1: 0.2975265383720398 (0.2975265383720398) + | > loss_disc_real_2: 0.2792969048023224 (0.2792969048023224) + | > loss_disc_real_3: 0.2279537320137024 (0.2279537320137024) + | > loss_disc_real_4: 0.20828746259212494 (0.20828746259212494) + | > loss_disc_real_5: 0.22571080923080444 (0.22571080923080444) + | > loss_0: 2.5271193981170654 (2.5271193981170654) + | > loss_gen: 2.409162998199463 (2.409162998199463) + | > loss_kl: 2.321434736251831 (2.321434736251831) + | > loss_feat: 6.522485733032227 (6.522485733032227) + | > loss_mel: 19.012008666992188 (19.012008666992188) + | > loss_duration: 1.6581920385360718 (1.6581920385360718) + | > loss_1: 31.92328453063965 (31.92328453063965) + + --> STEP: 2 + | > loss_disc: 2.5201265811920166 (2.523622989654541) + | > loss_disc_real_0: 0.13928014039993286 (0.14677052944898605) + | > loss_disc_real_1: 0.31009066104888916 (0.3038085997104645) + | > loss_disc_real_2: 0.25465425848960876 (0.2669755816459656) + | > loss_disc_real_3: 0.1886516660451889 (0.20830269902944565) + | > loss_disc_real_4: 0.17960263788700104 (0.193945050239563) + | > loss_disc_real_5: 0.23117978870868683 (0.22844529896974564) + | > loss_0: 2.5201265811920166 (2.523622989654541) + | > loss_gen: 2.269731044769287 (2.339447021484375) + | > loss_kl: 2.1339869499206543 (2.2277108430862427) + | > loss_feat: 8.034076690673828 (7.278281211853027) + | > loss_mel: 18.890905380249023 (18.951457023620605) + | > loss_duration: 1.5817415714263916 (1.6199668049812317) + | > loss_1: 32.91044235229492 (32.416863441467285) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003315567970275879 (-0.0001938343048095703) + | > avg_loss_disc: 2.523622989654541 (+0.05649411678314209) + | > avg_loss_disc_real_0: 0.14677052944898605 (+0.027955245226621628) + | > avg_loss_disc_real_1: 0.3038085997104645 (+0.040033116936683655) + | > avg_loss_disc_real_2: 0.2669755816459656 (+0.026339225471019745) + | > avg_loss_disc_real_3: 0.20830269902944565 (-0.027969785034656525) + | > avg_loss_disc_real_4: 0.193945050239563 (-0.06658749282360077) + | > avg_loss_disc_real_5: 0.22844529896974564 (-0.020587071776390076) + | > avg_loss_0: 2.523622989654541 (+0.05649411678314209) + | > avg_loss_gen: 2.339447021484375 (-0.16039752960205078) + | > avg_loss_kl: 2.2277108430862427 (+0.009504199028015137) + | > avg_loss_feat: 7.278281211853027 (-0.2926619052886963) + | > avg_loss_mel: 18.951457023620605 (-0.31245994567871094) + | > avg_loss_duration: 1.6199668049812317 (-0.00921851396560669) + | > avg_loss_1: 32.416863441467285 (-0.765233039855957) + + + > EPOCH: 33/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:33:08)  + + --> TIME: 2025-04-18 23:33:11 -- STEP: 7/142 -- GLOBAL_STEP: 383975 + | > loss_disc: 2.424504518508911 (2.4341656821114674) + | > loss_disc_real_0: 0.11775560677051544 (0.10998052677937917) + | > loss_disc_real_1: 0.24401743710041046 (0.21598743966647557) + | > loss_disc_real_2: 0.2668026387691498 (0.22801117386136735) + | > loss_disc_real_3: 0.2400946319103241 (0.22874202472823008) + | > loss_disc_real_4: 0.20517738163471222 (0.23022312351635524) + | > loss_disc_real_5: 0.21592850983142853 (0.23047583869525365) + | > loss_0: 2.424504518508911 (2.4341656821114674) + | > grad_norm_0: tensor(9.7326, device='cuda:0') (tensor(13.6301, device='cuda:0')) + | > loss_gen: 2.5915842056274414 (2.4654365267072404) + | > loss_kl: 1.9512890577316284 (2.013614399092538) + | > loss_feat: 7.73421049118042 (7.477612291063581) + | > loss_mel: 19.54470443725586 (18.54418999808175) + | > loss_duration: 1.4966106414794922 (1.5132662568773543) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.318397521972656 (32.01411928449358) + | > grad_norm_1: tensor(461.1144, device='cuda:0') (tensor(347.6062, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.3215 (0.325765745980399) + | > loader_time: 0.0039 (0.004177638462611607) + + + --> TIME: 2025-04-18 23:33:20 -- STEP: 32/142 -- GLOBAL_STEP: 384000 + | > loss_disc: 2.2663986682891846 (2.4430090561509132) + | > loss_disc_real_0: 0.0823899656534195 (0.11798009951598942) + | > loss_disc_real_1: 0.16431191563606262 (0.2082463689148426) + | > loss_disc_real_2: 0.20606032013893127 (0.23153324145823717) + | > loss_disc_real_3: 0.18179228901863098 (0.22757719596847892) + | > loss_disc_real_4: 0.21143396198749542 (0.22657750314101577) + | > loss_disc_real_5: 0.20005711913108826 (0.22976152319461107) + | > loss_0: 2.2663986682891846 (2.4430090561509132) + | > grad_norm_0: tensor(10.9798, device='cuda:0') (tensor(12.0905, device='cuda:0')) + | > loss_gen: 2.5081942081451416 (2.39800114184618) + | > loss_kl: 1.9928542375564575 (1.9717245884239674) + | > loss_feat: 8.113690376281738 (7.273769602179527) + | > loss_mel: 18.310012817382812 (17.976639956235886) + | > loss_duration: 1.4871101379394531 (1.5151457898318768) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.411861419677734 (31.135281205177307) + | > grad_norm_1: tensor(349.0155, device='cuda:0') (tensor(310.1426, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.3551 (0.33677559345960617) + | > loader_time: 0.0057 (0.004946082830429076) + + + --> TIME: 2025-04-18 23:33:29 -- STEP: 57/142 -- GLOBAL_STEP: 384025 + | > loss_disc: 2.5139517784118652 (2.4468605978447093) + | > loss_disc_real_0: 0.11580608040094376 (0.1230081864877751) + | > loss_disc_real_1: 0.19120265543460846 (0.2079219028615115) + | > loss_disc_real_2: 0.21769742667675018 (0.2267047647844281) + | > loss_disc_real_3: 0.22352197766304016 (0.22885699047331223) + | > loss_disc_real_4: 0.22988760471343994 (0.22822833610208412) + | > loss_disc_real_5: 0.22393108904361725 (0.2291790568514874) + | > loss_0: 2.5139517784118652 (2.4468605978447093) + | > grad_norm_0: tensor(8.9810, device='cuda:0') (tensor(12.4949, device='cuda:0')) + | > loss_gen: 2.3725926876068115 (2.393735149450469) + | > loss_kl: 1.9044917821884155 (1.96445660005536) + | > loss_feat: 7.285983085632324 (7.258581010918868) + | > loss_mel: 17.79149627685547 (18.108866356966786) + | > loss_duration: 1.5784492492675781 (1.523539382114745) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.933013916015625 (31.24917853505988) + | > grad_norm_1: tensor(315.0876, device='cuda:0') (tensor(303.5319, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.3761 (0.3480599135683294) + | > loader_time: 0.0062 (0.005245074891207511) + + + --> TIME: 2025-04-18 23:33:39 -- STEP: 82/142 -- GLOBAL_STEP: 384050 + | > loss_disc: 2.5164151191711426 (2.459902905836339) + | > loss_disc_real_0: 0.12262141704559326 (0.1243160241624204) + | > loss_disc_real_1: 0.2157360166311264 (0.20893045060518312) + | > loss_disc_real_2: 0.252410352230072 (0.22649951333679805) + | > loss_disc_real_3: 0.27350085973739624 (0.2302079019023151) + | > loss_disc_real_4: 0.2642989754676819 (0.22992533281808947) + | > loss_disc_real_5: 0.2798708975315094 (0.23074567972159968) + | > loss_0: 2.5164151191711426 (2.459902905836339) + | > grad_norm_0: tensor(9.1134, device='cuda:0') (tensor(12.9244, device='cuda:0')) + | > loss_gen: 2.538296699523926 (2.3837936680491376) + | > loss_kl: 2.254396677017212 (1.9773634701240352) + | > loss_feat: 7.216166019439697 (7.203505818436786) + | > loss_mel: 19.36054039001465 (18.117880495583137) + | > loss_duration: 1.6228206157684326 (1.5331079596426427) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.99222183227539 (31.215651372583903) + | > grad_norm_1: tensor(85.1758, device='cuda:0') (tensor(302.4632, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.3935 (0.36053293216519233) + | > loader_time: 0.0067 (0.0055168867111206055) + + + --> TIME: 2025-04-18 23:33:49 -- STEP: 107/142 -- GLOBAL_STEP: 384075 + | > loss_disc: 2.4624788761138916 (2.473481274096766) + | > loss_disc_real_0: 0.09188719093799591 (0.12803567361051793) + | > loss_disc_real_1: 0.2251146137714386 (0.20908374764094842) + | > loss_disc_real_2: 0.23087944090366364 (0.22809963889211138) + | > loss_disc_real_3: 0.19849203526973724 (0.23082638288212715) + | > loss_disc_real_4: 0.22707493603229523 (0.23063750384009887) + | > loss_disc_real_5: 0.20992815494537354 (0.2301585781518544) + | > loss_0: 2.4624788761138916 (2.473481274096766) + | > grad_norm_0: tensor(17.3880, device='cuda:0') (tensor(13.9933, device='cuda:0')) + | > loss_gen: 2.1559219360351562 (2.363831277205566) + | > loss_kl: 1.9247784614562988 (1.993185657207097) + | > loss_feat: 7.014006614685059 (7.111590679560866) + | > loss_mel: 17.093393325805664 (18.05139094201204) + | > loss_duration: 1.537890911102295 (1.5418369959447986) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.72599220275879 (31.061835493996877) + | > grad_norm_1: tensor(332.5970, device='cuda:0') (tensor(312.1021, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.4185 (0.37209695521916186) + | > loader_time: 0.0066 (0.005807056605258832) + + + --> TIME: 2025-04-18 23:34:01 -- STEP: 132/142 -- GLOBAL_STEP: 384100 + | > loss_disc: 2.4897031784057617 (2.4784904375220798) + | > loss_disc_real_0: 0.1288232058286667 (0.1304486388284149) + | > loss_disc_real_1: 0.22172969579696655 (0.20804890245199203) + | > loss_disc_real_2: 0.22155125439167023 (0.22766856183156822) + | > loss_disc_real_3: 0.21816343069076538 (0.2306188517673449) + | > loss_disc_real_4: 0.22248005867004395 (0.23144613646648146) + | > loss_disc_real_5: 0.2527960240840912 (0.23047102705547304) + | > loss_0: 2.4897031784057617 (2.4784904375220798) + | > grad_norm_0: tensor(19.6459, device='cuda:0') (tensor(14.4701, device='cuda:0')) + | > loss_gen: 2.223520040512085 (2.3537482890215795) + | > loss_kl: 2.1022939682006836 (2.0016466794591956) + | > loss_feat: 6.52768611907959 (7.015531240087567) + | > loss_mel: 18.262588500976562 (17.922885584108755) + | > loss_duration: 1.5393550395965576 (1.5466791553930803) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.65544319152832 (30.840490875822127) + | > grad_norm_1: tensor(234.1200, device='cuda:0') (tensor(307.6465, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.4463 (0.3842273842204701) + | > loader_time: 0.0077 (0.00607551589156642) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4687352180480957 (2.4687352180480957) + | > loss_disc_real_0: 0.2041931450366974 (0.2041931450366974) + | > loss_disc_real_1: 0.25270184874534607 (0.25270184874534607) + | > loss_disc_real_2: 0.20452065765857697 (0.20452065765857697) + | > loss_disc_real_3: 0.21437856554985046 (0.21437856554985046) + | > loss_disc_real_4: 0.24786978960037231 (0.24786978960037231) + | > loss_disc_real_5: 0.23828157782554626 (0.23828157782554626) + | > loss_0: 2.4687352180480957 (2.4687352180480957) + | > loss_gen: 2.45788311958313 (2.45788311958313) + | > loss_kl: 2.2218666076660156 (2.2218666076660156) + | > loss_feat: 6.588133811950684 (6.588133811950684) + | > loss_mel: 18.251178741455078 (18.251178741455078) + | > loss_duration: 1.6099028587341309 (1.6099028587341309) + | > loss_1: 31.12896728515625 (31.12896728515625) + + --> STEP: 1 + | > loss_disc: 2.4164905548095703 (2.4164905548095703) + | > loss_disc_real_0: 0.17331425845623016 (0.17331425845623016) + | > loss_disc_real_1: 0.237461119890213 (0.237461119890213) + | > loss_disc_real_2: 0.20791707932949066 (0.20791707932949066) + | > loss_disc_real_3: 0.2331186830997467 (0.2331186830997467) + | > loss_disc_real_4: 0.2168295681476593 (0.2168295681476593) + | > loss_disc_real_5: 0.18644000589847565 (0.18644000589847565) + | > loss_0: 2.4164905548095703 (2.4164905548095703) + | > loss_gen: 2.464857816696167 (2.464857816696167) + | > loss_kl: 2.2647483348846436 (2.2647483348846436) + | > loss_feat: 6.623405933380127 (6.623405933380127) + | > loss_mel: 18.60284996032715 (18.60284996032715) + | > loss_duration: 1.645310401916504 (1.645310401916504) + | > loss_1: 31.601173400878906 (31.601173400878906) + + --> STEP: 2 + | > loss_disc: 2.7704668045043945 (2.5934786796569824) + | > loss_disc_real_0: 0.279938668012619 (0.2266264632344246) + | > loss_disc_real_1: 0.253452867269516 (0.2454569935798645) + | > loss_disc_real_2: 0.20390911400318146 (0.20591309666633606) + | > loss_disc_real_3: 0.23773767054080963 (0.23542817682027817) + | > loss_disc_real_4: 0.29097306728363037 (0.25390131771564484) + | > loss_disc_real_5: 0.3083816170692444 (0.24741081148386002) + | > loss_0: 2.7704668045043945 (2.5934786796569824) + | > loss_gen: 2.4164364337921143 (2.4406471252441406) + | > loss_kl: 1.9668889045715332 (2.1158186197280884) + | > loss_feat: 5.921575546264648 (6.272490739822388) + | > loss_mel: 17.28043556213379 (17.94164276123047) + | > loss_duration: 1.5850872993469238 (1.6151988506317139) + | > loss_1: 29.17042350769043 (30.385798454284668) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037151575088500977 (+0.00039958953857421875) + | > avg_loss_disc: 2.5934786796569824 (+0.0698556900024414) + | > avg_loss_disc_real_0: 0.2266264632344246 (+0.07985593378543854) + | > avg_loss_disc_real_1: 0.2454569935798645 (-0.058351606130599976) + | > avg_loss_disc_real_2: 0.20591309666633606 (-0.06106248497962952) + | > avg_loss_disc_real_3: 0.23542817682027817 (+0.02712547779083252) + | > avg_loss_disc_real_4: 0.25390131771564484 (+0.05995626747608185) + | > avg_loss_disc_real_5: 0.24741081148386002 (+0.01896551251411438) + | > avg_loss_0: 2.5934786796569824 (+0.0698556900024414) + | > avg_loss_gen: 2.4406471252441406 (+0.10120010375976562) + | > avg_loss_kl: 2.1158186197280884 (-0.1118922233581543) + | > avg_loss_feat: 6.272490739822388 (-1.0057904720306396) + | > avg_loss_mel: 17.94164276123047 (-1.0098142623901367) + | > avg_loss_duration: 1.6151988506317139 (-0.004767954349517822) + | > avg_loss_1: 30.385798454284668 (-2.031064987182617) + + + > EPOCH: 34/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:34:07)  + + --> TIME: 2025-04-18 23:34:12 -- STEP: 15/142 -- GLOBAL_STEP: 384125 + | > loss_disc: 2.5265285968780518 (2.403875494003296) + | > loss_disc_real_0: 0.10526485741138458 (0.11073146810134252) + | > loss_disc_real_1: 0.21485650539398193 (0.1974918206532796) + | > loss_disc_real_2: 0.2307368963956833 (0.2271787812312444) + | > loss_disc_real_3: 0.2209433615207672 (0.22688130040963492) + | > loss_disc_real_4: 0.21267934143543243 (0.2195647845665614) + | > loss_disc_real_5: 0.21801449358463287 (0.22127757469813028) + | > loss_0: 2.5265285968780518 (2.403875494003296) + | > grad_norm_0: tensor(14.4530, device='cuda:0') (tensor(10.8264, device='cuda:0')) + | > loss_gen: 2.2841928005218506 (2.40417103767395) + | > loss_kl: 1.9590438604354858 (1.921382761001587) + | > loss_feat: 6.205435276031494 (7.215097077687582) + | > loss_mel: 15.36309814453125 (17.8880475362142) + | > loss_duration: 1.512570858001709 (1.5060671806335448) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.3243408203125 (30.934765243530272) + | > grad_norm_1: tensor(283.9397, device='cuda:0') (tensor(272.9103, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.3386 (0.3290578842163086) + | > loader_time: 0.0051 (0.004746103286743164) + + + --> TIME: 2025-04-18 23:34:21 -- STEP: 40/142 -- GLOBAL_STEP: 384150 + | > loss_disc: 2.3760533332824707 (2.4272808194160467) + | > loss_disc_real_0: 0.07504748553037643 (0.11766777336597442) + | > loss_disc_real_1: 0.19867248833179474 (0.20602843277156352) + | > loss_disc_real_2: 0.21051323413848877 (0.22707037180662154) + | > loss_disc_real_3: 0.21506312489509583 (0.22636259719729424) + | > loss_disc_real_4: 0.1758153736591339 (0.22479796558618545) + | > loss_disc_real_5: 0.23151929676532745 (0.2260094728320837) + | > loss_0: 2.3760533332824707 (2.4272808194160467) + | > grad_norm_0: tensor(24.3633, device='cuda:0') (tensor(9.9014, device='cuda:0')) + | > loss_gen: 2.295205593109131 (2.3872181713581084) + | > loss_kl: 1.981007695198059 (1.9494663178920746) + | > loss_feat: 7.1220383644104 (7.153633713722229) + | > loss_mel: 18.449626922607422 (17.999394750595094) + | > loss_duration: 1.492255687713623 (1.516091388463974) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.34013557434082 (31.00580439567566) + | > grad_norm_1: tensor(449.6136, device='cuda:0') (tensor(229.7806, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.359 (0.3411329030990601) + | > loader_time: 0.006 (0.005183255672454834) + + + --> TIME: 2025-04-18 23:34:31 -- STEP: 65/142 -- GLOBAL_STEP: 384175 + | > loss_disc: 2.604830741882324 (2.444686192732591) + | > loss_disc_real_0: 0.14435814321041107 (0.1231884533396134) + | > loss_disc_real_1: 0.17490892112255096 (0.20552261746846712) + | > loss_disc_real_2: 0.23038995265960693 (0.2271397508107699) + | > loss_disc_real_3: 0.24646809697151184 (0.22775847682586084) + | > loss_disc_real_4: 0.22880949079990387 (0.22921794056892394) + | > loss_disc_real_5: 0.2630384862422943 (0.22729981312384973) + | > loss_0: 2.604830741882324 (2.444686192732591) + | > grad_norm_0: tensor(6.8724, device='cuda:0') (tensor(9.7111, device='cuda:0')) + | > loss_gen: 2.1681370735168457 (2.3629082166231594) + | > loss_kl: 1.8939497470855713 (1.9670765656691331) + | > loss_feat: 6.539740085601807 (7.0525157928466795) + | > loss_mel: 16.278949737548828 (17.803108112628646) + | > loss_duration: 1.5417678356170654 (1.528331551185021) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.422542572021484 (30.7139403123122) + | > grad_norm_1: tensor(165.6325, device='cuda:0') (tensor(216.0020, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.3909 (0.35346569281357987) + | > loader_time: 0.0062 (0.005474083240215596) + + + --> TIME: 2025-04-18 23:34:41 -- STEP: 90/142 -- GLOBAL_STEP: 384200 + | > loss_disc: 2.4404971599578857 (2.4449092467625926) + | > loss_disc_real_0: 0.13008610904216766 (0.12603009947472146) + | > loss_disc_real_1: 0.20050765573978424 (0.20575872560342154) + | > loss_disc_real_2: 0.21331997215747833 (0.2260104798608356) + | > loss_disc_real_3: 0.2273082733154297 (0.22722873902983134) + | > loss_disc_real_4: 0.23225703835487366 (0.2289712420768208) + | > loss_disc_real_5: 0.21761858463287354 (0.22648688041501575) + | > loss_0: 2.4404971599578857 (2.4449092467625926) + | > grad_norm_0: tensor(5.3133, device='cuda:0') (tensor(8.8511, device='cuda:0')) + | > loss_gen: 2.2299513816833496 (2.356064324908787) + | > loss_kl: 2.045970916748047 (1.982382963763343) + | > loss_feat: 6.3453898429870605 (7.029688188764784) + | > loss_mel: 17.19106674194336 (17.798155784606934) + | > loss_duration: 1.5694406032562256 (1.5382309661971194) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.381820678710938 (30.704522344801163) + | > grad_norm_1: tensor(97.3376, device='cuda:0') (tensor(185.0589, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.4079 (0.36523349020216184) + | > loader_time: 0.0064 (0.0057463619444105335) + + + --> TIME: 2025-04-18 23:34:52 -- STEP: 115/142 -- GLOBAL_STEP: 384225 + | > loss_disc: 2.5347917079925537 (2.463134377935657) + | > loss_disc_real_0: 0.11474237591028214 (0.12877806625936333) + | > loss_disc_real_1: 0.23389317095279694 (0.20735448249008345) + | > loss_disc_real_2: 0.20669913291931152 (0.22745266919550688) + | > loss_disc_real_3: 0.20163780450820923 (0.22794672328492868) + | > loss_disc_real_4: 0.21112437546253204 (0.22935452098431794) + | > loss_disc_real_5: 0.17210552096366882 (0.2267977202716081) + | > loss_0: 2.5347917079925537 (2.463134377935657) + | > grad_norm_0: tensor(4.8214, device='cuda:0') (tensor(8.6825, device='cuda:0')) + | > loss_gen: 2.2906970977783203 (2.3338194225145434) + | > loss_kl: 2.0813164710998535 (1.9900121346763942) + | > loss_feat: 7.03938102722168 (6.92219125498896) + | > loss_mel: 16.635372161865234 (17.707729405942175) + | > loss_duration: 1.5587360858917236 (1.5430432444033413) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.60550308227539 (30.496795554783034) + | > grad_norm_1: tensor(98.0955, device='cuda:0') (tensor(165.4128, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.4285 (0.37636678944463303) + | > loader_time: 0.0074 (0.006009118453316064) + + + --> TIME: 2025-04-18 23:35:03 -- STEP: 140/142 -- GLOBAL_STEP: 384250 + | > loss_disc: 2.4218742847442627 (2.4633224793842854) + | > loss_disc_real_0: 0.1116069108247757 (0.12932886550469053) + | > loss_disc_real_1: 0.23414729535579681 (0.20818997312869342) + | > loss_disc_real_2: 0.2059841752052307 (0.22688615918159485) + | > loss_disc_real_3: 0.22195743024349213 (0.2284072534314224) + | > loss_disc_real_4: 0.21389764547348022 (0.22994208186864853) + | > loss_disc_real_5: 0.223099485039711 (0.22717306039163043) + | > loss_0: 2.4218742847442627 (2.4633224793842854) + | > grad_norm_0: tensor(8.1358, device='cuda:0') (tensor(8.2990, device='cuda:0')) + | > loss_gen: 2.3502748012542725 (2.33147338799068) + | > loss_kl: 2.004999876022339 (1.9883386296885355) + | > loss_feat: 6.685327053070068 (6.908138833727155) + | > loss_mel: 17.23992347717285 (17.71923148291452) + | > loss_duration: 1.614282488822937 (1.5487154679638997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.894807815551758 (30.49589787891933) + | > grad_norm_1: tensor(56.9200, device='cuda:0') (tensor(155.5519, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.4534 (0.3884288413184029) + | > loader_time: 0.0072 (0.006242176464625766) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3033361434936523 (2.3033361434936523) + | > loss_disc_real_0: 0.12053932249546051 (0.12053932249546051) + | > loss_disc_real_1: 0.17428433895111084 (0.17428433895111084) + | > loss_disc_real_2: 0.20728783309459686 (0.20728783309459686) + | > loss_disc_real_3: 0.2268185168504715 (0.2268185168504715) + | > loss_disc_real_4: 0.18777287006378174 (0.18777287006378174) + | > loss_disc_real_5: 0.2356407642364502 (0.2356407642364502) + | > loss_0: 2.3033361434936523 (2.3033361434936523) + | > loss_gen: 2.431452989578247 (2.431452989578247) + | > loss_kl: 2.327739953994751 (2.327739953994751) + | > loss_feat: 7.1828508377075195 (7.1828508377075195) + | > loss_mel: 18.633583068847656 (18.633583068847656) + | > loss_duration: 1.622852087020874 (1.622852087020874) + | > loss_1: 32.19847869873047 (32.19847869873047) + + --> STEP: 1 + | > loss_disc: 2.478968858718872 (2.478968858718872) + | > loss_disc_real_0: 0.13943412899971008 (0.13943412899971008) + | > loss_disc_real_1: 0.19572079181671143 (0.19572079181671143) + | > loss_disc_real_2: 0.21206483244895935 (0.21206483244895935) + | > loss_disc_real_3: 0.2128157764673233 (0.2128157764673233) + | > loss_disc_real_4: 0.22023144364356995 (0.22023144364356995) + | > loss_disc_real_5: 0.23464441299438477 (0.23464441299438477) + | > loss_0: 2.478968858718872 (2.478968858718872) + | > loss_gen: 2.1662821769714355 (2.1662821769714355) + | > loss_kl: 2.3858158588409424 (2.3858158588409424) + | > loss_feat: 6.457728385925293 (6.457728385925293) + | > loss_mel: 17.52181625366211 (17.52181625366211) + | > loss_duration: 1.6431195735931396 (1.6431195735931396) + | > loss_1: 30.174762725830078 (30.174762725830078) + + --> STEP: 2 + | > loss_disc: 2.362527370452881 (2.4207481145858765) + | > loss_disc_real_0: 0.11837837845087051 (0.1289062537252903) + | > loss_disc_real_1: 0.2037586271762848 (0.1997397094964981) + | > loss_disc_real_2: 0.25975343585014343 (0.2359091341495514) + | > loss_disc_real_3: 0.2032841145992279 (0.2080499455332756) + | > loss_disc_real_4: 0.2304646521806717 (0.22534804791212082) + | > loss_disc_real_5: 0.2270015925168991 (0.23082300275564194) + | > loss_0: 2.362527370452881 (2.4207481145858765) + | > loss_gen: 2.3995859622955322 (2.282934069633484) + | > loss_kl: 2.1535236835479736 (2.269669771194458) + | > loss_feat: 7.236826419830322 (6.847277402877808) + | > loss_mel: 18.615299224853516 (18.068557739257812) + | > loss_duration: 1.5676149129867554 (1.6053672432899475) + | > loss_1: 31.972850799560547 (31.073806762695312) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036363601684570312 (-7.87973403930664e-05) + | > avg_loss_disc: 2.4207481145858765 (-0.17273056507110596) + | > avg_loss_disc_real_0: 0.1289062537252903 (-0.09772020950913429) + | > avg_loss_disc_real_1: 0.1997397094964981 (-0.045717284083366394) + | > avg_loss_disc_real_2: 0.2359091341495514 (+0.029996037483215332) + | > avg_loss_disc_real_3: 0.2080499455332756 (-0.027378231287002563) + | > avg_loss_disc_real_4: 0.22534804791212082 (-0.028553269803524017) + | > avg_loss_disc_real_5: 0.23082300275564194 (-0.01658780872821808) + | > avg_loss_0: 2.4207481145858765 (-0.17273056507110596) + | > avg_loss_gen: 2.282934069633484 (-0.15771305561065674) + | > avg_loss_kl: 2.269669771194458 (+0.15385115146636963) + | > avg_loss_feat: 6.847277402877808 (+0.5747866630554199) + | > avg_loss_mel: 18.068557739257812 (+0.12691497802734375) + | > avg_loss_duration: 1.6053672432899475 (-0.009831607341766357) + | > avg_loss_1: 31.073806762695312 (+0.6880083084106445) + + + > EPOCH: 35/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:35:05)  + + --> TIME: 2025-04-18 23:35:14 -- STEP: 23/142 -- GLOBAL_STEP: 384275 + | > loss_disc: 2.462852954864502 (2.476496986720873) + | > loss_disc_real_0: 0.1365988403558731 (0.12505597070507382) + | > loss_disc_real_1: 0.21862399578094482 (0.20836196645446445) + | > loss_disc_real_2: 0.2238127440214157 (0.2283035873071007) + | > loss_disc_real_3: 0.24790209531784058 (0.23274075401865918) + | > loss_disc_real_4: 0.23160292208194733 (0.23675372224787009) + | > loss_disc_real_5: 0.20182672142982483 (0.2373797401137974) + | > loss_0: 2.462852954864502 (2.476496986720873) + | > grad_norm_0: tensor(9.1709, device='cuda:0') (tensor(10.7885, device='cuda:0')) + | > loss_gen: 2.2755017280578613 (2.3856026400690493) + | > loss_kl: 1.9506245851516724 (1.9155349368634431) + | > loss_feat: 7.1640753746032715 (7.168907828952955) + | > loss_mel: 18.226865768432617 (17.96116874528968) + | > loss_duration: 1.5354125499725342 (1.5058371917061184) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.15247917175293 (30.93705135843028) + | > grad_norm_1: tensor(479.5303, device='cuda:0') (tensor(264.1154, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.3422 (0.33120749307715375) + | > loader_time: 0.0055 (0.004846686902253524) + + + --> TIME: 2025-04-18 23:35:23 -- STEP: 48/142 -- GLOBAL_STEP: 384300 + | > loss_disc: 2.5411953926086426 (2.4759112596511836) + | > loss_disc_real_0: 0.27007293701171875 (0.1267200824804604) + | > loss_disc_real_1: 0.2621046006679535 (0.21027238791187605) + | > loss_disc_real_2: 0.2134401649236679 (0.227855968909959) + | > loss_disc_real_3: 0.2291198968887329 (0.23225078421334425) + | > loss_disc_real_4: 0.21101383864879608 (0.2343187409763535) + | > loss_disc_real_5: 0.24294184148311615 (0.23347781536479792) + | > loss_0: 2.5411953926086426 (2.4759112596511836) + | > grad_norm_0: tensor(39.5344, device='cuda:0') (tensor(14.3494, device='cuda:0')) + | > loss_gen: 2.2852063179016113 (2.38954803844293) + | > loss_kl: 1.9528932571411133 (1.9404279937346776) + | > loss_feat: 6.634064674377441 (7.021991680065791) + | > loss_mel: 17.86367416381836 (17.97025400400162) + | > loss_duration: 1.5010777711868286 (1.5145590553681056) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.236915588378906 (30.836780707041424) + | > grad_norm_1: tensor(154.6020, device='cuda:0') (tensor(305.8042, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.3628 (0.3434956570466359) + | > loader_time: 0.0059 (0.005273108681042989) + + + --> TIME: 2025-04-18 23:35:32 -- STEP: 73/142 -- GLOBAL_STEP: 384325 + | > loss_disc: 2.594510316848755 (2.469397838801553) + | > loss_disc_real_0: 0.22303052246570587 (0.12878679667841897) + | > loss_disc_real_1: 0.1824723333120346 (0.20776505298810463) + | > loss_disc_real_2: 0.2372288554906845 (0.22668775038359917) + | > loss_disc_real_3: 0.2619335353374481 (0.23162206558332052) + | > loss_disc_real_4: 0.28949758410453796 (0.23272687686632756) + | > loss_disc_real_5: 0.27793994545936584 (0.23272746209412404) + | > loss_0: 2.594510316848755 (2.469397838801553) + | > grad_norm_0: tensor(39.7137, device='cuda:0') (tensor(16.7409, device='cuda:0')) + | > loss_gen: 2.411396026611328 (2.4102196758740577) + | > loss_kl: 2.0534827709198 (1.9504416103232396) + | > loss_feat: 6.994862079620361 (7.037215180592995) + | > loss_mel: 16.695585250854492 (17.93616918015153) + | > loss_duration: 1.593211054801941 (1.5270486024961079) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.748537063598633 (30.861094148191686) + | > grad_norm_1: tensor(600.6802, device='cuda:0') (tensor(334.6579, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.3957 (0.3557660449041079) + | > loader_time: 0.0065 (0.005527826204691848) + + + --> TIME: 2025-04-18 23:35:43 -- STEP: 98/142 -- GLOBAL_STEP: 384350 + | > loss_disc: 2.519627571105957 (2.47996850646272) + | > loss_disc_real_0: 0.18030205368995667 (0.12764736918770536) + | > loss_disc_real_1: 0.20097240805625916 (0.210012608036703) + | > loss_disc_real_2: 0.23723535239696503 (0.22755761703058164) + | > loss_disc_real_3: 0.2059158831834793 (0.2321998081645187) + | > loss_disc_real_4: 0.2094847410917282 (0.23302740025885252) + | > loss_disc_real_5: 0.19546785950660706 (0.23234010031636881) + | > loss_0: 2.519627571105957 (2.47996850646272) + | > grad_norm_0: tensor(19.8543, device='cuda:0') (tensor(17.1446, device='cuda:0')) + | > loss_gen: 2.1607069969177246 (2.3905925458791306) + | > loss_kl: 1.9890490770339966 (1.9671966007777624) + | > loss_feat: 7.437979221343994 (7.024090844757703) + | > loss_mel: 17.23733901977539 (17.90798711776733) + | > loss_duration: 1.5241528749465942 (1.5366204879721816) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.349227905273438 (30.826487638512436) + | > grad_norm_1: tensor(382.3866, device='cuda:0') (tensor(347.6374, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.4142 (0.3672724183724851) + | > loader_time: 0.0068 (0.005782319574939964) + + + --> TIME: 2025-04-18 23:35:54 -- STEP: 123/142 -- GLOBAL_STEP: 384375 + | > loss_disc: 2.6555159091949463 (2.475471738877335) + | > loss_disc_real_0: 0.1660086065530777 (0.12840390514309813) + | > loss_disc_real_1: 0.20530857145786285 (0.2085595685776656) + | > loss_disc_real_2: 0.24966591596603394 (0.2273755808913611) + | > loss_disc_real_3: 0.23623712360858917 (0.231487303366506) + | > loss_disc_real_4: 0.24663126468658447 (0.2323300712476901) + | > loss_disc_real_5: 0.22228661179542542 (0.23013985350849184) + | > loss_0: 2.6555159091949463 (2.475471738877335) + | > grad_norm_0: tensor(10.9409, device='cuda:0') (tensor(15.7814, device='cuda:0')) + | > loss_gen: 2.3075320720672607 (2.37093719621984) + | > loss_kl: 1.9073821306228638 (1.9720155902025176) + | > loss_feat: 6.271392822265625 (6.942368406590408) + | > loss_mel: 15.468315124511719 (17.786689610985235) + | > loss_duration: 1.542822241783142 (1.5426716814196206) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.49744415283203 (30.61468251546224) + | > grad_norm_1: tensor(379.2628, device='cuda:0') (tensor(327.8381, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.436 (0.3796377356459455) + | > loader_time: 0.0071 (0.006096607301293352) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4275267124176025 (2.4275267124176025) + | > loss_disc_real_0: 0.1310182511806488 (0.1310182511806488) + | > loss_disc_real_1: 0.19519801437854767 (0.19519801437854767) + | > loss_disc_real_2: 0.19914965331554413 (0.19914965331554413) + | > loss_disc_real_3: 0.26282039284706116 (0.26282039284706116) + | > loss_disc_real_4: 0.22104719281196594 (0.22104719281196594) + | > loss_disc_real_5: 0.22810466587543488 (0.22810466587543488) + | > loss_0: 2.4275267124176025 (2.4275267124176025) + | > loss_gen: 2.290364980697632 (2.290364980697632) + | > loss_kl: 2.256333112716675 (2.256333112716675) + | > loss_feat: 6.737620830535889 (6.737620830535889) + | > loss_mel: 18.204010009765625 (18.204010009765625) + | > loss_duration: 1.6396760940551758 (1.6396760940551758) + | > loss_1: 31.128005981445312 (31.128005981445312) + + --> STEP: 1 + | > loss_disc: 2.5410664081573486 (2.5410664081573486) + | > loss_disc_real_0: 0.14343445003032684 (0.14343445003032684) + | > loss_disc_real_1: 0.23234345018863678 (0.23234345018863678) + | > loss_disc_real_2: 0.2444596141576767 (0.2444596141576767) + | > loss_disc_real_3: 0.24483241140842438 (0.24483241140842438) + | > loss_disc_real_4: 0.22149215638637543 (0.22149215638637543) + | > loss_disc_real_5: 0.19791404902935028 (0.19791404902935028) + | > loss_0: 2.5410664081573486 (2.5410664081573486) + | > loss_gen: 2.153785228729248 (2.153785228729248) + | > loss_kl: 2.391313314437866 (2.391313314437866) + | > loss_feat: 6.550372123718262 (6.550372123718262) + | > loss_mel: 19.08879280090332 (19.08879280090332) + | > loss_duration: 1.6501836776733398 (1.6501836776733398) + | > loss_1: 31.834449768066406 (31.834449768066406) + + --> STEP: 2 + | > loss_disc: 2.570314407348633 (2.5556904077529907) + | > loss_disc_real_0: 0.15708652138710022 (0.15026048570871353) + | > loss_disc_real_1: 0.2563972473144531 (0.24437034875154495) + | > loss_disc_real_2: 0.2244464010000229 (0.2344530075788498) + | > loss_disc_real_3: 0.2640969753265381 (0.25446469336748123) + | > loss_disc_real_4: 0.22793108224868774 (0.22471161931753159) + | > loss_disc_real_5: 0.20277126133441925 (0.20034265518188477) + | > loss_0: 2.570314407348633 (2.5556904077529907) + | > loss_gen: 2.2355170249938965 (2.1946511268615723) + | > loss_kl: 2.1808433532714844 (2.2860783338546753) + | > loss_feat: 6.884369850158691 (6.717370986938477) + | > loss_mel: 18.231739044189453 (18.660265922546387) + | > loss_duration: 1.576138973236084 (1.613161325454712) + | > loss_1: 31.10860824584961 (31.471529006958008) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003439784049987793 (-0.00019657611846923828) + | > avg_loss_disc: 2.5556904077529907 (+0.13494229316711426) + | > avg_loss_disc_real_0: 0.15026048570871353 (+0.021354231983423233) + | > avg_loss_disc_real_1: 0.24437034875154495 (+0.044630639255046844) + | > avg_loss_disc_real_2: 0.2344530075788498 (-0.0014561265707015991) + | > avg_loss_disc_real_3: 0.25446469336748123 (+0.04641474783420563) + | > avg_loss_disc_real_4: 0.22471161931753159 (-0.0006364285945892334) + | > avg_loss_disc_real_5: 0.20034265518188477 (-0.03048034757375717) + | > avg_loss_0: 2.5556904077529907 (+0.13494229316711426) + | > avg_loss_gen: 2.1946511268615723 (-0.08828294277191162) + | > avg_loss_kl: 2.2860783338546753 (+0.016408562660217285) + | > avg_loss_feat: 6.717370986938477 (-0.12990641593933105) + | > avg_loss_mel: 18.660265922546387 (+0.5917081832885742) + | > avg_loss_duration: 1.613161325454712 (+0.007794082164764404) + | > avg_loss_1: 31.471529006958008 (+0.3977222442626953) + + + > EPOCH: 36/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:36:04)  + + --> TIME: 2025-04-18 23:36:07 -- STEP: 6/142 -- GLOBAL_STEP: 384400 + | > loss_disc: 2.4882702827453613 (2.4337671995162964) + | > loss_disc_real_0: 0.17174585163593292 (0.12044098228216171) + | > loss_disc_real_1: 0.208990678191185 (0.21082549542188644) + | > loss_disc_real_2: 0.21818678081035614 (0.21852691719929376) + | > loss_disc_real_3: 0.23679815232753754 (0.2317394738396009) + | > loss_disc_real_4: 0.23112696409225464 (0.23194044331709543) + | > loss_disc_real_5: 0.23273442685604095 (0.2171644518772761) + | > loss_0: 2.4882702827453613 (2.4337671995162964) + | > grad_norm_0: tensor(5.7793, device='cuda:0') (tensor(9.0591, device='cuda:0')) + | > loss_gen: 2.3827950954437256 (2.3965899546941123) + | > loss_kl: 1.870152235031128 (1.9519639015197754) + | > loss_feat: 6.366119384765625 (7.148113250732422) + | > loss_mel: 16.238059997558594 (17.774193445841473) + | > loss_duration: 1.4972681999206543 (1.5132020910580952) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.354394912719727 (30.784062385559082) + | > grad_norm_1: tensor(172.2316, device='cuda:0') (tensor(161.5703, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.3264 (0.3228786786397298) + | > loader_time: 0.0039 (0.004165331522623698) + + + --> TIME: 2025-04-18 23:36:15 -- STEP: 31/142 -- GLOBAL_STEP: 384425 + | > loss_disc: 2.6025571823120117 (2.443980224670903) + | > loss_disc_real_0: 0.1251545250415802 (0.1184783217407042) + | > loss_disc_real_1: 0.22704999148845673 (0.20796145739093905) + | > loss_disc_real_2: 0.2459316998720169 (0.22819579320569192) + | > loss_disc_real_3: 0.23818983137607574 (0.22916930240969505) + | > loss_disc_real_4: 0.2208302766084671 (0.22860412972588692) + | > loss_disc_real_5: 0.2554760277271271 (0.22510421708706888) + | > loss_0: 2.6025571823120117 (2.443980224670903) + | > grad_norm_0: tensor(16.8162, device='cuda:0') (tensor(8.3693, device='cuda:0')) + | > loss_gen: 2.213549852371216 (2.3636391162872314) + | > loss_kl: 1.899794101715088 (1.938673646219315) + | > loss_feat: 6.143081188201904 (7.132048576108871) + | > loss_mel: 16.21485137939453 (17.943975264026275) + | > loss_duration: 1.4908170700073242 (1.5081140725843367) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.962093353271484 (30.886450429116525) + | > grad_norm_1: tensor(495.6941, device='cuda:0') (tensor(221.9346, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.352 (0.3351273613591348) + | > loader_time: 0.0059 (0.004997068835842995) + + + --> TIME: 2025-04-18 23:36:24 -- STEP: 56/142 -- GLOBAL_STEP: 384450 + | > loss_disc: 2.438080310821533 (2.444573278938021) + | > loss_disc_real_0: 0.07588031142950058 (0.12111241849405426) + | > loss_disc_real_1: 0.23852549493312836 (0.20783855712839536) + | > loss_disc_real_2: 0.2305177003145218 (0.22794803950403417) + | > loss_disc_real_3: 0.2648144066333771 (0.22963134332426957) + | > loss_disc_real_4: 0.27925780415534973 (0.23015549512846128) + | > loss_disc_real_5: 0.2611285448074341 (0.22750594333878585) + | > loss_0: 2.438080310821533 (2.444573278938021) + | > grad_norm_0: tensor(11.2264, device='cuda:0') (tensor(8.7952, device='cuda:0')) + | > loss_gen: 2.474640369415283 (2.3716151288577487) + | > loss_kl: 2.07901930809021 (1.946484706231526) + | > loss_feat: 6.992532730102539 (7.09542464358466) + | > loss_mel: 17.85446548461914 (17.93454844611032) + | > loss_duration: 1.5460482835769653 (1.514420170869146) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.946706771850586 (30.862492970057897) + | > grad_norm_1: tensor(257.3798, device='cuda:0') (tensor(228.5354, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.3736 (0.3473084441253117) + | > loader_time: 0.0059 (0.005365920918328423) + + + --> TIME: 2025-04-18 23:36:34 -- STEP: 81/142 -- GLOBAL_STEP: 384475 + | > loss_disc: 2.5018513202667236 (2.450666872071632) + | > loss_disc_real_0: 0.09469887614250183 (0.12497925592793359) + | > loss_disc_real_1: 0.19188947975635529 (0.20779195538273565) + | > loss_disc_real_2: 0.204305037856102 (0.22751978665222355) + | > loss_disc_real_3: 0.20316195487976074 (0.22769260296115168) + | > loss_disc_real_4: 0.22789381444454193 (0.22841242066136114) + | > loss_disc_real_5: 0.20420469343662262 (0.22743245775316967) + | > loss_0: 2.5018513202667236 (2.450666872071632) + | > grad_norm_0: tensor(10.7270, device='cuda:0') (tensor(9.1749, device='cuda:0')) + | > loss_gen: 2.1839511394500732 (2.357053288707027) + | > loss_kl: 1.929999828338623 (1.95782599920108) + | > loss_feat: 7.131109714508057 (7.029701250570793) + | > loss_mel: 18.235044479370117 (17.903559555242087) + | > loss_duration: 1.5326521396636963 (1.5253652348930453) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.01275634765625 (30.77350515200768) + | > grad_norm_1: tensor(265.6441, device='cuda:0') (tensor(229.7194, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.3943 (0.3600943559481774) + | > loader_time: 0.0069 (0.005718958230666173) + + + --> TIME: 2025-04-18 23:36:45 -- STEP: 106/142 -- GLOBAL_STEP: 384500 + | > loss_disc: 2.5366525650024414 (2.4696299197538845) + | > loss_disc_real_0: 0.10138034075498581 (0.12828897431773964) + | > loss_disc_real_1: 0.20822910964488983 (0.2096020021247414) + | > loss_disc_real_2: 0.25502443313598633 (0.2284656749019083) + | > loss_disc_real_3: 0.2290341854095459 (0.22843168521264814) + | > loss_disc_real_4: 0.25189319252967834 (0.2299581192574411) + | > loss_disc_real_5: 0.2037210911512375 (0.23004579333201894) + | > loss_0: 2.5366525650024414 (2.4696299197538845) + | > grad_norm_0: tensor(14.2776, device='cuda:0') (tensor(9.8104, device='cuda:0')) + | > loss_gen: 2.2201311588287354 (2.343662320442919) + | > loss_kl: 2.1159348487854004 (1.9836577615647946) + | > loss_feat: 6.482026100158691 (6.980413369412693) + | > loss_mel: 17.91009521484375 (17.984056598735304) + | > loss_duration: 1.5584220886230469 (1.5356415836316235) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.286609649658203 (30.82743151682728) + | > grad_norm_1: tensor(324.7760, device='cuda:0') (tensor(234.1105, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.413 (0.37152528312971006) + | > loader_time: 0.0071 (0.006001800860998765) + + + --> TIME: 2025-04-18 23:36:56 -- STEP: 131/142 -- GLOBAL_STEP: 384525 + | > loss_disc: 2.3944449424743652 (2.4700685876016397) + | > loss_disc_real_0: 0.08383702486753464 (0.1289341495806024) + | > loss_disc_real_1: 0.20170335471630096 (0.2091120466703677) + | > loss_disc_real_2: 0.20748351514339447 (0.2272469194335792) + | > loss_disc_real_3: 0.21651458740234375 (0.227603466797421) + | > loss_disc_real_4: 0.2238084077835083 (0.22978432101147775) + | > loss_disc_real_5: 0.22178271412849426 (0.23027325245260283) + | > loss_0: 2.3944449424743652 (2.4700685876016397) + | > grad_norm_0: tensor(11.0116, device='cuda:0') (tensor(9.9569, device='cuda:0')) + | > loss_gen: 2.473137855529785 (2.3327314853668204) + | > loss_kl: 2.0859487056732178 (1.9920446608812754) + | > loss_feat: 7.334355354309082 (6.917799745807212) + | > loss_mel: 19.16541290283203 (17.911706007163936) + | > loss_duration: 1.5460841655731201 (1.5406468915575333) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.60493850708008 (30.69492870068732) + | > grad_norm_1: tensor(242.0335, device='cuda:0') (tensor(231.2252, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.4475 (0.38361269463109604) + | > loader_time: 0.0079 (0.0063098423353588305) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.483497381210327 (2.483497381210327) + | > loss_disc_real_0: 0.15711325407028198 (0.15711325407028198) + | > loss_disc_real_1: 0.2661420404911041 (0.2661420404911041) + | > loss_disc_real_2: 0.2953404486179352 (0.2953404486179352) + | > loss_disc_real_3: 0.28736788034439087 (0.28736788034439087) + | > loss_disc_real_4: 0.19980189204216003 (0.19980189204216003) + | > loss_disc_real_5: 0.26050472259521484 (0.26050472259521484) + | > loss_0: 2.483497381210327 (2.483497381210327) + | > loss_gen: 2.5883331298828125 (2.5883331298828125) + | > loss_kl: 2.1858127117156982 (2.1858127117156982) + | > loss_feat: 6.944924831390381 (6.944924831390381) + | > loss_mel: 17.837629318237305 (17.837629318237305) + | > loss_duration: 1.631482481956482 (1.631482481956482) + | > loss_1: 31.188182830810547 (31.188182830810547) + + --> STEP: 1 + | > loss_disc: 2.620213270187378 (2.620213270187378) + | > loss_disc_real_0: 0.16630558669567108 (0.16630558669567108) + | > loss_disc_real_1: 0.2910638749599457 (0.2910638749599457) + | > loss_disc_real_2: 0.2765433192253113 (0.2765433192253113) + | > loss_disc_real_3: 0.26221370697021484 (0.26221370697021484) + | > loss_disc_real_4: 0.26183778047561646 (0.26183778047561646) + | > loss_disc_real_5: 0.25869235396385193 (0.25869235396385193) + | > loss_0: 2.620213270187378 (2.620213270187378) + | > loss_gen: 2.3300929069519043 (2.3300929069519043) + | > loss_kl: 2.479708671569824 (2.479708671569824) + | > loss_feat: 5.588687896728516 (5.588687896728516) + | > loss_mel: 15.658893585205078 (15.658893585205078) + | > loss_duration: 1.6597992181777954 (1.6597992181777954) + | > loss_1: 27.71718406677246 (27.71718406677246) + + --> STEP: 2 + | > loss_disc: 2.568960189819336 (2.594586730003357) + | > loss_disc_real_0: 0.18026304244995117 (0.17328431457281113) + | > loss_disc_real_1: 0.23337244987487793 (0.2622181624174118) + | > loss_disc_real_2: 0.2948061227798462 (0.28567472100257874) + | > loss_disc_real_3: 0.2532567083835602 (0.2577352076768875) + | > loss_disc_real_4: 0.2611016631126404 (0.2614697217941284) + | > loss_disc_real_5: 0.2512332797050476 (0.25496281683444977) + | > loss_0: 2.568960189819336 (2.594586730003357) + | > loss_gen: 2.2869503498077393 (2.3085216283798218) + | > loss_kl: 2.0285584926605225 (2.2541335821151733) + | > loss_feat: 5.770852565765381 (5.679770231246948) + | > loss_mel: 17.099273681640625 (16.37908363342285) + | > loss_duration: 1.5991740226745605 (1.629486620426178) + | > loss_1: 28.78481101989746 (28.25099754333496) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003932476043701172 (+0.0004926919937133789) + | > avg_loss_disc: 2.594586730003357 (+0.03889632225036621) + | > avg_loss_disc_real_0: 0.17328431457281113 (+0.023023828864097595) + | > avg_loss_disc_real_1: 0.2622181624174118 (+0.017847813665866852) + | > avg_loss_disc_real_2: 0.28567472100257874 (+0.05122171342372894) + | > avg_loss_disc_real_3: 0.2577352076768875 (+0.0032705143094062805) + | > avg_loss_disc_real_4: 0.2614697217941284 (+0.03675810247659683) + | > avg_loss_disc_real_5: 0.25496281683444977 (+0.054620161652565) + | > avg_loss_0: 2.594586730003357 (+0.03889632225036621) + | > avg_loss_gen: 2.3085216283798218 (+0.11387050151824951) + | > avg_loss_kl: 2.2541335821151733 (-0.03194475173950195) + | > avg_loss_feat: 5.679770231246948 (-1.0376007556915283) + | > avg_loss_mel: 16.37908363342285 (-2.281182289123535) + | > avg_loss_duration: 1.629486620426178 (+0.016325294971466064) + | > avg_loss_1: 28.25099754333496 (-3.220531463623047) + + + > EPOCH: 37/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:37:02)  + + --> TIME: 2025-04-18 23:37:08 -- STEP: 14/142 -- GLOBAL_STEP: 384550 + | > loss_disc: 2.4529504776000977 (2.4463891983032227) + | > loss_disc_real_0: 0.10263305902481079 (0.11410580256155559) + | > loss_disc_real_1: 0.22221286594867706 (0.20316563653094427) + | > loss_disc_real_2: 0.17388714849948883 (0.2201350639973368) + | > loss_disc_real_3: 0.18542085587978363 (0.2275144351380212) + | > loss_disc_real_4: 0.2488323599100113 (0.2385172556553568) + | > loss_disc_real_5: 0.22742317616939545 (0.23518975291933333) + | > loss_0: 2.4529504776000977 (2.4463891983032227) + | > grad_norm_0: tensor(6.9071, device='cuda:0') (tensor(10.7833, device='cuda:0')) + | > loss_gen: 2.291261911392212 (2.376302650996617) + | > loss_kl: 2.00396990776062 (1.9867522716522217) + | > loss_feat: 7.663395404815674 (7.345926114491054) + | > loss_mel: 17.644840240478516 (18.08706624167306) + | > loss_duration: 1.497445821762085 (1.4963232619421822) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.100914001464844 (31.29237106868199) + | > grad_norm_1: tensor(183.4538, device='cuda:0') (tensor(287.0029, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.3322 (0.3271562031337193) + | > loader_time: 0.0048 (0.004504748753138951) + + + --> TIME: 2025-04-18 23:37:17 -- STEP: 39/142 -- GLOBAL_STEP: 384575 + | > loss_disc: 2.4484410285949707 (2.444553234638311) + | > loss_disc_real_0: 0.11647863686084747 (0.11806376488545002) + | > loss_disc_real_1: 0.20007462799549103 (0.20307831962903342) + | > loss_disc_real_2: 0.24403995275497437 (0.22771453246092185) + | > loss_disc_real_3: 0.22397851943969727 (0.22749496308656839) + | > loss_disc_real_4: 0.25180116295814514 (0.22833582911735925) + | > loss_disc_real_5: 0.2400064915418625 (0.2343414639815306) + | > loss_0: 2.4484410285949707 (2.444553234638311) + | > grad_norm_0: tensor(17.7748, device='cuda:0') (tensor(11.4433, device='cuda:0')) + | > loss_gen: 2.243628978729248 (2.3714752319531556) + | > loss_kl: 1.9552909135818481 (2.0011154199257875) + | > loss_feat: 6.5181732177734375 (7.193753315852239) + | > loss_mel: 17.659832000732422 (18.023885384584087) + | > loss_duration: 1.5452415943145752 (1.510621929780031) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.922168731689453 (31.100851205679085) + | > grad_norm_1: tensor(360.1167, device='cuda:0') (tensor(284.4581, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.3659 (0.3444688748090695) + | > loader_time: 0.0059 (0.005073156112279646) + + + --> TIME: 2025-04-18 23:37:27 -- STEP: 64/142 -- GLOBAL_STEP: 384600 + | > loss_disc: 2.510251998901367 (2.4561229199171057) + | > loss_disc_real_0: 0.12693984806537628 (0.12418852373957634) + | > loss_disc_real_1: 0.19817236065864563 (0.20399209531024098) + | > loss_disc_real_2: 0.23657646775245667 (0.226133729564026) + | > loss_disc_real_3: 0.25123175978660583 (0.22896596137434244) + | > loss_disc_real_4: 0.272877961397171 (0.2299795567523688) + | > loss_disc_real_5: 0.23283414542675018 (0.23411458753980696) + | > loss_0: 2.510251998901367 (2.4561229199171057) + | > grad_norm_0: tensor(6.8061, device='cuda:0') (tensor(14.1768, device='cuda:0')) + | > loss_gen: 2.3613712787628174 (2.3816775418817997) + | > loss_kl: 2.0068206787109375 (1.9998800698667765) + | > loss_feat: 5.903940677642822 (7.095959760248661) + | > loss_mel: 16.27185821533203 (17.919636979699128) + | > loss_duration: 1.5658575296401978 (1.5222873371094467) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.109848022460938 (30.919441521167755) + | > grad_norm_1: tensor(236.7627, device='cuda:0') (tensor(286.5546, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.3942 (0.3586926348507405) + | > loader_time: 0.0067 (0.00541827455163002) + + + --> TIME: 2025-04-18 23:37:37 -- STEP: 89/142 -- GLOBAL_STEP: 384625 + | > loss_disc: 2.409883737564087 (2.4595631347613374) + | > loss_disc_real_0: 0.052288975566625595 (0.12588128288475314) + | > loss_disc_real_1: 0.23701153695583344 (0.2051423918665125) + | > loss_disc_real_2: 0.219148650765419 (0.2255858773930689) + | > loss_disc_real_3: 0.24413010478019714 (0.2278828244195895) + | > loss_disc_real_4: 0.19592459499835968 (0.22957324395688733) + | > loss_disc_real_5: 0.22324110567569733 (0.23359779494532038) + | > loss_0: 2.409883737564087 (2.4595631347613374) + | > grad_norm_0: tensor(22.0868, device='cuda:0') (tensor(16.0875, device='cuda:0')) + | > loss_gen: 2.327624797821045 (2.390098143159674) + | > loss_kl: 1.982621192932129 (2.005001207415977) + | > loss_feat: 7.460583686828613 (7.0615411179789) + | > loss_mel: 18.150856018066406 (17.947041340088575) + | > loss_duration: 1.537182092666626 (1.531627641634995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.4588680267334 (30.935309335087123) + | > grad_norm_1: tensor(447.3513, device='cuda:0') (tensor(314.1681, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.4132 (0.37199113342199447) + | > loader_time: 0.0065 (0.005696840500563715) + + + --> TIME: 2025-04-18 23:37:48 -- STEP: 114/142 -- GLOBAL_STEP: 384650 + | > loss_disc: 2.487302541732788 (2.4711691371181534) + | > loss_disc_real_0: 0.09141969680786133 (0.1283090371769249) + | > loss_disc_real_1: 0.20565150678157806 (0.20606169901918947) + | > loss_disc_real_2: 0.2206221967935562 (0.2262760998125662) + | > loss_disc_real_3: 0.21545624732971191 (0.22848944771185256) + | > loss_disc_real_4: 0.19568374752998352 (0.22944931280717515) + | > loss_disc_real_5: 0.21240206062793732 (0.23342822635905786) + | > loss_0: 2.487302541732788 (2.4711691371181534) + | > grad_norm_0: tensor(9.5910, device='cuda:0') (tensor(16.2708, device='cuda:0')) + | > loss_gen: 2.252883195877075 (2.3712009965327754) + | > loss_kl: 1.9716898202896118 (2.007838483442339) + | > loss_feat: 7.279932975769043 (7.0083448803215695) + | > loss_mel: 18.288394927978516 (17.95099326183921) + | > loss_duration: 1.5526504516601562 (1.5374004830393877) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.345552444458008 (30.87577796400639) + | > grad_norm_1: tensor(315.3636, device='cuda:0') (tensor(316.4977, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.4358 (0.38388657569885265) + | > loader_time: 0.0069 (0.0059489199989720375) + + + --> TIME: 2025-04-18 23:38:00 -- STEP: 139/142 -- GLOBAL_STEP: 384675 + | > loss_disc: 2.405341386795044 (2.4731299431203926) + | > loss_disc_real_0: 0.1389605551958084 (0.12892511671931622) + | > loss_disc_real_1: 0.19590811431407928 (0.20623431070674236) + | > loss_disc_real_2: 0.20876498520374298 (0.22626406393891615) + | > loss_disc_real_3: 0.2156905084848404 (0.22861531108832187) + | > loss_disc_real_4: 0.21537204086780548 (0.2293399827514621) + | > loss_disc_real_5: 0.1977851390838623 (0.23505458128538062) + | > loss_0: 2.405341386795044 (2.4731299431203926) + | > grad_norm_0: tensor(5.3209, device='cuda:0') (tensor(15.0512, device='cuda:0')) + | > loss_gen: 2.2815098762512207 (2.3544507901445573) + | > loss_kl: 2.0254361629486084 (2.010823853581929) + | > loss_feat: 6.367195129394531 (6.946038551467786) + | > loss_mel: 16.49080467224121 (17.879517369990722) + | > loss_duration: 1.5691773891448975 (1.5424638103238113) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.73412322998047 (30.73329424686569) + | > grad_norm_1: tensor(149.4266, device='cuda:0') (tensor(290.6846, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.4614 (0.39730435652698554) + | > loader_time: 0.0079 (0.006229378336625132) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3162002563476562 (2.3162002563476562) + | > loss_disc_real_0: 0.07247857004404068 (0.07247857004404068) + | > loss_disc_real_1: 0.1657845675945282 (0.1657845675945282) + | > loss_disc_real_2: 0.2589346170425415 (0.2589346170425415) + | > loss_disc_real_3: 0.2563970685005188 (0.2563970685005188) + | > loss_disc_real_4: 0.23439331352710724 (0.23439331352710724) + | > loss_disc_real_5: 0.25254666805267334 (0.25254666805267334) + | > loss_0: 2.3162002563476562 (2.3162002563476562) + | > loss_gen: 2.360842704772949 (2.360842704772949) + | > loss_kl: 2.359679937362671 (2.359679937362671) + | > loss_feat: 7.975888729095459 (7.975888729095459) + | > loss_mel: 20.001628875732422 (20.001628875732422) + | > loss_duration: 1.614483118057251 (1.614483118057251) + | > loss_1: 34.312522888183594 (34.312522888183594) + + --> STEP: 1 + | > loss_disc: 2.506791114807129 (2.506791114807129) + | > loss_disc_real_0: 0.1282588094472885 (0.1282588094472885) + | > loss_disc_real_1: 0.20142963528633118 (0.20142963528633118) + | > loss_disc_real_2: 0.2371145635843277 (0.2371145635843277) + | > loss_disc_real_3: 0.2410614788532257 (0.2410614788532257) + | > loss_disc_real_4: 0.23493164777755737 (0.23493164777755737) + | > loss_disc_real_5: 0.2436482459306717 (0.2436482459306717) + | > loss_0: 2.506791114807129 (2.506791114807129) + | > loss_gen: 2.126213550567627 (2.126213550567627) + | > loss_kl: 2.3064119815826416 (2.3064119815826416) + | > loss_feat: 6.012286186218262 (6.012286186218262) + | > loss_mel: 16.942659378051758 (16.942659378051758) + | > loss_duration: 1.6449371576309204 (1.6449371576309204) + | > loss_1: 29.032506942749023 (29.032506942749023) + + --> STEP: 2 + | > loss_disc: 2.6415810585021973 (2.574186086654663) + | > loss_disc_real_0: 0.11839085817337036 (0.12332483381032944) + | > loss_disc_real_1: 0.2310502678155899 (0.21623995155096054) + | > loss_disc_real_2: 0.2581196427345276 (0.24761710315942764) + | > loss_disc_real_3: 0.26075950264930725 (0.2509104907512665) + | > loss_disc_real_4: 0.24147270619869232 (0.23820217698812485) + | > loss_disc_real_5: 0.27512744069099426 (0.259387843310833) + | > loss_0: 2.6415810585021973 (2.574186086654663) + | > loss_gen: 2.0471596717834473 (2.086686611175537) + | > loss_kl: 2.1740353107452393 (2.2402236461639404) + | > loss_feat: 5.1579179763793945 (5.585102081298828) + | > loss_mel: 15.697970390319824 (16.32031488418579) + | > loss_duration: 1.5903010368347168 (1.6176190972328186) + | > loss_1: 26.667383193969727 (27.849945068359375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035496950149536133 (-0.0003827810287475586) + | > avg_loss_disc: 2.574186086654663 (-0.020400643348693848) + | > avg_loss_disc_real_0: 0.12332483381032944 (-0.04995948076248169) + | > avg_loss_disc_real_1: 0.21623995155096054 (-0.04597821086645126) + | > avg_loss_disc_real_2: 0.24761710315942764 (-0.03805761784315109) + | > avg_loss_disc_real_3: 0.2509104907512665 (-0.006824716925621033) + | > avg_loss_disc_real_4: 0.23820217698812485 (-0.02326754480600357) + | > avg_loss_disc_real_5: 0.259387843310833 (+0.004425026476383209) + | > avg_loss_0: 2.574186086654663 (-0.020400643348693848) + | > avg_loss_gen: 2.086686611175537 (-0.22183501720428467) + | > avg_loss_kl: 2.2402236461639404 (-0.01390993595123291) + | > avg_loss_feat: 5.585102081298828 (-0.09466814994812012) + | > avg_loss_mel: 16.32031488418579 (-0.05876874923706055) + | > avg_loss_duration: 1.6176190972328186 (-0.011867523193359375) + | > avg_loss_1: 27.849945068359375 (-0.40105247497558594) + + + > EPOCH: 38/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:38:02)  + + --> TIME: 2025-04-18 23:38:10 -- STEP: 22/142 -- GLOBAL_STEP: 384700 + | > loss_disc: 2.4522745609283447 (2.4506661241704766) + | > loss_disc_real_0: 0.16045686602592468 (0.12526417049494656) + | > loss_disc_real_1: 0.21248646080493927 (0.20656803114847702) + | > loss_disc_real_2: 0.21203412115573883 (0.22593194314024664) + | > loss_disc_real_3: 0.23708826303482056 (0.229548188095743) + | > loss_disc_real_4: 0.20688894391059875 (0.22858986258506775) + | > loss_disc_real_5: 0.27298998832702637 (0.23207231001420456) + | > loss_0: 2.4522745609283447 (2.4506661241704766) + | > grad_norm_0: tensor(8.3226, device='cuda:0') (tensor(6.3520, device='cuda:0')) + | > loss_gen: 2.4163928031921387 (2.361656589941545) + | > loss_kl: 1.7784287929534912 (1.9265108867125078) + | > loss_feat: 7.204204082489014 (7.0236569101160224) + | > loss_mel: 18.491939544677734 (17.508396408774637) + | > loss_duration: 1.4760303497314453 (1.4960646466775374) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.366994857788086 (30.31628556685014) + | > grad_norm_1: tensor(178.1856, device='cuda:0') (tensor(128.1691, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.3509 (0.33164418827403674) + | > loader_time: 0.0054 (0.004879344593394886) + + + --> TIME: 2025-04-18 23:38:20 -- STEP: 47/142 -- GLOBAL_STEP: 384725 + | > loss_disc: 2.414412260055542 (2.480464813557077) + | > loss_disc_real_0: 0.11009187996387482 (0.12662378143756947) + | > loss_disc_real_1: 0.20199276506900787 (0.20778699567977418) + | > loss_disc_real_2: 0.23151744902133942 (0.23031731171810882) + | > loss_disc_real_3: 0.21500444412231445 (0.2318312386248974) + | > loss_disc_real_4: 0.21317307651042938 (0.22999414611370006) + | > loss_disc_real_5: 0.1987663358449936 (0.2347368173142697) + | > loss_0: 2.414412260055542 (2.480464813557077) + | > grad_norm_0: tensor(12.4925, device='cuda:0') (tensor(7.4592, device='cuda:0')) + | > loss_gen: 2.328065872192383 (2.336803096406003) + | > loss_kl: 1.8921877145767212 (1.942305605462257) + | > loss_feat: 6.157105922698975 (6.9405133064757) + | > loss_mel: 15.89834976196289 (17.70413313520715) + | > loss_duration: 1.4994823932647705 (1.507120005627896) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.775192260742188 (30.43087521005184) + | > grad_norm_1: tensor(165.2280, device='cuda:0') (tensor(125.1858, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.3721 (0.3483258460430389) + | > loader_time: 0.0057 (0.005379484054890085) + + + --> TIME: 2025-04-18 23:38:30 -- STEP: 72/142 -- GLOBAL_STEP: 384750 + | > loss_disc: 2.583711624145508 (2.4816757672362857) + | > loss_disc_real_0: 0.14791272580623627 (0.12740503034243977) + | > loss_disc_real_1: 0.21499879658222198 (0.2073793967978822) + | > loss_disc_real_2: 0.19528385996818542 (0.22920006120370495) + | > loss_disc_real_3: 0.24139554798603058 (0.23172080827256045) + | > loss_disc_real_4: 0.2209652215242386 (0.2323634899738762) + | > loss_disc_real_5: 0.22456665337085724 (0.23388593685295847) + | > loss_0: 2.583711624145508 (2.4816757672362857) + | > grad_norm_0: tensor(9.5784, device='cuda:0') (tensor(7.2879, device='cuda:0')) + | > loss_gen: 2.155707836151123 (2.327673912048339) + | > loss_kl: 1.9805173873901367 (1.9467931787172954) + | > loss_feat: 7.169870376586914 (6.904938850137922) + | > loss_mel: 18.326208114624023 (17.84697874387105) + | > loss_duration: 1.5138275623321533 (1.5187908146116469) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.146133422851562 (30.545175525877212) + | > grad_norm_1: tensor(171.0448, device='cuda:0') (tensor(130.5591, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.4026 (0.36268152793248487) + | > loader_time: 0.0067 (0.005732533004548814) + + + --> TIME: 2025-04-18 23:38:40 -- STEP: 97/142 -- GLOBAL_STEP: 384775 + | > loss_disc: 2.548553466796875 (2.4896243380517076) + | > loss_disc_real_0: 0.15321753919124603 (0.12966614539168533) + | > loss_disc_real_1: 0.2231062352657318 (0.2089087638965587) + | > loss_disc_real_2: 0.21808302402496338 (0.22861193704236413) + | > loss_disc_real_3: 0.23196077346801758 (0.23135455751541964) + | > loss_disc_real_4: 0.22353826463222504 (0.232151217374605) + | > loss_disc_real_5: 0.21510201692581177 (0.23425958712690884) + | > loss_0: 2.548553466796875 (2.4896243380517076) + | > grad_norm_0: tensor(17.1563, device='cuda:0') (tensor(7.6614, device='cuda:0')) + | > loss_gen: 2.262524366378784 (2.3151695654564284) + | > loss_kl: 1.9143993854522705 (1.9604373039658536) + | > loss_feat: 5.796909332275391 (6.866691805652736) + | > loss_mel: 14.939751625061035 (17.815911194712847) + | > loss_duration: 1.5676250457763672 (1.5303140030693763) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.48120880126953 (30.488523876544125) + | > grad_norm_1: tensor(305.4943, device='cuda:0') (tensor(139.3860, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.4226 (0.3753954184424016) + | > loader_time: 0.0072 (0.006004109825055624) + + + --> TIME: 2025-04-18 23:38:51 -- STEP: 122/142 -- GLOBAL_STEP: 384800 + | > loss_disc: 2.610342264175415 (2.4963205388334933) + | > loss_disc_real_0: 0.11278247833251953 (0.13170694755237597) + | > loss_disc_real_1: 0.21658270061016083 (0.20942521315129076) + | > loss_disc_real_2: 0.19555820524692535 (0.22797165786633727) + | > loss_disc_real_3: 0.21774423122406006 (0.23080678113171313) + | > loss_disc_real_4: 0.23972710967063904 (0.23217554378216385) + | > loss_disc_real_5: 0.20255333185195923 (0.2334403473822797) + | > loss_0: 2.610342264175415 (2.4963205388334933) + | > grad_norm_0: tensor(6.8411, device='cuda:0') (tensor(8.5566, device='cuda:0')) + | > loss_gen: 2.289794921875 (2.3059508663709027) + | > loss_kl: 1.8532743453979492 (1.9671240869115612) + | > loss_feat: 6.874856948852539 (6.829020898850238) + | > loss_mel: 18.14862060546875 (17.821976388087045) + | > loss_duration: 1.5341055393218994 (1.5364829174807815) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.700651168823242 (30.46055510786713) + | > grad_norm_1: tensor(283.5663, device='cuda:0') (tensor(156.1070, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.4623 (0.38991100866286477) + | > loader_time: 0.0071 (0.006237471689943408) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.401059865951538 (2.401059865951538) + | > loss_disc_real_0: 0.1733274757862091 (0.1733274757862091) + | > loss_disc_real_1: 0.1828782707452774 (0.1828782707452774) + | > loss_disc_real_2: 0.22281405329704285 (0.22281405329704285) + | > loss_disc_real_3: 0.19043084979057312 (0.19043084979057312) + | > loss_disc_real_4: 0.2113560289144516 (0.2113560289144516) + | > loss_disc_real_5: 0.20662938058376312 (0.20662938058376312) + | > loss_0: 2.401059865951538 (2.401059865951538) + | > loss_gen: 2.2384848594665527 (2.2384848594665527) + | > loss_kl: 2.311098337173462 (2.311098337173462) + | > loss_feat: 6.707700252532959 (6.707700252532959) + | > loss_mel: 16.99724769592285 (16.99724769592285) + | > loss_duration: 1.626811146736145 (1.626811146736145) + | > loss_1: 29.881343841552734 (29.881343841552734) + + --> STEP: 1 + | > loss_disc: 2.4630091190338135 (2.4630091190338135) + | > loss_disc_real_0: 0.17068949341773987 (0.17068949341773987) + | > loss_disc_real_1: 0.20751680433750153 (0.20751680433750153) + | > loss_disc_real_2: 0.24024666845798492 (0.24024666845798492) + | > loss_disc_real_3: 0.23971380293369293 (0.23971380293369293) + | > loss_disc_real_4: 0.203253835439682 (0.203253835439682) + | > loss_disc_real_5: 0.23065869510173798 (0.23065869510173798) + | > loss_0: 2.4630091190338135 (2.4630091190338135) + | > loss_gen: 2.384185314178467 (2.384185314178467) + | > loss_kl: 2.194861650466919 (2.194861650466919) + | > loss_feat: 7.783769130706787 (7.783769130706787) + | > loss_mel: 19.224124908447266 (19.224124908447266) + | > loss_duration: 1.6450724601745605 (1.6450724601745605) + | > loss_1: 33.23201370239258 (33.23201370239258) + + --> STEP: 2 + | > loss_disc: 2.5614631175994873 (2.5122361183166504) + | > loss_disc_real_0: 0.21252796053886414 (0.191608726978302) + | > loss_disc_real_1: 0.18365438282489777 (0.19558559358119965) + | > loss_disc_real_2: 0.2143733948469162 (0.22731003165245056) + | > loss_disc_real_3: 0.2280426323413849 (0.2338782176375389) + | > loss_disc_real_4: 0.2124686986207962 (0.2078612670302391) + | > loss_disc_real_5: 0.248732790350914 (0.239695742726326) + | > loss_0: 2.5614631175994873 (2.5122361183166504) + | > loss_gen: 2.1836583614349365 (2.2839218378067017) + | > loss_kl: 2.030303478240967 (2.112582564353943) + | > loss_feat: 6.2217698097229 (7.002769470214844) + | > loss_mel: 17.653181076049805 (18.438652992248535) + | > loss_duration: 1.5982829332351685 (1.6216776967048645) + | > loss_1: 29.687196731567383 (31.45960521697998) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033693313598632812 (-0.00018036365509033203) + | > avg_loss_disc: 2.5122361183166504 (-0.061949968338012695) + | > avg_loss_disc_real_0: 0.191608726978302 (+0.06828389316797256) + | > avg_loss_disc_real_1: 0.19558559358119965 (-0.020654357969760895) + | > avg_loss_disc_real_2: 0.22731003165245056 (-0.02030707150697708) + | > avg_loss_disc_real_3: 0.2338782176375389 (-0.01703227311372757) + | > avg_loss_disc_real_4: 0.2078612670302391 (-0.030340909957885742) + | > avg_loss_disc_real_5: 0.239695742726326 (-0.01969210058450699) + | > avg_loss_0: 2.5122361183166504 (-0.061949968338012695) + | > avg_loss_gen: 2.2839218378067017 (+0.19723522663116455) + | > avg_loss_kl: 2.112582564353943 (-0.12764108180999756) + | > avg_loss_feat: 7.002769470214844 (+1.4176673889160156) + | > avg_loss_mel: 18.438652992248535 (+2.118338108062744) + | > avg_loss_duration: 1.6216776967048645 (+0.0040585994720458984) + | > avg_loss_1: 31.45960521697998 (+3.6096601486206055) + + + > EPOCH: 39/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:39:02)  + + --> TIME: 2025-04-18 23:39:04 -- STEP: 5/142 -- GLOBAL_STEP: 384825 + | > loss_disc: 2.366222858428955 (2.392404556274414) + | > loss_disc_real_0: 0.12371983379125595 (0.12552317529916762) + | > loss_disc_real_1: 0.14211708307266235 (0.19655734300613403) + | > loss_disc_real_2: 0.18115456402301788 (0.22445571422576904) + | > loss_disc_real_3: 0.2299807369709015 (0.21991785168647765) + | > loss_disc_real_4: 0.21816915273666382 (0.23617051243782045) + | > loss_disc_real_5: 0.23815102875232697 (0.22318878769874573) + | > loss_0: 2.366222858428955 (2.392404556274414) + | > grad_norm_0: tensor(10.5655, device='cuda:0') (tensor(11.0147, device='cuda:0')) + | > loss_gen: 2.3692586421966553 (2.379207420349121) + | > loss_kl: 1.9842244386672974 (1.9726586580276488) + | > loss_feat: 7.3546953201293945 (7.118666553497315) + | > loss_mel: 18.202707290649414 (17.78846092224121) + | > loss_duration: 1.4996987581253052 (1.5113563537597656) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.41058349609375 (30.7703498840332) + | > grad_norm_1: tensor(323.6842, device='cuda:0') (tensor(256.6199, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.3252 (0.32180113792419435) + | > loader_time: 0.0041 (0.004368019104003906) + + + --> TIME: 2025-04-18 23:39:13 -- STEP: 30/142 -- GLOBAL_STEP: 384850 + | > loss_disc: 2.439661741256714 (2.4381930430730185) + | > loss_disc_real_0: 0.13795481622219086 (0.12059642026821772) + | > loss_disc_real_1: 0.20689280331134796 (0.20595863709847131) + | > loss_disc_real_2: 0.24175472557544708 (0.22790458897749583) + | > loss_disc_real_3: 0.24539916217327118 (0.22480942060550055) + | > loss_disc_real_4: 0.23184555768966675 (0.22925685594479242) + | > loss_disc_real_5: 0.26014477014541626 (0.22683273206154506) + | > loss_0: 2.439661741256714 (2.4381930430730185) + | > grad_norm_0: tensor(6.8815, device='cuda:0') (tensor(10.8536, device='cuda:0')) + | > loss_gen: 2.483071804046631 (2.3917805592219037) + | > loss_kl: 1.9872632026672363 (1.9495319485664369) + | > loss_feat: 7.129326820373535 (7.156321716308594) + | > loss_mel: 18.311220169067383 (17.796302477518726) + | > loss_duration: 1.507519245147705 (1.5097755352656044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.41840171813965 (30.803712590535483) + | > grad_norm_1: tensor(208.1153, device='cuda:0') (tensor(267.1579, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.3576 (0.3385301431020101) + | > loader_time: 0.0059 (0.005221446355183919) + + + --> TIME: 2025-04-18 23:39:23 -- STEP: 55/142 -- GLOBAL_STEP: 384875 + | > loss_disc: 2.4394326210021973 (2.4507854374972267) + | > loss_disc_real_0: 0.19816285371780396 (0.12474624826149507) + | > loss_disc_real_1: 0.19578486680984497 (0.2091121500188654) + | > loss_disc_real_2: 0.19975310564041138 (0.22730191918936643) + | > loss_disc_real_3: 0.228643536567688 (0.22732276781038804) + | > loss_disc_real_4: 0.19636811316013336 (0.22851319692351602) + | > loss_disc_real_5: 0.23997065424919128 (0.2291555334221233) + | > loss_0: 2.4394326210021973 (2.4507854374972267) + | > grad_norm_0: tensor(6.3448, device='cuda:0') (tensor(9.3487, device='cuda:0')) + | > loss_gen: 2.5255441665649414 (2.3794298778880725) + | > loss_kl: 2.075808525085449 (1.9675457239151002) + | > loss_feat: 7.860564231872559 (7.176400436054577) + | > loss_mel: 19.068161010742188 (17.969644165039064) + | > loss_duration: 1.5560212135314941 (1.5135929974642663) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.08610153198242 (31.006613332575018) + | > grad_norm_1: tensor(106.1124, device='cuda:0') (tensor(216.2988, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.3824 (0.3530767700888894) + | > loader_time: 0.0061 (0.005596668070012873) + + + --> TIME: 2025-04-18 23:39:33 -- STEP: 80/142 -- GLOBAL_STEP: 384900 + | > loss_disc: 2.49106502532959 (2.4546176224946987) + | > loss_disc_real_0: 0.11947856098413467 (0.12514439933001995) + | > loss_disc_real_1: 0.22766512632369995 (0.20929824523627757) + | > loss_disc_real_2: 0.20590858161449432 (0.2263882804661989) + | > loss_disc_real_3: 0.21625493466854095 (0.22638376988470554) + | > loss_disc_real_4: 0.22383442521095276 (0.2300749931484461) + | > loss_disc_real_5: 0.2681756019592285 (0.22918215487152338) + | > loss_0: 2.49106502532959 (2.4546176224946987) + | > grad_norm_0: tensor(13.9762, device='cuda:0') (tensor(9.7998, device='cuda:0')) + | > loss_gen: 2.3818700313568115 (2.37352243065834) + | > loss_kl: 2.086962938308716 (1.9791827708482743) + | > loss_feat: 7.744423866271973 (7.0999518752098085) + | > loss_mel: 18.81218719482422 (17.963442170619963) + | > loss_duration: 1.5513722896575928 (1.5240281015634536) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.57681655883789 (30.940127468109132) + | > grad_norm_1: tensor(401.9861, device='cuda:0') (tensor(241.0920, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.4219 (0.36688487231731415) + | > loader_time: 0.0066 (0.00585706830024719) + + + --> TIME: 2025-04-18 23:39:43 -- STEP: 105/142 -- GLOBAL_STEP: 384925 + | > loss_disc: 2.4830081462860107 (2.469959917522612) + | > loss_disc_real_0: 0.1359769105911255 (0.12775228775682906) + | > loss_disc_real_1: 0.20731821656227112 (0.2107500216790608) + | > loss_disc_real_2: 0.2293807864189148 (0.22710110530966804) + | > loss_disc_real_3: 0.19944995641708374 (0.22709026989482697) + | > loss_disc_real_4: 0.24930910766124725 (0.2305199804760161) + | > loss_disc_real_5: 0.23938848078250885 (0.2306263364496685) + | > loss_0: 2.4830081462860107 (2.469959917522612) + | > grad_norm_0: tensor(17.1385, device='cuda:0') (tensor(11.0619, device='cuda:0')) + | > loss_gen: 2.282463312149048 (2.3538756529490152) + | > loss_kl: 2.1430017948150635 (1.990164650054205) + | > loss_feat: 7.116428852081299 (6.986301399412609) + | > loss_mel: 17.833263397216797 (17.87371908823649) + | > loss_duration: 1.5818424224853516 (1.5339202233723233) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.956998825073242 (30.73798101515997) + | > grad_norm_1: tensor(246.5266, device='cuda:0') (tensor(254.8797, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.4308 (0.3785308656238374) + | > loader_time: 0.0071 (0.006112146377563474) + + + --> TIME: 2025-04-18 23:39:55 -- STEP: 130/142 -- GLOBAL_STEP: 384950 + | > loss_disc: 2.5328946113586426 (2.467740535736084) + | > loss_disc_real_0: 0.1174125000834465 (0.12804561962301908) + | > loss_disc_real_1: 0.22709105908870697 (0.21020927658447852) + | > loss_disc_real_2: 0.23832763731479645 (0.22650579500656862) + | > loss_disc_real_3: 0.2387244701385498 (0.2275380539206358) + | > loss_disc_real_4: 0.2435763031244278 (0.23081974433018612) + | > loss_disc_real_5: 0.2711507976055145 (0.2300739084298794) + | > loss_0: 2.5328946113586426 (2.467740535736084) + | > grad_norm_0: tensor(4.5984, device='cuda:0') (tensor(11.2225, device='cuda:0')) + | > loss_gen: 2.3265390396118164 (2.350338171078609) + | > loss_kl: 1.9283636808395386 (1.9917861241560715) + | > loss_feat: 7.238842964172363 (6.9580984812516435) + | > loss_mel: 19.190725326538086 (17.865906840104326) + | > loss_duration: 1.5330523252487183 (1.5381692473704998) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.21752166748047 (30.704298841036284) + | > grad_norm_1: tensor(102.5142, device='cuda:0') (tensor(254.9952, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.4591 (0.39113633999457725) + | > loader_time: 0.0076 (0.0063701262840857855) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4848783016204834 (2.4848783016204834) + | > loss_disc_real_0: 0.16652660071849823 (0.16652660071849823) + | > loss_disc_real_1: 0.24556434154510498 (0.24556434154510498) + | > loss_disc_real_2: 0.2427344173192978 (0.2427344173192978) + | > loss_disc_real_3: 0.27282676100730896 (0.27282676100730896) + | > loss_disc_real_4: 0.21848559379577637 (0.21848559379577637) + | > loss_disc_real_5: 0.23411604762077332 (0.23411604762077332) + | > loss_0: 2.4848783016204834 (2.4848783016204834) + | > loss_gen: 2.4293274879455566 (2.4293274879455566) + | > loss_kl: 2.302427053451538 (2.302427053451538) + | > loss_feat: 5.933521270751953 (5.933521270751953) + | > loss_mel: 16.69855308532715 (16.69855308532715) + | > loss_duration: 1.6247093677520752 (1.6247093677520752) + | > loss_1: 28.98853874206543 (28.98853874206543) + + --> STEP: 1 + | > loss_disc: 2.5140113830566406 (2.5140113830566406) + | > loss_disc_real_0: 0.1648208349943161 (0.1648208349943161) + | > loss_disc_real_1: 0.2740015387535095 (0.2740015387535095) + | > loss_disc_real_2: 0.26041606068611145 (0.26041606068611145) + | > loss_disc_real_3: 0.27075183391571045 (0.27075183391571045) + | > loss_disc_real_4: 0.25198498368263245 (0.25198498368263245) + | > loss_disc_real_5: 0.23588712513446808 (0.23588712513446808) + | > loss_0: 2.5140113830566406 (2.5140113830566406) + | > loss_gen: 2.592524528503418 (2.592524528503418) + | > loss_kl: 2.494612693786621 (2.494612693786621) + | > loss_feat: 7.821938991546631 (7.821938991546631) + | > loss_mel: 18.829940795898438 (18.829940795898438) + | > loss_duration: 1.6533173322677612 (1.6533173322677612) + | > loss_1: 33.392333984375 (33.392333984375) + + --> STEP: 2 + | > loss_disc: 2.5629634857177734 (2.538487434387207) + | > loss_disc_real_0: 0.14091670513153076 (0.15286877006292343) + | > loss_disc_real_1: 0.25917479395866394 (0.26658816635608673) + | > loss_disc_real_2: 0.2529480457305908 (0.25668205320835114) + | > loss_disc_real_3: 0.27881962060928345 (0.27478572726249695) + | > loss_disc_real_4: 0.2571021318435669 (0.25454355776309967) + | > loss_disc_real_5: 0.22095556557178497 (0.22842134535312653) + | > loss_0: 2.5629634857177734 (2.538487434387207) + | > loss_gen: 2.269413471221924 (2.430968999862671) + | > loss_kl: 1.9877452850341797 (2.2411789894104004) + | > loss_feat: 6.010205268859863 (6.916072130203247) + | > loss_mel: 17.800060272216797 (18.315000534057617) + | > loss_duration: 1.5911250114440918 (1.6222211718559265) + | > loss_1: 29.658550262451172 (31.525442123413086) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003165006637573242 (-0.00020432472229003906) + | > avg_loss_disc: 2.538487434387207 (+0.02625131607055664) + | > avg_loss_disc_real_0: 0.15286877006292343 (-0.03873995691537857) + | > avg_loss_disc_real_1: 0.26658816635608673 (+0.07100257277488708) + | > avg_loss_disc_real_2: 0.25668205320835114 (+0.029372021555900574) + | > avg_loss_disc_real_3: 0.27478572726249695 (+0.04090750962495804) + | > avg_loss_disc_real_4: 0.25454355776309967 (+0.046682290732860565) + | > avg_loss_disc_real_5: 0.22842134535312653 (-0.011274397373199463) + | > avg_loss_0: 2.538487434387207 (+0.02625131607055664) + | > avg_loss_gen: 2.430968999862671 (+0.14704716205596924) + | > avg_loss_kl: 2.2411789894104004 (+0.12859642505645752) + | > avg_loss_feat: 6.916072130203247 (-0.08669734001159668) + | > avg_loss_mel: 18.315000534057617 (-0.12365245819091797) + | > avg_loss_duration: 1.6222211718559265 (+0.0005434751510620117) + | > avg_loss_1: 31.525442123413086 (+0.06583690643310547) + + + > EPOCH: 40/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:40:02)  + + --> TIME: 2025-04-18 23:40:07 -- STEP: 13/142 -- GLOBAL_STEP: 384975 + | > loss_disc: 2.5024752616882324 (2.4273886680603027) + | > loss_disc_real_0: 0.1246776208281517 (0.12120408965991093) + | > loss_disc_real_1: 0.2602928876876831 (0.2050155527316607) + | > loss_disc_real_2: 0.2525995969772339 (0.21629582001612738) + | > loss_disc_real_3: 0.21495497226715088 (0.2226907633818113) + | > loss_disc_real_4: 0.24976353347301483 (0.22868551657750055) + | > loss_disc_real_5: 0.2595011293888092 (0.22728194296360016) + | > loss_0: 2.5024752616882324 (2.4273886680603027) + | > grad_norm_0: tensor(8.7201, device='cuda:0') (tensor(16.9002, device='cuda:0')) + | > loss_gen: 2.3072333335876465 (2.393729650057279) + | > loss_kl: 1.9619197845458984 (1.9403489919809194) + | > loss_feat: 7.058341979980469 (7.314808918879582) + | > loss_mel: 17.67296028137207 (18.216080005352314) + | > loss_duration: 1.5155216455459595 (1.4969095633580134) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.51597785949707 (31.361876707810623) + | > grad_norm_1: tensor(61.0448, device='cuda:0') (tensor(330.6009, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.3275 (0.3303719117091252) + | > loader_time: 0.0049 (0.004666566848754883) + + + --> TIME: 2025-04-18 23:40:16 -- STEP: 38/142 -- GLOBAL_STEP: 385000 + | > loss_disc: 2.4398584365844727 (2.4315560554203235) + | > loss_disc_real_0: 0.1340251863002777 (0.12451883621121708) + | > loss_disc_real_1: 0.18421673774719238 (0.20199619586530485) + | > loss_disc_real_2: 0.25420689582824707 (0.22418111131379478) + | > loss_disc_real_3: 0.2708370089530945 (0.22673732787370682) + | > loss_disc_real_4: 0.26100265979766846 (0.2282492636065734) + | > loss_disc_real_5: 0.23412615060806274 (0.22494792467669436) + | > loss_0: 2.4398584365844727 (2.4315560554203235) + | > grad_norm_0: tensor(20.9179, device='cuda:0') (tensor(15.3668, device='cuda:0')) + | > loss_gen: 2.6563103199005127 (2.380228557084736) + | > loss_kl: 1.9475436210632324 (1.9588775854361684) + | > loss_feat: 7.364452362060547 (7.066431371789229) + | > loss_mel: 17.846500396728516 (17.91439874548661) + | > loss_duration: 1.53302001953125 (1.5041358502287614) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.347827911376953 (30.824071783768503) + | > grad_norm_1: tensor(442.3928, device='cuda:0') (tensor(330.3865, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.3603 (0.34491219018634994) + | > loader_time: 0.006 (0.005228506891350997) + + + --> TIME: 2025-04-18 23:40:26 -- STEP: 63/142 -- GLOBAL_STEP: 385025 + | > loss_disc: 2.356757640838623 (2.4550844866131976) + | > loss_disc_real_0: 0.1165148988366127 (0.12531906543743046) + | > loss_disc_real_1: 0.20837630331516266 (0.2052144424782859) + | > loss_disc_real_2: 0.21919207274913788 (0.22513855284168607) + | > loss_disc_real_3: 0.21268711984157562 (0.22797132389886038) + | > loss_disc_real_4: 0.20394070446491241 (0.23106419898214794) + | > loss_disc_real_5: 0.1956796497106552 (0.22830461959044138) + | > loss_0: 2.356757640838623 (2.4550844866131976) + | > grad_norm_0: tensor(6.9175, device='cuda:0') (tensor(14.2883, device='cuda:0')) + | > loss_gen: 2.452338933944702 (2.3599397492787193) + | > loss_kl: 1.928602695465088 (1.961148595053052) + | > loss_feat: 6.9653239250183105 (7.008054188319615) + | > loss_mel: 18.582212448120117 (17.983338128952752) + | > loss_duration: 1.5775595903396606 (1.5154999759462144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.506038665771484 (30.82798070756216) + | > grad_norm_1: tensor(242.4914, device='cuda:0') (tensor(322.6459, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.4037 (0.35814180449833943) + | > loader_time: 0.0063 (0.0055955856565445185) + + + --> TIME: 2025-04-18 23:40:36 -- STEP: 88/142 -- GLOBAL_STEP: 385050 + | > loss_disc: 2.5492517948150635 (2.4631155214526435) + | > loss_disc_real_0: 0.1290438026189804 (0.12742419828745455) + | > loss_disc_real_1: 0.20792458951473236 (0.2049482258206064) + | > loss_disc_real_2: 0.21139046549797058 (0.2261005203154954) + | > loss_disc_real_3: 0.20854942500591278 (0.22933059490539812) + | > loss_disc_real_4: 0.23520168662071228 (0.2308056169951504) + | > loss_disc_real_5: 0.2550477087497711 (0.2302350483157418) + | > loss_0: 2.5492517948150635 (2.4631155214526435) + | > grad_norm_0: tensor(12.0355, device='cuda:0') (tensor(14.0097, device='cuda:0')) + | > loss_gen: 2.2581286430358887 (2.3520432656461527) + | > loss_kl: 2.181903600692749 (1.9789193895730106) + | > loss_feat: 6.428979873657227 (6.967035840858113) + | > loss_mel: 17.37993049621582 (17.990216406908896) + | > loss_duration: 1.6233965158462524 (1.5261036198247562) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.872339248657227 (30.81431859189814) + | > grad_norm_1: tensor(217.6467, device='cuda:0') (tensor(317.4807, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.4054 (0.37339348684657686) + | > loader_time: 0.0067 (0.005911781029267746) + + + --> TIME: 2025-04-18 23:40:47 -- STEP: 113/142 -- GLOBAL_STEP: 385075 + | > loss_disc: 2.5495429039001465 (2.4671297094463243) + | > loss_disc_real_0: 0.12955377995967865 (0.1280434690086188) + | > loss_disc_real_1: 0.22376833856105804 (0.2062486567064724) + | > loss_disc_real_2: 0.20397113263607025 (0.22584414983217696) + | > loss_disc_real_3: 0.2092580497264862 (0.229062717440909) + | > loss_disc_real_4: 0.21147574484348297 (0.23109388166824274) + | > loss_disc_real_5: 0.2319837510585785 (0.2305998783723443) + | > loss_0: 2.5495429039001465 (2.4671297094463243) + | > grad_norm_0: tensor(10.6280, device='cuda:0') (tensor(13.8943, device='cuda:0')) + | > loss_gen: 2.2117035388946533 (2.348824431410932) + | > loss_kl: 1.9498100280761719 (1.9877395883070683) + | > loss_feat: 6.191699504852295 (6.952983332946237) + | > loss_mel: 16.588346481323242 (17.95826883231643) + | > loss_duration: 1.5610556602478027 (1.5325179764654784) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.50261688232422 (30.780334185709997) + | > grad_norm_1: tensor(397.5316, device='cuda:0') (tensor(310.8272, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.4262 (0.38273629467044246) + | > loader_time: 0.0073 (0.006177990837434753) + + + --> TIME: 2025-04-18 23:40:58 -- STEP: 138/142 -- GLOBAL_STEP: 385100 + | > loss_disc: 2.510450601577759 (2.464954554170802) + | > loss_disc_real_0: 0.11256267875432968 (0.1287475817877314) + | > loss_disc_real_1: 0.2331925332546234 (0.2053247374900873) + | > loss_disc_real_2: 0.2522018253803253 (0.22627162846966067) + | > loss_disc_real_3: 0.23736317455768585 (0.2282946469343227) + | > loss_disc_real_4: 0.2566973567008972 (0.23082917766726535) + | > loss_disc_real_5: 0.2564077377319336 (0.23085315914257712) + | > loss_0: 2.510450601577759 (2.464954554170802) + | > grad_norm_0: tensor(28.9097, device='cuda:0') (tensor(14.8835, device='cuda:0')) + | > loss_gen: 2.21010422706604 (2.356159581654313) + | > loss_kl: 2.1002755165100098 (1.993322801762733) + | > loss_feat: 6.694252967834473 (6.955730023591414) + | > loss_mel: 19.00362777709961 (17.929387216982626) + | > loss_duration: 1.599212646484375 (1.5381175340085789) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.607473373413086 (30.772717157999676) + | > grad_norm_1: tensor(443.6868, device='cuda:0') (tensor(314.3543, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.4588 (0.3935479012088498) + | > loader_time: 0.0072 (0.006410688593767692) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.730252504348755 (2.730252504348755) + | > loss_disc_real_0: 0.3996589481830597 (0.3996589481830597) + | > loss_disc_real_1: 0.24341246485710144 (0.24341246485710144) + | > loss_disc_real_2: 0.2549556493759155 (0.2549556493759155) + | > loss_disc_real_3: 0.24401889741420746 (0.24401889741420746) + | > loss_disc_real_4: 0.24009954929351807 (0.24009954929351807) + | > loss_disc_real_5: 0.3113606572151184 (0.3113606572151184) + | > loss_0: 2.730252504348755 (2.730252504348755) + | > loss_gen: 2.8017547130584717 (2.8017547130584717) + | > loss_kl: 2.296218156814575 (2.296218156814575) + | > loss_feat: 6.877542495727539 (6.877542495727539) + | > loss_mel: 20.227991104125977 (20.227991104125977) + | > loss_duration: 1.645266056060791 (1.645266056060791) + | > loss_1: 33.84877395629883 (33.84877395629883) + + --> STEP: 1 + | > loss_disc: 2.549807548522949 (2.549807548522949) + | > loss_disc_real_0: 0.3326652944087982 (0.3326652944087982) + | > loss_disc_real_1: 0.2587857246398926 (0.2587857246398926) + | > loss_disc_real_2: 0.21938133239746094 (0.21938133239746094) + | > loss_disc_real_3: 0.19866879284381866 (0.19866879284381866) + | > loss_disc_real_4: 0.23349319398403168 (0.23349319398403168) + | > loss_disc_real_5: 0.21923640370368958 (0.21923640370368958) + | > loss_0: 2.549807548522949 (2.549807548522949) + | > loss_gen: 2.530782461166382 (2.530782461166382) + | > loss_kl: 2.4321141242980957 (2.4321141242980957) + | > loss_feat: 6.593305587768555 (6.593305587768555) + | > loss_mel: 19.079626083374023 (19.079626083374023) + | > loss_duration: 1.6484508514404297 (1.6484508514404297) + | > loss_1: 32.284278869628906 (32.284278869628906) + + --> STEP: 2 + | > loss_disc: 2.6787843704223633 (2.6142959594726562) + | > loss_disc_real_0: 0.39389142394065857 (0.3632783591747284) + | > loss_disc_real_1: 0.22340857982635498 (0.24109715223312378) + | > loss_disc_real_2: 0.20802848041057587 (0.2137049064040184) + | > loss_disc_real_3: 0.23307612538337708 (0.21587245911359787) + | > loss_disc_real_4: 0.2820942997932434 (0.25779374688863754) + | > loss_disc_real_5: 0.2475755512714386 (0.2334059774875641) + | > loss_0: 2.6787843704223633 (2.6142959594726562) + | > loss_gen: 2.7991156578063965 (2.664949059486389) + | > loss_kl: 2.061286687850952 (2.246700406074524) + | > loss_feat: 6.842508316040039 (6.717906951904297) + | > loss_mel: 19.56450080871582 (19.322063446044922) + | > loss_duration: 1.5680153369903564 (1.608233094215393) + | > loss_1: 32.835426330566406 (32.559852600097656) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003290534019470215 (+0.00012552738189697266) + | > avg_loss_disc: 2.6142959594726562 (+0.07580852508544922) + | > avg_loss_disc_real_0: 0.3632783591747284 (+0.21040958911180496) + | > avg_loss_disc_real_1: 0.24109715223312378 (-0.02549101412296295) + | > avg_loss_disc_real_2: 0.2137049064040184 (-0.04297714680433273) + | > avg_loss_disc_real_3: 0.21587245911359787 (-0.05891326814889908) + | > avg_loss_disc_real_4: 0.25779374688863754 (+0.0032501891255378723) + | > avg_loss_disc_real_5: 0.2334059774875641 (+0.004984632134437561) + | > avg_loss_0: 2.6142959594726562 (+0.07580852508544922) + | > avg_loss_gen: 2.664949059486389 (+0.23398005962371826) + | > avg_loss_kl: 2.246700406074524 (+0.005521416664123535) + | > avg_loss_feat: 6.717906951904297 (-0.1981651782989502) + | > avg_loss_mel: 19.322063446044922 (+1.0070629119873047) + | > avg_loss_duration: 1.608233094215393 (-0.013988077640533447) + | > avg_loss_1: 32.559852600097656 (+1.0344104766845703) + + + > EPOCH: 41/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:41:01)  + + --> TIME: 2025-04-18 23:41:09 -- STEP: 21/142 -- GLOBAL_STEP: 385125 + | > loss_disc: 2.5465269088745117 (2.4316984244755337) + | > loss_disc_real_0: 0.11356937885284424 (0.11455241768133073) + | > loss_disc_real_1: 0.2543396055698395 (0.20471846773510888) + | > loss_disc_real_2: 0.25736191868782043 (0.22568433128652118) + | > loss_disc_real_3: 0.24668079614639282 (0.23266364137331644) + | > loss_disc_real_4: 0.26157984137535095 (0.23435800061339424) + | > loss_disc_real_5: 0.2773068845272064 (0.23394631346066794) + | > loss_0: 2.5465269088745117 (2.4316984244755337) + | > grad_norm_0: tensor(13.2459, device='cuda:0') (tensor(14.0834, device='cuda:0')) + | > loss_gen: 2.3536272048950195 (2.435951130730765) + | > loss_kl: 2.0091819763183594 (1.9601441848845709) + | > loss_feat: 6.9530487060546875 (7.3304234232221335) + | > loss_mel: 17.821931838989258 (18.31469708397275) + | > loss_duration: 1.5125634670257568 (1.499117680958339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.650354385375977 (31.540333429972332) + | > grad_norm_1: tensor(392.8853, device='cuda:0') (tensor(250.4715, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.3435 (0.3307360467456636) + | > loader_time: 0.0053 (0.004671675818307059) + + + --> TIME: 2025-04-18 23:41:18 -- STEP: 46/142 -- GLOBAL_STEP: 385150 + | > loss_disc: 2.390803813934326 (2.4396601241567866) + | > loss_disc_real_0: 0.10633260011672974 (0.1192922047946764) + | > loss_disc_real_1: 0.17511659860610962 (0.20233879659486853) + | > loss_disc_real_2: 0.1929188370704651 (0.22285658380259638) + | > loss_disc_real_3: 0.18991902470588684 (0.22985622545947199) + | > loss_disc_real_4: 0.22677727043628693 (0.23000617487275082) + | > loss_disc_real_5: 0.2165375053882599 (0.23062361096558365) + | > loss_0: 2.390803813934326 (2.4396601241567866) + | > grad_norm_0: tensor(9.4300, device='cuda:0') (tensor(12.9845, device='cuda:0')) + | > loss_gen: 2.455349922180176 (2.3722466385882823) + | > loss_kl: 1.9564027786254883 (1.9850690287092458) + | > loss_feat: 7.551503658294678 (7.142341261324675) + | > loss_mel: 18.30577850341797 (17.927896520365845) + | > loss_duration: 1.5063230991363525 (1.5059769024019656) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.775358200073242 (30.933530019677203) + | > grad_norm_1: tensor(363.2910, device='cuda:0') (tensor(275.5063, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.3641 (0.3431492370107899) + | > loader_time: 0.0064 (0.005135660586149795) + + + --> TIME: 2025-04-18 23:41:28 -- STEP: 71/142 -- GLOBAL_STEP: 385175 + | > loss_disc: 2.4860432147979736 (2.4580309357441656) + | > loss_disc_real_0: 0.09908058494329453 (0.1249840861894715) + | > loss_disc_real_1: 0.23296190798282623 (0.20569173425016268) + | > loss_disc_real_2: 0.23763564229011536 (0.22576562514607335) + | > loss_disc_real_3: 0.24454616010189056 (0.23111211581968924) + | > loss_disc_real_4: 0.2640080153942108 (0.23027148674911177) + | > loss_disc_real_5: 0.22368232905864716 (0.23125330264299687) + | > loss_0: 2.4860432147979736 (2.4580309357441656) + | > grad_norm_0: tensor(6.6212, device='cuda:0') (tensor(12.7512, device='cuda:0')) + | > loss_gen: 2.372115135192871 (2.3569631979499057) + | > loss_kl: 2.0661556720733643 (1.9943765149989598) + | > loss_feat: 6.593777656555176 (7.054444024260615) + | > loss_mel: 18.335784912109375 (17.920903125279388) + | > loss_duration: 1.541437029838562 (1.5154959248824857) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.909269332885742 (30.842182454928547) + | > grad_norm_1: tensor(37.2997, device='cuda:0') (tensor(258.1234, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.3888 (0.35509315007169484) + | > loader_time: 0.0062 (0.005440889949529942) + + + --> TIME: 2025-04-18 23:41:38 -- STEP: 96/142 -- GLOBAL_STEP: 385200 + | > loss_disc: 2.5279338359832764 (2.474929357568423) + | > loss_disc_real_0: 0.14044250547885895 (0.12560767556230232) + | > loss_disc_real_1: 0.2349996119737625 (0.20773735859741768) + | > loss_disc_real_2: 0.21761032938957214 (0.22819428083797297) + | > loss_disc_real_3: 0.24256879091262817 (0.23077974685778221) + | > loss_disc_real_4: 0.2559601068496704 (0.23087776141862074) + | > loss_disc_real_5: 0.19773495197296143 (0.23203645925968885) + | > loss_0: 2.5279338359832764 (2.474929357568423) + | > grad_norm_0: tensor(4.4310, device='cuda:0') (tensor(12.0096, device='cuda:0')) + | > loss_gen: 2.3911848068237305 (2.3386270751555767) + | > loss_kl: 1.9771264791488647 (2.0009098139901957) + | > loss_feat: 7.118313789367676 (6.978617926438649) + | > loss_mel: 18.044078826904297 (17.958403180042904) + | > loss_duration: 1.5576889514923096 (1.526948186258475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.088394165039062 (30.803505957126617) + | > grad_norm_1: tensor(91.5730, device='cuda:0') (tensor(254.0697, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.4116 (0.3673681939641635) + | > loader_time: 0.0067 (0.005737721920013428) + + + --> TIME: 2025-04-18 23:41:49 -- STEP: 121/142 -- GLOBAL_STEP: 385225 + | > loss_disc: 2.530973196029663 (2.4824907858509664) + | > loss_disc_real_0: 0.14111828804016113 (0.12878783582902162) + | > loss_disc_real_1: 0.25799664855003357 (0.20846376106266148) + | > loss_disc_real_2: 0.1962605118751526 (0.2279730360370037) + | > loss_disc_real_3: 0.2558179795742035 (0.23139809281372828) + | > loss_disc_real_4: 0.19505536556243896 (0.2314819423866666) + | > loss_disc_real_5: 0.276837557554245 (0.23316101803759898) + | > loss_0: 2.530973196029663 (2.4824907858509664) + | > grad_norm_0: tensor(11.8637, device='cuda:0') (tensor(11.8797, device='cuda:0')) + | > loss_gen: 2.355071783065796 (2.3268424676469537) + | > loss_kl: 2.171865463256836 (2.0046040391133846) + | > loss_feat: 6.7292022705078125 (6.893931329742936) + | > loss_mel: 18.892637252807617 (17.844250600200056) + | > loss_duration: 1.6216130256652832 (1.534076614813371) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.770389556884766 (30.60370489388458) + | > grad_norm_1: tensor(340.8412, device='cuda:0') (tensor(247.8408, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.4389 (0.3789730288765647) + | > loader_time: 0.0071 (0.0060013168114275975) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3985636234283447 (2.3985636234283447) + | > loss_disc_real_0: 0.1534777134656906 (0.1534777134656906) + | > loss_disc_real_1: 0.20736144483089447 (0.20736144483089447) + | > loss_disc_real_2: 0.2301366627216339 (0.2301366627216339) + | > loss_disc_real_3: 0.22142937779426575 (0.22142937779426575) + | > loss_disc_real_4: 0.20328199863433838 (0.20328199863433838) + | > loss_disc_real_5: 0.22398260235786438 (0.22398260235786438) + | > loss_0: 2.3985636234283447 (2.3985636234283447) + | > loss_gen: 2.3159890174865723 (2.3159890174865723) + | > loss_kl: 2.5228052139282227 (2.5228052139282227) + | > loss_feat: 5.850731372833252 (5.850731372833252) + | > loss_mel: 16.92897605895996 (16.92897605895996) + | > loss_duration: 1.6412556171417236 (1.6412556171417236) + | > loss_1: 29.25975799560547 (29.25975799560547) + + --> STEP: 1 + | > loss_disc: 2.5404319763183594 (2.5404319763183594) + | > loss_disc_real_0: 0.1399335414171219 (0.1399335414171219) + | > loss_disc_real_1: 0.2285691201686859 (0.2285691201686859) + | > loss_disc_real_2: 0.24946056306362152 (0.24946056306362152) + | > loss_disc_real_3: 0.2014448195695877 (0.2014448195695877) + | > loss_disc_real_4: 0.24097375571727753 (0.24097375571727753) + | > loss_disc_real_5: 0.19863756000995636 (0.19863756000995636) + | > loss_0: 2.5404319763183594 (2.5404319763183594) + | > loss_gen: 2.1277883052825928 (2.1277883052825928) + | > loss_kl: 2.5259151458740234 (2.5259151458740234) + | > loss_feat: 6.109384536743164 (6.109384536743164) + | > loss_mel: 17.524667739868164 (17.524667739868164) + | > loss_duration: 1.6386229991912842 (1.6386229991912842) + | > loss_1: 29.92637825012207 (29.92637825012207) + + --> STEP: 2 + | > loss_disc: 2.5259809494018555 (2.5332064628601074) + | > loss_disc_real_0: 0.22777225077152252 (0.1838528960943222) + | > loss_disc_real_1: 0.23937861621379852 (0.23397386819124222) + | > loss_disc_real_2: 0.2432018369436264 (0.24633120000362396) + | > loss_disc_real_3: 0.21214117109775543 (0.20679299533367157) + | > loss_disc_real_4: 0.2580985724925995 (0.2495361641049385) + | > loss_disc_real_5: 0.23045077919960022 (0.2145441696047783) + | > loss_0: 2.5259809494018555 (2.5332064628601074) + | > loss_gen: 2.265639305114746 (2.1967138051986694) + | > loss_kl: 1.9551039934158325 (2.240509569644928) + | > loss_feat: 5.497178077697754 (5.803281307220459) + | > loss_mel: 15.780915260314941 (16.652791500091553) + | > loss_duration: 1.5873916149139404 (1.6130073070526123) + | > loss_1: 27.086227416992188 (28.50630283355713) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031713247299194336 (-0.00011920928955078125) + | > avg_loss_disc: 2.5332064628601074 (-0.08108949661254883) + | > avg_loss_disc_real_0: 0.1838528960943222 (-0.1794254630804062) + | > avg_loss_disc_real_1: 0.23397386819124222 (-0.007123284041881561) + | > avg_loss_disc_real_2: 0.24633120000362396 (+0.03262629359960556) + | > avg_loss_disc_real_3: 0.20679299533367157 (-0.0090794637799263) + | > avg_loss_disc_real_4: 0.2495361641049385 (-0.008257582783699036) + | > avg_loss_disc_real_5: 0.2145441696047783 (-0.018861807882785797) + | > avg_loss_0: 2.5332064628601074 (-0.08108949661254883) + | > avg_loss_gen: 2.1967138051986694 (-0.4682352542877197) + | > avg_loss_kl: 2.240509569644928 (-0.006190836429595947) + | > avg_loss_feat: 5.803281307220459 (-0.9146256446838379) + | > avg_loss_mel: 16.652791500091553 (-2.669271945953369) + | > avg_loss_duration: 1.6130073070526123 (+0.004774212837219238) + | > avg_loss_1: 28.50630283355713 (-4.053549766540527) + + + > EPOCH: 42/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:42:00)  + + --> TIME: 2025-04-18 23:42:02 -- STEP: 4/142 -- GLOBAL_STEP: 385250 + | > loss_disc: 2.3274316787719727 (2.345372498035431) + | > loss_disc_real_0: 0.1014583483338356 (0.10323760472238064) + | > loss_disc_real_1: 0.20187534391880035 (0.18992431461811066) + | > loss_disc_real_2: 0.20234104990959167 (0.21271593496203423) + | > loss_disc_real_3: 0.22711805999279022 (0.21683677285909653) + | > loss_disc_real_4: 0.17933468520641327 (0.19793444126844406) + | > loss_disc_real_5: 0.21974803507328033 (0.22312740981578827) + | > loss_0: 2.3274316787719727 (2.345372498035431) + | > grad_norm_0: tensor(11.2281, device='cuda:0') (tensor(8.1255, device='cuda:0')) + | > loss_gen: 2.4564218521118164 (2.42476886510849) + | > loss_kl: 2.0087149143218994 (1.9535563588142395) + | > loss_feat: 7.978118896484375 (7.496294379234314) + | > loss_mel: 18.24056625366211 (18.20364999771118) + | > loss_duration: 1.5315033197402954 (1.5103779435157776) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.21532440185547 (31.58864688873291) + | > grad_norm_1: tensor(389.7835, device='cuda:0') (tensor(271.7457, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.3199 (0.3222951889038086) + | > loader_time: 0.0056 (0.004462122917175293) + + + --> TIME: 2025-04-18 23:42:11 -- STEP: 29/142 -- GLOBAL_STEP: 385275 + | > loss_disc: 2.428864002227783 (2.4709982460942763) + | > loss_disc_real_0: 0.07962857931852341 (0.122520017726668) + | > loss_disc_real_1: 0.20293763279914856 (0.20940379952562266) + | > loss_disc_real_2: 0.23606882989406586 (0.23036546439960084) + | > loss_disc_real_3: 0.24788030982017517 (0.2295792868425106) + | > loss_disc_real_4: 0.24294628202915192 (0.22751172503520703) + | > loss_disc_real_5: 0.23566877841949463 (0.23192008353512863) + | > loss_0: 2.428864002227783 (2.4709982460942763) + | > grad_norm_0: tensor(13.4714, device='cuda:0') (tensor(10.5724, device='cuda:0')) + | > loss_gen: 2.3563685417175293 (2.363869445077304) + | > loss_kl: 1.9518821239471436 (1.939308474803793) + | > loss_feat: 7.543673038482666 (7.1204184663706815) + | > loss_mel: 19.105403900146484 (18.037752315915863) + | > loss_duration: 1.4855287075042725 (1.504046082496643) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.44285583496094 (30.9653948422136) + | > grad_norm_1: tensor(249.1477, device='cuda:0') (tensor(254.4080, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.3453 (0.336702823638916) + | > loader_time: 0.005 (0.0048306646018192664) + + + --> TIME: 2025-04-18 23:42:20 -- STEP: 54/142 -- GLOBAL_STEP: 385300 + | > loss_disc: 2.438302993774414 (2.4623324606153694) + | > loss_disc_real_0: 0.12751458585262299 (0.12437290118800269) + | > loss_disc_real_1: 0.22733311355113983 (0.2080206796526909) + | > loss_disc_real_2: 0.25057101249694824 (0.2282861849224126) + | > loss_disc_real_3: 0.23060745000839233 (0.2274821568970327) + | > loss_disc_real_4: 0.2006784826517105 (0.2260569800381307) + | > loss_disc_real_5: 0.21228668093681335 (0.23052358930861508) + | > loss_0: 2.438302993774414 (2.4623324606153694) + | > grad_norm_0: tensor(19.5765, device='cuda:0') (tensor(11.2917, device='cuda:0')) + | > loss_gen: 2.3706068992614746 (2.3573272625605264) + | > loss_kl: 1.928471565246582 (1.9519004512716223) + | > loss_feat: 7.633669376373291 (7.050452815161811) + | > loss_mel: 17.88692855834961 (17.97172120765403) + | > loss_duration: 1.519270896911621 (1.5088800765849923) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.338947296142578 (30.840281769081397) + | > grad_norm_1: tensor(397.6714, device='cuda:0') (tensor(263.7875, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.3682 (0.3477346234851412) + | > loader_time: 0.0056 (0.0051649676428900835) + + + --> TIME: 2025-04-18 23:42:30 -- STEP: 79/142 -- GLOBAL_STEP: 385325 + | > loss_disc: 2.6346707344055176 (2.4764708657807937) + | > loss_disc_real_0: 0.11839727312326431 (0.12610899694735492) + | > loss_disc_real_1: 0.24377861618995667 (0.20878836776636825) + | > loss_disc_real_2: 0.24559660255908966 (0.2291244249555129) + | > loss_disc_real_3: 0.27776193618774414 (0.22886269831959205) + | > loss_disc_real_4: 0.2823779284954071 (0.22756577235988423) + | > loss_disc_real_5: 0.30167287588119507 (0.23296773414823074) + | > loss_0: 2.6346707344055176 (2.4764708657807937) + | > grad_norm_0: tensor(9.8828, device='cuda:0') (tensor(12.0741, device='cuda:0')) + | > loss_gen: 2.4534597396850586 (2.353462880170798) + | > loss_kl: 2.0669422149658203 (1.9746166120601605) + | > loss_feat: 6.651112079620361 (7.012447502039656) + | > loss_mel: 18.4626407623291 (18.006447888627836) + | > loss_duration: 1.5836455821990967 (1.5201524299911302) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.21780014038086 (30.867127358158932) + | > grad_norm_1: tensor(291.7246, device='cuda:0') (tensor(265.8981, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.3891 (0.3598960290981243) + | > loader_time: 0.0069 (0.0055075023747697665) + + + --> TIME: 2025-04-18 23:42:40 -- STEP: 104/142 -- GLOBAL_STEP: 385350 + | > loss_disc: 2.569316864013672 (2.4774726766806383) + | > loss_disc_real_0: 0.12520615756511688 (0.12775147835222572) + | > loss_disc_real_1: 0.18018318712711334 (0.2084243715955661) + | > loss_disc_real_2: 0.24420350790023804 (0.2288410339791041) + | > loss_disc_real_3: 0.25586867332458496 (0.2286842340746751) + | > loss_disc_real_4: 0.2647514343261719 (0.22937052109493658) + | > loss_disc_real_5: 0.319955050945282 (0.23303018180796733) + | > loss_0: 2.569316864013672 (2.4774726766806383) + | > grad_norm_0: tensor(6.2824, device='cuda:0') (tensor(11.4479, device='cuda:0')) + | > loss_gen: 2.248871326446533 (2.343540508013506) + | > loss_kl: 2.0812017917633057 (1.9879524203447194) + | > loss_feat: 6.130624294281006 (6.955769332555624) + | > loss_mel: 17.18803596496582 (17.945933021031898) + | > loss_duration: 1.5800158977508545 (1.5299498828557818) + | > amp_scaler: 512.0 (275.6923076923077) + | > loss_1: 29.228748321533203 (30.763145190018875) + | > grad_norm_1: tensor(168.5839, device='cuda:0') (tensor(258.6757, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.413 (0.37134758784220756) + | > loader_time: 0.0066 (0.005765983691582315) + + + --> TIME: 2025-04-18 23:42:51 -- STEP: 129/142 -- GLOBAL_STEP: 385375 + | > loss_disc: 2.508620500564575 (2.4805410684541216) + | > loss_disc_real_0: 0.15933609008789062 (0.1284007668610691) + | > loss_disc_real_1: 0.16455644369125366 (0.208940362976503) + | > loss_disc_real_2: 0.2156250923871994 (0.22854566585647967) + | > loss_disc_real_3: 0.2397085428237915 (0.2282399566367615) + | > loss_disc_real_4: 0.22919954359531403 (0.22985993568287338) + | > loss_disc_real_5: 0.22410491108894348 (0.2322198035874108) + | > loss_0: 2.508620500564575 (2.4805410684541216) + | > grad_norm_0: tensor(9.4268, device='cuda:0') (tensor(11.0140, device='cuda:0')) + | > loss_gen: 2.315581798553467 (2.3355073023212056) + | > loss_kl: 2.0321102142333984 (1.9870507513829905) + | > loss_feat: 6.982356548309326 (6.919557427251061) + | > loss_mel: 18.411991119384766 (17.895173279814024) + | > loss_duration: 1.5414659976959229 (1.5354832179786622) + | > amp_scaler: 512.0 (321.48837209302326) + | > loss_1: 31.283506393432617 (30.672771986140763) + | > grad_norm_1: tensor(247.8564, device='cuda:0') (tensor(249.2441, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.4451 (0.38319181656652634) + | > loader_time: 0.0071 (0.00600841987964719) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5533950328826904 (2.5533950328826904) + | > loss_disc_real_0: 0.13528014719486237 (0.13528014719486237) + | > loss_disc_real_1: 0.21060144901275635 (0.21060144901275635) + | > loss_disc_real_2: 0.2544463276863098 (0.2544463276863098) + | > loss_disc_real_3: 0.24094392359256744 (0.24094392359256744) + | > loss_disc_real_4: 0.24088986217975616 (0.24088986217975616) + | > loss_disc_real_5: 0.27142298221588135 (0.27142298221588135) + | > loss_0: 2.5533950328826904 (2.5533950328826904) + | > loss_gen: 2.3368589878082275 (2.3368589878082275) + | > loss_kl: 2.3247063159942627 (2.3247063159942627) + | > loss_feat: 6.190424919128418 (6.190424919128418) + | > loss_mel: 17.416303634643555 (17.416303634643555) + | > loss_duration: 1.628676176071167 (1.628676176071167) + | > loss_1: 29.896968841552734 (29.896968841552734) + + --> STEP: 1 + | > loss_disc: 2.424382209777832 (2.424382209777832) + | > loss_disc_real_0: 0.1450258195400238 (0.1450258195400238) + | > loss_disc_real_1: 0.19927340745925903 (0.19927340745925903) + | > loss_disc_real_2: 0.2059478461742401 (0.2059478461742401) + | > loss_disc_real_3: 0.21880337595939636 (0.21880337595939636) + | > loss_disc_real_4: 0.22399559617042542 (0.22399559617042542) + | > loss_disc_real_5: 0.2563225328922272 (0.2563225328922272) + | > loss_0: 2.424382209777832 (2.424382209777832) + | > loss_gen: 2.3188517093658447 (2.3188517093658447) + | > loss_kl: 2.4972171783447266 (2.4972171783447266) + | > loss_feat: 6.487673282623291 (6.487673282623291) + | > loss_mel: 17.97003746032715 (17.97003746032715) + | > loss_duration: 1.6605699062347412 (1.6605699062347412) + | > loss_1: 30.934349060058594 (30.934349060058594) + + --> STEP: 2 + | > loss_disc: 2.2904067039489746 (2.3573944568634033) + | > loss_disc_real_0: 0.12004829943180084 (0.13253705948591232) + | > loss_disc_real_1: 0.1890774518251419 (0.19417542964220047) + | > loss_disc_real_2: 0.19178323447704315 (0.19886554032564163) + | > loss_disc_real_3: 0.22113680839538574 (0.21997009217739105) + | > loss_disc_real_4: 0.208571657538414 (0.2162836268544197) + | > loss_disc_real_5: 0.24806226789951324 (0.2521924003958702) + | > loss_0: 2.2904067039489746 (2.3573944568634033) + | > loss_gen: 2.4130425453186035 (2.365947127342224) + | > loss_kl: 2.076704502105713 (2.2869608402252197) + | > loss_feat: 7.996173858642578 (7.241923570632935) + | > loss_mel: 19.815160751342773 (18.89259910583496) + | > loss_duration: 1.5932022333145142 (1.6268860697746277) + | > loss_1: 33.894283294677734 (32.414316177368164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038088560104370117 (+0.0006375312805175781) + | > avg_loss_disc: 2.3573944568634033 (-0.1758120059967041) + | > avg_loss_disc_real_0: 0.13253705948591232 (-0.05131583660840988) + | > avg_loss_disc_real_1: 0.19417542964220047 (-0.03979843854904175) + | > avg_loss_disc_real_2: 0.19886554032564163 (-0.04746565967798233) + | > avg_loss_disc_real_3: 0.21997009217739105 (+0.013177096843719482) + | > avg_loss_disc_real_4: 0.2162836268544197 (-0.0332525372505188) + | > avg_loss_disc_real_5: 0.2521924003958702 (+0.03764823079109192) + | > avg_loss_0: 2.3573944568634033 (-0.1758120059967041) + | > avg_loss_gen: 2.365947127342224 (+0.1692333221435547) + | > avg_loss_kl: 2.2869608402252197 (+0.04645127058029175) + | > avg_loss_feat: 7.241923570632935 (+1.4386422634124756) + | > avg_loss_mel: 18.89259910583496 (+2.239807605743408) + | > avg_loss_duration: 1.6268860697746277 (+0.01387876272201538) + | > avg_loss_1: 32.414316177368164 (+3.908013343811035) + + + > EPOCH: 43/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:42:59)  + + --> TIME: 2025-04-18 23:43:03 -- STEP: 12/142 -- GLOBAL_STEP: 385400 + | > loss_disc: 2.446629762649536 (2.35966690381368) + | > loss_disc_real_0: 0.11176134645938873 (0.11605428345501423) + | > loss_disc_real_1: 0.1896962821483612 (0.20223539198438326) + | > loss_disc_real_2: 0.2520250678062439 (0.21827392652630806) + | > loss_disc_real_3: 0.2505842447280884 (0.21647669126590094) + | > loss_disc_real_4: 0.24535565078258514 (0.21401099736491838) + | > loss_disc_real_5: 0.26726412773132324 (0.2235985609392325) + | > loss_0: 2.446629762649536 (2.35966690381368) + | > grad_norm_0: tensor(8.8464, device='cuda:0') (tensor(7.0484, device='cuda:0')) + | > loss_gen: 2.4893243312835693 (2.439307789007823) + | > loss_kl: 1.765702724456787 (1.8856967786947887) + | > loss_feat: 7.068329811096191 (7.566529472668965) + | > loss_mel: 17.47503662109375 (18.46034526824951) + | > loss_duration: 1.494847297668457 (1.4886725544929504) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.29323959350586 (31.84055169423421) + | > grad_norm_1: tensor(157.5244, device='cuda:0') (tensor(157.0445, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.3283 (0.3257318536440531) + | > loader_time: 0.0048 (0.0045952995618184405) + + + --> TIME: 2025-04-18 23:43:12 -- STEP: 37/142 -- GLOBAL_STEP: 385425 + | > loss_disc: 2.5233206748962402 (2.442176122923155) + | > loss_disc_real_0: 0.12074065208435059 (0.12278422874373358) + | > loss_disc_real_1: 0.1687394231557846 (0.20562875794397817) + | > loss_disc_real_2: 0.22891977429389954 (0.2272953362883748) + | > loss_disc_real_3: 0.2207525074481964 (0.22790311719920184) + | > loss_disc_real_4: 0.2164333611726761 (0.22584483752379547) + | > loss_disc_real_5: 0.23954445123672485 (0.23022119580088435) + | > loss_0: 2.5233206748962402 (2.442176122923155) + | > grad_norm_0: tensor(6.1256, device='cuda:0') (tensor(8.7104, device='cuda:0')) + | > loss_gen: 2.3674581050872803 (2.375054816941957) + | > loss_kl: 2.0839476585388184 (1.9249737230507102) + | > loss_feat: 7.332619667053223 (7.162193079252501) + | > loss_mel: 17.703001022338867 (17.909087155316328) + | > loss_duration: 1.5239431858062744 (1.4992791285385956) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.010969161987305 (30.870587993312526) + | > grad_norm_1: tensor(160.9412, device='cuda:0') (tensor(207.6364, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.3507 (0.3380638779820623) + | > loader_time: 0.0057 (0.0049919179967931795) + + + --> TIME: 2025-04-18 23:43:21 -- STEP: 62/142 -- GLOBAL_STEP: 385450 + | > loss_disc: 2.4492807388305664 (2.4589105690679247) + | > loss_disc_real_0: 0.12002884596586227 (0.12230911802860998) + | > loss_disc_real_1: 0.2081948220729828 (0.20903190874284314) + | > loss_disc_real_2: 0.23169736564159393 (0.22670247189460263) + | > loss_disc_real_3: 0.23834964632987976 (0.22929660999967205) + | > loss_disc_real_4: 0.2191663384437561 (0.22877484055296068) + | > loss_disc_real_5: 0.22270287573337555 (0.23300329043019202) + | > loss_0: 2.4492807388305664 (2.4589105690679247) + | > grad_norm_0: tensor(10.1498, device='cuda:0') (tensor(9.7497, device='cuda:0')) + | > loss_gen: 2.3215222358703613 (2.3758154646042855) + | > loss_kl: 1.9689562320709229 (1.9546695293918732) + | > loss_feat: 6.881107330322266 (7.03236690644295) + | > loss_mel: 17.044870376586914 (17.883170466269213) + | > loss_duration: 1.572737455368042 (1.5101231644230504) + | > amp_scaler: 256.0 (450.06451612903226) + | > loss_1: 29.78919219970703 (30.75614566187705) + | > grad_norm_1: tensor(286.1678, device='cuda:0') (tensor(228.3476, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.3844 (0.3495004138638897) + | > loader_time: 0.006 (0.005373343344657651) + + + --> TIME: 2025-04-18 23:43:31 -- STEP: 87/142 -- GLOBAL_STEP: 385475 + | > loss_disc: 2.534292697906494 (2.4727560679117837) + | > loss_disc_real_0: 0.17675483226776123 (0.12396783330317201) + | > loss_disc_real_1: 0.21607394516468048 (0.21156732463973693) + | > loss_disc_real_2: 0.2344997674226761 (0.22785443615639348) + | > loss_disc_real_3: 0.2239290177822113 (0.22956816218365197) + | > loss_disc_real_4: 0.2522944509983063 (0.23036317630060787) + | > loss_disc_real_5: 0.2686116695404053 (0.23238546324187312) + | > loss_0: 2.534292697906494 (2.4727560679117837) + | > grad_norm_0: tensor(26.3384, device='cuda:0') (tensor(11.7963, device='cuda:0')) + | > loss_gen: 2.290130376815796 (2.3617749707452194) + | > loss_kl: 2.0242581367492676 (1.9809075936503795) + | > loss_feat: 7.139536380767822 (6.92724102392964) + | > loss_mel: 18.81041717529297 (17.903991819798257) + | > loss_duration: 1.575340986251831 (1.522672658679129) + | > amp_scaler: 256.0 (394.2988505747126) + | > loss_1: 31.839683532714844 (30.696588121611498) + | > grad_norm_1: tensor(515.5660, device='cuda:0') (tensor(254.4804, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.4047 (0.3619807917496254) + | > loader_time: 0.0066 (0.0057239505066268735) + + + --> TIME: 2025-04-18 23:43:42 -- STEP: 112/142 -- GLOBAL_STEP: 385500 + | > loss_disc: 2.434011459350586 (2.4770634919404984) + | > loss_disc_real_0: 0.08208545297384262 (0.12552940512874294) + | > loss_disc_real_1: 0.21480654180049896 (0.2112766850207533) + | > loss_disc_real_2: 0.1899501234292984 (0.22724943315344198) + | > loss_disc_real_3: 0.22238996624946594 (0.2295987531542778) + | > loss_disc_real_4: 0.21936672925949097 (0.2305674556908863) + | > loss_disc_real_5: 0.23488564789295197 (0.23262886463531426) + | > loss_0: 2.434011459350586 (2.4770634919404984) + | > grad_norm_0: tensor(26.7083, device='cuda:0') (tensor(13.3663, device='cuda:0')) + | > loss_gen: 2.354480743408203 (2.3537045674664654) + | > loss_kl: 1.8876550197601318 (1.9869082325271197) + | > loss_feat: 7.045603275299072 (6.892233282327652) + | > loss_mel: 17.666873931884766 (17.847363122871943) + | > loss_duration: 1.514375925064087 (1.5298228721533502) + | > amp_scaler: 256.0 (363.42857142857144) + | > loss_1: 30.4689884185791 (30.610032149723597) + | > grad_norm_1: tensor(534.3978, device='cuda:0') (tensor(270.3746, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.429 (0.3733493174825396) + | > loader_time: 0.0075 (0.005987286567687987) + + + --> TIME: 2025-04-18 23:43:53 -- STEP: 137/142 -- GLOBAL_STEP: 385525 + | > loss_disc: 2.5843849182128906 (2.479241144918177) + | > loss_disc_real_0: 0.15366844832897186 (0.12650047112120333) + | > loss_disc_real_1: 0.2140468955039978 (0.21095982966196797) + | > loss_disc_real_2: 0.26285627484321594 (0.22717965878274318) + | > loss_disc_real_3: 0.23443932831287384 (0.22901976750279868) + | > loss_disc_real_4: 0.2863575220108032 (0.23106972449017268) + | > loss_disc_real_5: 0.23340576887130737 (0.23213649470440662) + | > loss_0: 2.5843849182128906 (2.479241144918177) + | > grad_norm_0: tensor(5.0922, device='cuda:0') (tensor(13.7565, device='cuda:0')) + | > loss_gen: 2.2824008464813232 (2.3558086217754965) + | > loss_kl: 1.9592323303222656 (1.9915060909995197) + | > loss_feat: 6.571602821350098 (6.918074684421511) + | > loss_mel: 18.862449645996094 (17.884387698486773) + | > loss_duration: 1.5477579832077026 (1.5347093047886868) + | > amp_scaler: 256.0 (343.82481751824815) + | > loss_1: 31.22344207763672 (30.684486444849167) + | > grad_norm_1: tensor(136.3136, device='cuda:0') (tensor(279.7888, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.4542 (0.3854367889627053) + | > loader_time: 0.0076 (0.0062749855709772024) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4059898853302 (2.4059898853302) + | > loss_disc_real_0: 0.14983348548412323 (0.14983348548412323) + | > loss_disc_real_1: 0.2532123923301697 (0.2532123923301697) + | > loss_disc_real_2: 0.2168463170528412 (0.2168463170528412) + | > loss_disc_real_3: 0.21663545072078705 (0.21663545072078705) + | > loss_disc_real_4: 0.23955635726451874 (0.23955635726451874) + | > loss_disc_real_5: 0.24192224442958832 (0.24192224442958832) + | > loss_0: 2.4059898853302 (2.4059898853302) + | > loss_gen: 2.419650077819824 (2.419650077819824) + | > loss_kl: 2.0623037815093994 (2.0623037815093994) + | > loss_feat: 6.362257480621338 (6.362257480621338) + | > loss_mel: 17.24102020263672 (17.24102020263672) + | > loss_duration: 1.619107961654663 (1.619107961654663) + | > loss_1: 29.7043399810791 (29.7043399810791) + + --> STEP: 1 + | > loss_disc: 2.3401620388031006 (2.3401620388031006) + | > loss_disc_real_0: 0.09757047146558762 (0.09757047146558762) + | > loss_disc_real_1: 0.2209928333759308 (0.2209928333759308) + | > loss_disc_real_2: 0.2189740687608719 (0.2189740687608719) + | > loss_disc_real_3: 0.16851919889450073 (0.16851919889450073) + | > loss_disc_real_4: 0.20363400876522064 (0.20363400876522064) + | > loss_disc_real_5: 0.21022929251194 (0.21022929251194) + | > loss_0: 2.3401620388031006 (2.3401620388031006) + | > loss_gen: 2.2320070266723633 (2.2320070266723633) + | > loss_kl: 2.125678777694702 (2.125678777694702) + | > loss_feat: 6.673452377319336 (6.673452377319336) + | > loss_mel: 18.1999454498291 (18.1999454498291) + | > loss_duration: 1.6561046838760376 (1.6561046838760376) + | > loss_1: 30.887187957763672 (30.887187957763672) + + --> STEP: 2 + | > loss_disc: 2.405390977859497 (2.372776508331299) + | > loss_disc_real_0: 0.1484040468931198 (0.12298725917935371) + | > loss_disc_real_1: 0.25099050998687744 (0.2359916716814041) + | > loss_disc_real_2: 0.20632000267505646 (0.21264703571796417) + | > loss_disc_real_3: 0.18370968103408813 (0.17611443996429443) + | > loss_disc_real_4: 0.2072593718767166 (0.20544669032096863) + | > loss_disc_real_5: 0.21309563517570496 (0.21166246384382248) + | > loss_0: 2.405390977859497 (2.372776508331299) + | > loss_gen: 2.3168251514434814 (2.2744160890579224) + | > loss_kl: 2.2781484127044678 (2.201913595199585) + | > loss_feat: 7.373142719268799 (7.023297548294067) + | > loss_mel: 18.805953979492188 (18.502949714660645) + | > loss_duration: 1.5839102268218994 (1.6200074553489685) + | > loss_1: 32.35798263549805 (31.62258529663086) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00371551513671875 (-9.334087371826172e-05) + | > avg_loss_disc: 2.372776508331299 (+0.015382051467895508) + | > avg_loss_disc_real_0: 0.12298725917935371 (-0.009549800306558609) + | > avg_loss_disc_real_1: 0.2359916716814041 (+0.041816242039203644) + | > avg_loss_disc_real_2: 0.21264703571796417 (+0.01378149539232254) + | > avg_loss_disc_real_3: 0.17611443996429443 (-0.04385565221309662) + | > avg_loss_disc_real_4: 0.20544669032096863 (-0.01083693653345108) + | > avg_loss_disc_real_5: 0.21166246384382248 (-0.04052993655204773) + | > avg_loss_0: 2.372776508331299 (+0.015382051467895508) + | > avg_loss_gen: 2.2744160890579224 (-0.09153103828430176) + | > avg_loss_kl: 2.201913595199585 (-0.08504724502563477) + | > avg_loss_feat: 7.023297548294067 (-0.2186260223388672) + | > avg_loss_mel: 18.502949714660645 (-0.3896493911743164) + | > avg_loss_duration: 1.6200074553489685 (-0.00687861442565918) + | > avg_loss_1: 31.62258529663086 (-0.7917308807373047) + + + > EPOCH: 44/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:43:57)  + + --> TIME: 2025-04-18 23:44:05 -- STEP: 20/142 -- GLOBAL_STEP: 385550 + | > loss_disc: 2.5163869857788086 (2.427673029899597) + | > loss_disc_real_0: 0.10294961929321289 (0.11801175288856029) + | > loss_disc_real_1: 0.1856071799993515 (0.2104871928691864) + | > loss_disc_real_2: 0.22983525693416595 (0.22903972044587134) + | > loss_disc_real_3: 0.20124360918998718 (0.2258606694638729) + | > loss_disc_real_4: 0.19316193461418152 (0.22573304921388626) + | > loss_disc_real_5: 0.22756662964820862 (0.23006792962551117) + | > loss_0: 2.5163869857788086 (2.427673029899597) + | > grad_norm_0: tensor(7.7043, device='cuda:0') (tensor(8.1695, device='cuda:0')) + | > loss_gen: 2.2695913314819336 (2.365245890617371) + | > loss_kl: 1.9457253217697144 (1.973031622171402) + | > loss_feat: 6.18930721282959 (7.084851837158203) + | > loss_mel: 16.14008903503418 (17.73201789855957) + | > loss_duration: 1.5339628458023071 (1.4875746548175812) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.07867431640625 (30.64272184371948) + | > grad_norm_1: tensor(81.9060, device='cuda:0') (tensor(170.1696, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.3409 (0.33562875986099244) + | > loader_time: 0.005 (0.004665756225585937) + + + --> TIME: 2025-04-18 23:44:14 -- STEP: 45/142 -- GLOBAL_STEP: 385575 + | > loss_disc: 2.401862382888794 (2.4481848398844406) + | > loss_disc_real_0: 0.08004386723041534 (0.12185046904616886) + | > loss_disc_real_1: 0.1842830628156662 (0.20899515052636464) + | > loss_disc_real_2: 0.22551696002483368 (0.22894079784552257) + | > loss_disc_real_3: 0.2153596431016922 (0.22674549983607398) + | > loss_disc_real_4: 0.1809225082397461 (0.22785723076926337) + | > loss_disc_real_5: 0.17000459134578705 (0.22638360891077253) + | > loss_0: 2.401862382888794 (2.4481848398844406) + | > grad_norm_0: tensor(10.2663, device='cuda:0') (tensor(7.6809, device='cuda:0')) + | > loss_gen: 2.167545795440674 (2.3300306638081874) + | > loss_kl: 1.986008882522583 (1.9745215601391262) + | > loss_feat: 6.728644371032715 (6.964661290910509) + | > loss_mel: 17.365917205810547 (17.639430046081543) + | > loss_duration: 1.5343289375305176 (1.4985773642857874) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.78244400024414 (30.407221009996203) + | > grad_norm_1: tensor(253.2666, device='cuda:0') (tensor(148.4918, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.3661 (0.3451247109307184) + | > loader_time: 0.0064 (0.0051607555813259565) + + + --> TIME: 2025-04-18 23:44:23 -- STEP: 70/142 -- GLOBAL_STEP: 385600 + | > loss_disc: 2.416930913925171 (2.460290741920472) + | > loss_disc_real_0: 0.11098586767911911 (0.12674755475350788) + | > loss_disc_real_1: 0.22309957444667816 (0.20921497281108584) + | > loss_disc_real_2: 0.21969611942768097 (0.23038978087050574) + | > loss_disc_real_3: 0.2247425615787506 (0.22774141345705304) + | > loss_disc_real_4: 0.24530825018882751 (0.22924596858876092) + | > loss_disc_real_5: 0.21387410163879395 (0.22807501086166926) + | > loss_0: 2.416930913925171 (2.460290741920472) + | > grad_norm_0: tensor(7.7067, device='cuda:0') (tensor(8.9540, device='cuda:0')) + | > loss_gen: 2.2746336460113525 (2.336753908225469) + | > loss_kl: 2.084210157394409 (1.9810178944042751) + | > loss_feat: 7.655962944030762 (7.003259461266654) + | > loss_mel: 18.85439682006836 (17.745906366620744) + | > loss_duration: 1.550071120262146 (1.5077927981104169) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.419273376464844 (30.574730627877372) + | > grad_norm_1: tensor(119.0198, device='cuda:0') (tensor(178.1865, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.3898 (0.35711225441523964) + | > loader_time: 0.0059 (0.005434155464172364) + + + --> TIME: 2025-04-18 23:44:33 -- STEP: 95/142 -- GLOBAL_STEP: 385625 + | > loss_disc: 2.5845699310302734 (2.464953442623741) + | > loss_disc_real_0: 0.1221957728266716 (0.12863390790788748) + | > loss_disc_real_1: 0.18164369463920593 (0.20733502318984584) + | > loss_disc_real_2: 0.2021777182817459 (0.2282539792751011) + | > loss_disc_real_3: 0.22334879636764526 (0.2264328271150589) + | > loss_disc_real_4: 0.2373565286397934 (0.22972204246019062) + | > loss_disc_real_5: 0.22366847097873688 (0.2283388492308165) + | > loss_0: 2.5845699310302734 (2.464953442623741) + | > grad_norm_0: tensor(14.8755, device='cuda:0') (tensor(9.1484, device='cuda:0')) + | > loss_gen: 2.148559331893921 (2.3355291379125513) + | > loss_kl: 2.0854620933532715 (1.9948891238162392) + | > loss_feat: 5.588196754455566 (6.988749308335153) + | > loss_mel: 16.276193618774414 (17.74770623257286) + | > loss_duration: 1.5762887001037598 (1.519045160946093) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.674699783325195 (30.585919129221065) + | > grad_norm_1: tensor(236.0403, device='cuda:0') (tensor(170.8970, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.4046 (0.3674583861702367) + | > loader_time: 0.0062 (0.005704545974731445) + + + --> TIME: 2025-04-18 23:44:44 -- STEP: 120/142 -- GLOBAL_STEP: 385650 + | > loss_disc: 2.4399614334106445 (2.471767771244049) + | > loss_disc_real_0: 0.11666610091924667 (0.12912451835970087) + | > loss_disc_real_1: 0.20217953622341156 (0.20824870231250922) + | > loss_disc_real_2: 0.22916129231452942 (0.22850799895823) + | > loss_disc_real_3: 0.2286304086446762 (0.22898387536406517) + | > loss_disc_real_4: 0.2339749038219452 (0.23034309471646944) + | > loss_disc_real_5: 0.19788430631160736 (0.229428202410539) + | > loss_0: 2.4399614334106445 (2.471767771244049) + | > grad_norm_0: tensor(8.8750, device='cuda:0') (tensor(9.4719, device='cuda:0')) + | > loss_gen: 2.2515125274658203 (2.332168874144555) + | > loss_kl: 1.9524370431900024 (1.9966551313797634) + | > loss_feat: 5.93655252456665 (6.930736680825551) + | > loss_mel: 16.80621337890625 (17.680309947331747) + | > loss_duration: 1.5670554637908936 (1.5254150787989298) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.513769149780273 (30.465285873413087) + | > grad_norm_1: tensor(107.3176, device='cuda:0') (tensor(185.1017, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.4357 (0.37823720971743263) + | > loader_time: 0.0069 (0.005954523881276448) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.481642484664917 (2.481642484664917) + | > loss_disc_real_0: 0.13271956145763397 (0.13271956145763397) + | > loss_disc_real_1: 0.272576242685318 (0.272576242685318) + | > loss_disc_real_2: 0.26027482748031616 (0.26027482748031616) + | > loss_disc_real_3: 0.2661392092704773 (0.2661392092704773) + | > loss_disc_real_4: 0.250760555267334 (0.250760555267334) + | > loss_disc_real_5: 0.24436300992965698 (0.24436300992965698) + | > loss_0: 2.481642484664917 (2.481642484664917) + | > loss_gen: 2.5620408058166504 (2.5620408058166504) + | > loss_kl: 2.2153725624084473 (2.2153725624084473) + | > loss_feat: 6.856759548187256 (6.856759548187256) + | > loss_mel: 19.08658218383789 (19.08658218383789) + | > loss_duration: 1.6199884414672852 (1.6199884414672852) + | > loss_1: 32.34074401855469 (32.34074401855469) + + --> STEP: 1 + | > loss_disc: 2.576352596282959 (2.576352596282959) + | > loss_disc_real_0: 0.1544526368379593 (0.1544526368379593) + | > loss_disc_real_1: 0.22309048473834991 (0.22309048473834991) + | > loss_disc_real_2: 0.26303336024284363 (0.26303336024284363) + | > loss_disc_real_3: 0.2634328603744507 (0.2634328603744507) + | > loss_disc_real_4: 0.29431843757629395 (0.29431843757629395) + | > loss_disc_real_5: 0.2743736803531647 (0.2743736803531647) + | > loss_0: 2.576352596282959 (2.576352596282959) + | > loss_gen: 2.3944852352142334 (2.3944852352142334) + | > loss_kl: 2.4431934356689453 (2.4431934356689453) + | > loss_feat: 5.730869293212891 (5.730869293212891) + | > loss_mel: 17.496252059936523 (17.496252059936523) + | > loss_duration: 1.660374402999878 (1.660374402999878) + | > loss_1: 29.725173950195312 (29.725173950195312) + + --> STEP: 2 + | > loss_disc: 2.5425913333892822 (2.5594719648361206) + | > loss_disc_real_0: 0.17575818300247192 (0.1651054099202156) + | > loss_disc_real_1: 0.2379613220691681 (0.230525903403759) + | > loss_disc_real_2: 0.24562516808509827 (0.25432926416397095) + | > loss_disc_real_3: 0.22363631427288055 (0.24353458732366562) + | > loss_disc_real_4: 0.27960941195487976 (0.28696392476558685) + | > loss_disc_real_5: 0.24290305376052856 (0.2586383670568466) + | > loss_0: 2.5425913333892822 (2.5594719648361206) + | > loss_gen: 2.3407840728759766 (2.367634654045105) + | > loss_kl: 2.0915448665618896 (2.2673691511154175) + | > loss_feat: 4.423906326293945 (5.077387809753418) + | > loss_mel: 13.980717658996582 (15.738484859466553) + | > loss_duration: 1.6014962196350098 (1.6309353113174438) + | > loss_1: 24.43844985961914 (27.081811904907227) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003797292709350586 (+8.177757263183594e-05) + | > avg_loss_disc: 2.5594719648361206 (+0.18669545650482178) + | > avg_loss_disc_real_0: 0.1651054099202156 (+0.04211815074086189) + | > avg_loss_disc_real_1: 0.230525903403759 (-0.005465768277645111) + | > avg_loss_disc_real_2: 0.25432926416397095 (+0.041682228446006775) + | > avg_loss_disc_real_3: 0.24353458732366562 (+0.06742014735937119) + | > avg_loss_disc_real_4: 0.28696392476558685 (+0.08151723444461823) + | > avg_loss_disc_real_5: 0.2586383670568466 (+0.04697590321302414) + | > avg_loss_0: 2.5594719648361206 (+0.18669545650482178) + | > avg_loss_gen: 2.367634654045105 (+0.09321856498718262) + | > avg_loss_kl: 2.2673691511154175 (+0.06545555591583252) + | > avg_loss_feat: 5.077387809753418 (-1.9459097385406494) + | > avg_loss_mel: 15.738484859466553 (-2.764464855194092) + | > avg_loss_duration: 1.6309353113174438 (+0.010927855968475342) + | > avg_loss_1: 27.081811904907227 (-4.540773391723633) + + + > EPOCH: 45/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:44:55)  + + --> TIME: 2025-04-18 23:44:57 -- STEP: 3/142 -- GLOBAL_STEP: 385675 + | > loss_disc: 2.371481418609619 (2.365407705307007) + | > loss_disc_real_0: 0.09610994905233383 (0.08649939546982448) + | > loss_disc_real_1: 0.19162432849407196 (0.18207615613937378) + | > loss_disc_real_2: 0.21865491569042206 (0.22014267245928446) + | > loss_disc_real_3: 0.22081097960472107 (0.2230473111073176) + | > loss_disc_real_4: 0.22348198294639587 (0.23195850352446237) + | > loss_disc_real_5: 0.2881844639778137 (0.2467933644851049) + | > loss_0: 2.371481418609619 (2.365407705307007) + | > grad_norm_0: tensor(11.8041, device='cuda:0') (tensor(7.1946, device='cuda:0')) + | > loss_gen: 2.5522327423095703 (2.425458033879598) + | > loss_kl: 2.145749092102051 (2.018807530403137) + | > loss_feat: 7.3490519523620605 (7.483701388041179) + | > loss_mel: 18.999420166015625 (18.453509012858074) + | > loss_duration: 1.5068292617797852 (1.4943196773529053) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.55328369140625 (31.87579600016276) + | > grad_norm_1: tensor(323.6629, device='cuda:0') (tensor(172.9133, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.3201 (0.3229631582895915) + | > loader_time: 0.0045 (0.004677136739095052) + + + --> TIME: 2025-04-18 23:45:06 -- STEP: 28/142 -- GLOBAL_STEP: 385700 + | > loss_disc: 2.3658719062805176 (2.4708772812570845) + | > loss_disc_real_0: 0.142001673579216 (0.12249703279563359) + | > loss_disc_real_1: 0.25948217511177063 (0.20972689134734018) + | > loss_disc_real_2: 0.23484061658382416 (0.22929507972938673) + | > loss_disc_real_3: 0.23258773982524872 (0.22950028628110886) + | > loss_disc_real_4: 0.2452702671289444 (0.23427200370601245) + | > loss_disc_real_5: 0.24583151936531067 (0.23607248812913895) + | > loss_0: 2.3658719062805176 (2.4708772812570845) + | > grad_norm_0: tensor(11.6656, device='cuda:0') (tensor(7.1184, device='cuda:0')) + | > loss_gen: 2.658745288848877 (2.3409689153943742) + | > loss_kl: 1.961722731590271 (1.938578303371157) + | > loss_feat: 7.067166805267334 (7.133634652410235) + | > loss_mel: 19.47853660583496 (18.13026564461844) + | > loss_duration: 1.5326212644577026 (1.4960413660321916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.69879150390625 (31.03948872429984) + | > grad_norm_1: tensor(329.4649, device='cuda:0') (tensor(157.4330, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.3441 (0.334158684526171) + | > loader_time: 0.0055 (0.00511836154120309) + + + --> TIME: 2025-04-18 23:45:15 -- STEP: 53/142 -- GLOBAL_STEP: 385725 + | > loss_disc: 2.435300827026367 (2.454179678323134) + | > loss_disc_real_0: 0.067616306245327 (0.12444019092703767) + | > loss_disc_real_1: 0.13583581149578094 (0.20830054721742305) + | > loss_disc_real_2: 0.1815980076789856 (0.22608326881561638) + | > loss_disc_real_3: 0.17714443802833557 (0.22493717552356) + | > loss_disc_real_4: 0.23792003095149994 (0.22787889753872492) + | > loss_disc_real_5: 0.1972050666809082 (0.23114555033872713) + | > loss_0: 2.435300827026367 (2.454179678323134) + | > grad_norm_0: tensor(13.1962, device='cuda:0') (tensor(9.6403, device='cuda:0')) + | > loss_gen: 2.4457476139068604 (2.351958495266032) + | > loss_kl: 1.8840141296386719 (1.9354533946739052) + | > loss_feat: 7.902737617492676 (7.071743047462319) + | > loss_mel: 18.031686782836914 (18.023252523170335) + | > loss_duration: 1.4846794605255127 (1.5017837465934034) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.74886703491211 (30.884191153184425) + | > grad_norm_1: tensor(344.3353, device='cuda:0') (tensor(222.6436, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.3663 (0.34606130617969433) + | > loader_time: 0.0057 (0.005432016444656084) + + + --> TIME: 2025-04-18 23:45:25 -- STEP: 78/142 -- GLOBAL_STEP: 385750 + | > loss_disc: 2.569633722305298 (2.4748286559031563) + | > loss_disc_real_0: 0.12330396473407745 (0.12798068958979383) + | > loss_disc_real_1: 0.20081815123558044 (0.20975386523283446) + | > loss_disc_real_2: 0.22829456627368927 (0.22631279436441568) + | > loss_disc_real_3: 0.2475338578224182 (0.22938439135368055) + | > loss_disc_real_4: 0.24878865480422974 (0.23159715151175475) + | > loss_disc_real_5: 0.2689729928970337 (0.2334084485967954) + | > loss_0: 2.569633722305298 (2.4748286559031563) + | > grad_norm_0: tensor(14.3880, device='cuda:0') (tensor(11.1609, device='cuda:0')) + | > loss_gen: 2.231084108352661 (2.354453707352662) + | > loss_kl: 2.061028242111206 (1.9617843948877776) + | > loss_feat: 7.127639293670654 (6.985806715794099) + | > loss_mel: 18.667173385620117 (17.970597034845603) + | > loss_duration: 1.5509860515594482 (1.512550678008642) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.637910842895508 (30.785192636343147) + | > grad_norm_1: tensor(250.2899, device='cuda:0') (tensor(226.9873, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.3943 (0.35811224350562454) + | > loader_time: 0.0066 (0.005747455817002516) + + + --> TIME: 2025-04-18 23:45:35 -- STEP: 103/142 -- GLOBAL_STEP: 385775 + | > loss_disc: 2.4266183376312256 (2.4719897867406466) + | > loss_disc_real_0: 0.07298814505338669 (0.12660861811012888) + | > loss_disc_real_1: 0.18415236473083496 (0.20976015954341704) + | > loss_disc_real_2: 0.22024591267108917 (0.22635779623846405) + | > loss_disc_real_3: 0.21106229722499847 (0.22954173805644212) + | > loss_disc_real_4: 0.21677812933921814 (0.23169689025115042) + | > loss_disc_real_5: 0.2497701793909073 (0.2316414587995381) + | > loss_0: 2.4266183376312256 (2.4719897867406466) + | > grad_norm_0: tensor(15.8875, device='cuda:0') (tensor(11.6129, device='cuda:0')) + | > loss_gen: 2.311941146850586 (2.3476928970188777) + | > loss_kl: 2.1515512466430664 (1.9860221807239125) + | > loss_feat: 6.542541980743408 (6.945328244885195) + | > loss_mel: 18.672866821289062 (18.015308870852582) + | > loss_duration: 1.6067405939102173 (1.5246883815932049) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.285642623901367 (30.819040631784976) + | > grad_norm_1: tensor(280.1866, device='cuda:0') (tensor(247.9563, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.4101 (0.36888504491269) + | > loader_time: 0.0066 (0.006030888233369994) + + + --> TIME: 2025-04-18 23:45:46 -- STEP: 128/142 -- GLOBAL_STEP: 385800 + | > loss_disc: 2.5364573001861572 (2.4690331351012) + | > loss_disc_real_0: 0.1623104065656662 (0.12756370526039967) + | > loss_disc_real_1: 0.2588271498680115 (0.20872889959719032) + | > loss_disc_real_2: 0.2604900598526001 (0.2259440820198506) + | > loss_disc_real_3: 0.2726045250892639 (0.22919189848471433) + | > loss_disc_real_4: 0.2692156732082367 (0.2310945614008233) + | > loss_disc_real_5: 0.2481500208377838 (0.23203038179781288) + | > loss_0: 2.5364573001861572 (2.4690331351012) + | > grad_norm_0: tensor(11.3070, device='cuda:0') (tensor(11.8705, device='cuda:0')) + | > loss_gen: 2.483572483062744 (2.3531353380531077) + | > loss_kl: 1.9471943378448486 (1.9929583286866546) + | > loss_feat: 6.8726887702941895 (6.963429991155863) + | > loss_mel: 19.20374298095703 (18.015545606613163) + | > loss_duration: 1.555668592453003 (1.5308607239276175) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.0628662109375 (30.85592994093895) + | > grad_norm_1: tensor(330.1245, device='cuda:0') (tensor(257.3448, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.4389 (0.38079125620424736) + | > loader_time: 0.0071 (0.006273847073316574) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.462893486022949 (2.462893486022949) + | > loss_disc_real_0: 0.15775184333324432 (0.15775184333324432) + | > loss_disc_real_1: 0.22940894961357117 (0.22940894961357117) + | > loss_disc_real_2: 0.24291470646858215 (0.24291470646858215) + | > loss_disc_real_3: 0.2349921464920044 (0.2349921464920044) + | > loss_disc_real_4: 0.2608708143234253 (0.2608708143234253) + | > loss_disc_real_5: 0.247965008020401 (0.247965008020401) + | > loss_0: 2.462893486022949 (2.462893486022949) + | > loss_gen: 2.4496798515319824 (2.4496798515319824) + | > loss_kl: 2.3115146160125732 (2.3115146160125732) + | > loss_feat: 6.378883361816406 (6.378883361816406) + | > loss_mel: 17.531932830810547 (17.531932830810547) + | > loss_duration: 1.6323411464691162 (1.6323411464691162) + | > loss_1: 30.304353713989258 (30.304353713989258) + + --> STEP: 1 + | > loss_disc: 2.582216739654541 (2.582216739654541) + | > loss_disc_real_0: 0.15645432472229004 (0.15645432472229004) + | > loss_disc_real_1: 0.2437657117843628 (0.2437657117843628) + | > loss_disc_real_2: 0.2368430197238922 (0.2368430197238922) + | > loss_disc_real_3: 0.2184719741344452 (0.2184719741344452) + | > loss_disc_real_4: 0.2656741142272949 (0.2656741142272949) + | > loss_disc_real_5: 0.23053187131881714 (0.23053187131881714) + | > loss_0: 2.582216739654541 (2.582216739654541) + | > loss_gen: 2.165708541870117 (2.165708541870117) + | > loss_kl: 2.3633604049682617 (2.3633604049682617) + | > loss_feat: 6.701265811920166 (6.701265811920166) + | > loss_mel: 18.125272750854492 (18.125272750854492) + | > loss_duration: 1.6425933837890625 (1.6425933837890625) + | > loss_1: 30.998201370239258 (30.998201370239258) + + --> STEP: 2 + | > loss_disc: 2.323072671890259 (2.4526447057724) + | > loss_disc_real_0: 0.10553888231515884 (0.13099660351872444) + | > loss_disc_real_1: 0.23345570266246796 (0.23861070722341537) + | > loss_disc_real_2: 0.19733409583568573 (0.21708855777978897) + | > loss_disc_real_3: 0.16724970936775208 (0.19286084175109863) + | > loss_disc_real_4: 0.21292971074581146 (0.2393019124865532) + | > loss_disc_real_5: 0.19088251888751984 (0.2107071951031685) + | > loss_0: 2.323072671890259 (2.4526447057724) + | > loss_gen: 2.280524492263794 (2.2231165170669556) + | > loss_kl: 2.0685675144195557 (2.2159639596939087) + | > loss_feat: 8.018967628479004 (7.360116720199585) + | > loss_mel: 19.406845092773438 (18.766058921813965) + | > loss_duration: 1.5983171463012695 (1.620455265045166) + | > loss_1: 33.37322235107422 (32.18571186065674) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003281712532043457 (-0.0005155801773071289) + | > avg_loss_disc: 2.4526447057724 (-0.1068272590637207) + | > avg_loss_disc_real_0: 0.13099660351872444 (-0.034108806401491165) + | > avg_loss_disc_real_1: 0.23861070722341537 (+0.008084803819656372) + | > avg_loss_disc_real_2: 0.21708855777978897 (-0.037240706384181976) + | > avg_loss_disc_real_3: 0.19286084175109863 (-0.050673745572566986) + | > avg_loss_disc_real_4: 0.2393019124865532 (-0.04766201227903366) + | > avg_loss_disc_real_5: 0.2107071951031685 (-0.04793117195367813) + | > avg_loss_0: 2.4526447057724 (-0.1068272590637207) + | > avg_loss_gen: 2.2231165170669556 (-0.14451813697814941) + | > avg_loss_kl: 2.2159639596939087 (-0.05140519142150879) + | > avg_loss_feat: 7.360116720199585 (+2.282728910446167) + | > avg_loss_mel: 18.766058921813965 (+3.027574062347412) + | > avg_loss_duration: 1.620455265045166 (-0.010480046272277832) + | > avg_loss_1: 32.18571186065674 (+5.103899955749512) + + + > EPOCH: 46/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:45:54)  + + --> TIME: 2025-04-18 23:45:58 -- STEP: 11/142 -- GLOBAL_STEP: 385825 + | > loss_disc: 2.487330913543701 (2.433652010830966) + | > loss_disc_real_0: 0.11715970188379288 (0.11587531864643097) + | > loss_disc_real_1: 0.21130938827991486 (0.20676823231306943) + | > loss_disc_real_2: 0.2350333333015442 (0.22712588039311496) + | > loss_disc_real_3: 0.23636142909526825 (0.2240502048622478) + | > loss_disc_real_4: 0.21873147785663605 (0.2267089757052335) + | > loss_disc_real_5: 0.21795806288719177 (0.22963061928749084) + | > loss_0: 2.487330913543701 (2.433652010830966) + | > grad_norm_0: tensor(4.6770, device='cuda:0') (tensor(11.1934, device='cuda:0')) + | > loss_gen: 2.3603527545928955 (2.355439619584517) + | > loss_kl: 1.8391456604003906 (1.9308963038704612) + | > loss_feat: 6.1515889167785645 (6.900592240420255) + | > loss_mel: 16.064491271972656 (17.47805265946822) + | > loss_duration: 1.482616662979126 (1.4880099513313987) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.898195266723633 (30.152990687977184) + | > grad_norm_1: tensor(127.4678, device='cuda:0') (tensor(330.2422, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.3315 (0.32548759200356225) + | > loader_time: 0.0049 (0.004481012170965021) + + + --> TIME: 2025-04-18 23:46:07 -- STEP: 36/142 -- GLOBAL_STEP: 385850 + | > loss_disc: 2.391399383544922 (2.4263468384742737) + | > loss_disc_real_0: 0.1033870205283165 (0.11328511498868465) + | > loss_disc_real_1: 0.24088336527347565 (0.2055460454689132) + | > loss_disc_real_2: 0.20569761097431183 (0.2252487718231148) + | > loss_disc_real_3: 0.23071803152561188 (0.22802417683932516) + | > loss_disc_real_4: 0.23354095220565796 (0.22660977724525663) + | > loss_disc_real_5: 0.2553826570510864 (0.23251006421115664) + | > loss_0: 2.391399383544922 (2.4263468384742737) + | > grad_norm_0: tensor(10.9911, device='cuda:0') (tensor(12.3999, device='cuda:0')) + | > loss_gen: 2.5452041625976562 (2.3924145102500916) + | > loss_kl: 2.0135889053344727 (1.9361006518205006) + | > loss_feat: 7.011058330535889 (7.138628549045986) + | > loss_mel: 17.05345916748047 (17.893795649210613) + | > loss_duration: 1.490707516670227 (1.4995190865463681) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.114017486572266 (30.860458427005344) + | > grad_norm_1: tensor(299.2285, device='cuda:0') (tensor(286.7568, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.3559 (0.3384662800365024) + | > loader_time: 0.0055 (0.005071169800228543) + + + --> TIME: 2025-04-18 23:46:17 -- STEP: 61/142 -- GLOBAL_STEP: 385875 + | > loss_disc: 2.4217681884765625 (2.4339765759765126) + | > loss_disc_real_0: 0.1258246749639511 (0.11619413815072326) + | > loss_disc_real_1: 0.20553679764270782 (0.20417112185329686) + | > loss_disc_real_2: 0.22067686915397644 (0.22703444127176628) + | > loss_disc_real_3: 0.19675306975841522 (0.22707897107132147) + | > loss_disc_real_4: 0.207403302192688 (0.22795571582239182) + | > loss_disc_real_5: 0.21298043429851532 (0.2319230070856751) + | > loss_0: 2.4217681884765625 (2.4339765759765126) + | > grad_norm_0: tensor(11.5897, device='cuda:0') (tensor(11.2730, device='cuda:0')) + | > loss_gen: 2.319005012512207 (2.383414577265255) + | > loss_kl: 2.0441322326660156 (1.9432954143305294) + | > loss_feat: 7.0151166915893555 (7.128812305262832) + | > loss_mel: 17.45448875427246 (17.987155945574646) + | > loss_duration: 1.5266883373260498 (1.5087797680839163) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35942840576172 (30.951457977294922) + | > grad_norm_1: tensor(317.0140, device='cuda:0') (tensor(265.7885, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.3841 (0.35022650781224984) + | > loader_time: 0.0069 (0.005397421414734887) + + + --> TIME: 2025-04-18 23:46:27 -- STEP: 86/142 -- GLOBAL_STEP: 385900 + | > loss_disc: 2.3849523067474365 (2.4485947725384736) + | > loss_disc_real_0: 0.11346576362848282 (0.11976864622082821) + | > loss_disc_real_1: 0.16112639009952545 (0.20534459692101145) + | > loss_disc_real_2: 0.20945899188518524 (0.22899984620338262) + | > loss_disc_real_3: 0.2034299373626709 (0.22802913604780686) + | > loss_disc_real_4: 0.2266494780778885 (0.23027593885050263) + | > loss_disc_real_5: 0.22306174039840698 (0.23189266060673913) + | > loss_0: 2.3849523067474365 (2.4485947725384736) + | > grad_norm_0: tensor(9.0294, device='cuda:0') (tensor(10.6188, device='cuda:0')) + | > loss_gen: 2.368319272994995 (2.3747685205104743) + | > loss_kl: 1.9869673252105713 (1.955676254838012) + | > loss_feat: 7.200867176055908 (7.058648763700973) + | > loss_mel: 18.214345932006836 (17.988173861836277) + | > loss_duration: 1.5557026863098145 (1.5193992548210675) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.326200485229492 (30.896666637686796) + | > grad_norm_1: tensor(226.7792, device='cuda:0') (tensor(240.7796, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.3989 (0.36237185500388924) + | > loader_time: 0.0062 (0.005659250325934832) + + + --> TIME: 2025-04-18 23:46:37 -- STEP: 111/142 -- GLOBAL_STEP: 385925 + | > loss_disc: 2.5545077323913574 (2.4560955107749045) + | > loss_disc_real_0: 0.15128977596759796 (0.12292992759932268) + | > loss_disc_real_1: 0.20881938934326172 (0.20677776151412242) + | > loss_disc_real_2: 0.2169760763645172 (0.22920415248419787) + | > loss_disc_real_3: 0.2514045238494873 (0.22936996951833502) + | > loss_disc_real_4: 0.2698591947555542 (0.22979298140014615) + | > loss_disc_real_5: 0.27140533924102783 (0.23091631080653216) + | > loss_0: 2.5545077323913574 (2.4560955107749045) + | > grad_norm_0: tensor(4.8991, device='cuda:0') (tensor(10.1001, device='cuda:0')) + | > loss_gen: 2.306422710418701 (2.365718023197072) + | > loss_kl: 1.967677354812622 (1.9634746968209207) + | > loss_feat: 6.491738796234131 (7.003056500409101) + | > loss_mel: 18.920961380004883 (17.952095074696583) + | > loss_duration: 1.578960657119751 (1.5263250346656319) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.265762329101562 (30.810669374895525) + | > grad_norm_1: tensor(292.5151, device='cuda:0') (tensor(225.9233, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.4219 (0.37314446981962734) + | > loader_time: 0.007 (0.00586864540168831) + + + --> TIME: 2025-04-18 23:46:48 -- STEP: 136/142 -- GLOBAL_STEP: 385950 + | > loss_disc: 2.5234274864196777 (2.4618446195826813) + | > loss_disc_real_0: 0.13442446291446686 (0.12564128368873806) + | > loss_disc_real_1: 0.18971194326877594 (0.20688052102923393) + | > loss_disc_real_2: 0.2091277539730072 (0.22823906448834083) + | > loss_disc_real_3: 0.18824143707752228 (0.22840343218515902) + | > loss_disc_real_4: 0.18205228447914124 (0.23080830462276936) + | > loss_disc_real_5: 0.18358489871025085 (0.2301036101930282) + | > loss_0: 2.5234274864196777 (2.4618446195826813) + | > grad_norm_0: tensor(10.2230, device='cuda:0') (tensor(9.6228, device='cuda:0')) + | > loss_gen: 2.2713234424591064 (2.3569876814589787) + | > loss_kl: 1.9536861181259155 (1.970726563650019) + | > loss_feat: 7.5252861976623535 (6.976232391946456) + | > loss_mel: 18.227060317993164 (17.915836453437805) + | > loss_duration: 1.5581423044204712 (1.5312128715655382) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.535497665405273 (30.750995972577265) + | > grad_norm_1: tensor(360.0184, device='cuda:0') (tensor(217.0109, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.4492 (0.3849297516486224) + | > loader_time: 0.0076 (0.0061320767683141365) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5465104579925537 (2.5465104579925537) + | > loss_disc_real_0: 0.1524628847837448 (0.1524628847837448) + | > loss_disc_real_1: 0.2078564316034317 (0.2078564316034317) + | > loss_disc_real_2: 0.24750465154647827 (0.24750465154647827) + | > loss_disc_real_3: 0.2897077202796936 (0.2897077202796936) + | > loss_disc_real_4: 0.26857009530067444 (0.26857009530067444) + | > loss_disc_real_5: 0.247120201587677 (0.247120201587677) + | > loss_0: 2.5465104579925537 (2.5465104579925537) + | > loss_gen: 2.354043960571289 (2.354043960571289) + | > loss_kl: 2.0130438804626465 (2.0130438804626465) + | > loss_feat: 6.629127502441406 (6.629127502441406) + | > loss_mel: 18.101898193359375 (18.101898193359375) + | > loss_duration: 1.6089341640472412 (1.6089341640472412) + | > loss_1: 30.707048416137695 (30.707048416137695) + + --> STEP: 1 + | > loss_disc: 2.478327751159668 (2.478327751159668) + | > loss_disc_real_0: 0.11517031490802765 (0.11517031490802765) + | > loss_disc_real_1: 0.21207508444786072 (0.21207508444786072) + | > loss_disc_real_2: 0.2705543339252472 (0.2705543339252472) + | > loss_disc_real_3: 0.277946412563324 (0.277946412563324) + | > loss_disc_real_4: 0.25778597593307495 (0.25778597593307495) + | > loss_disc_real_5: 0.23301982879638672 (0.23301982879638672) + | > loss_0: 2.478327751159668 (2.478327751159668) + | > loss_gen: 2.3509039878845215 (2.3509039878845215) + | > loss_kl: 2.2675023078918457 (2.2675023078918457) + | > loss_feat: 6.379678726196289 (6.379678726196289) + | > loss_mel: 19.019773483276367 (19.019773483276367) + | > loss_duration: 1.6535427570343018 (1.6535427570343018) + | > loss_1: 31.671401977539062 (31.671401977539062) + + --> STEP: 2 + | > loss_disc: 2.6812500953674316 (2.57978892326355) + | > loss_disc_real_0: 0.22338369488716125 (0.16927700489759445) + | > loss_disc_real_1: 0.23492524027824402 (0.22350016236305237) + | > loss_disc_real_2: 0.24448375403881073 (0.25751904398202896) + | > loss_disc_real_3: 0.24315665662288666 (0.2605515345931053) + | > loss_disc_real_4: 0.2622639238834381 (0.26002494990825653) + | > loss_disc_real_5: 0.2525385320186615 (0.2427791804075241) + | > loss_0: 2.6812500953674316 (2.57978892326355) + | > loss_gen: 2.2756428718566895 (2.3132734298706055) + | > loss_kl: 2.009058713912964 (2.138280510902405) + | > loss_feat: 6.869817733764648 (6.624748229980469) + | > loss_mel: 16.443580627441406 (17.731677055358887) + | > loss_duration: 1.5905742645263672 (1.6220585107803345) + | > loss_1: 29.18867301940918 (30.43003749847412) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003939509391784668 (+0.0006577968597412109) + | > avg_loss_disc: 2.57978892326355 (+0.1271442174911499) + | > avg_loss_disc_real_0: 0.16927700489759445 (+0.03828040137887001) + | > avg_loss_disc_real_1: 0.22350016236305237 (-0.015110544860363007) + | > avg_loss_disc_real_2: 0.25751904398202896 (+0.04043048620223999) + | > avg_loss_disc_real_3: 0.2605515345931053 (+0.06769069284200668) + | > avg_loss_disc_real_4: 0.26002494990825653 (+0.02072303742170334) + | > avg_loss_disc_real_5: 0.2427791804075241 (+0.03207198530435562) + | > avg_loss_0: 2.57978892326355 (+0.1271442174911499) + | > avg_loss_gen: 2.3132734298706055 (+0.0901569128036499) + | > avg_loss_kl: 2.138280510902405 (-0.0776834487915039) + | > avg_loss_feat: 6.624748229980469 (-0.7353684902191162) + | > avg_loss_mel: 17.731677055358887 (-1.0343818664550781) + | > avg_loss_duration: 1.6220585107803345 (+0.001603245735168457) + | > avg_loss_1: 30.43003749847412 (-1.7556743621826172) + + + > EPOCH: 47/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:46:52)  + + --> TIME: 2025-04-18 23:46:59 -- STEP: 19/142 -- GLOBAL_STEP: 385975 + | > loss_disc: 2.4560348987579346 (2.483368246178878) + | > loss_disc_real_0: 0.11758103221654892 (0.1207434005643192) + | > loss_disc_real_1: 0.1640939861536026 (0.21243281897745633) + | > loss_disc_real_2: 0.2066369503736496 (0.2326968014240265) + | > loss_disc_real_3: 0.19398167729377747 (0.22582352082980306) + | > loss_disc_real_4: 0.21143975853919983 (0.22758104063962636) + | > loss_disc_real_5: 0.24759377539157867 (0.23336016504388107) + | > loss_0: 2.4560348987579346 (2.483368246178878) + | > grad_norm_0: tensor(9.8545, device='cuda:0') (tensor(12.5570, device='cuda:0')) + | > loss_gen: 2.300147533416748 (2.364006343640779) + | > loss_kl: 2.1064577102661133 (1.8891597170578807) + | > loss_feat: 7.150257110595703 (7.188379538686652) + | > loss_mel: 18.238323211669922 (18.247523709347373) + | > loss_duration: 1.481583833694458 (1.4817373250660144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.276769638061523 (31.170806884765625) + | > grad_norm_1: tensor(255.5201, device='cuda:0') (tensor(265.4500, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.337 (0.3294271042472438) + | > loader_time: 0.0064 (0.004794484690616005) + + + --> TIME: 2025-04-18 23:47:08 -- STEP: 44/142 -- GLOBAL_STEP: 386000 + | > loss_disc: 2.5042545795440674 (2.4579331224614935) + | > loss_disc_real_0: 0.12315521389245987 (0.11956694535911083) + | > loss_disc_real_1: 0.206248477101326 (0.20776299251751465) + | > loss_disc_real_2: 0.22998745739459991 (0.22870196131142703) + | > loss_disc_real_3: 0.20807649195194244 (0.2277215187522498) + | > loss_disc_real_4: 0.19107942283153534 (0.22839233076030557) + | > loss_disc_real_5: 0.22307062149047852 (0.230987453663891) + | > loss_0: 2.5042545795440674 (2.4579331224614935) + | > grad_norm_0: tensor(12.8062, device='cuda:0') (tensor(12.4981, device='cuda:0')) + | > loss_gen: 2.230243682861328 (2.392625776204196) + | > loss_kl: 2.020845413208008 (1.9190222024917603) + | > loss_feat: 7.214320182800293 (7.22277785431255) + | > loss_mel: 18.54862403869629 (18.165882305665455) + | > loss_duration: 1.523932933807373 (1.495428199117834) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.537967681884766 (31.195736278187145) + | > grad_norm_1: tensor(371.7343, device='cuda:0') (tensor(287.7053, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.3564 (0.3419662876562639) + | > loader_time: 0.0053 (0.005190860141407359) + + + --> TIME: 2025-04-18 23:47:18 -- STEP: 69/142 -- GLOBAL_STEP: 386025 + | > loss_disc: 2.4955873489379883 (2.4592947130617895) + | > loss_disc_real_0: 0.12876754999160767 (0.1205806503261345) + | > loss_disc_real_1: 0.19236278533935547 (0.20875653775705807) + | > loss_disc_real_2: 0.2770036458969116 (0.22792058340881183) + | > loss_disc_real_3: 0.24921874701976776 (0.2270662590213444) + | > loss_disc_real_4: 0.2572936415672302 (0.22908479104871335) + | > loss_disc_real_5: 0.23850475251674652 (0.23366104476693747) + | > loss_0: 2.4955873489379883 (2.4592947130617895) + | > grad_norm_0: tensor(12.7213, device='cuda:0') (tensor(13.2180, device='cuda:0')) + | > loss_gen: 2.3152945041656494 (2.3854062902754634) + | > loss_kl: 2.036677360534668 (1.93485382847164) + | > loss_feat: 6.035078525543213 (7.1511638821035195) + | > loss_mel: 16.13597297668457 (18.09983382017716) + | > loss_duration: 1.5862611532211304 (1.505832072617351) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.109285354614258 (31.07708986254706) + | > grad_norm_1: tensor(379.8477, device='cuda:0') (tensor(312.7424, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.3961 (0.35733886387037195) + | > loader_time: 0.0063 (0.005487189776655556) + + + --> TIME: 2025-04-18 23:47:28 -- STEP: 94/142 -- GLOBAL_STEP: 386050 + | > loss_disc: 2.4683897495269775 (2.464288222028855) + | > loss_disc_real_0: 0.1865580528974533 (0.12338162887286633) + | > loss_disc_real_1: 0.2110414206981659 (0.20695763017902982) + | > loss_disc_real_2: 0.2224290817975998 (0.22689696464766848) + | > loss_disc_real_3: 0.24357937276363373 (0.22761432303393142) + | > loss_disc_real_4: 0.2155660092830658 (0.22996447314607335) + | > loss_disc_real_5: 0.24502569437026978 (0.23501213386337808) + | > loss_0: 2.4683897495269775 (2.464288222028855) + | > grad_norm_0: tensor(6.2721, device='cuda:0') (tensor(13.7296, device='cuda:0')) + | > loss_gen: 2.4533462524414062 (2.3742334107135212) + | > loss_kl: 1.9790081977844238 (1.9469431740172365) + | > loss_feat: 6.828605651855469 (7.037035779750093) + | > loss_mel: 18.054767608642578 (17.998958323864223) + | > loss_duration: 1.5455840826034546 (1.5167158943541512) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.861310958862305 (30.873886534508237) + | > grad_norm_1: tensor(171.4097, device='cuda:0') (tensor(310.0345, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.4025 (0.3681909403902418) + | > loader_time: 0.0066 (0.005768593321455286) + + + --> TIME: 2025-04-18 23:47:39 -- STEP: 119/142 -- GLOBAL_STEP: 386075 + | > loss_disc: 2.5184967517852783 (2.468386802352778) + | > loss_disc_real_0: 0.10999321937561035 (0.1248717431385978) + | > loss_disc_real_1: 0.23861509561538696 (0.20818325363788284) + | > loss_disc_real_2: 0.22028040885925293 (0.2269022165476775) + | > loss_disc_real_3: 0.21614323556423187 (0.22780700002898688) + | > loss_disc_real_4: 0.23060546815395355 (0.22964374996533915) + | > loss_disc_real_5: 0.21334655582904816 (0.23447748442657856) + | > loss_0: 2.5184967517852783 (2.468386802352778) + | > grad_norm_0: tensor(12.3146, device='cuda:0') (tensor(13.3737, device='cuda:0')) + | > loss_gen: 2.2404253482818604 (2.359632944860379) + | > loss_kl: 2.1164214611053467 (1.9517434294484242) + | > loss_feat: 6.297124862670898 (6.978706684433112) + | > loss_mel: 17.09857940673828 (17.955927912928463) + | > loss_duration: 1.6200017929077148 (1.5231461745350305) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.372554779052734 (30.769157073077032) + | > grad_norm_1: tensor(148.0232, device='cuda:0') (tensor(295.1840, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.4292 (0.3797801402436584) + | > loader_time: 0.0067 (0.006037860357460855) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.7242233753204346 (2.7242233753204346) + | > loss_disc_real_0: 0.3358273506164551 (0.3358273506164551) + | > loss_disc_real_1: 0.24589458107948303 (0.24589458107948303) + | > loss_disc_real_2: 0.2339998185634613 (0.2339998185634613) + | > loss_disc_real_3: 0.2626878619194031 (0.2626878619194031) + | > loss_disc_real_4: 0.27889660000801086 (0.27889660000801086) + | > loss_disc_real_5: 0.29538580775260925 (0.29538580775260925) + | > loss_0: 2.7242233753204346 (2.7242233753204346) + | > loss_gen: 2.6482949256896973 (2.6482949256896973) + | > loss_kl: 2.068631887435913 (2.068631887435913) + | > loss_feat: 4.939968109130859 (4.939968109130859) + | > loss_mel: 16.64921760559082 (16.64921760559082) + | > loss_duration: 1.6329152584075928 (1.6329152584075928) + | > loss_1: 27.939027786254883 (27.939027786254883) + + --> STEP: 1 + | > loss_disc: 2.5857343673706055 (2.5857343673706055) + | > loss_disc_real_0: 0.2727999687194824 (0.2727999687194824) + | > loss_disc_real_1: 0.24149765074253082 (0.24149765074253082) + | > loss_disc_real_2: 0.23575513064861298 (0.23575513064861298) + | > loss_disc_real_3: 0.252987265586853 (0.252987265586853) + | > loss_disc_real_4: 0.25969383120536804 (0.25969383120536804) + | > loss_disc_real_5: 0.20686453580856323 (0.20686453580856323) + | > loss_0: 2.5857343673706055 (2.5857343673706055) + | > loss_gen: 2.5659234523773193 (2.5659234523773193) + | > loss_kl: 2.3012540340423584 (2.3012540340423584) + | > loss_feat: 6.454357147216797 (6.454357147216797) + | > loss_mel: 16.93771743774414 (16.93771743774414) + | > loss_duration: 1.6505341529846191 (1.6505341529846191) + | > loss_1: 29.909786224365234 (29.909786224365234) + + --> STEP: 2 + | > loss_disc: 2.7029869556427 (2.644360661506653) + | > loss_disc_real_0: 0.373171865940094 (0.3229859173297882) + | > loss_disc_real_1: 0.3011813461780548 (0.2713394984602928) + | > loss_disc_real_2: 0.2710956633090973 (0.25342539697885513) + | > loss_disc_real_3: 0.2737918198108673 (0.26338954269886017) + | > loss_disc_real_4: 0.2778506577014923 (0.2687722444534302) + | > loss_disc_real_5: 0.21763911843299866 (0.21225182712078094) + | > loss_0: 2.7029869556427 (2.644360661506653) + | > loss_gen: 2.89693021774292 (2.7314268350601196) + | > loss_kl: 2.0203466415405273 (2.160800337791443) + | > loss_feat: 6.971485137939453 (6.712921142578125) + | > loss_mel: 18.458850860595703 (17.698284149169922) + | > loss_duration: 1.586900234222412 (1.6187171936035156) + | > loss_1: 31.93451499938965 (30.92215061187744) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003775358200073242 (-0.00016415119171142578) + | > avg_loss_disc: 2.644360661506653 (+0.06457173824310303) + | > avg_loss_disc_real_0: 0.3229859173297882 (+0.15370891243219376) + | > avg_loss_disc_real_1: 0.2713394984602928 (+0.04783933609724045) + | > avg_loss_disc_real_2: 0.25342539697885513 (-0.004093647003173828) + | > avg_loss_disc_real_3: 0.26338954269886017 (+0.0028380081057548523) + | > avg_loss_disc_real_4: 0.2687722444534302 (+0.008747294545173645) + | > avg_loss_disc_real_5: 0.21225182712078094 (-0.030527353286743164) + | > avg_loss_0: 2.644360661506653 (+0.06457173824310303) + | > avg_loss_gen: 2.7314268350601196 (+0.41815340518951416) + | > avg_loss_kl: 2.160800337791443 (+0.022519826889038086) + | > avg_loss_feat: 6.712921142578125 (+0.08817291259765625) + | > avg_loss_mel: 17.698284149169922 (-0.033392906188964844) + | > avg_loss_duration: 1.6187171936035156 (-0.0033413171768188477) + | > avg_loss_1: 30.92215061187744 (+0.4921131134033203) + + + > EPOCH: 48/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:47:51)  + + --> TIME: 2025-04-18 23:47:52 -- STEP: 2/142 -- GLOBAL_STEP: 386100 + | > loss_disc: 2.477689504623413 (2.4834362268447876) + | > loss_disc_real_0: 0.10051341354846954 (0.11896450817584991) + | > loss_disc_real_1: 0.2192389816045761 (0.21264398097991943) + | > loss_disc_real_2: 0.24633611738681793 (0.24720966815948486) + | > loss_disc_real_3: 0.24763064086437225 (0.244243323802948) + | > loss_disc_real_4: 0.2559286653995514 (0.2535158842802048) + | > loss_disc_real_5: 0.24962207674980164 (0.24342061579227448) + | > loss_0: 2.477689504623413 (2.4834362268447876) + | > grad_norm_0: tensor(18.7190, device='cuda:0') (tensor(12.5837, device='cuda:0')) + | > loss_gen: 2.2950923442840576 (2.324972987174988) + | > loss_kl: 2.073763132095337 (1.983894944190979) + | > loss_feat: 7.434603214263916 (7.6833813190460205) + | > loss_mel: 17.654109954833984 (17.853351593017578) + | > loss_duration: 1.517464518547058 (1.4852407574653625) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.97503089904785 (31.330841064453125) + | > grad_norm_1: tensor(357.8736, device='cuda:0') (tensor(318.4399, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.3221 (0.3251241445541382) + | > loader_time: 0.004 (0.004001975059509277) + + + --> TIME: 2025-04-18 23:48:01 -- STEP: 27/142 -- GLOBAL_STEP: 386125 + | > loss_disc: 2.471123695373535 (2.461827772635001) + | > loss_disc_real_0: 0.11496908962726593 (0.11982571264659916) + | > loss_disc_real_1: 0.22599123418331146 (0.20973006553120083) + | > loss_disc_real_2: 0.23213663697242737 (0.22918201762217064) + | > loss_disc_real_3: 0.26767322421073914 (0.22547239672254632) + | > loss_disc_real_4: 0.23125159740447998 (0.22998464162702914) + | > loss_disc_real_5: 0.26266491413116455 (0.24007902046044668) + | > loss_0: 2.471123695373535 (2.461827772635001) + | > grad_norm_0: tensor(12.6572, device='cuda:0') (tensor(12.9442, device='cuda:0')) + | > loss_gen: 2.374019145965576 (2.3778064957371465) + | > loss_kl: 1.9055964946746826 (1.9104569443949946) + | > loss_feat: 6.807297229766846 (7.032574953856291) + | > loss_mel: 17.867473602294922 (17.96887927585178) + | > loss_duration: 1.504629373550415 (1.4917650796748974) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.459016799926758 (30.781483332316082) + | > grad_norm_1: tensor(329.9533, device='cuda:0') (tensor(276.3158, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.348 (0.33390288882785374) + | > loader_time: 0.0054 (0.004977376372725874) + + + --> TIME: 2025-04-18 23:48:10 -- STEP: 52/142 -- GLOBAL_STEP: 386150 + | > loss_disc: 2.6109108924865723 (2.4537338018417354) + | > loss_disc_real_0: 0.17368687689304352 (0.12253287971879427) + | > loss_disc_real_1: 0.21768143773078918 (0.2051876339201744) + | > loss_disc_real_2: 0.2564507722854614 (0.22610359438336813) + | > loss_disc_real_3: 0.24942225217819214 (0.22780097992374346) + | > loss_disc_real_4: 0.2343159317970276 (0.22899935853022796) + | > loss_disc_real_5: 0.2751421332359314 (0.23542811606939024) + | > loss_0: 2.6109108924865723 (2.4537338018417354) + | > grad_norm_0: tensor(16.9533, device='cuda:0') (tensor(12.4773, device='cuda:0')) + | > loss_gen: 2.325509548187256 (2.369829237461089) + | > loss_kl: 2.015167236328125 (1.9124558178278117) + | > loss_feat: 6.6942572593688965 (7.016955641599802) + | > loss_mel: 18.615371704101562 (17.96051935049204) + | > loss_duration: 1.5671151876449585 (1.4996124597696157) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.217422485351562 (30.759373151339016) + | > grad_norm_1: tensor(363.5817, device='cuda:0') (tensor(293.0172, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.3679 (0.3456655557339008) + | > loader_time: 0.006 (0.005338733012859637) + + + --> TIME: 2025-04-18 23:48:20 -- STEP: 77/142 -- GLOBAL_STEP: 386175 + | > loss_disc: 2.4449071884155273 (2.463326051637723) + | > loss_disc_real_0: 0.07977057993412018 (0.12346354412374558) + | > loss_disc_real_1: 0.18931248784065247 (0.20539418520865502) + | > loss_disc_real_2: 0.2499121129512787 (0.22788771174170755) + | > loss_disc_real_3: 0.24415425956249237 (0.2285364523723528) + | > loss_disc_real_4: 0.23050828278064728 (0.2302982710398637) + | > loss_disc_real_5: 0.21120159327983856 (0.23599072594147225) + | > loss_0: 2.4449071884155273 (2.463326051637723) + | > grad_norm_0: tensor(10.5771, device='cuda:0') (tensor(12.9349, device='cuda:0')) + | > loss_gen: 2.498478889465332 (2.3662572835947007) + | > loss_kl: 1.911178469657898 (1.930104568407133) + | > loss_feat: 7.275547504425049 (7.017379748356807) + | > loss_mel: 18.88887596130371 (17.92477740250625) + | > loss_duration: 1.5078418254852295 (1.5097912039075576) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.08192443847656 (30.748310658838843) + | > grad_norm_1: tensor(311.2151, device='cuda:0') (tensor(291.4837, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.393 (0.3585229384434688) + | > loader_time: 0.0064 (0.005658899034772601) + + + --> TIME: 2025-04-18 23:48:30 -- STEP: 102/142 -- GLOBAL_STEP: 386200 + | > loss_disc: 2.644045829772949 (2.476236651925478) + | > loss_disc_real_0: 0.178150936961174 (0.12697907628528046) + | > loss_disc_real_1: 0.215039923787117 (0.20627661662943222) + | > loss_disc_real_2: 0.23549014329910278 (0.22877966700231328) + | > loss_disc_real_3: 0.26463907957077026 (0.22876395036776861) + | > loss_disc_real_4: 0.29737380146980286 (0.23248201769356633) + | > loss_disc_real_5: 0.31821444630622864 (0.23598026571904912) + | > loss_0: 2.644045829772949 (2.476236651925478) + | > grad_norm_0: tensor(24.6060, device='cuda:0') (tensor(14.2610, device='cuda:0')) + | > loss_gen: 2.37977933883667 (2.3664364335583694) + | > loss_kl: 1.9779672622680664 (1.9381088111914841) + | > loss_feat: 6.5016655921936035 (6.9627363635044475) + | > loss_mel: 17.77073097229004 (17.885909650840023) + | > loss_duration: 1.535632848739624 (1.5209408694622562) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.1657772064209 (30.674132496702907) + | > grad_norm_1: tensor(54.5452, device='cuda:0') (tensor(307.4875, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.4291 (0.3701170238794065) + | > loader_time: 0.0076 (0.005907425693437165) + + + --> TIME: 2025-04-18 23:48:41 -- STEP: 127/142 -- GLOBAL_STEP: 386225 + | > loss_disc: 2.4116554260253906 (2.4791963701173065) + | > loss_disc_real_0: 0.16468964517116547 (0.1296850152196378) + | > loss_disc_real_1: 0.1989029049873352 (0.2067945728621145) + | > loss_disc_real_2: 0.2127590775489807 (0.22788843453869106) + | > loss_disc_real_3: 0.23967130482196808 (0.22820604422430354) + | > loss_disc_real_4: 0.2144089639186859 (0.2317625062437508) + | > loss_disc_real_5: 0.24163560569286346 (0.23612161356163777) + | > loss_0: 2.4116554260253906 (2.4791963701173065) + | > grad_norm_0: tensor(19.0084, device='cuda:0') (tensor(15.3907, device='cuda:0')) + | > loss_gen: 2.439871311187744 (2.363832547908692) + | > loss_kl: 1.982226848602295 (1.9384473900156698) + | > loss_feat: 7.42596960067749 (6.9310842574112055) + | > loss_mel: 18.79022789001465 (17.883442360585136) + | > loss_duration: 1.5483477115631104 (1.5272270957315999) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.1866455078125 (30.644033942635605) + | > grad_norm_1: tensor(487.4050, device='cuda:0') (tensor(316.8495, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.438 (0.3820369525218573) + | > loader_time: 0.0075 (0.006154801901869887) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4784998893737793 (2.4784998893737793) + | > loss_disc_real_0: 0.13022662699222565 (0.13022662699222565) + | > loss_disc_real_1: 0.20328646898269653 (0.20328646898269653) + | > loss_disc_real_2: 0.28382718563079834 (0.28382718563079834) + | > loss_disc_real_3: 0.24715104699134827 (0.24715104699134827) + | > loss_disc_real_4: 0.2570568323135376 (0.2570568323135376) + | > loss_disc_real_5: 0.23291108012199402 (0.23291108012199402) + | > loss_0: 2.4784998893737793 (2.4784998893737793) + | > loss_gen: 2.4018068313598633 (2.4018068313598633) + | > loss_kl: 2.052591323852539 (2.052591323852539) + | > loss_feat: 6.7781662940979 (6.7781662940979) + | > loss_mel: 19.384862899780273 (19.384862899780273) + | > loss_duration: 1.611862301826477 (1.611862301826477) + | > loss_1: 32.22929000854492 (32.22929000854492) + + --> STEP: 1 + | > loss_disc: 2.3322043418884277 (2.3322043418884277) + | > loss_disc_real_0: 0.13709130883216858 (0.13709130883216858) + | > loss_disc_real_1: 0.21743963658809662 (0.21743963658809662) + | > loss_disc_real_2: 0.2684134244918823 (0.2684134244918823) + | > loss_disc_real_3: 0.2523807883262634 (0.2523807883262634) + | > loss_disc_real_4: 0.22397373616695404 (0.22397373616695404) + | > loss_disc_real_5: 0.22871334850788116 (0.22871334850788116) + | > loss_0: 2.3322043418884277 (2.3322043418884277) + | > loss_gen: 2.5977485179901123 (2.5977485179901123) + | > loss_kl: 2.1834144592285156 (2.1834144592285156) + | > loss_feat: 6.873528003692627 (6.873528003692627) + | > loss_mel: 19.390316009521484 (19.390316009521484) + | > loss_duration: 1.6408295631408691 (1.6408295631408691) + | > loss_1: 32.68583679199219 (32.68583679199219) + + --> STEP: 2 + | > loss_disc: 2.5873830318450928 (2.4597936868667603) + | > loss_disc_real_0: 0.16866543889045715 (0.15287837386131287) + | > loss_disc_real_1: 0.19989261031150818 (0.2086661234498024) + | > loss_disc_real_2: 0.2575710713863373 (0.2629922479391098) + | > loss_disc_real_3: 0.23713718354701996 (0.2447589859366417) + | > loss_disc_real_4: 0.2039075344800949 (0.21394063532352448) + | > loss_disc_real_5: 0.2418821156024933 (0.23529773205518723) + | > loss_0: 2.5873830318450928 (2.4597936868667603) + | > loss_gen: 2.178727388381958 (2.388237953186035) + | > loss_kl: 2.0716989040374756 (2.1275566816329956) + | > loss_feat: 7.175739765167236 (7.024633884429932) + | > loss_mel: 18.95194435119629 (19.171130180358887) + | > loss_duration: 1.586138367652893 (1.613483965396881) + | > loss_1: 31.964248657226562 (32.325042724609375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003693103790283203 (-8.225440979003906e-05) + | > avg_loss_disc: 2.4597936868667603 (-0.18456697463989258) + | > avg_loss_disc_real_0: 0.15287837386131287 (-0.17010754346847534) + | > avg_loss_disc_real_1: 0.2086661234498024 (-0.06267337501049042) + | > avg_loss_disc_real_2: 0.2629922479391098 (+0.00956685096025467) + | > avg_loss_disc_real_3: 0.2447589859366417 (-0.018630556762218475) + | > avg_loss_disc_real_4: 0.21394063532352448 (-0.0548316091299057) + | > avg_loss_disc_real_5: 0.23529773205518723 (+0.02304590493440628) + | > avg_loss_0: 2.4597936868667603 (-0.18456697463989258) + | > avg_loss_gen: 2.388237953186035 (-0.3431888818740845) + | > avg_loss_kl: 2.1275566816329956 (-0.033243656158447266) + | > avg_loss_feat: 7.024633884429932 (+0.31171274185180664) + | > avg_loss_mel: 19.171130180358887 (+1.4728460311889648) + | > avg_loss_duration: 1.613483965396881 (-0.0052332282066345215) + | > avg_loss_1: 32.325042724609375 (+1.4028921127319336) + + + > EPOCH: 49/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:48:50)  + + --> TIME: 2025-04-18 23:48:54 -- STEP: 10/142 -- GLOBAL_STEP: 386250 + | > loss_disc: 2.3652749061584473 (2.449764060974121) + | > loss_disc_real_0: 0.07904507219791412 (0.11753612160682678) + | > loss_disc_real_1: 0.19547760486602783 (0.2076393783092499) + | > loss_disc_real_2: 0.2326432168483734 (0.22776215374469758) + | > loss_disc_real_3: 0.23412179946899414 (0.22980601191520691) + | > loss_disc_real_4: 0.24977533519268036 (0.2330484241247177) + | > loss_disc_real_5: 0.22997915744781494 (0.2288404330611229) + | > loss_0: 2.3652749061584473 (2.449764060974121) + | > grad_norm_0: tensor(4.6141, device='cuda:0') (tensor(7.8187, device='cuda:0')) + | > loss_gen: 2.519909620285034 (2.3589187622070313) + | > loss_kl: 2.072524070739746 (1.9157716631889343) + | > loss_feat: 7.112188816070557 (7.038153457641601) + | > loss_mel: 18.303905487060547 (17.91390495300293) + | > loss_duration: 1.4574095010757446 (1.483139455318451) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.4659366607666 (30.709888076782228) + | > grad_norm_1: tensor(74.7724, device='cuda:0') (tensor(216.7809, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.3348 (0.32397868633270266) + | > loader_time: 0.005 (0.0046389102935791016) + + + --> TIME: 2025-04-18 23:49:03 -- STEP: 35/142 -- GLOBAL_STEP: 386275 + | > loss_disc: 2.4101104736328125 (2.448912709099906) + | > loss_disc_real_0: 0.15784324705600739 (0.12287825686591013) + | > loss_disc_real_1: 0.2265699952840805 (0.20580282509326936) + | > loss_disc_real_2: 0.20773433148860931 (0.22644496517521995) + | > loss_disc_real_3: 0.20841741561889648 (0.22912998838084084) + | > loss_disc_real_4: 0.1818774938583374 (0.22986114536012922) + | > loss_disc_real_5: 0.19816803932189941 (0.23112199817384993) + | > loss_0: 2.4101104736328125 (2.448912709099906) + | > grad_norm_0: tensor(9.2042, device='cuda:0') (tensor(6.7338, device='cuda:0')) + | > loss_gen: 2.402773857116699 (2.350996174131121) + | > loss_kl: 2.0019514560699463 (1.9107357672282628) + | > loss_feat: 7.477067947387695 (7.090697615487235) + | > loss_mel: 18.888879776000977 (17.976989800589426) + | > loss_duration: 1.5136594772338867 (1.4936003071921213) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.284332275390625 (30.82301902770996) + | > grad_norm_1: tensor(134.4395, device='cuda:0') (tensor(164.1168, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.3561 (0.3449657576424735) + | > loader_time: 0.0054 (0.005073540551321847) + + + --> TIME: 2025-04-18 23:49:12 -- STEP: 60/142 -- GLOBAL_STEP: 386300 + | > loss_disc: 2.4813475608825684 (2.4462859551111866) + | > loss_disc_real_0: 0.11004190891981125 (0.1254191675533851) + | > loss_disc_real_1: 0.23936404287815094 (0.2049018104871114) + | > loss_disc_real_2: 0.21002866327762604 (0.22634650841355325) + | > loss_disc_real_3: 0.22108305990695953 (0.22924917961160343) + | > loss_disc_real_4: 0.1997305303812027 (0.22695778533816338) + | > loss_disc_real_5: 0.19507573544979095 (0.23135308027267457) + | > loss_0: 2.4813475608825684 (2.4462859551111866) + | > grad_norm_0: tensor(13.6997, device='cuda:0') (tensor(7.9205, device='cuda:0')) + | > loss_gen: 2.284392833709717 (2.346202850341797) + | > loss_kl: 1.973447322845459 (1.925376433134079) + | > loss_feat: 6.431297779083252 (7.033125416437785) + | > loss_mel: 16.51331901550293 (17.893196392059334) + | > loss_duration: 1.5505154132843018 (1.5036468823750815) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.752973556518555 (30.70154759089152) + | > grad_norm_1: tensor(219.6816, device='cuda:0') (tensor(188.8439, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.374 (0.35375020106633503) + | > loader_time: 0.0067 (0.0054369886716206855) + + + --> TIME: 2025-04-18 23:49:22 -- STEP: 85/142 -- GLOBAL_STEP: 386325 + | > loss_disc: 2.5417423248291016 (2.4661622804753933) + | > loss_disc_real_0: 0.1619199961423874 (0.12909483147018097) + | > loss_disc_real_1: 0.20168718695640564 (0.20754586475736955) + | > loss_disc_real_2: 0.2581690847873688 (0.22798909124206093) + | > loss_disc_real_3: 0.22481586039066315 (0.2301247710690779) + | > loss_disc_real_4: 0.23996232450008392 (0.22981234420748317) + | > loss_disc_real_5: 0.2270970493555069 (0.23186688949080075) + | > loss_0: 2.5417423248291016 (2.4661622804753933) + | > grad_norm_0: tensor(8.0475, device='cuda:0') (tensor(8.6620, device='cuda:0')) + | > loss_gen: 2.2133376598358154 (2.3398417108199174) + | > loss_kl: 1.9820737838745117 (1.9420114026350133) + | > loss_feat: 6.963667869567871 (6.985942363739014) + | > loss_mel: 18.3826961517334 (17.827143590590538) + | > loss_duration: 1.5346840620040894 (1.5153553710264318) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.076457977294922 (30.61029416252585) + | > grad_norm_1: tensor(174.7308, device='cuda:0') (tensor(200.2181, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.3991 (0.36534553135142595) + | > loader_time: 0.0063 (0.0057244497186997355) + + + --> TIME: 2025-04-18 23:49:33 -- STEP: 110/142 -- GLOBAL_STEP: 386350 + | > loss_disc: 2.4675631523132324 (2.4790759043260056) + | > loss_disc_real_0: 0.13300175964832306 (0.13037518242543403) + | > loss_disc_real_1: 0.17369444668293 (0.20860204547643663) + | > loss_disc_real_2: 0.22157207131385803 (0.2295402999628674) + | > loss_disc_real_3: 0.24826127290725708 (0.23043831329454076) + | > loss_disc_real_4: 0.21150127053260803 (0.2311524594371969) + | > loss_disc_real_5: 0.21437379717826843 (0.23108773895285345) + | > loss_0: 2.4675631523132324 (2.4790759043260056) + | > grad_norm_0: tensor(7.2080, device='cuda:0') (tensor(8.6025, device='cuda:0')) + | > loss_gen: 2.2827794551849365 (2.3236642339012845) + | > loss_kl: 2.1672768592834473 (1.9432283574884588) + | > loss_feat: 6.243143558502197 (6.925487683036111) + | > loss_mel: 17.577838897705078 (17.764205533807925) + | > loss_duration: 1.5352530479431152 (1.522209673578089) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.806291580200195 (30.47879522497004) + | > grad_norm_1: tensor(78.1386, device='cuda:0') (tensor(182.7257, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.4447 (0.37640263817527064) + | > loader_time: 0.0075 (0.006007777560840953) + + + --> TIME: 2025-04-18 23:49:44 -- STEP: 135/142 -- GLOBAL_STEP: 386375 + | > loss_disc: 2.4524354934692383 (2.4761279741923015) + | > loss_disc_real_0: 0.17871369421482086 (0.13228611476995328) + | > loss_disc_real_1: 0.24253787100315094 (0.20765018661816914) + | > loss_disc_real_2: 0.24206015467643738 (0.2283226552936766) + | > loss_disc_real_3: 0.26211124658584595 (0.22974855822545512) + | > loss_disc_real_4: 0.22358815371990204 (0.23045904415625112) + | > loss_disc_real_5: 0.22452907264232635 (0.2304079818504828) + | > loss_0: 2.4524354934692383 (2.4761279741923015) + | > grad_norm_0: tensor(8.0733, device='cuda:0') (tensor(8.6366, device='cuda:0')) + | > loss_gen: 2.364041328430176 (2.320802877567433) + | > loss_kl: 1.8400763273239136 (1.9482139410795989) + | > loss_feat: 6.871031284332275 (6.887510702345106) + | > loss_mel: 17.048187255859375 (17.702541464346425) + | > loss_duration: 1.5246026515960693 (1.5274173886687668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.647939682006836 (30.386486180623372) + | > grad_norm_1: tensor(268.4142, device='cuda:0') (tensor(178.1730, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.4689 (0.3895481604116933) + | > loader_time: 0.0076 (0.00624474772700557) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.466266393661499 (2.466266393661499) + | > loss_disc_real_0: 0.1609407663345337 (0.1609407663345337) + | > loss_disc_real_1: 0.20447205007076263 (0.20447205007076263) + | > loss_disc_real_2: 0.2584627866744995 (0.2584627866744995) + | > loss_disc_real_3: 0.3152025640010834 (0.3152025640010834) + | > loss_disc_real_4: 0.23515655100345612 (0.23515655100345612) + | > loss_disc_real_5: 0.3227600157260895 (0.3227600157260895) + | > loss_0: 2.466266393661499 (2.466266393661499) + | > loss_gen: 2.6654906272888184 (2.6654906272888184) + | > loss_kl: 2.169154644012451 (2.169154644012451) + | > loss_feat: 6.63864803314209 (6.63864803314209) + | > loss_mel: 18.331436157226562 (18.331436157226562) + | > loss_duration: 1.6407954692840576 (1.6407954692840576) + | > loss_1: 31.445524215698242 (31.445524215698242) + + --> STEP: 1 + | > loss_disc: 2.5386078357696533 (2.5386078357696533) + | > loss_disc_real_0: 0.17364540696144104 (0.17364540696144104) + | > loss_disc_real_1: 0.22643661499023438 (0.22643661499023438) + | > loss_disc_real_2: 0.2689597010612488 (0.2689597010612488) + | > loss_disc_real_3: 0.2587333619594574 (0.2587333619594574) + | > loss_disc_real_4: 0.28561028838157654 (0.28561028838157654) + | > loss_disc_real_5: 0.22068274021148682 (0.22068274021148682) + | > loss_0: 2.5386078357696533 (2.5386078357696533) + | > loss_gen: 2.4130358695983887 (2.4130358695983887) + | > loss_kl: 2.4554941654205322 (2.4554941654205322) + | > loss_feat: 6.373882293701172 (6.373882293701172) + | > loss_mel: 18.36765480041504 (18.36765480041504) + | > loss_duration: 1.6481525897979736 (1.6481525897979736) + | > loss_1: 31.258220672607422 (31.258220672607422) + + --> STEP: 2 + | > loss_disc: 2.5090181827545166 (2.523813009262085) + | > loss_disc_real_0: 0.2185916006565094 (0.19611850380897522) + | > loss_disc_real_1: 0.20299147069454193 (0.21471404284238815) + | > loss_disc_real_2: 0.2167486697435379 (0.24285418540239334) + | > loss_disc_real_3: 0.27915704250335693 (0.26894520223140717) + | > loss_disc_real_4: 0.24184803664684296 (0.26372916251420975) + | > loss_disc_real_5: 0.26738205552101135 (0.24403239786624908) + | > loss_0: 2.5090181827545166 (2.523813009262085) + | > loss_gen: 2.5874876976013184 (2.5002617835998535) + | > loss_kl: 2.121840715408325 (2.2886674404144287) + | > loss_feat: 6.32839822769165 (6.351140260696411) + | > loss_mel: 17.49188232421875 (17.929768562316895) + | > loss_duration: 1.5847910642623901 (1.6164718270301819) + | > loss_1: 30.11440086364746 (30.68631076812744) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003968358039855957 (+0.0002752542495727539) + | > avg_loss_disc: 2.523813009262085 (+0.06401932239532471) + | > avg_loss_disc_real_0: 0.19611850380897522 (+0.043240129947662354) + | > avg_loss_disc_real_1: 0.21471404284238815 (+0.006047919392585754) + | > avg_loss_disc_real_2: 0.24285418540239334 (-0.02013806253671646) + | > avg_loss_disc_real_3: 0.26894520223140717 (+0.024186216294765472) + | > avg_loss_disc_real_4: 0.26372916251420975 (+0.04978852719068527) + | > avg_loss_disc_real_5: 0.24403239786624908 (+0.008734665811061859) + | > avg_loss_0: 2.523813009262085 (+0.06401932239532471) + | > avg_loss_gen: 2.5002617835998535 (+0.11202383041381836) + | > avg_loss_kl: 2.2886674404144287 (+0.1611107587814331) + | > avg_loss_feat: 6.351140260696411 (-0.6734936237335205) + | > avg_loss_mel: 17.929768562316895 (-1.2413616180419922) + | > avg_loss_duration: 1.6164718270301819 (+0.0029878616333007812) + | > avg_loss_1: 30.68631076812744 (-1.6387319564819336) + + + > EPOCH: 50/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:49:49)  + + --> TIME: 2025-04-18 23:49:56 -- STEP: 18/142 -- GLOBAL_STEP: 386400 + | > loss_disc: 2.464240550994873 (2.4564322895473905) + | > loss_disc_real_0: 0.11804597824811935 (0.12191799614164564) + | > loss_disc_real_1: 0.22650489211082458 (0.21143660032086903) + | > loss_disc_real_2: 0.21366862952709198 (0.22574595279163784) + | > loss_disc_real_3: 0.25822409987449646 (0.22719327939881218) + | > loss_disc_real_4: 0.20832142233848572 (0.22762449002928203) + | > loss_disc_real_5: 0.2206551879644394 (0.23377067512936062) + | > loss_0: 2.464240550994873 (2.4564322895473905) + | > grad_norm_0: tensor(8.4860, device='cuda:0') (tensor(7.3059, device='cuda:0')) + | > loss_gen: 2.426741600036621 (2.3493763075934515) + | > loss_kl: 1.8480167388916016 (1.9456541670693293) + | > loss_feat: 7.72652006149292 (7.181896501117283) + | > loss_mel: 19.307743072509766 (18.086169666714138) + | > loss_duration: 1.4775454998016357 (1.48414628373252) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.78656768798828 (31.047242800394695) + | > grad_norm_1: tensor(88.5086, device='cuda:0') (tensor(113.3785, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.3382 (0.33120456006791854) + | > loader_time: 0.005 (0.004689123895433214) + + + --> TIME: 2025-04-18 23:50:05 -- STEP: 43/142 -- GLOBAL_STEP: 386425 + | > loss_disc: 2.4510414600372314 (2.4683591543241987) + | > loss_disc_real_0: 0.1029849424958229 (0.1189419271987538) + | > loss_disc_real_1: 0.2337181717157364 (0.20832911653574127) + | > loss_disc_real_2: 0.21012771129608154 (0.22665162211240725) + | > loss_disc_real_3: 0.22256535291671753 (0.23208963316540385) + | > loss_disc_real_4: 0.2015421986579895 (0.23049378048541935) + | > loss_disc_real_5: 0.2394721657037735 (0.23867907392424206) + | > loss_0: 2.4510414600372314 (2.4683591543241987) + | > grad_norm_0: tensor(9.7133, device='cuda:0') (tensor(7.9186, device='cuda:0')) + | > loss_gen: 2.254082441329956 (2.350436798361845) + | > loss_kl: 1.9447038173675537 (1.92946481427481) + | > loss_feat: 7.08459997177124 (7.077132003251896) + | > loss_mel: 17.74009895324707 (17.941722315411244) + | > loss_duration: 1.5046281814575195 (1.492312231729197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.52811050415039 (30.79106805490893) + | > grad_norm_1: tensor(230.6375, device='cuda:0') (tensor(181.8602, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.3613 (0.34318326240362124) + | > loader_time: 0.0054 (0.005156627921170966) + + + --> TIME: 2025-04-18 23:50:14 -- STEP: 68/142 -- GLOBAL_STEP: 386450 + | > loss_disc: 2.371779441833496 (2.459697053712957) + | > loss_disc_real_0: 0.09940407425165176 (0.11910950852667584) + | > loss_disc_real_1: 0.17109154164791107 (0.20587554070002895) + | > loss_disc_real_2: 0.22599895298480988 (0.2260726631125983) + | > loss_disc_real_3: 0.21124453842639923 (0.23035173096201) + | > loss_disc_real_4: 0.2157929539680481 (0.23165811883176074) + | > loss_disc_real_5: 0.21993699669837952 (0.2328483488629846) + | > loss_0: 2.371779441833496 (2.459697053712957) + | > grad_norm_0: tensor(11.5578, device='cuda:0') (tensor(9.2568, device='cuda:0')) + | > loss_gen: 2.4434051513671875 (2.364312175442191) + | > loss_kl: 1.9882190227508545 (1.9340276314931757) + | > loss_feat: 7.9832234382629395 (7.112053275108337) + | > loss_mel: 19.077959060668945 (18.01308703422547) + | > loss_duration: 1.507596731185913 (1.5032030133640064) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.00040054321289 (30.92668311736163) + | > grad_norm_1: tensor(253.5423, device='cuda:0') (tensor(222.0251, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.3879 (0.3546368479728699) + | > loader_time: 0.0064 (0.005502010093015782) + + + --> TIME: 2025-04-18 23:50:24 -- STEP: 93/142 -- GLOBAL_STEP: 386475 + | > loss_disc: 2.649657726287842 (2.466051283703055) + | > loss_disc_real_0: 0.1330885887145996 (0.12278729901518873) + | > loss_disc_real_1: 0.2845098078250885 (0.20781750191924392) + | > loss_disc_real_2: 0.20920149981975555 (0.22633263484765126) + | > loss_disc_real_3: 0.23628786206245422 (0.23129123225007006) + | > loss_disc_real_4: 0.2613171339035034 (0.23189264984541041) + | > loss_disc_real_5: 0.26516464352607727 (0.23285394955065944) + | > loss_0: 2.649657726287842 (2.466051283703055) + | > grad_norm_0: tensor(27.3398, device='cuda:0') (tensor(11.0767, device='cuda:0')) + | > loss_gen: 2.122422933578491 (2.368050744456629) + | > loss_kl: 2.0722851753234863 (1.9480320894589989) + | > loss_feat: 6.3947062492370605 (7.072359777265979) + | > loss_mel: 18.623401641845703 (18.0126800947292) + | > loss_duration: 1.5294790267944336 (1.5151154264327018) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.742294311523438 (30.91623814900716) + | > grad_norm_1: tensor(230.9696, device='cuda:0') (tensor(248.2479, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.4032 (0.3663187590978479) + | > loader_time: 0.0064 (0.005765620098319104) + + + --> TIME: 2025-04-18 23:50:35 -- STEP: 118/142 -- GLOBAL_STEP: 386500 + | > loss_disc: 2.584794759750366 (2.4669987670445845) + | > loss_disc_real_0: 0.08837499469518661 (0.12468169616945722) + | > loss_disc_real_1: 0.20571695268154144 (0.20891888060812222) + | > loss_disc_real_2: 0.22406317293643951 (0.22663775237940126) + | > loss_disc_real_3: 0.21620620787143707 (0.23006499179844128) + | > loss_disc_real_4: 0.24606157839298248 (0.2308467083563239) + | > loss_disc_real_5: 0.25905418395996094 (0.2327517816575907) + | > loss_0: 2.584794759750366 (2.4669987670445845) + | > grad_norm_0: tensor(26.7376, device='cuda:0') (tensor(13.2918, device='cuda:0')) + | > loss_gen: 2.157210111618042 (2.3748841891854493) + | > loss_kl: 1.9821184873580933 (1.9565757007922155) + | > loss_feat: 6.19742488861084 (7.049928442906525) + | > loss_mel: 17.690610885620117 (17.99671913405596) + | > loss_duration: 1.5114670991897583 (1.5211913535150432) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.53883171081543 (30.8992988457114) + | > grad_norm_1: tensor(89.9440, device='cuda:0') (tensor(274.8369, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.4663 (0.3783836081876593) + | > loader_time: 0.0075 (0.0060629238516597414) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5877299308776855 (2.5877299308776855) + | > loss_disc_real_0: 0.2186841517686844 (0.2186841517686844) + | > loss_disc_real_1: 0.22853460907936096 (0.22853460907936096) + | > loss_disc_real_2: 0.25511881709098816 (0.25511881709098816) + | > loss_disc_real_3: 0.2843727767467499 (0.2843727767467499) + | > loss_disc_real_4: 0.23072843253612518 (0.23072843253612518) + | > loss_disc_real_5: 0.2582927346229553 (0.2582927346229553) + | > loss_0: 2.5877299308776855 (2.5877299308776855) + | > loss_gen: 2.428250551223755 (2.428250551223755) + | > loss_kl: 2.198420524597168 (2.198420524597168) + | > loss_feat: 6.871164798736572 (6.871164798736572) + | > loss_mel: 19.5133056640625 (19.5133056640625) + | > loss_duration: 1.6176440715789795 (1.6176440715789795) + | > loss_1: 32.628787994384766 (32.628787994384766) + + --> STEP: 1 + | > loss_disc: 2.5760083198547363 (2.5760083198547363) + | > loss_disc_real_0: 0.23871217668056488 (0.23871217668056488) + | > loss_disc_real_1: 0.27282974123954773 (0.27282974123954773) + | > loss_disc_real_2: 0.2500087320804596 (0.2500087320804596) + | > loss_disc_real_3: 0.27431169152259827 (0.27431169152259827) + | > loss_disc_real_4: 0.26405882835388184 (0.26405882835388184) + | > loss_disc_real_5: 0.22044706344604492 (0.22044706344604492) + | > loss_0: 2.5760083198547363 (2.5760083198547363) + | > loss_gen: 2.528535842895508 (2.528535842895508) + | > loss_kl: 2.4420554637908936 (2.4420554637908936) + | > loss_feat: 7.190724849700928 (7.190724849700928) + | > loss_mel: 18.745229721069336 (18.745229721069336) + | > loss_duration: 1.661041498184204 (1.661041498184204) + | > loss_1: 32.567588806152344 (32.567588806152344) + + --> STEP: 2 + | > loss_disc: 2.5447115898132324 (2.5603599548339844) + | > loss_disc_real_0: 0.19795072078704834 (0.2183314487338066) + | > loss_disc_real_1: 0.2334766834974289 (0.2531532123684883) + | > loss_disc_real_2: 0.2351551204919815 (0.24258192628622055) + | > loss_disc_real_3: 0.24960528314113617 (0.2619584873318672) + | > loss_disc_real_4: 0.24927526712417603 (0.25666704773902893) + | > loss_disc_real_5: 0.22181877493858337 (0.22113291919231415) + | > loss_0: 2.5447115898132324 (2.5603599548339844) + | > loss_gen: 2.4177780151367188 (2.4731569290161133) + | > loss_kl: 2.027935743331909 (2.2349956035614014) + | > loss_feat: 7.040301322937012 (7.11551308631897) + | > loss_mel: 18.60515785217285 (18.675193786621094) + | > loss_duration: 1.6177074909210205 (1.6393744945526123) + | > loss_1: 31.708881378173828 (32.138235092163086) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004057765007019043 (+8.940696716308594e-05) + | > avg_loss_disc: 2.5603599548339844 (+0.036546945571899414) + | > avg_loss_disc_real_0: 0.2183314487338066 (+0.02221294492483139) + | > avg_loss_disc_real_1: 0.2531532123684883 (+0.03843916952610016) + | > avg_loss_disc_real_2: 0.24258192628622055 (-0.00027225911617279053) + | > avg_loss_disc_real_3: 0.2619584873318672 (-0.0069867148995399475) + | > avg_loss_disc_real_4: 0.25666704773902893 (-0.007062114775180817) + | > avg_loss_disc_real_5: 0.22113291919231415 (-0.022899478673934937) + | > avg_loss_0: 2.5603599548339844 (+0.036546945571899414) + | > avg_loss_gen: 2.4731569290161133 (-0.027104854583740234) + | > avg_loss_kl: 2.2349956035614014 (-0.053671836853027344) + | > avg_loss_feat: 7.11551308631897 (+0.7643728256225586) + | > avg_loss_mel: 18.675193786621094 (+0.7454252243041992) + | > avg_loss_duration: 1.6393744945526123 (+0.02290266752243042) + | > avg_loss_1: 32.138235092163086 (+1.4519243240356445) + + + > EPOCH: 51/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:50:48)  + + --> TIME: 2025-04-18 23:50:49 -- STEP: 1/142 -- GLOBAL_STEP: 386525 + | > loss_disc: 2.4043548107147217 (2.4043548107147217) + | > loss_disc_real_0: 0.16912828385829926 (0.16912828385829926) + | > loss_disc_real_1: 0.18940392136573792 (0.18940392136573792) + | > loss_disc_real_2: 0.23381368815898895 (0.23381368815898895) + | > loss_disc_real_3: 0.2692843973636627 (0.2692843973636627) + | > loss_disc_real_4: 0.2252085655927658 (0.2252085655927658) + | > loss_disc_real_5: 0.211573988199234 (0.211573988199234) + | > loss_0: 2.4043548107147217 (2.4043548107147217) + | > grad_norm_0: tensor(29.6559, device='cuda:0') (tensor(29.6559, device='cuda:0')) + | > loss_gen: 2.563734769821167 (2.563734769821167) + | > loss_kl: 2.0211870670318604 (2.0211870670318604) + | > loss_feat: 7.4553022384643555 (7.4553022384643555) + | > loss_mel: 18.233489990234375 (18.233489990234375) + | > loss_duration: 1.442819356918335 (1.442819356918335) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.716533660888672 (31.716533660888672) + | > grad_norm_1: tensor(583.7663, device='cuda:0') (tensor(583.7663, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.3239 (0.3238706588745117) + | > loader_time: 0.0052 (0.005163669586181641) + + + --> TIME: 2025-04-18 23:50:57 -- STEP: 26/142 -- GLOBAL_STEP: 386550 + | > loss_disc: 2.5113866329193115 (2.4481586584678063) + | > loss_disc_real_0: 0.13402937352657318 (0.12154237639445525) + | > loss_disc_real_1: 0.18571996688842773 (0.20412188883011156) + | > loss_disc_real_2: 0.22422078251838684 (0.22539231123832557) + | > loss_disc_real_3: 0.23310936987400055 (0.22965069287098372) + | > loss_disc_real_4: 0.24289312958717346 (0.2309866977425722) + | > loss_disc_real_5: 0.2428043931722641 (0.2312278260405247) + | > loss_0: 2.5113866329193115 (2.4481586584678063) + | > grad_norm_0: tensor(6.7112, device='cuda:0') (tensor(9.9103, device='cuda:0')) + | > loss_gen: 2.2449121475219727 (2.3391828261888947) + | > loss_kl: 1.9500740766525269 (1.9007627322123601) + | > loss_feat: 6.298891067504883 (7.084297106816218) + | > loss_mel: 17.028295516967773 (17.89610070448655) + | > loss_duration: 1.4911141395568848 (1.4908161346728985) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.01328468322754 (30.71115933931791) + | > grad_norm_1: tensor(150.4799, device='cuda:0') (tensor(207.9463, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.3476 (0.3340279322404128) + | > loader_time: 0.0053 (0.004916301140418421) + + + --> TIME: 2025-04-18 23:51:06 -- STEP: 51/142 -- GLOBAL_STEP: 386575 + | > loss_disc: 2.662750720977783 (2.438505345699834) + | > loss_disc_real_0: 0.1557677835226059 (0.12011103551177417) + | > loss_disc_real_1: 0.18392661213874817 (0.20297327725326314) + | > loss_disc_real_2: 0.24223089218139648 (0.22396815933433234) + | > loss_disc_real_3: 0.23385274410247803 (0.22815975371529074) + | > loss_disc_real_4: 0.2641628384590149 (0.22912510206886366) + | > loss_disc_real_5: 0.22911426424980164 (0.2329704527761422) + | > loss_0: 2.662750720977783 (2.438505345699834) + | > grad_norm_0: tensor(5.0842, device='cuda:0') (tensor(8.4663, device='cuda:0')) + | > loss_gen: 2.1488871574401855 (2.3549935396979835) + | > loss_kl: 1.9664701223373413 (1.9105082937315399) + | > loss_feat: 6.641567707061768 (7.113185209386489) + | > loss_mel: 15.228856086730957 (17.901993377535952) + | > loss_duration: 1.4877219200134277 (1.4966611160951502) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.4735050201416 (30.77734173045439) + | > grad_norm_1: tensor(115.1243, device='cuda:0') (tensor(179.6570, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.367 (0.34676902434405166) + | > loader_time: 0.006 (0.0053429182837991165) + + + --> TIME: 2025-04-18 23:51:16 -- STEP: 76/142 -- GLOBAL_STEP: 386600 + | > loss_disc: 2.4977455139160156 (2.4578595475146647) + | > loss_disc_real_0: 0.1511608213186264 (0.12390853650867939) + | > loss_disc_real_1: 0.22527599334716797 (0.20560032835132194) + | > loss_disc_real_2: 0.25863778591156006 (0.22598845923417493) + | > loss_disc_real_3: 0.23828186094760895 (0.2286040484905243) + | > loss_disc_real_4: 0.261948823928833 (0.23207056502762594) + | > loss_disc_real_5: 0.2526175379753113 (0.23293888706125712) + | > loss_0: 2.4977455139160156 (2.4578595475146647) + | > grad_norm_0: tensor(6.2393, device='cuda:0') (tensor(8.3373, device='cuda:0')) + | > loss_gen: 2.4603893756866455 (2.3479728259538355) + | > loss_kl: 2.1232850551605225 (1.931685353580274) + | > loss_feat: 6.457124710083008 (7.073437979346828) + | > loss_mel: 16.727386474609375 (17.87793755531311) + | > loss_duration: 1.551850438117981 (1.5075879567547847) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.320037841796875 (30.738621812117728) + | > grad_norm_1: tensor(121.9612, device='cuda:0') (tensor(173.9609, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.3964 (0.3586930758074711) + | > loader_time: 0.006 (0.005721597295058401) + + + --> TIME: 2025-04-18 23:51:27 -- STEP: 101/142 -- GLOBAL_STEP: 386625 + | > loss_disc: 2.568955898284912 (2.466100435445804) + | > loss_disc_real_0: 0.1986749917268753 (0.12907552239623393) + | > loss_disc_real_1: 0.23204374313354492 (0.20552556482282017) + | > loss_disc_real_2: 0.24290849268436432 (0.22574583728714745) + | > loss_disc_real_3: 0.24234679341316223 (0.2295160438164626) + | > loss_disc_real_4: 0.2303309589624405 (0.23146344706563665) + | > loss_disc_real_5: 0.2291748970746994 (0.2317436697459457) + | > loss_0: 2.568955898284912 (2.466100435445804) + | > grad_norm_0: tensor(12.2019, device='cuda:0') (tensor(8.2808, device='cuda:0')) + | > loss_gen: 2.2770862579345703 (2.339163928928942) + | > loss_kl: 1.9979759454727173 (1.9490731930968785) + | > loss_feat: 6.42636775970459 (6.996893434241267) + | > loss_mel: 17.375186920166016 (17.887938357815887) + | > loss_duration: 1.5632624626159668 (1.5189418037339013) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63987922668457 (30.69201086063196) + | > grad_norm_1: tensor(126.5144, device='cuda:0') (tensor(165.8711, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.4173 (0.3697524094345547) + | > loader_time: 0.0075 (0.006004043144754844) + + + --> TIME: 2025-04-18 23:51:38 -- STEP: 126/142 -- GLOBAL_STEP: 386650 + | > loss_disc: 2.5493710041046143 (2.4690132462789136) + | > loss_disc_real_0: 0.1735561341047287 (0.1310265602336989) + | > loss_disc_real_1: 0.20123358070850372 (0.20533015744553673) + | > loss_disc_real_2: 0.23240455985069275 (0.22667844759093392) + | > loss_disc_real_3: 0.21837447583675385 (0.2296164869552567) + | > loss_disc_real_4: 0.2374071627855301 (0.2318143213079089) + | > loss_disc_real_5: 0.22021760046482086 (0.23118973972778473) + | > loss_0: 2.5493710041046143 (2.4690132462789136) + | > grad_norm_0: tensor(6.2673, device='cuda:0') (tensor(8.0097, device='cuda:0')) + | > loss_gen: 2.1079769134521484 (2.3332083830757746) + | > loss_kl: 2.001845359802246 (1.958772201386709) + | > loss_feat: 5.640955448150635 (6.968180891067263) + | > loss_mel: 14.276700973510742 (17.864288322509278) + | > loss_duration: 1.5378015041351318 (1.5256020975491358) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.56528091430664 (30.65005199492924) + | > grad_norm_1: tensor(45.2469, device='cuda:0') (tensor(153.0849, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.4538 (0.38242180952950133) + | > loader_time: 0.008 (0.00625421887352353) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6159956455230713 (2.6159956455230713) + | > loss_disc_real_0: 0.28115344047546387 (0.28115344047546387) + | > loss_disc_real_1: 0.23086312413215637 (0.23086312413215637) + | > loss_disc_real_2: 0.21808044612407684 (0.21808044612407684) + | > loss_disc_real_3: 0.25651347637176514 (0.25651347637176514) + | > loss_disc_real_4: 0.2124899923801422 (0.2124899923801422) + | > loss_disc_real_5: 0.23228690028190613 (0.23228690028190613) + | > loss_0: 2.6159956455230713 (2.6159956455230713) + | > loss_gen: 2.427708148956299 (2.427708148956299) + | > loss_kl: 2.22879958152771 (2.22879958152771) + | > loss_feat: 6.9210381507873535 (6.9210381507873535) + | > loss_mel: 17.474687576293945 (17.474687576293945) + | > loss_duration: 1.603289008140564 (1.603289008140564) + | > loss_1: 30.655521392822266 (30.655521392822266) + + --> STEP: 1 + | > loss_disc: 2.303722620010376 (2.303722620010376) + | > loss_disc_real_0: 0.1672980785369873 (0.1672980785369873) + | > loss_disc_real_1: 0.21639032661914825 (0.21639032661914825) + | > loss_disc_real_2: 0.20711620151996613 (0.20711620151996613) + | > loss_disc_real_3: 0.19202691316604614 (0.19202691316604614) + | > loss_disc_real_4: 0.19123464822769165 (0.19123464822769165) + | > loss_disc_real_5: 0.15208569169044495 (0.15208569169044495) + | > loss_0: 2.303722620010376 (2.303722620010376) + | > loss_gen: 2.455183506011963 (2.455183506011963) + | > loss_kl: 2.4944698810577393 (2.4944698810577393) + | > loss_feat: 7.649144649505615 (7.649144649505615) + | > loss_mel: 18.825315475463867 (18.825315475463867) + | > loss_duration: 1.6701467037200928 (1.6701467037200928) + | > loss_1: 33.09425735473633 (33.09425735473633) + + --> STEP: 2 + | > loss_disc: 2.5148754119873047 (2.4092990159988403) + | > loss_disc_real_0: 0.268417626619339 (0.21785785257816315) + | > loss_disc_real_1: 0.23002883791923523 (0.22320958226919174) + | > loss_disc_real_2: 0.20822453498840332 (0.20767036825418472) + | > loss_disc_real_3: 0.23143018782138824 (0.2117285504937172) + | > loss_disc_real_4: 0.21302784979343414 (0.2021312490105629) + | > loss_disc_real_5: 0.22474370896816254 (0.18841470032930374) + | > loss_0: 2.5148754119873047 (2.4092990159988403) + | > loss_gen: 2.5120797157287598 (2.4836316108703613) + | > loss_kl: 1.912102460861206 (2.2032861709594727) + | > loss_feat: 6.063032150268555 (6.856088399887085) + | > loss_mel: 16.548831939697266 (17.687073707580566) + | > loss_duration: 1.595465898513794 (1.6328063011169434) + | > loss_1: 28.631513595581055 (30.86288547515869) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034689903259277344 (-0.0005887746810913086) + | > avg_loss_disc: 2.4092990159988403 (-0.15106093883514404) + | > avg_loss_disc_real_0: 0.21785785257816315 (-0.00047359615564346313) + | > avg_loss_disc_real_1: 0.22320958226919174 (-0.02994363009929657) + | > avg_loss_disc_real_2: 0.20767036825418472 (-0.03491155803203583) + | > avg_loss_disc_real_3: 0.2117285504937172 (-0.050229936838150024) + | > avg_loss_disc_real_4: 0.2021312490105629 (-0.054535798728466034) + | > avg_loss_disc_real_5: 0.18841470032930374 (-0.032718218863010406) + | > avg_loss_0: 2.4092990159988403 (-0.15106093883514404) + | > avg_loss_gen: 2.4836316108703613 (+0.010474681854248047) + | > avg_loss_kl: 2.2032861709594727 (-0.03170943260192871) + | > avg_loss_feat: 6.856088399887085 (-0.25942468643188477) + | > avg_loss_mel: 17.687073707580566 (-0.9881200790405273) + | > avg_loss_duration: 1.6328063011169434 (-0.006568193435668945) + | > avg_loss_1: 30.86288547515869 (-1.2753496170043945) + + + > EPOCH: 52/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:51:46)  + + --> TIME: 2025-04-18 23:51:50 -- STEP: 9/142 -- GLOBAL_STEP: 386675 + | > loss_disc: 2.599605083465576 (2.475385003619724) + | > loss_disc_real_0: 0.1084791049361229 (0.11398858080307643) + | > loss_disc_real_1: 0.2295340597629547 (0.21393304069836935) + | > loss_disc_real_2: 0.24427199363708496 (0.2292226039701038) + | > loss_disc_real_3: 0.22379615902900696 (0.23462028139167362) + | > loss_disc_real_4: 0.22583650052547455 (0.2376188553041882) + | > loss_disc_real_5: 0.28593388199806213 (0.2541448606385125) + | > loss_0: 2.599605083465576 (2.475385003619724) + | > grad_norm_0: tensor(10.6289, device='cuda:0') (tensor(9.0092, device='cuda:0')) + | > loss_gen: 2.314866065979004 (2.3752666314442954) + | > loss_kl: 2.0084612369537354 (1.9788118733300104) + | > loss_feat: 6.236729145050049 (7.17710706922743) + | > loss_mel: 17.56733512878418 (18.16391224331326) + | > loss_duration: 1.4531922340393066 (1.4861944648954604) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.580583572387695 (31.181292639838325) + | > grad_norm_1: tensor(136.7256, device='cuda:0') (tensor(186.8829, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.3257 (0.3233349588182237) + | > loader_time: 0.0047 (0.004751920700073242) + + + --> TIME: 2025-04-18 23:51:59 -- STEP: 34/142 -- GLOBAL_STEP: 386700 + | > loss_disc: 2.46856951713562 (2.4567555960486906) + | > loss_disc_real_0: 0.1311168521642685 (0.12166471954654245) + | > loss_disc_real_1: 0.15586347877979279 (0.2089116029879626) + | > loss_disc_real_2: 0.22412776947021484 (0.22830859440214493) + | > loss_disc_real_3: 0.20615899562835693 (0.227766756187467) + | > loss_disc_real_4: 0.21224097907543182 (0.22963827760780559) + | > loss_disc_real_5: 0.22754819691181183 (0.23454590974485173) + | > loss_0: 2.46856951713562 (2.4567555960486906) + | > grad_norm_0: tensor(7.6899, device='cuda:0') (tensor(7.6635, device='cuda:0')) + | > loss_gen: 2.234236240386963 (2.3574927414164826) + | > loss_kl: 1.9157662391662598 (1.9493791496052462) + | > loss_feat: 7.29097318649292 (7.2360208595500275) + | > loss_mel: 18.15418243408203 (18.126300643472113) + | > loss_duration: 1.5203015804290771 (1.4943859997917623) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.115459442138672 (31.16357926761403) + | > grad_norm_1: tensor(74.8120, device='cuda:0') (tensor(173.6624, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.3648 (0.3401521654690014) + | > loader_time: 0.0053 (0.0051645671620088466) + + + --> TIME: 2025-04-18 23:52:08 -- STEP: 59/142 -- GLOBAL_STEP: 386725 + | > loss_disc: 2.5716261863708496 (2.464966689125965) + | > loss_disc_real_0: 0.11180663108825684 (0.12443323004043709) + | > loss_disc_real_1: 0.24412232637405396 (0.20870577076734123) + | > loss_disc_real_2: 0.2396090030670166 (0.22839951161610877) + | > loss_disc_real_3: 0.2395174354314804 (0.22800683545864234) + | > loss_disc_real_4: 0.24002671241760254 (0.23059617892160253) + | > loss_disc_real_5: 0.29323098063468933 (0.23376780773623515) + | > loss_0: 2.5716261863708496 (2.464966689125965) + | > grad_norm_0: tensor(7.1507, device='cuda:0') (tensor(8.5440, device='cuda:0')) + | > loss_gen: 2.3378727436065674 (2.3455297259961143) + | > loss_kl: 2.0240914821624756 (1.9546796851238961) + | > loss_feat: 6.883548736572266 (7.035466638662047) + | > loss_mel: 16.601442337036133 (17.911444405377924) + | > loss_duration: 1.5124061107635498 (1.501870353343123) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.35936164855957 (30.748990689293812) + | > grad_norm_1: tensor(259.1442, device='cuda:0') (tensor(173.2397, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.3851 (0.35463348485655705) + | > loader_time: 0.0064 (0.005636251578896733) + + + --> TIME: 2025-04-18 23:52:19 -- STEP: 84/142 -- GLOBAL_STEP: 386750 + | > loss_disc: 2.5333757400512695 (2.4792320983750478) + | > loss_disc_real_0: 0.10139503329992294 (0.12743501640146682) + | > loss_disc_real_1: 0.23451238870620728 (0.2103829277413232) + | > loss_disc_real_2: 0.23410893976688385 (0.22930096817158518) + | > loss_disc_real_3: 0.23137910664081573 (0.22895763566096625) + | > loss_disc_real_4: 0.21923036873340607 (0.23154585365028607) + | > loss_disc_real_5: 0.2200915664434433 (0.23251432144925707) + | > loss_0: 2.5333757400512695 (2.4792320983750478) + | > grad_norm_0: tensor(15.3384, device='cuda:0') (tensor(9.7994, device='cuda:0')) + | > loss_gen: 2.3530595302581787 (2.338397057283492) + | > loss_kl: 2.075439691543579 (1.961292857215518) + | > loss_feat: 6.860545635223389 (6.980516257740202) + | > loss_mel: 17.42399024963379 (17.901612838109344) + | > loss_duration: 1.598290205001831 (1.5141312238715945) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.311325073242188 (30.69595028105236) + | > grad_norm_1: tensor(348.8036, device='cuda:0') (tensor(199.9845, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.4045 (0.36838600181397946) + | > loader_time: 0.0063 (0.005915743964059013) + + + --> TIME: 2025-04-18 23:52:29 -- STEP: 109/142 -- GLOBAL_STEP: 386775 + | > loss_disc: 2.5185937881469727 (2.4860438425606546) + | > loss_disc_real_0: 0.1571337878704071 (0.12961710743401028) + | > loss_disc_real_1: 0.2102307379245758 (0.2095786762346915) + | > loss_disc_real_2: 0.2516373097896576 (0.22814174010119306) + | > loss_disc_real_3: 0.2734792232513428 (0.22972222082658644) + | > loss_disc_real_4: 0.272643119096756 (0.23128788555980823) + | > loss_disc_real_5: 0.28080543875694275 (0.23304646715111688) + | > loss_0: 2.5185937881469727 (2.4860438425606546) + | > grad_norm_0: tensor(12.7945, device='cuda:0') (tensor(10.7684, device='cuda:0')) + | > loss_gen: 2.4265990257263184 (2.3261329121545904) + | > loss_kl: 1.9385981559753418 (1.9698659498757178) + | > loss_feat: 6.379180908203125 (6.877987231683294) + | > loss_mel: 17.03050422668457 (17.84165678986716) + | > loss_duration: 1.5402967929840088 (1.5214407126837914) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.3151798248291 (30.537083634542764) + | > grad_norm_1: tensor(319.0394, device='cuda:0') (tensor(224.0193, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.4247 (0.38038803459307474) + | > loader_time: 0.0073 (0.00614262939593114) + + + --> TIME: 2025-04-18 23:52:41 -- STEP: 134/142 -- GLOBAL_STEP: 386800 + | > loss_disc: 2.4799444675445557 (2.484757199216245) + | > loss_disc_real_0: 0.11773297935724258 (0.12946508585739494) + | > loss_disc_real_1: 0.2060161679983139 (0.20982741709075756) + | > loss_disc_real_2: 0.2284046709537506 (0.22844557597566006) + | > loss_disc_real_3: 0.25369808077812195 (0.2299701610385482) + | > loss_disc_real_4: 0.22711794078350067 (0.23100007025163566) + | > loss_disc_real_5: 0.2481021136045456 (0.23337519958392897) + | > loss_0: 2.4799444675445557 (2.484757199216245) + | > grad_norm_0: tensor(21.3461, device='cuda:0') (tensor(11.6190, device='cuda:0')) + | > loss_gen: 2.335963726043701 (2.3234988237494845) + | > loss_kl: 2.085819959640503 (1.9787713262572217) + | > loss_feat: 7.048593997955322 (6.850676771420152) + | > loss_mel: 18.32126235961914 (17.74231455931023) + | > loss_duration: 1.586847186088562 (1.526542561267739) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.378488540649414 (30.421804015316177) + | > grad_norm_1: tensor(402.8022, device='cuda:0') (tensor(234.2785, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.4525 (0.3931830996897684) + | > loader_time: 0.0073 (0.006378440714594144) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.389270067214966 (2.389270067214966) + | > loss_disc_real_0: 0.11736853420734406 (0.11736853420734406) + | > loss_disc_real_1: 0.16902977228164673 (0.16902977228164673) + | > loss_disc_real_2: 0.20804040133953094 (0.20804040133953094) + | > loss_disc_real_3: 0.19230619072914124 (0.19230619072914124) + | > loss_disc_real_4: 0.16888296604156494 (0.16888296604156494) + | > loss_disc_real_5: 0.20404081046581268 (0.20404081046581268) + | > loss_0: 2.389270067214966 (2.389270067214966) + | > loss_gen: 2.119642734527588 (2.119642734527588) + | > loss_kl: 2.323683023452759 (2.323683023452759) + | > loss_feat: 6.548598766326904 (6.548598766326904) + | > loss_mel: 19.25177574157715 (19.25177574157715) + | > loss_duration: 1.626582384109497 (1.626582384109497) + | > loss_1: 31.870283126831055 (31.870283126831055) + + --> STEP: 1 + | > loss_disc: 2.404395580291748 (2.404395580291748) + | > loss_disc_real_0: 0.15400439500808716 (0.15400439500808716) + | > loss_disc_real_1: 0.2022865265607834 (0.2022865265607834) + | > loss_disc_real_2: 0.20537586510181427 (0.20537586510181427) + | > loss_disc_real_3: 0.18701313436031342 (0.18701313436031342) + | > loss_disc_real_4: 0.2230202555656433 (0.2230202555656433) + | > loss_disc_real_5: 0.15668120980262756 (0.15668120980262756) + | > loss_0: 2.404395580291748 (2.404395580291748) + | > loss_gen: 2.1838810443878174 (2.1838810443878174) + | > loss_kl: 2.2241945266723633 (2.2241945266723633) + | > loss_feat: 6.8092360496521 (6.8092360496521) + | > loss_mel: 16.939966201782227 (16.939966201782227) + | > loss_duration: 1.6675503253936768 (1.6675503253936768) + | > loss_1: 29.8248291015625 (29.8248291015625) + + --> STEP: 2 + | > loss_disc: 2.4390528202056885 (2.4217242002487183) + | > loss_disc_real_0: 0.10477626323699951 (0.12939032912254333) + | > loss_disc_real_1: 0.22724968194961548 (0.21476810425519943) + | > loss_disc_real_2: 0.18305954337120056 (0.19421770423650742) + | > loss_disc_real_3: 0.18708446621894836 (0.1870488002896309) + | > loss_disc_real_4: 0.18065117299556732 (0.20183571428060532) + | > loss_disc_real_5: 0.21102741360664368 (0.18385431170463562) + | > loss_0: 2.4390528202056885 (2.4217242002487183) + | > loss_gen: 2.2191038131713867 (2.201492428779602) + | > loss_kl: 2.1198244094848633 (2.1720094680786133) + | > loss_feat: 7.637421607971191 (7.2233288288116455) + | > loss_mel: 19.066802978515625 (18.003384590148926) + | > loss_duration: 1.6048771142959595 (1.6362137198448181) + | > loss_1: 32.64802932739258 (31.23642921447754) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0039664506912231445 (+0.0004974603652954102) + | > avg_loss_disc: 2.4217242002487183 (+0.01242518424987793) + | > avg_loss_disc_real_0: 0.12939032912254333 (-0.08846752345561981) + | > avg_loss_disc_real_1: 0.21476810425519943 (-0.00844147801399231) + | > avg_loss_disc_real_2: 0.19421770423650742 (-0.013452664017677307) + | > avg_loss_disc_real_3: 0.1870488002896309 (-0.024679750204086304) + | > avg_loss_disc_real_4: 0.20183571428060532 (-0.00029553472995758057) + | > avg_loss_disc_real_5: 0.18385431170463562 (-0.004560388624668121) + | > avg_loss_0: 2.4217242002487183 (+0.01242518424987793) + | > avg_loss_gen: 2.201492428779602 (-0.2821391820907593) + | > avg_loss_kl: 2.1720094680786133 (-0.031276702880859375) + | > avg_loss_feat: 7.2233288288116455 (+0.36724042892456055) + | > avg_loss_mel: 18.003384590148926 (+0.3163108825683594) + | > avg_loss_duration: 1.6362137198448181 (+0.003407418727874756) + | > avg_loss_1: 31.23642921447754 (+0.37354373931884766) + + + > EPOCH: 53/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:52:46)  + + --> TIME: 2025-04-18 23:52:53 -- STEP: 17/142 -- GLOBAL_STEP: 386825 + | > loss_disc: 2.5348682403564453 (2.4574213588938996) + | > loss_disc_real_0: 0.1240622028708458 (0.1279071700923583) + | > loss_disc_real_1: 0.22273989021778107 (0.21039839264224558) + | > loss_disc_real_2: 0.22527755796909332 (0.22311344041543849) + | > loss_disc_real_3: 0.22122527658939362 (0.2228025329463622) + | > loss_disc_real_4: 0.25481483340263367 (0.2280130509067984) + | > loss_disc_real_5: 0.23850172758102417 (0.2234412142459084) + | > loss_0: 2.5348682403564453 (2.4574213588938996) + | > grad_norm_0: tensor(14.3614, device='cuda:0') (tensor(12.0850, device='cuda:0')) + | > loss_gen: 2.2083568572998047 (2.404929090948666) + | > loss_kl: 1.8926664590835571 (1.9360174291274126) + | > loss_feat: 7.10280704498291 (7.36922306172988) + | > loss_mel: 18.32328224182129 (18.13751927544089) + | > loss_duration: 1.4835171699523926 (1.4826475942836088) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.010629653930664 (31.330336402444278) + | > grad_norm_1: tensor(151.5877, device='cuda:0') (tensor(286.9794, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.3362 (0.328061580657959) + | > loader_time: 0.0052 (0.004656721563900218) + + + --> TIME: 2025-04-18 23:53:02 -- STEP: 42/142 -- GLOBAL_STEP: 386850 + | > loss_disc: 2.417745351791382 (2.4516287871769498) + | > loss_disc_real_0: 0.10451149195432663 (0.12374389810221535) + | > loss_disc_real_1: 0.26958680152893066 (0.20953324543578283) + | > loss_disc_real_2: 0.26322653889656067 (0.22461058979942686) + | > loss_disc_real_3: 0.23829491436481476 (0.22795630069006056) + | > loss_disc_real_4: 0.24175453186035156 (0.22740836689869562) + | > loss_disc_real_5: 0.22409534454345703 (0.23063522506327855) + | > loss_0: 2.417745351791382 (2.4516287871769498) + | > grad_norm_0: tensor(12.2314, device='cuda:0') (tensor(9.2781, device='cuda:0')) + | > loss_gen: 2.5115444660186768 (2.3792219786416915) + | > loss_kl: 1.7724956274032593 (1.9409331849643163) + | > loss_feat: 7.364816188812256 (7.249900761104765) + | > loss_mel: 18.659717559814453 (18.0375638235183) + | > loss_duration: 1.5269197225570679 (1.4903701118060522) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.835494995117188 (31.097989808945428) + | > grad_norm_1: tensor(236.5105, device='cuda:0') (tensor(209.9162, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.3629 (0.34108803385779973) + | > loader_time: 0.0055 (0.005141201473417737) + + + --> TIME: 2025-04-18 23:53:11 -- STEP: 67/142 -- GLOBAL_STEP: 386875 + | > loss_disc: 2.425663471221924 (2.4536320095631616) + | > loss_disc_real_0: 0.11633725464344025 (0.12409486821783122) + | > loss_disc_real_1: 0.18926213681697845 (0.20643976686605767) + | > loss_disc_real_2: 0.21488310396671295 (0.22569713040963926) + | > loss_disc_real_3: 0.21756193041801453 (0.2275755394305756) + | > loss_disc_real_4: 0.19143357872962952 (0.2269931924876882) + | > loss_disc_real_5: 0.2021714597940445 (0.2301575096685495) + | > loss_0: 2.425663471221924 (2.4536320095631616) + | > grad_norm_0: tensor(10.8150, device='cuda:0') (tensor(10.7492, device='cuda:0')) + | > loss_gen: 2.4453067779541016 (2.3607348684054696) + | > loss_kl: 2.123201608657837 (1.9567309030845983) + | > loss_feat: 7.551199436187744 (7.155823714697539) + | > loss_mel: 18.702714920043945 (17.95935317651549) + | > loss_duration: 1.4522281885147095 (1.5011212665643265) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.27465057373047 (30.93376387411089) + | > grad_norm_1: tensor(549.9227, device='cuda:0') (tensor(234.1041, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.3813 (0.3529821082727233) + | > loader_time: 0.0066 (0.005446060379939294) + + + --> TIME: 2025-04-18 23:53:21 -- STEP: 92/142 -- GLOBAL_STEP: 386900 + | > loss_disc: 2.444086790084839 (2.4527001795561416) + | > loss_disc_real_0: 0.11621935665607452 (0.12398723172752754) + | > loss_disc_real_1: 0.19174917042255402 (0.20501252992645555) + | > loss_disc_real_2: 0.2686783969402313 (0.22580502700546515) + | > loss_disc_real_3: 0.25718972086906433 (0.22822919973860617) + | > loss_disc_real_4: 0.21715664863586426 (0.2278961466382379) + | > loss_disc_real_5: 0.24827930331230164 (0.23199798539280891) + | > loss_0: 2.444086790084839 (2.4527001795561416) + | > grad_norm_0: tensor(4.4074, device='cuda:0') (tensor(9.6980, device='cuda:0')) + | > loss_gen: 2.478297710418701 (2.360737074976383) + | > loss_kl: 1.9167847633361816 (1.9708291849364405) + | > loss_feat: 7.396467208862305 (7.110688390939132) + | > loss_mel: 18.309879302978516 (17.968195355456807) + | > loss_duration: 1.4817826747894287 (1.5140957443610485) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.58321189880371 (30.924545702726945) + | > grad_norm_1: tensor(131.7264, device='cuda:0') (tensor(209.8016, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.4095 (0.36426161683124036) + | > loader_time: 0.0069 (0.005716196868730631) + + + --> TIME: 2025-04-18 23:53:32 -- STEP: 117/142 -- GLOBAL_STEP: 386925 + | > loss_disc: 2.4403865337371826 (2.4607785176008177) + | > loss_disc_real_0: 0.1269567459821701 (0.1264434213566984) + | > loss_disc_real_1: 0.18495608866214752 (0.20593902570569617) + | > loss_disc_real_2: 0.21366554498672485 (0.22563573055797154) + | > loss_disc_real_3: 0.21716929972171783 (0.22843901303588834) + | > loss_disc_real_4: 0.2489921599626541 (0.22874287522246695) + | > loss_disc_real_5: 0.24512284994125366 (0.23106574872110644) + | > loss_0: 2.4403865337371826 (2.4607785176008177) + | > grad_norm_0: tensor(5.9416, device='cuda:0') (tensor(9.1745, device='cuda:0')) + | > loss_gen: 2.2850630283355713 (2.341998336661576) + | > loss_kl: 1.8881053924560547 (1.9782247981454573) + | > loss_feat: 6.326371669769287 (6.999569607596112) + | > loss_mel: 17.206174850463867 (17.86617737142448) + | > loss_duration: 1.5245693922042847 (1.5198520564625408) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.230283737182617 (30.70582207247742) + | > grad_norm_1: tensor(110.2335, device='cuda:0') (tensor(194.0352, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.4321 (0.375174173941979) + | > loader_time: 0.007 (0.00598577964000213) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.428645133972168 (2.428645133972168) + | > loss_disc_real_0: 0.1389778107404709 (0.1389778107404709) + | > loss_disc_real_1: 0.23481957614421844 (0.23481957614421844) + | > loss_disc_real_2: 0.24207797646522522 (0.24207797646522522) + | > loss_disc_real_3: 0.25046253204345703 (0.25046253204345703) + | > loss_disc_real_4: 0.22169868648052216 (0.22169868648052216) + | > loss_disc_real_5: 0.3053613305091858 (0.3053613305091858) + | > loss_0: 2.428645133972168 (2.428645133972168) + | > loss_gen: 2.585038661956787 (2.585038661956787) + | > loss_kl: 2.38489031791687 (2.38489031791687) + | > loss_feat: 6.851371765136719 (6.851371765136719) + | > loss_mel: 19.21893882751465 (19.21893882751465) + | > loss_duration: 1.6374452114105225 (1.6374452114105225) + | > loss_1: 32.67768478393555 (32.67768478393555) + + --> STEP: 1 + | > loss_disc: 2.5311503410339355 (2.5311503410339355) + | > loss_disc_real_0: 0.12759023904800415 (0.12759023904800415) + | > loss_disc_real_1: 0.28280484676361084 (0.28280484676361084) + | > loss_disc_real_2: 0.22915898263454437 (0.22915898263454437) + | > loss_disc_real_3: 0.28444579243659973 (0.28444579243659973) + | > loss_disc_real_4: 0.21303457021713257 (0.21303457021713257) + | > loss_disc_real_5: 0.2703012228012085 (0.2703012228012085) + | > loss_0: 2.5311503410339355 (2.5311503410339355) + | > loss_gen: 2.4501700401306152 (2.4501700401306152) + | > loss_kl: 2.46671986579895 (2.46671986579895) + | > loss_feat: 7.298254013061523 (7.298254013061523) + | > loss_mel: 18.43990707397461 (18.43990707397461) + | > loss_duration: 1.6576274633407593 (1.6576274633407593) + | > loss_1: 32.312679290771484 (32.312679290771484) + + --> STEP: 2 + | > loss_disc: 2.330181837081909 (2.4306660890579224) + | > loss_disc_real_0: 0.1853182315826416 (0.15645423531532288) + | > loss_disc_real_1: 0.2215915024280548 (0.2521981745958328) + | > loss_disc_real_2: 0.21828508377075195 (0.22372203320264816) + | > loss_disc_real_3: 0.26770642399787903 (0.2760761082172394) + | > loss_disc_real_4: 0.21758608520030975 (0.21531032770872116) + | > loss_disc_real_5: 0.23000401258468628 (0.2501526176929474) + | > loss_0: 2.330181837081909 (2.4306660890579224) + | > loss_gen: 2.7262027263641357 (2.5881863832473755) + | > loss_kl: 2.1247072219848633 (2.2957135438919067) + | > loss_feat: 8.771607398986816 (8.03493070602417) + | > loss_mel: 19.778608322143555 (19.109257698059082) + | > loss_duration: 1.5738435983657837 (1.6157355308532715) + | > loss_1: 34.974971771240234 (33.64382553100586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035637617111206055 (-0.00040268898010253906) + | > avg_loss_disc: 2.4306660890579224 (+0.008941888809204102) + | > avg_loss_disc_real_0: 0.15645423531532288 (+0.02706390619277954) + | > avg_loss_disc_real_1: 0.2521981745958328 (+0.03743007034063339) + | > avg_loss_disc_real_2: 0.22372203320264816 (+0.029504328966140747) + | > avg_loss_disc_real_3: 0.2760761082172394 (+0.08902730792760849) + | > avg_loss_disc_real_4: 0.21531032770872116 (+0.013474613428115845) + | > avg_loss_disc_real_5: 0.2501526176929474 (+0.06629830598831177) + | > avg_loss_0: 2.4306660890579224 (+0.008941888809204102) + | > avg_loss_gen: 2.5881863832473755 (+0.38669395446777344) + | > avg_loss_kl: 2.2957135438919067 (+0.12370407581329346) + | > avg_loss_feat: 8.03493070602417 (+0.8116018772125244) + | > avg_loss_mel: 19.109257698059082 (+1.1058731079101562) + | > avg_loss_duration: 1.6157355308532715 (-0.02047818899154663) + | > avg_loss_1: 33.64382553100586 (+2.4073963165283203) + + + > EPOCH: 54/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:53:45)  + + --> TIME: 2025-04-18 23:53:45 -- STEP: 0/142 -- GLOBAL_STEP: 386950 + | > loss_disc: 2.4278767108917236 (2.4278767108917236) + | > loss_disc_real_0: 0.1058606281876564 (0.1058606281876564) + | > loss_disc_real_1: 0.2483464926481247 (0.2483464926481247) + | > loss_disc_real_2: 0.24700753390789032 (0.24700753390789032) + | > loss_disc_real_3: 0.275860995054245 (0.275860995054245) + | > loss_disc_real_4: 0.23055118322372437 (0.23055118322372437) + | > loss_disc_real_5: 0.27855345606803894 (0.27855345606803894) + | > loss_0: 2.4278767108917236 (2.4278767108917236) + | > grad_norm_0: tensor(4.1092, device='cuda:0') (tensor(4.1092, device='cuda:0')) + | > loss_gen: 2.6108181476593018 (2.6108181476593018) + | > loss_kl: 1.7968857288360596 (1.7968857288360596) + | > loss_feat: 7.482381343841553 (7.482381343841553) + | > loss_mel: 19.094079971313477 (19.094079971313477) + | > loss_duration: 1.5084655284881592 (1.5084655284881592) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.49263381958008 (32.49263381958008) + | > grad_norm_1: tensor(48.5190, device='cuda:0') (tensor(48.5190, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.3392 (0.33915019035339355) + | > loader_time: 0.3085 (0.30853700637817383) + + + --> TIME: 2025-04-18 23:53:54 -- STEP: 25/142 -- GLOBAL_STEP: 386975 + | > loss_disc: 2.4664435386657715 (2.4623760318756105) + | > loss_disc_real_0: 0.11931788921356201 (0.12427327930927279) + | > loss_disc_real_1: 0.2172020822763443 (0.20316994428634644) + | > loss_disc_real_2: 0.23014479875564575 (0.22905610144138336) + | > loss_disc_real_3: 0.247224822640419 (0.2244754511117935) + | > loss_disc_real_4: 0.21947123110294342 (0.2294225251674652) + | > loss_disc_real_5: 0.23898328840732574 (0.2408764863014221) + | > loss_0: 2.4664435386657715 (2.4623760318756105) + | > grad_norm_0: tensor(10.0223, device='cuda:0') (tensor(9.6340, device='cuda:0')) + | > loss_gen: 2.5638675689697266 (2.3943545913696287) + | > loss_kl: 1.9471906423568726 (1.9813277339935302) + | > loss_feat: 7.974145889282227 (7.084254398345947) + | > loss_mel: 18.519269943237305 (17.67753570556641) + | > loss_duration: 1.5244288444519043 (1.487115092277527) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.528900146484375 (30.624587173461915) + | > grad_norm_1: tensor(485.7534, device='cuda:0') (tensor(208.8331, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.343 (0.33196481704711917) + | > loader_time: 0.0053 (0.005033655166625977) + + + --> TIME: 2025-04-18 23:54:03 -- STEP: 50/142 -- GLOBAL_STEP: 387000 + | > loss_disc: 2.4938511848449707 (2.4796854305267337) + | > loss_disc_real_0: 0.07660859078168869 (0.12836365997791296) + | > loss_disc_real_1: 0.21540406346321106 (0.20692256063222886) + | > loss_disc_real_2: 0.21686704456806183 (0.23056446075439452) + | > loss_disc_real_3: 0.2351376712322235 (0.22615336239337921) + | > loss_disc_real_4: 0.2544902563095093 (0.2315627482533455) + | > loss_disc_real_5: 0.2514592707157135 (0.23644297808408737) + | > loss_0: 2.4938511848449707 (2.4796854305267337) + | > grad_norm_0: tensor(14.9520, device='cuda:0') (tensor(13.4495, device='cuda:0')) + | > loss_gen: 2.363891124725342 (2.3786402654647834) + | > loss_kl: 1.9637024402618408 (1.9634389781951904) + | > loss_feat: 7.3983612060546875 (7.096745738983153) + | > loss_mel: 18.945444107055664 (17.972746200561527) + | > loss_duration: 1.5377116203308105 (1.4937667274475097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.209110260009766 (30.905337715148924) + | > grad_norm_1: tensor(355.3470, device='cuda:0') (tensor(264.8201, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.375 (0.3483397817611694) + | > loader_time: 0.0061 (0.005370860099792481) + + + --> TIME: 2025-04-18 23:54:13 -- STEP: 75/142 -- GLOBAL_STEP: 387025 + | > loss_disc: 2.412348747253418 (2.4703901481628416) + | > loss_disc_real_0: 0.08410856127738953 (0.1251009851694107) + | > loss_disc_real_1: 0.2082648128271103 (0.20745219389597574) + | > loss_disc_real_2: 0.24892586469650269 (0.22969296157360078) + | > loss_disc_real_3: 0.21875540912151337 (0.2264134434858958) + | > loss_disc_real_4: 0.19354233145713806 (0.22913700143496196) + | > loss_disc_real_5: 0.19061703979969025 (0.23545406858126322) + | > loss_0: 2.412348747253418 (2.4703901481628416) + | > grad_norm_0: tensor(10.8200, device='cuda:0') (tensor(13.9389, device='cuda:0')) + | > loss_gen: 2.3210251331329346 (2.389469223022461) + | > loss_kl: 1.951051115989685 (1.9832974036534627) + | > loss_feat: 7.439439296722412 (7.084450861612954) + | > loss_mel: 17.3950138092041 (17.936316223144534) + | > loss_duration: 1.4823665618896484 (1.5041748174031575) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.588895797729492 (30.897708409627278) + | > grad_norm_1: tensor(279.8405, device='cuda:0') (tensor(292.8290, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.3983 (0.36250468889872234) + | > loader_time: 0.0064 (0.005705712636311849) + + + --> TIME: 2025-04-18 23:54:24 -- STEP: 100/142 -- GLOBAL_STEP: 387050 + | > loss_disc: 2.52828311920166 (2.47180434703827) + | > loss_disc_real_0: 0.13236390054225922 (0.12514699295163156) + | > loss_disc_real_1: 0.1916775107383728 (0.20840954259037972) + | > loss_disc_real_2: 0.23526450991630554 (0.22898395121097564) + | > loss_disc_real_3: 0.2359180897474289 (0.22688162654638291) + | > loss_disc_real_4: 0.223799929022789 (0.22953685745596886) + | > loss_disc_real_5: 0.21197178959846497 (0.2340717789530754) + | > loss_0: 2.52828311920166 (2.47180434703827) + | > grad_norm_0: tensor(16.5841, device='cuda:0') (tensor(13.5685, device='cuda:0')) + | > loss_gen: 2.1913561820983887 (2.3724402189254774) + | > loss_kl: 2.022970676422119 (1.9910323083400727) + | > loss_feat: 6.9323039054870605 (7.011081299781798) + | > loss_mel: 16.673969268798828 (17.892354679107676) + | > loss_duration: 1.5423744916915894 (1.5153716683387757) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.362974166870117 (30.782280063629152) + | > grad_norm_1: tensor(292.2627, device='cuda:0') (tensor(291.6832, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.417 (0.3761992383003235) + | > loader_time: 0.0073 (0.005981032848358154) + + + --> TIME: 2025-04-18 23:54:35 -- STEP: 125/142 -- GLOBAL_STEP: 387075 + | > loss_disc: 2.4825661182403564 (2.470492031097413) + | > loss_disc_real_0: 0.10433275997638702 (0.1263372588157654) + | > loss_disc_real_1: 0.17377729713916779 (0.20795514333248138) + | > loss_disc_real_2: 0.2764129340648651 (0.22813785314559937) + | > loss_disc_real_3: 0.24343241751194 (0.22717581021785735) + | > loss_disc_real_4: 0.27808308601379395 (0.22953177869319916) + | > loss_disc_real_5: 0.2409292608499527 (0.2340887453556061) + | > loss_0: 2.4825661182403564 (2.470492031097413) + | > grad_norm_0: tensor(16.9698, device='cuda:0') (tensor(13.0335, device='cuda:0')) + | > loss_gen: 2.3800783157348633 (2.36792428970337) + | > loss_kl: 1.9418281316757202 (2.0014054660797123) + | > loss_feat: 6.209160327911377 (6.990081298828124) + | > loss_mel: 17.40962791442871 (17.906505592346203) + | > loss_duration: 1.5677931308746338 (1.5222299766540528) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.508485794067383 (30.788146530151366) + | > grad_norm_1: tensor(317.2092, device='cuda:0') (tensor(285.9006, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.4381 (0.387575008392334) + | > loader_time: 0.0076 (0.00621963119506836) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.512305498123169 (2.512305498123169) + | > loss_disc_real_0: 0.13305701315402985 (0.13305701315402985) + | > loss_disc_real_1: 0.24678881466388702 (0.24678881466388702) + | > loss_disc_real_2: 0.2209501564502716 (0.2209501564502716) + | > loss_disc_real_3: 0.23348011076450348 (0.23348011076450348) + | > loss_disc_real_4: 0.20744113624095917 (0.20744113624095917) + | > loss_disc_real_5: 0.2507133483886719 (0.2507133483886719) + | > loss_0: 2.512305498123169 (2.512305498123169) + | > loss_gen: 2.2485647201538086 (2.2485647201538086) + | > loss_kl: 2.3558716773986816 (2.3558716773986816) + | > loss_feat: 7.2899017333984375 (7.2899017333984375) + | > loss_mel: 18.784868240356445 (18.784868240356445) + | > loss_duration: 1.6211148500442505 (1.6211148500442505) + | > loss_1: 32.300323486328125 (32.300323486328125) + + --> STEP: 1 + | > loss_disc: 2.469266176223755 (2.469266176223755) + | > loss_disc_real_0: 0.14246295392513275 (0.14246295392513275) + | > loss_disc_real_1: 0.25076600909233093 (0.25076600909233093) + | > loss_disc_real_2: 0.2689397633075714 (0.2689397633075714) + | > loss_disc_real_3: 0.21355237066745758 (0.21355237066745758) + | > loss_disc_real_4: 0.24326525628566742 (0.24326525628566742) + | > loss_disc_real_5: 0.2018841952085495 (0.2018841952085495) + | > loss_0: 2.469266176223755 (2.469266176223755) + | > loss_gen: 2.458179235458374 (2.458179235458374) + | > loss_kl: 2.3916378021240234 (2.3916378021240234) + | > loss_feat: 6.797206401824951 (6.797206401824951) + | > loss_mel: 19.204158782958984 (19.204158782958984) + | > loss_duration: 1.6415784358978271 (1.6415784358978271) + | > loss_1: 32.492759704589844 (32.492759704589844) + + --> STEP: 2 + | > loss_disc: 2.487847089767456 (2.4785566329956055) + | > loss_disc_real_0: 0.1362769454717636 (0.13936994969844818) + | > loss_disc_real_1: 0.2791799008846283 (0.2649729549884796) + | > loss_disc_real_2: 0.23925669491291046 (0.25409822911024094) + | > loss_disc_real_3: 0.1936282217502594 (0.2035902962088585) + | > loss_disc_real_4: 0.2005065679550171 (0.22188591212034225) + | > loss_disc_real_5: 0.21803098917007446 (0.20995759218931198) + | > loss_0: 2.487847089767456 (2.4785566329956055) + | > loss_gen: 2.2879655361175537 (2.373072385787964) + | > loss_kl: 2.101135492324829 (2.2463866472244263) + | > loss_feat: 6.323437213897705 (6.560321807861328) + | > loss_mel: 15.70870304107666 (17.456430912017822) + | > loss_duration: 1.5925791263580322 (1.6170787811279297) + | > loss_1: 28.01382064819336 (30.2532901763916) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003767728805541992 (+0.00020396709442138672) + | > avg_loss_disc: 2.4785566329956055 (+0.047890543937683105) + | > avg_loss_disc_real_0: 0.13936994969844818 (-0.017084285616874695) + | > avg_loss_disc_real_1: 0.2649729549884796 (+0.01277478039264679) + | > avg_loss_disc_real_2: 0.25409822911024094 (+0.030376195907592773) + | > avg_loss_disc_real_3: 0.2035902962088585 (-0.07248581200838089) + | > avg_loss_disc_real_4: 0.22188591212034225 (+0.006575584411621094) + | > avg_loss_disc_real_5: 0.20995759218931198 (-0.040195025503635406) + | > avg_loss_0: 2.4785566329956055 (+0.047890543937683105) + | > avg_loss_gen: 2.373072385787964 (-0.21511399745941162) + | > avg_loss_kl: 2.2463866472244263 (-0.04932689666748047) + | > avg_loss_feat: 6.560321807861328 (-1.4746088981628418) + | > avg_loss_mel: 17.456430912017822 (-1.6528267860412598) + | > avg_loss_duration: 1.6170787811279297 (+0.0013432502746582031) + | > avg_loss_1: 30.2532901763916 (-3.390535354614258) + + + > EPOCH: 55/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:54:44)  + + --> TIME: 2025-04-18 23:54:47 -- STEP: 8/142 -- GLOBAL_STEP: 387100 + | > loss_disc: 2.5476107597351074 (2.4748949110507965) + | > loss_disc_real_0: 0.12412577122449875 (0.12477487232536077) + | > loss_disc_real_1: 0.18165074288845062 (0.21580531261861324) + | > loss_disc_real_2: 0.2159772664308548 (0.22923428192734718) + | > loss_disc_real_3: 0.26532983779907227 (0.21737801656126976) + | > loss_disc_real_4: 0.21128572523593903 (0.233523141592741) + | > loss_disc_real_5: 0.22917227447032928 (0.23434689827263355) + | > loss_0: 2.5476107597351074 (2.4748949110507965) + | > grad_norm_0: tensor(8.8963, device='cuda:0') (tensor(7.4139, device='cuda:0')) + | > loss_gen: 2.3023829460144043 (2.3554161489009857) + | > loss_kl: 1.8868716955184937 (1.888020396232605) + | > loss_feat: 7.154900550842285 (7.018515408039093) + | > loss_mel: 17.1012020111084 (18.013988494873047) + | > loss_duration: 1.4856714010238647 (1.4835078567266464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9310302734375 (30.759448766708374) + | > grad_norm_1: tensor(245.7108, device='cuda:0') (tensor(137.9982, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.3235 (0.3266107439994812) + | > loader_time: 0.0048 (0.0043070316314697266) + + + --> TIME: 2025-04-18 23:54:56 -- STEP: 33/142 -- GLOBAL_STEP: 387125 + | > loss_disc: 2.3888232707977295 (2.4704089164733887) + | > loss_disc_real_0: 0.16210341453552246 (0.12422103344491034) + | > loss_disc_real_1: 0.20676907896995544 (0.21828236814701196) + | > loss_disc_real_2: 0.18939298391342163 (0.2288763148315025) + | > loss_disc_real_3: 0.2157064974308014 (0.2239621380964915) + | > loss_disc_real_4: 0.188881054520607 (0.23106757528854138) + | > loss_disc_real_5: 0.20875097811222076 (0.23179541663690048) + | > loss_0: 2.3888232707977295 (2.4704089164733887) + | > grad_norm_0: tensor(13.2798, device='cuda:0') (tensor(8.9872, device='cuda:0')) + | > loss_gen: 2.3607871532440186 (2.3635954929120615) + | > loss_kl: 1.8698259592056274 (1.8954066435496013) + | > loss_feat: 6.984609603881836 (7.13339376449585) + | > loss_mel: 18.30577278137207 (18.111990437363133) + | > loss_duration: 1.48046875 (1.4877654639157383) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.00146484375 (30.992151607166637) + | > grad_norm_1: tensor(357.7741, device='cuda:0') (tensor(190.7434, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.3535 (0.3398073875542843) + | > loader_time: 0.0057 (0.0050732728206750125) + + + --> TIME: 2025-04-18 23:55:05 -- STEP: 58/142 -- GLOBAL_STEP: 387150 + | > loss_disc: 2.528515338897705 (2.479993725645131) + | > loss_disc_real_0: 0.11870958656072617 (0.1268416861521787) + | > loss_disc_real_1: 0.18049213290214539 (0.21251795286762304) + | > loss_disc_real_2: 0.18654409050941467 (0.2273226783707224) + | > loss_disc_real_3: 0.21293340623378754 (0.22560588861333913) + | > loss_disc_real_4: 0.18929213285446167 (0.2303879826233305) + | > loss_disc_real_5: 0.2778400182723999 (0.2344976825446918) + | > loss_0: 2.528515338897705 (2.479993725645131) + | > grad_norm_0: tensor(12.0093, device='cuda:0') (tensor(10.2677, device='cuda:0')) + | > loss_gen: 2.2133450508117676 (2.344674883217647) + | > loss_kl: 2.043086290359497 (1.924770986211711) + | > loss_feat: 6.400383949279785 (6.992660974634105) + | > loss_mel: 17.23410987854004 (17.955899485226336) + | > loss_duration: 1.5451788902282715 (1.49818309422197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.43610382080078 (30.716189285804486) + | > grad_norm_1: tensor(140.9946, device='cuda:0') (tensor(227.6580, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.3761 (0.3503346360962966) + | > loader_time: 0.0064 (0.005422518171113113) + + + --> TIME: 2025-04-18 23:55:15 -- STEP: 83/142 -- GLOBAL_STEP: 387175 + | > loss_disc: 2.5099220275878906 (2.482989974768765) + | > loss_disc_real_0: 0.10854698717594147 (0.1277955634048186) + | > loss_disc_real_1: 0.1931658685207367 (0.21197971809341246) + | > loss_disc_real_2: 0.23659299314022064 (0.22716859270291156) + | > loss_disc_real_3: 0.19804400205612183 (0.2276171593062849) + | > loss_disc_real_4: 0.2175958752632141 (0.23188499334346818) + | > loss_disc_real_5: 0.21917618811130524 (0.23530519062496094) + | > loss_0: 2.5099220275878906 (2.482989974768765) + | > grad_norm_0: tensor(10.8585, device='cuda:0') (tensor(9.8851, device='cuda:0')) + | > loss_gen: 2.3307738304138184 (2.34602187340518) + | > loss_kl: 1.973073124885559 (1.9509617908891426) + | > loss_feat: 6.820317268371582 (6.976732443614178) + | > loss_mel: 17.77615737915039 (17.979320146951327) + | > loss_duration: 1.6194796562194824 (1.5102429404316182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.51980209350586 (30.76327912204237) + | > grad_norm_1: tensor(312.3914, device='cuda:0') (tensor(220.9527, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.398 (0.36202936861888463) + | > loader_time: 0.007 (0.005748504615691772) + + + --> TIME: 2025-04-18 23:55:26 -- STEP: 108/142 -- GLOBAL_STEP: 387200 + | > loss_disc: 2.3957531452178955 (2.483067558871374) + | > loss_disc_real_0: 0.11868629604578018 (0.12976259327735057) + | > loss_disc_real_1: 0.21113388240337372 (0.21133123269235646) + | > loss_disc_real_2: 0.24423718452453613 (0.22722979055510628) + | > loss_disc_real_3: 0.22886650264263153 (0.22719725925061438) + | > loss_disc_real_4: 0.2041528970003128 (0.23111015641027027) + | > loss_disc_real_5: 0.20940952003002167 (0.23430081788036558) + | > loss_0: 2.3957531452178955 (2.483067558871374) + | > grad_norm_0: tensor(10.7995, device='cuda:0') (tensor(10.5340, device='cuda:0')) + | > loss_gen: 2.371737480163574 (2.335595221431166) + | > loss_kl: 1.959439992904663 (1.9612559565791376) + | > loss_feat: 6.771662712097168 (6.935830191329673) + | > loss_mel: 18.5991153717041 (17.949748931107692) + | > loss_duration: 1.5401504039764404 (1.518906436584614) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.242107391357422 (30.701336666389746) + | > grad_norm_1: tensor(296.0620, device='cuda:0') (tensor(231.8266, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.424 (0.3734418948491414) + | > loader_time: 0.0073 (0.0060208174917433) + + + --> TIME: 2025-04-18 23:55:37 -- STEP: 133/142 -- GLOBAL_STEP: 387225 + | > loss_disc: 2.4019951820373535 (2.482690103071972) + | > loss_disc_real_0: 0.07612469047307968 (0.12972577131892504) + | > loss_disc_real_1: 0.20520161092281342 (0.21043063274451665) + | > loss_disc_real_2: 0.19271047413349152 (0.22737811561813928) + | > loss_disc_real_3: 0.22605293989181519 (0.22834150280271256) + | > loss_disc_real_4: 0.20223039388656616 (0.23114292390812607) + | > loss_disc_real_5: 0.23096971213817596 (0.23465295953858167) + | > loss_0: 2.4019951820373535 (2.482690103071972) + | > grad_norm_0: tensor(28.3877, device='cuda:0') (tensor(11.5560, device='cuda:0')) + | > loss_gen: 2.2375240325927734 (2.3316025698095326) + | > loss_kl: 2.1690921783447266 (1.9678904759256464) + | > loss_feat: 6.537908554077148 (6.894773895579173) + | > loss_mel: 17.506765365600586 (17.900533919943893) + | > loss_duration: 1.6163634061813354 (1.5247472282639125) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.06765365600586 (30.619548023195193) + | > grad_norm_1: tensor(292.5515, device='cuda:0') (tensor(245.3126, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.4451 (0.38707828700990604) + | > loader_time: 0.0077 (0.006315661552257107) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5645580291748047 (2.5645580291748047) + | > loss_disc_real_0: 0.16124755144119263 (0.16124755144119263) + | > loss_disc_real_1: 0.2899552881717682 (0.2899552881717682) + | > loss_disc_real_2: 0.21399061381816864 (0.21399061381816864) + | > loss_disc_real_3: 0.22729116678237915 (0.22729116678237915) + | > loss_disc_real_4: 0.20935019850730896 (0.20935019850730896) + | > loss_disc_real_5: 0.18766246736049652 (0.18766246736049652) + | > loss_0: 2.5645580291748047 (2.5645580291748047) + | > loss_gen: 2.1276416778564453 (2.1276416778564453) + | > loss_kl: 2.324970245361328 (2.324970245361328) + | > loss_feat: 6.0194783210754395 (6.0194783210754395) + | > loss_mel: 17.562015533447266 (17.562015533447266) + | > loss_duration: 1.6252756118774414 (1.6252756118774414) + | > loss_1: 29.659381866455078 (29.659381866455078) + + --> STEP: 1 + | > loss_disc: 2.352491855621338 (2.352491855621338) + | > loss_disc_real_0: 0.14161048829555511 (0.14161048829555511) + | > loss_disc_real_1: 0.20978358387947083 (0.20978358387947083) + | > loss_disc_real_2: 0.19296948611736298 (0.19296948611736298) + | > loss_disc_real_3: 0.20804817974567413 (0.20804817974567413) + | > loss_disc_real_4: 0.1865915060043335 (0.1865915060043335) + | > loss_disc_real_5: 0.2347501963376999 (0.2347501963376999) + | > loss_0: 2.352491855621338 (2.352491855621338) + | > loss_gen: 2.331960678100586 (2.331960678100586) + | > loss_kl: 2.4306201934814453 (2.4306201934814453) + | > loss_feat: 7.039794921875 (7.039794921875) + | > loss_mel: 18.87804412841797 (18.87804412841797) + | > loss_duration: 1.6522750854492188 (1.6522750854492188) + | > loss_1: 32.33269500732422 (32.33269500732422) + + --> STEP: 2 + | > loss_disc: 2.46701979637146 (2.409755825996399) + | > loss_disc_real_0: 0.1327650249004364 (0.13718775659799576) + | > loss_disc_real_1: 0.209186390042305 (0.2094849869608879) + | > loss_disc_real_2: 0.21056778728961945 (0.2017686367034912) + | > loss_disc_real_3: 0.23979370296001434 (0.22392094135284424) + | > loss_disc_real_4: 0.22357015311717987 (0.20508082956075668) + | > loss_disc_real_5: 0.18030618131160736 (0.20752818882465363) + | > loss_0: 2.46701979637146 (2.409755825996399) + | > loss_gen: 2.191168785095215 (2.2615647315979004) + | > loss_kl: 2.1954405307769775 (2.3130303621292114) + | > loss_feat: 6.998226642608643 (7.019010782241821) + | > loss_mel: 18.5206241607666 (18.699334144592285) + | > loss_duration: 1.5945391654968262 (1.6234071254730225) + | > loss_1: 31.5 (31.91634750366211) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035420656204223633 (-0.0002256631851196289) + | > avg_loss_disc: 2.409755825996399 (-0.06880080699920654) + | > avg_loss_disc_real_0: 0.13718775659799576 (-0.002182193100452423) + | > avg_loss_disc_real_1: 0.2094849869608879 (-0.055487968027591705) + | > avg_loss_disc_real_2: 0.2017686367034912 (-0.052329592406749725) + | > avg_loss_disc_real_3: 0.22392094135284424 (+0.02033064514398575) + | > avg_loss_disc_real_4: 0.20508082956075668 (-0.01680508255958557) + | > avg_loss_disc_real_5: 0.20752818882465363 (-0.0024294033646583557) + | > avg_loss_0: 2.409755825996399 (-0.06880080699920654) + | > avg_loss_gen: 2.2615647315979004 (-0.11150765419006348) + | > avg_loss_kl: 2.3130303621292114 (+0.06664371490478516) + | > avg_loss_feat: 7.019010782241821 (+0.45868897438049316) + | > avg_loss_mel: 18.699334144592285 (+1.242903232574463) + | > avg_loss_duration: 1.6234071254730225 (+0.0063283443450927734) + | > avg_loss_1: 31.91634750366211 (+1.6630573272705078) + + + > EPOCH: 56/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:55:43)  + + --> TIME: 2025-04-18 23:55:49 -- STEP: 16/142 -- GLOBAL_STEP: 387250 + | > loss_disc: 2.392174482345581 (2.4357616007328033) + | > loss_disc_real_0: 0.07323556393384933 (0.12340898951515554) + | > loss_disc_real_1: 0.18761295080184937 (0.20093511044979095) + | > loss_disc_real_2: 0.18999841809272766 (0.22555816173553467) + | > loss_disc_real_3: 0.21458563208580017 (0.22778095677495003) + | > loss_disc_real_4: 0.24701084196567535 (0.2289904123172164) + | > loss_disc_real_5: 0.22419030964374542 (0.2314495500177145) + | > loss_0: 2.392174482345581 (2.4357616007328033) + | > grad_norm_0: tensor(16.8716, device='cuda:0') (tensor(19.9584, device='cuda:0')) + | > loss_gen: 2.6211276054382324 (2.46285143494606) + | > loss_kl: 1.7693159580230713 (1.8907185345888138) + | > loss_feat: 7.850430965423584 (7.338027536869049) + | > loss_mel: 17.75008773803711 (17.906004786491394) + | > loss_duration: 1.4917237758636475 (1.4833466336131096) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.48268699645996 (31.08094894886017) + | > grad_norm_1: tensor(344.4717, device='cuda:0') (tensor(342.0310, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.3492 (0.3284722715616226) + | > loader_time: 0.0049 (0.004878818988800049) + + + --> TIME: 2025-04-18 23:55:58 -- STEP: 41/142 -- GLOBAL_STEP: 387275 + | > loss_disc: 2.4206695556640625 (2.437693113233985) + | > loss_disc_real_0: 0.09791786968708038 (0.12052132607233233) + | > loss_disc_real_1: 0.17713114619255066 (0.2027037536225668) + | > loss_disc_real_2: 0.1945299655199051 (0.22664891837573634) + | > loss_disc_real_3: 0.24990472197532654 (0.22689438047932414) + | > loss_disc_real_4: 0.24202880263328552 (0.22797916211733005) + | > loss_disc_real_5: 0.26254451274871826 (0.2349305131086489) + | > loss_0: 2.4206695556640625 (2.437693113233985) + | > grad_norm_0: tensor(4.5758, device='cuda:0') (tensor(15.4260, device='cuda:0')) + | > loss_gen: 2.3285200595855713 (2.4100779382193958) + | > loss_kl: 1.9348301887512207 (1.9053124974413616) + | > loss_feat: 7.092094898223877 (7.120857459742848) + | > loss_mel: 18.15485954284668 (17.78613662719726) + | > loss_duration: 1.4594659805297852 (1.4926551202448404) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.969772338867188 (30.715039509098705) + | > grad_norm_1: tensor(57.2815, device='cuda:0') (tensor(337.3518, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.3697 (0.34548496036994747) + | > loader_time: 0.0061 (0.00533258624193145) + + + --> TIME: 2025-04-18 23:56:08 -- STEP: 66/142 -- GLOBAL_STEP: 387300 + | > loss_disc: 2.4418673515319824 (2.444843949693622) + | > loss_disc_real_0: 0.10404353588819504 (0.12111824288061171) + | > loss_disc_real_1: 0.2337479293346405 (0.20604464727820773) + | > loss_disc_real_2: 0.20137621462345123 (0.2269185810829654) + | > loss_disc_real_3: 0.19493716955184937 (0.22625780083013303) + | > loss_disc_real_4: 0.17205111682415009 (0.22580628390565063) + | > loss_disc_real_5: 0.24150903522968292 (0.23365619530280432) + | > loss_0: 2.4418673515319824 (2.444843949693622) + | > grad_norm_0: tensor(8.5456, device='cuda:0') (tensor(14.3782, device='cuda:0')) + | > loss_gen: 2.247480869293213 (2.388749382712624) + | > loss_kl: 1.969377040863037 (1.9314554062756626) + | > loss_feat: 7.330502510070801 (7.074631300839511) + | > loss_mel: 18.906320571899414 (17.789027893181995) + | > loss_duration: 1.4906351566314697 (1.5021831411303896) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.94431495666504 (30.686047004930902) + | > grad_norm_1: tensor(250.5233, device='cuda:0') (tensor(325.2788, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.3985 (0.36206180037874164) + | > loader_time: 0.0061 (0.005686019406174169) + + + --> TIME: 2025-04-18 23:56:18 -- STEP: 91/142 -- GLOBAL_STEP: 387325 + | > loss_disc: 2.504425525665283 (2.45860406592652) + | > loss_disc_real_0: 0.13779400289058685 (0.12392712556398831) + | > loss_disc_real_1: 0.20419572293758392 (0.20671788267381902) + | > loss_disc_real_2: 0.21939529478549957 (0.2271518879211866) + | > loss_disc_real_3: 0.21369622647762299 (0.22673023483910404) + | > loss_disc_real_4: 0.272966206073761 (0.2293148535293537) + | > loss_disc_real_5: 0.20710013806819916 (0.2332465232728602) + | > loss_0: 2.504425525665283 (2.45860406592652) + | > grad_norm_0: tensor(12.3788, device='cuda:0') (tensor(13.9984, device='cuda:0')) + | > loss_gen: 2.3516035079956055 (2.375477848472176) + | > loss_kl: 1.889612078666687 (1.9523168776061508) + | > loss_feat: 6.488879203796387 (7.039881470439198) + | > loss_mel: 16.62873649597168 (17.794397155007154) + | > loss_duration: 1.5439062118530273 (1.5130822304840923) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.902740478515625 (30.67515540908981) + | > grad_norm_1: tensor(285.8938, device='cuda:0') (tensor(314.6637, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.4051 (0.3717430266705189) + | > loader_time: 0.0068 (0.0059445690322708295) + + + --> TIME: 2025-04-18 23:56:29 -- STEP: 116/142 -- GLOBAL_STEP: 387350 + | > loss_disc: 2.4830873012542725 (2.462836306670616) + | > loss_disc_real_0: 0.1200854554772377 (0.12392396537651276) + | > loss_disc_real_1: 0.21217042207717896 (0.20689865574240687) + | > loss_disc_real_2: 0.240116149187088 (0.22703786669620152) + | > loss_disc_real_3: 0.2500295639038086 (0.22753780463646198) + | > loss_disc_real_4: 0.2656958997249603 (0.22945317561770306) + | > loss_disc_real_5: 0.3031195104122162 (0.23515869188925315) + | > loss_0: 2.4830873012542725 (2.462836306670616) + | > grad_norm_0: tensor(11.2359, device='cuda:0') (tensor(13.5663, device='cuda:0')) + | > loss_gen: 2.491384744644165 (2.3638171183651893) + | > loss_kl: 1.8864291906356812 (1.961287647485733) + | > loss_feat: 6.492291450500488 (6.981893066702218) + | > loss_mel: 17.502696990966797 (17.80119477469345) + | > loss_duration: 1.5174899101257324 (1.5185423565321954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.890291213989258 (30.626734766466864) + | > grad_norm_1: tensor(359.8293, device='cuda:0') (tensor(307.6187, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.4348 (0.3822571121413133) + | > loader_time: 0.0073 (0.006136080314373148) + + + --> TIME: 2025-04-18 23:56:40 -- STEP: 141/142 -- GLOBAL_STEP: 387375 + | > loss_disc: 2.206144094467163 (2.465822923267986) + | > loss_disc_real_0: 0.14384794235229492 (0.12596542837349237) + | > loss_disc_real_1: 0.1993316411972046 (0.20772518270404627) + | > loss_disc_real_2: 0.24991700053215027 (0.2276618610671226) + | > loss_disc_real_3: 0.24113653600215912 (0.22681043016995098) + | > loss_disc_real_4: 0.2050766944885254 (0.2289175255924252) + | > loss_disc_real_5: 0.18466101586818695 (0.23417063120831835) + | > loss_0: 2.206144094467163 (2.465822923267986) + | > grad_norm_0: tensor(7.3254, device='cuda:0') (tensor(13.1292, device='cuda:0')) + | > loss_gen: 2.5806801319122314 (2.359240465975822) + | > loss_kl: 1.820827841758728 (1.9664139603892117) + | > loss_feat: 7.552015781402588 (6.976005689472172) + | > loss_mel: 17.523426055908203 (17.83598399331383) + | > loss_duration: 1.4076553583145142 (1.5238541085669333) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.884607315063477 (30.661498076526833) + | > grad_norm_1: tensor(155.5464, device='cuda:0') (tensor(293.0848, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.2042 (0.3933722736142206) + | > loader_time: 0.0026 (0.00630775918351843) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5083954334259033 (2.5083954334259033) + | > loss_disc_real_0: 0.13566671311855316 (0.13566671311855316) + | > loss_disc_real_1: 0.26679563522338867 (0.26679563522338867) + | > loss_disc_real_2: 0.23635731637477875 (0.23635731637477875) + | > loss_disc_real_3: 0.2462526112794876 (0.2462526112794876) + | > loss_disc_real_4: 0.2326149344444275 (0.2326149344444275) + | > loss_disc_real_5: 0.25586968660354614 (0.25586968660354614) + | > loss_0: 2.5083954334259033 (2.5083954334259033) + | > loss_gen: 2.299736499786377 (2.299736499786377) + | > loss_kl: 2.3093466758728027 (2.3093466758728027) + | > loss_feat: 6.372378826141357 (6.372378826141357) + | > loss_mel: 18.691835403442383 (18.691835403442383) + | > loss_duration: 1.6489499807357788 (1.6489499807357788) + | > loss_1: 31.322248458862305 (31.322248458862305) + + --> STEP: 1 + | > loss_disc: 2.3595876693725586 (2.3595876693725586) + | > loss_disc_real_0: 0.11174675077199936 (0.11174675077199936) + | > loss_disc_real_1: 0.2509787976741791 (0.2509787976741791) + | > loss_disc_real_2: 0.18300096690654755 (0.18300096690654755) + | > loss_disc_real_3: 0.20111162960529327 (0.20111162960529327) + | > loss_disc_real_4: 0.2466767132282257 (0.2466767132282257) + | > loss_disc_real_5: 0.22455444931983948 (0.22455444931983948) + | > loss_0: 2.3595876693725586 (2.3595876693725586) + | > loss_gen: 2.4003384113311768 (2.4003384113311768) + | > loss_kl: 2.588620185852051 (2.588620185852051) + | > loss_feat: 7.903141021728516 (7.903141021728516) + | > loss_mel: 19.357419967651367 (19.357419967651367) + | > loss_duration: 1.6643266677856445 (1.6643266677856445) + | > loss_1: 33.91384506225586 (33.91384506225586) + + --> STEP: 2 + | > loss_disc: 2.506847381591797 (2.4332175254821777) + | > loss_disc_real_0: 0.11308020353317261 (0.11241347715258598) + | > loss_disc_real_1: 0.23243945837020874 (0.2417091280221939) + | > loss_disc_real_2: 0.1656707376241684 (0.17433585226535797) + | > loss_disc_real_3: 0.2043565809726715 (0.2027341052889824) + | > loss_disc_real_4: 0.24650858342647552 (0.24659264832735062) + | > loss_disc_real_5: 0.2587879002094269 (0.24167117476463318) + | > loss_0: 2.506847381591797 (2.4332175254821777) + | > loss_gen: 2.008699655532837 (2.204519033432007) + | > loss_kl: 2.1710004806518555 (2.379810333251953) + | > loss_feat: 5.608103275299072 (6.755622148513794) + | > loss_mel: 15.77011489868164 (17.563767433166504) + | > loss_duration: 1.6051615476608276 (1.634744107723236) + | > loss_1: 27.1630802154541 (30.53846263885498) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003103494644165039 (-0.0004385709762573242) + | > avg_loss_disc: 2.4332175254821777 (+0.02346169948577881) + | > avg_loss_disc_real_0: 0.11241347715258598 (-0.024774279445409775) + | > avg_loss_disc_real_1: 0.2417091280221939 (+0.032224141061306) + | > avg_loss_disc_real_2: 0.17433585226535797 (-0.02743278443813324) + | > avg_loss_disc_real_3: 0.2027341052889824 (-0.021186836063861847) + | > avg_loss_disc_real_4: 0.24659264832735062 (+0.04151181876659393) + | > avg_loss_disc_real_5: 0.24167117476463318 (+0.03414298593997955) + | > avg_loss_0: 2.4332175254821777 (+0.02346169948577881) + | > avg_loss_gen: 2.204519033432007 (-0.057045698165893555) + | > avg_loss_kl: 2.379810333251953 (+0.0667799711227417) + | > avg_loss_feat: 6.755622148513794 (-0.26338863372802734) + | > avg_loss_mel: 17.563767433166504 (-1.1355667114257812) + | > avg_loss_duration: 1.634744107723236 (+0.011336982250213623) + | > avg_loss_1: 30.53846263885498 (-1.377884864807129) + + + > EPOCH: 57/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:56:42)  + + --> TIME: 2025-04-18 23:56:51 -- STEP: 24/142 -- GLOBAL_STEP: 387400 + | > loss_disc: 2.5803394317626953 (2.4794343610604606) + | > loss_disc_real_0: 0.10396835952997208 (0.12190096918493509) + | > loss_disc_real_1: 0.18870317935943604 (0.2099662981927395) + | > loss_disc_real_2: 0.20028406381607056 (0.2318849222113689) + | > loss_disc_real_3: 0.2535478472709656 (0.23175832132498422) + | > loss_disc_real_4: 0.22041960060596466 (0.2307669793566068) + | > loss_disc_real_5: 0.22572174668312073 (0.23854431261618933) + | > loss_0: 2.5803394317626953 (2.4794343610604606) + | > grad_norm_0: tensor(8.7961, device='cuda:0') (tensor(9.1570, device='cuda:0')) + | > loss_gen: 2.1691017150878906 (2.3524955610434213) + | > loss_kl: 2.0344791412353516 (1.8932950794696808) + | > loss_feat: 6.195010662078857 (6.98433792591095) + | > loss_mel: 16.77456283569336 (17.767120798428852) + | > loss_duration: 1.5345752239227295 (1.482466275493304) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.70772933959961 (30.47971526781718) + | > grad_norm_1: tensor(201.5018, device='cuda:0') (tensor(232.5756, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.3456 (0.33328900734583544) + | > loader_time: 0.0054 (0.004965007305145264) + + + --> TIME: 2025-04-18 23:57:00 -- STEP: 49/142 -- GLOBAL_STEP: 387425 + | > loss_disc: 2.5896341800689697 (2.4724434979107914) + | > loss_disc_real_0: 0.09809362888336182 (0.12433961642031767) + | > loss_disc_real_1: 0.206557035446167 (0.20735190595899308) + | > loss_disc_real_2: 0.22907647490501404 (0.22998730047624938) + | > loss_disc_real_3: 0.24426059424877167 (0.2299741725532376) + | > loss_disc_real_4: 0.24968773126602173 (0.2302769653651179) + | > loss_disc_real_5: 0.23251163959503174 (0.23617981100568966) + | > loss_0: 2.5896341800689697 (2.4724434979107914) + | > grad_norm_0: tensor(22.9330, device='cuda:0') (tensor(11.4400, device='cuda:0')) + | > loss_gen: 2.1405673027038574 (2.340219273859141) + | > loss_kl: 1.987023115158081 (1.9207787027164382) + | > loss_feat: 6.446250915527344 (6.963577786270453) + | > loss_mel: 17.451435089111328 (17.69098457025021) + | > loss_duration: 1.5107429027557373 (1.4905430565074997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.53601837158203 (30.40610305630431) + | > grad_norm_1: tensor(340.0610, device='cuda:0') (tensor(261.1471, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.3717 (0.3458631622548006) + | > loader_time: 0.0058 (0.005358793297592475) + + + --> TIME: 2025-04-18 23:57:10 -- STEP: 74/142 -- GLOBAL_STEP: 387450 + | > loss_disc: 2.4959945678710938 (2.479309156134322) + | > loss_disc_real_0: 0.13516445457935333 (0.12794890697743438) + | > loss_disc_real_1: 0.21589425206184387 (0.2074445049102242) + | > loss_disc_real_2: 0.19492273032665253 (0.22807048724309817) + | > loss_disc_real_3: 0.26419737935066223 (0.2299968314331931) + | > loss_disc_real_4: 0.21438300609588623 (0.23066296690219157) + | > loss_disc_real_5: 0.2643657326698303 (0.2359581474919577) + | > loss_0: 2.4959945678710938 (2.479309156134322) + | > grad_norm_0: tensor(25.3921, device='cuda:0') (tensor(12.7962, device='cuda:0')) + | > loss_gen: 2.28696608543396 (2.3476801466297466) + | > loss_kl: 1.9454025030136108 (1.9395375493410472) + | > loss_feat: 6.756075859069824 (6.944566688022098) + | > loss_mel: 18.296384811401367 (17.76185649150126) + | > loss_duration: 1.5334221124649048 (1.503738247059487) + | > amp_scaler: 256.0 (304.4324324324325) + | > loss_1: 30.818252563476562 (30.497378942128773) + | > grad_norm_1: 0 (tensor(272.4367, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.3778 (0.35779263844361175) + | > loader_time: 0.0068 (0.0056906874115402635) + + + --> TIME: 2025-04-18 23:57:20 -- STEP: 99/142 -- GLOBAL_STEP: 387475 + | > loss_disc: 2.5582470893859863 (2.4878543603299854) + | > loss_disc_real_0: 0.106447234749794 (0.1285219159391191) + | > loss_disc_real_1: 0.24825775623321533 (0.2082067576020655) + | > loss_disc_real_2: 0.23504705727100372 (0.22848607208391633) + | > loss_disc_real_3: 0.2545275092124939 (0.23133431134199856) + | > loss_disc_real_4: 0.236869677901268 (0.23090585313662135) + | > loss_disc_real_5: 0.22755123674869537 (0.23481752083759116) + | > loss_0: 2.5582470893859863 (2.4878543603299854) + | > grad_norm_0: tensor(15.1640, device='cuda:0') (tensor(14.0609, device='cuda:0')) + | > loss_gen: 2.4524085521698 (2.347395511588666) + | > loss_kl: 1.9472101926803589 (1.9460947007843943) + | > loss_feat: 7.321455478668213 (6.914092179500695) + | > loss_mel: 18.585277557373047 (17.778326862990244) + | > loss_duration: 1.551656723022461 (1.5145291391045157) + | > amp_scaler: 256.0 (292.2020202020203) + | > loss_1: 31.858007431030273 (30.500438189265704) + | > grad_norm_1: tensor(478.9009, device='cuda:0') (tensor(290.6030, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.4078 (0.36893113935836636) + | > loader_time: 0.0069 (0.005974497457947394) + + + --> TIME: 2025-04-18 23:57:31 -- STEP: 124/142 -- GLOBAL_STEP: 387500 + | > loss_disc: 2.457030773162842 (2.4859007981515706) + | > loss_disc_real_0: 0.09576942026615143 (0.13063827691779983) + | > loss_disc_real_1: 0.15259358286857605 (0.20799823790307967) + | > loss_disc_real_2: 0.2391444742679596 (0.22781810344707581) + | > loss_disc_real_3: 0.23733744025230408 (0.23115699473888643) + | > loss_disc_real_4: 0.24785196781158447 (0.23145737883544737) + | > loss_disc_real_5: 0.25898653268814087 (0.23353525118962412) + | > loss_0: 2.457030773162842 (2.4859007981515706) + | > grad_norm_0: tensor(15.6438, device='cuda:0') (tensor(14.2532, device='cuda:0')) + | > loss_gen: 2.3412692546844482 (2.3424189340683736) + | > loss_kl: 2.0306029319763184 (1.947868674993515) + | > loss_feat: 7.729831218719482 (6.884691138421336) + | > loss_mel: 19.60218048095703 (17.77947208958286) + | > loss_duration: 1.5805585384368896 (1.5211868122700722) + | > amp_scaler: 256.0 (284.90322580645176) + | > loss_1: 33.28443908691406 (30.475637512822306) + | > grad_norm_1: tensor(400.3356, device='cuda:0') (tensor(296.8341, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.4501 (0.38169615499434933) + | > loader_time: 0.0074 (0.006255192141379079) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4722900390625 (2.4722900390625) + | > loss_disc_real_0: 0.1443069577217102 (0.1443069577217102) + | > loss_disc_real_1: 0.23271815478801727 (0.23271815478801727) + | > loss_disc_real_2: 0.2477094680070877 (0.2477094680070877) + | > loss_disc_real_3: 0.20369158685207367 (0.20369158685207367) + | > loss_disc_real_4: 0.2013276070356369 (0.2013276070356369) + | > loss_disc_real_5: 0.22930140793323517 (0.22930140793323517) + | > loss_0: 2.4722900390625 (2.4722900390625) + | > loss_gen: 2.2409422397613525 (2.2409422397613525) + | > loss_kl: 2.2429986000061035 (2.2429986000061035) + | > loss_feat: 6.259562969207764 (6.259562969207764) + | > loss_mel: 17.26146697998047 (17.26146697998047) + | > loss_duration: 1.645315408706665 (1.645315408706665) + | > loss_1: 29.650285720825195 (29.650285720825195) + + --> STEP: 1 + | > loss_disc: 2.5784120559692383 (2.5784120559692383) + | > loss_disc_real_0: 0.20704208314418793 (0.20704208314418793) + | > loss_disc_real_1: 0.22296877205371857 (0.22296877205371857) + | > loss_disc_real_2: 0.22339123487472534 (0.22339123487472534) + | > loss_disc_real_3: 0.20449267327785492 (0.20449267327785492) + | > loss_disc_real_4: 0.2336171418428421 (0.2336171418428421) + | > loss_disc_real_5: 0.20189397037029266 (0.20189397037029266) + | > loss_0: 2.5784120559692383 (2.5784120559692383) + | > loss_gen: 2.12119722366333 (2.12119722366333) + | > loss_kl: 2.5684640407562256 (2.5684640407562256) + | > loss_feat: 6.5370001792907715 (6.5370001792907715) + | > loss_mel: 16.606510162353516 (16.606510162353516) + | > loss_duration: 1.6525784730911255 (1.6525784730911255) + | > loss_1: 29.485750198364258 (29.485750198364258) + + --> STEP: 2 + | > loss_disc: 2.557009220123291 (2.5677106380462646) + | > loss_disc_real_0: 0.24246862530708313 (0.22475535422563553) + | > loss_disc_real_1: 0.26501619815826416 (0.24399248510599136) + | > loss_disc_real_2: 0.25807127356529236 (0.24073125422000885) + | > loss_disc_real_3: 0.1983395367860794 (0.20141610503196716) + | > loss_disc_real_4: 0.21479402482509613 (0.22420558333396912) + | > loss_disc_real_5: 0.25586995482444763 (0.22888196259737015) + | > loss_0: 2.557009220123291 (2.5677106380462646) + | > loss_gen: 2.3151297569274902 (2.21816349029541) + | > loss_kl: 1.9972410202026367 (2.282852530479431) + | > loss_feat: 5.459643840789795 (5.998322010040283) + | > loss_mel: 15.24340534210205 (15.924957752227783) + | > loss_duration: 1.5990052223205566 (1.625791847705841) + | > loss_1: 26.614425659179688 (28.050087928771973) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036067962646484375 (+0.0005033016204833984) + | > avg_loss_disc: 2.5677106380462646 (+0.13449311256408691) + | > avg_loss_disc_real_0: 0.22475535422563553 (+0.11234187707304955) + | > avg_loss_disc_real_1: 0.24399248510599136 (+0.002283357083797455) + | > avg_loss_disc_real_2: 0.24073125422000885 (+0.06639540195465088) + | > avg_loss_disc_real_3: 0.20141610503196716 (-0.0013180002570152283) + | > avg_loss_disc_real_4: 0.22420558333396912 (-0.0223870649933815) + | > avg_loss_disc_real_5: 0.22888196259737015 (-0.012789212167263031) + | > avg_loss_0: 2.5677106380462646 (+0.13449311256408691) + | > avg_loss_gen: 2.21816349029541 (+0.01364445686340332) + | > avg_loss_kl: 2.282852530479431 (-0.09695780277252197) + | > avg_loss_feat: 5.998322010040283 (-0.7573001384735107) + | > avg_loss_mel: 15.924957752227783 (-1.6388096809387207) + | > avg_loss_duration: 1.625791847705841 (-0.00895226001739502) + | > avg_loss_1: 28.050087928771973 (-2.488374710083008) + + + > EPOCH: 58/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:57:41)  + + --> TIME: 2025-04-18 23:57:44 -- STEP: 7/142 -- GLOBAL_STEP: 387525 + | > loss_disc: 2.4983389377593994 (2.3916495527539934) + | > loss_disc_real_0: 0.1457156240940094 (0.12285317587000984) + | > loss_disc_real_1: 0.22376315295696259 (0.2036688072340829) + | > loss_disc_real_2: 0.25743991136550903 (0.22539264389446803) + | > loss_disc_real_3: 0.21588267385959625 (0.22256895686898912) + | > loss_disc_real_4: 0.2109401971101761 (0.21663935482501984) + | > loss_disc_real_5: 0.2774452865123749 (0.22425921899931772) + | > loss_0: 2.4983389377593994 (2.3916495527539934) + | > grad_norm_0: tensor(17.8419, device='cuda:0') (tensor(13.5353, device='cuda:0')) + | > loss_gen: 2.248819351196289 (2.466301543372018) + | > loss_kl: 1.9184504747390747 (1.8938733850206648) + | > loss_feat: 6.858754634857178 (7.210968290056501) + | > loss_mel: 18.06180763244629 (17.777029309953964) + | > loss_duration: 1.453173279762268 (1.485931396484375) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.541004180908203 (30.834103448050364) + | > grad_norm_1: tensor(122.1796, device='cuda:0') (tensor(264.5565, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.3227 (0.3246283531188965) + | > loader_time: 0.0045 (0.004666669028145927) + + + --> TIME: 2025-04-18 23:57:53 -- STEP: 32/142 -- GLOBAL_STEP: 387550 + | > loss_disc: 2.5264527797698975 (2.451924592256546) + | > loss_disc_real_0: 0.08112165331840515 (0.12242140632588416) + | > loss_disc_real_1: 0.19306230545043945 (0.20661032805219293) + | > loss_disc_real_2: 0.20234999060630798 (0.22711734054610133) + | > loss_disc_real_3: 0.19881397485733032 (0.22366106091067195) + | > loss_disc_real_4: 0.25895369052886963 (0.2259585722349584) + | > loss_disc_real_5: 0.23904791474342346 (0.23048059362918139) + | > loss_0: 2.5264527797698975 (2.451924592256546) + | > grad_norm_0: tensor(9.6405, device='cuda:0') (tensor(17.8300, device='cuda:0')) + | > loss_gen: 2.1902637481689453 (2.393654361367225) + | > loss_kl: 1.954547643661499 (1.901279240846634) + | > loss_feat: 6.872906684875488 (7.137658417224884) + | > loss_mel: 17.796003341674805 (17.898155212402347) + | > loss_duration: 1.4715216159820557 (1.4844768568873405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.28524398803711 (30.81522434949875) + | > grad_norm_1: tensor(225.8690, device='cuda:0') (tensor(330.0207, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.3539 (0.33715367317199707) + | > loader_time: 0.0052 (0.005091212689876556) + + + --> TIME: 2025-04-18 23:58:02 -- STEP: 57/142 -- GLOBAL_STEP: 387575 + | > loss_disc: 2.3406529426574707 (2.449848635154858) + | > loss_disc_real_0: 0.09141359478235245 (0.12288659490775644) + | > loss_disc_real_1: 0.15106046199798584 (0.20693085015865795) + | > loss_disc_real_2: 0.23410269618034363 (0.2282291940952602) + | > loss_disc_real_3: 0.26626813411712646 (0.22541863824191846) + | > loss_disc_real_4: 0.2289499193429947 (0.22785276122260512) + | > loss_disc_real_5: 0.2232609987258911 (0.229782368268883) + | > loss_0: 2.3406529426574707 (2.449848635154858) + | > grad_norm_0: tensor(7.4785, device='cuda:0') (tensor(14.9685, device='cuda:0')) + | > loss_gen: 2.4381141662597656 (2.3798236805096007) + | > loss_kl: 1.815125584602356 (1.9186562404297947) + | > loss_feat: 7.275378704071045 (7.103395286359285) + | > loss_mel: 18.846054077148438 (17.891931818242657) + | > loss_duration: 1.546715259552002 (1.4921578503491586) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.92138671875 (30.785965032744826) + | > grad_norm_1: tensor(222.8692, device='cuda:0') (tensor(318.8755, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.3777 (0.3488976202513042) + | > loader_time: 0.006 (0.005419417431480006) + + + --> TIME: 2025-04-18 23:58:12 -- STEP: 82/142 -- GLOBAL_STEP: 387600 + | > loss_disc: 2.4559807777404785 (2.448123289317619) + | > loss_disc_real_0: 0.09059413522481918 (0.12156162551808648) + | > loss_disc_real_1: 0.1962396800518036 (0.20662322258803903) + | > loss_disc_real_2: 0.22790969908237457 (0.2271662218178191) + | > loss_disc_real_3: 0.2252543717622757 (0.22482998959901856) + | > loss_disc_real_4: 0.22156845033168793 (0.22876805684915402) + | > loss_disc_real_5: 0.24810025095939636 (0.22817787437177287) + | > loss_0: 2.4559807777404785 (2.448123289317619) + | > grad_norm_0: tensor(8.5189, device='cuda:0') (tensor(14.4890, device='cuda:0')) + | > loss_gen: 2.3154239654541016 (2.3733006744849967) + | > loss_kl: 2.0322916507720947 (1.9338706033985789) + | > loss_feat: 6.823018550872803 (7.050926574846593) + | > loss_mel: 18.9333438873291 (17.929345468195475) + | > loss_duration: 1.6070919036865234 (1.503687214560625) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.711170196533203 (30.791130647426698) + | > grad_norm_1: tensor(267.7722, device='cuda:0') (tensor(318.2611, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.3924 (0.36119431111870737) + | > loader_time: 0.0064 (0.005681535092795767) + + + --> TIME: 2025-04-18 23:58:23 -- STEP: 107/142 -- GLOBAL_STEP: 387625 + | > loss_disc: 2.4748921394348145 (2.4542991335146893) + | > loss_disc_real_0: 0.09605573117733002 (0.12250007713370234) + | > loss_disc_real_1: 0.20205940306186676 (0.20698397056521656) + | > loss_disc_real_2: 0.2432519495487213 (0.22724061897981948) + | > loss_disc_real_3: 0.2268860638141632 (0.2260863630571098) + | > loss_disc_real_4: 0.23358780145645142 (0.22832258073526007) + | > loss_disc_real_5: 0.21149303019046783 (0.23000376177168338) + | > loss_0: 2.4748921394348145 (2.4542991335146893) + | > grad_norm_0: tensor(18.8424, device='cuda:0') (tensor(14.6860, device='cuda:0')) + | > loss_gen: 2.2674314975738525 (2.3648489546553004) + | > loss_kl: 2.0305118560791016 (1.9453823209923005) + | > loss_feat: 6.080435752868652 (7.001115602867626) + | > loss_mel: 15.803829193115234 (17.879475700521027) + | > loss_duration: 1.5226510763168335 (1.5140605173378348) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.704858779907227 (30.704883183274315) + | > grad_norm_1: tensor(131.1008, device='cuda:0') (tensor(320.3529, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.4191 (0.37239342092353594) + | > loader_time: 0.0072 (0.0059661976644925984) + + + --> TIME: 2025-04-18 23:58:34 -- STEP: 132/142 -- GLOBAL_STEP: 387650 + | > loss_disc: 2.5359861850738525 (2.461968304532947) + | > loss_disc_real_0: 0.12418293952941895 (0.12338802830852343) + | > loss_disc_real_1: 0.19474489986896515 (0.20747733533833967) + | > loss_disc_real_2: 0.24030068516731262 (0.22784475541927598) + | > loss_disc_real_3: 0.23801924288272858 (0.2269054235834064) + | > loss_disc_real_4: 0.23057816922664642 (0.2291620789152203) + | > loss_disc_real_5: 0.23906633257865906 (0.23032743178985335) + | > loss_0: 2.5359861850738525 (2.461968304532947) + | > grad_norm_0: tensor(12.9943, device='cuda:0') (tensor(14.3887, device='cuda:0')) + | > loss_gen: 2.256173610687256 (2.360400440114918) + | > loss_kl: 2.0758373737335205 (1.950953118728869) + | > loss_feat: 6.8294806480407715 (6.951628356268912) + | > loss_mel: 18.207813262939453 (17.806657971757836) + | > loss_duration: 1.5166714191436768 (1.519795966870857) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.88597869873047 (30.58943595308246) + | > grad_norm_1: tensor(354.7912, device='cuda:0') (tensor(317.1627, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.4404 (0.38561774203271565) + | > loader_time: 0.0072 (0.006223358891227029) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4007577896118164 (2.4007577896118164) + | > loss_disc_real_0: 0.1415906846523285 (0.1415906846523285) + | > loss_disc_real_1: 0.19385281205177307 (0.19385281205177307) + | > loss_disc_real_2: 0.2638539671897888 (0.2638539671897888) + | > loss_disc_real_3: 0.2637096643447876 (0.2637096643447876) + | > loss_disc_real_4: 0.24113428592681885 (0.24113428592681885) + | > loss_disc_real_5: 0.20425407588481903 (0.20425407588481903) + | > loss_0: 2.4007577896118164 (2.4007577896118164) + | > loss_gen: 2.353724479675293 (2.353724479675293) + | > loss_kl: 2.3866658210754395 (2.3866658210754395) + | > loss_feat: 5.5028581619262695 (5.5028581619262695) + | > loss_mel: 15.556683540344238 (15.556683540344238) + | > loss_duration: 1.6448050737380981 (1.6448050737380981) + | > loss_1: 27.44473648071289 (27.44473648071289) + + --> STEP: 1 + | > loss_disc: 2.5884647369384766 (2.5884647369384766) + | > loss_disc_real_0: 0.1892225593328476 (0.1892225593328476) + | > loss_disc_real_1: 0.20161740481853485 (0.20161740481853485) + | > loss_disc_real_2: 0.2531970143318176 (0.2531970143318176) + | > loss_disc_real_3: 0.25954046845436096 (0.25954046845436096) + | > loss_disc_real_4: 0.2490045428276062 (0.2490045428276062) + | > loss_disc_real_5: 0.24466997385025024 (0.24466997385025024) + | > loss_0: 2.5884647369384766 (2.5884647369384766) + | > loss_gen: 2.236616373062134 (2.236616373062134) + | > loss_kl: 2.274644613265991 (2.274644613265991) + | > loss_feat: 6.486235618591309 (6.486235618591309) + | > loss_mel: 18.07105827331543 (18.07105827331543) + | > loss_duration: 1.6909801959991455 (1.6909801959991455) + | > loss_1: 30.759536743164062 (30.759536743164062) + + --> STEP: 2 + | > loss_disc: 2.5970041751861572 (2.592734456062317) + | > loss_disc_real_0: 0.1579548865556717 (0.17358872294425964) + | > loss_disc_real_1: 0.18949247896671295 (0.1955549418926239) + | > loss_disc_real_2: 0.25261449813842773 (0.2529057562351227) + | > loss_disc_real_3: 0.2716214954853058 (0.2655809819698334) + | > loss_disc_real_4: 0.20688660442829132 (0.22794557362794876) + | > loss_disc_real_5: 0.2646609842777252 (0.25466547906398773) + | > loss_0: 2.5970041751861572 (2.592734456062317) + | > loss_gen: 2.0609002113342285 (2.148758292198181) + | > loss_kl: 2.103893518447876 (2.1892690658569336) + | > loss_feat: 6.1444525718688965 (6.3153440952301025) + | > loss_mel: 17.079513549804688 (17.57528591156006) + | > loss_duration: 1.5957307815551758 (1.6433554887771606) + | > loss_1: 28.98448944091797 (29.872013092041016) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038480758666992188 (+0.00024127960205078125) + | > avg_loss_disc: 2.592734456062317 (+0.025023818016052246) + | > avg_loss_disc_real_0: 0.17358872294425964 (-0.051166631281375885) + | > avg_loss_disc_real_1: 0.1955549418926239 (-0.04843754321336746) + | > avg_loss_disc_real_2: 0.2529057562351227 (+0.01217450201511383) + | > avg_loss_disc_real_3: 0.2655809819698334 (+0.06416487693786621) + | > avg_loss_disc_real_4: 0.22794557362794876 (+0.0037399902939796448) + | > avg_loss_disc_real_5: 0.25466547906398773 (+0.025783516466617584) + | > avg_loss_0: 2.592734456062317 (+0.025023818016052246) + | > avg_loss_gen: 2.148758292198181 (-0.069405198097229) + | > avg_loss_kl: 2.1892690658569336 (-0.09358346462249756) + | > avg_loss_feat: 6.3153440952301025 (+0.31702208518981934) + | > avg_loss_mel: 17.57528591156006 (+1.6503281593322754) + | > avg_loss_duration: 1.6433554887771606 (+0.01756364107131958) + | > avg_loss_1: 29.872013092041016 (+1.821925163269043) + + + > EPOCH: 59/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:58:40)  + + --> TIME: 2025-04-18 23:58:46 -- STEP: 15/142 -- GLOBAL_STEP: 387675 + | > loss_disc: 2.351914405822754 (2.3701258977254227) + | > loss_disc_real_0: 0.09540919214487076 (0.10913753112157186) + | > loss_disc_real_1: 0.22371631860733032 (0.19805329144001008) + | > loss_disc_real_2: 0.17595231533050537 (0.22022991279760998) + | > loss_disc_real_3: 0.18138499557971954 (0.22872668703397114) + | > loss_disc_real_4: 0.20658983290195465 (0.2170249064763387) + | > loss_disc_real_5: 0.18333251774311066 (0.23028771976629894) + | > loss_0: 2.351914405822754 (2.3701258977254227) + | > grad_norm_0: tensor(3.5604, device='cuda:0') (tensor(8.2940, device='cuda:0')) + | > loss_gen: 2.350715160369873 (2.3890785853068035) + | > loss_kl: 1.9288042783737183 (1.8552817900975545) + | > loss_feat: 7.0206828117370605 (7.129193623860677) + | > loss_mel: 18.45506477355957 (17.922388712565105) + | > loss_duration: 1.4704359769821167 (1.4771965742111206) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.225704193115234 (30.773139572143556) + | > grad_norm_1: tensor(309.0166, device='cuda:0') (tensor(246.5979, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.3338 (0.32674736976623536) + | > loader_time: 0.0054 (0.0047234217325846355) + + + --> TIME: 2025-04-18 23:58:55 -- STEP: 40/142 -- GLOBAL_STEP: 387700 + | > loss_disc: 2.428260087966919 (2.4230968117713925) + | > loss_disc_real_0: 0.1407063752412796 (0.11718535926193) + | > loss_disc_real_1: 0.18361647427082062 (0.20527531318366524) + | > loss_disc_real_2: 0.19710379838943481 (0.2245078705251217) + | > loss_disc_real_3: 0.24697527289390564 (0.23035758063197137) + | > loss_disc_real_4: 0.21952764689922333 (0.22522761821746826) + | > loss_disc_real_5: 0.20345595479011536 (0.2298271030187607) + | > loss_0: 2.428260087966919 (2.4230968117713925) + | > grad_norm_0: tensor(15.7290, device='cuda:0') (tensor(10.4534, device='cuda:0')) + | > loss_gen: 2.3167102336883545 (2.3755833446979526) + | > loss_kl: 1.925284743309021 (1.8911711931228639) + | > loss_feat: 7.488926410675049 (7.066844868659973) + | > loss_mel: 18.291332244873047 (17.748832702636715) + | > loss_duration: 1.4637513160705566 (1.4860649585723877) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.486003875732422 (30.568496990203858) + | > grad_norm_1: tensor(515.0632, device='cuda:0') (tensor(289.9116, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.3718 (0.3440062999725341) + | > loader_time: 0.0062 (0.005218696594238281) + + + --> TIME: 2025-04-18 23:59:05 -- STEP: 65/142 -- GLOBAL_STEP: 387725 + | > loss_disc: 2.4772753715515137 (2.442360236094548) + | > loss_disc_real_0: 0.08535703271627426 (0.12141769792024906) + | > loss_disc_real_1: 0.24462160468101501 (0.20633713075747853) + | > loss_disc_real_2: 0.24634255468845367 (0.22668612026251278) + | > loss_disc_real_3: 0.22584974765777588 (0.22906906650616574) + | > loss_disc_real_4: 0.22789935767650604 (0.2281306436428657) + | > loss_disc_real_5: 0.2165464162826538 (0.22912443578243255) + | > loss_0: 2.4772753715515137 (2.442360236094548) + | > grad_norm_0: tensor(6.5432, device='cuda:0') (tensor(11.0873, device='cuda:0')) + | > loss_gen: 2.3610308170318604 (2.363703335248507) + | > loss_kl: 2.0089287757873535 (1.9058873048195473) + | > loss_feat: 7.299295902252197 (7.033359021406907) + | > loss_mel: 18.508338928222656 (17.78582574404202) + | > loss_duration: 1.513207197189331 (1.4992814467503475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.690799713134766 (30.58805671105018) + | > grad_norm_1: tensor(383.0414, device='cuda:0') (tensor(284.5075, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.4123 (0.36032311366154596) + | > loader_time: 0.0063 (0.005576896667480471) + + + --> TIME: 2025-04-18 23:59:15 -- STEP: 90/142 -- GLOBAL_STEP: 387750 + | > loss_disc: 2.3851988315582275 (2.465827168358697) + | > loss_disc_real_0: 0.14535817503929138 (0.1266969913409816) + | > loss_disc_real_1: 0.16522391140460968 (0.20738923086060418) + | > loss_disc_real_2: 0.22616246342658997 (0.2271584454509947) + | > loss_disc_real_3: 0.19622309505939484 (0.2304459434416559) + | > loss_disc_real_4: 0.24227085709571838 (0.23160579237673018) + | > loss_disc_real_5: 0.2142646163702011 (0.2309158398045434) + | > loss_0: 2.3851988315582275 (2.465827168358697) + | > grad_norm_0: tensor(31.2957, device='cuda:0') (tensor(13.1304, device='cuda:0')) + | > loss_gen: 2.2653117179870605 (2.359812243779501) + | > loss_kl: 2.05814790725708 (1.9310957815912035) + | > loss_feat: 7.543731689453125 (7.020417992273966) + | > loss_mel: 19.198118209838867 (17.805671575334326) + | > loss_duration: 1.554013729095459 (1.5111930198139614) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.61932373046875 (30.628190528021918) + | > grad_norm_1: tensor(276.2369, device='cuda:0') (tensor(309.5844, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.4208 (0.37294236024220784) + | > loader_time: 0.0073 (0.005883068508572049) + + + --> TIME: 2025-04-18 23:59:26 -- STEP: 115/142 -- GLOBAL_STEP: 387775 + | > loss_disc: 2.402207612991333 (2.469072211307028) + | > loss_disc_real_0: 0.07884740084409714 (0.127190667261248) + | > loss_disc_real_1: 0.2065979689359665 (0.20790704346221425) + | > loss_disc_real_2: 0.19372448325157166 (0.22672435421010723) + | > loss_disc_real_3: 0.1923503577709198 (0.22960183503835097) + | > loss_disc_real_4: 0.1958388239145279 (0.231600326558818) + | > loss_disc_real_5: 0.16649141907691956 (0.23112668174764384) + | > loss_0: 2.402207612991333 (2.469072211307028) + | > grad_norm_0: tensor(12.4094, device='cuda:0') (tensor(14.9677, device='cuda:0')) + | > loss_gen: 2.242556571960449 (2.3590862357098112) + | > loss_kl: 1.9436181783676147 (1.9338538657064024) + | > loss_feat: 7.163872241973877 (6.980518233257791) + | > loss_mel: 16.689647674560547 (17.795203565514598) + | > loss_duration: 1.5601829290390015 (1.5172301935112995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.599878311157227 (30.585892022174338) + | > grad_norm_1: tensor(324.9576, device='cuda:0') (tensor(320.6298, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.4448 (0.38468514318051544) + | > loader_time: 0.0073 (0.006137240451315175) + + + --> TIME: 2025-04-18 23:59:38 -- STEP: 140/142 -- GLOBAL_STEP: 387800 + | > loss_disc: 2.5574135780334473 (2.467772490637643) + | > loss_disc_real_0: 0.09028250724077225 (0.12737474861953932) + | > loss_disc_real_1: 0.19481270015239716 (0.2074879697390965) + | > loss_disc_real_2: 0.22944292426109314 (0.22666193523577283) + | > loss_disc_real_3: 0.23310837149620056 (0.22915435912353652) + | > loss_disc_real_4: 0.26298925280570984 (0.23112690363611493) + | > loss_disc_real_5: 0.2259800285100937 (0.2312417578484331) + | > loss_0: 2.5574135780334473 (2.467772490637643) + | > grad_norm_0: tensor(8.8076, device='cuda:0') (tensor(15.1795, device='cuda:0')) + | > loss_gen: 2.303091287612915 (2.358020155770437) + | > loss_kl: 2.002443313598633 (1.937886907373156) + | > loss_feat: 7.064291000366211 (6.9715068714959285) + | > loss_mel: 17.93015480041504 (17.78427877426147) + | > loss_duration: 1.5917375087738037 (1.5232773848942347) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.8917179107666 (30.574969986506872) + | > grad_norm_1: tensor(316.2116, device='cuda:0') (tensor(321.8723, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.4655 (0.39707143817629137) + | > loader_time: 0.0074 (0.006381411211831228) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.381925106048584 (2.381925106048584) + | > loss_disc_real_0: 0.09349250793457031 (0.09349250793457031) + | > loss_disc_real_1: 0.17800192534923553 (0.17800192534923553) + | > loss_disc_real_2: 0.22298531234264374 (0.22298531234264374) + | > loss_disc_real_3: 0.26228874921798706 (0.26228874921798706) + | > loss_disc_real_4: 0.23462237417697906 (0.23462237417697906) + | > loss_disc_real_5: 0.24042420089244843 (0.24042420089244843) + | > loss_0: 2.381925106048584 (2.381925106048584) + | > loss_gen: 2.3557562828063965 (2.3557562828063965) + | > loss_kl: 2.3113179206848145 (2.3113179206848145) + | > loss_feat: 7.45481538772583 (7.45481538772583) + | > loss_mel: 19.478134155273438 (19.478134155273438) + | > loss_duration: 1.6092504262924194 (1.6092504262924194) + | > loss_1: 33.20927429199219 (33.20927429199219) + + --> STEP: 1 + | > loss_disc: 2.432723045349121 (2.432723045349121) + | > loss_disc_real_0: 0.11313813924789429 (0.11313813924789429) + | > loss_disc_real_1: 0.20235170423984528 (0.20235170423984528) + | > loss_disc_real_2: 0.1994462013244629 (0.1994462013244629) + | > loss_disc_real_3: 0.23357270658016205 (0.23357270658016205) + | > loss_disc_real_4: 0.2037927657365799 (0.2037927657365799) + | > loss_disc_real_5: 0.31874340772628784 (0.31874340772628784) + | > loss_0: 2.432723045349121 (2.432723045349121) + | > loss_gen: 2.359754800796509 (2.359754800796509) + | > loss_kl: 2.4034178256988525 (2.4034178256988525) + | > loss_feat: 7.1355438232421875 (7.1355438232421875) + | > loss_mel: 18.95474624633789 (18.95474624633789) + | > loss_duration: 1.6578792333602905 (1.6578792333602905) + | > loss_1: 32.5113410949707 (32.5113410949707) + + --> STEP: 2 + | > loss_disc: 2.5289885997772217 (2.4808558225631714) + | > loss_disc_real_0: 0.13929609954357147 (0.12621711939573288) + | > loss_disc_real_1: 0.2112709879875183 (0.2068113461136818) + | > loss_disc_real_2: 0.23246844112873077 (0.21595732122659683) + | > loss_disc_real_3: 0.25838255882263184 (0.24597763270139694) + | > loss_disc_real_4: 0.26137053966522217 (0.23258165270090103) + | > loss_disc_real_5: 0.23893818259239197 (0.2788407951593399) + | > loss_0: 2.5289885997772217 (2.4808558225631714) + | > loss_gen: 2.2023160457611084 (2.2810354232788086) + | > loss_kl: 2.1072025299072266 (2.2553101778030396) + | > loss_feat: 6.653051376342773 (6.8942975997924805) + | > loss_mel: 18.01096534729004 (18.482855796813965) + | > loss_duration: 1.6072521209716797 (1.632565677165985) + | > loss_1: 30.580787658691406 (31.546064376831055) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003478527069091797 (-0.0003695487976074219) + | > avg_loss_disc: 2.4808558225631714 (-0.11187863349914551) + | > avg_loss_disc_real_0: 0.12621711939573288 (-0.047371603548526764) + | > avg_loss_disc_real_1: 0.2068113461136818 (+0.011256404221057892) + | > avg_loss_disc_real_2: 0.21595732122659683 (-0.03694843500852585) + | > avg_loss_disc_real_3: 0.24597763270139694 (-0.019603349268436432) + | > avg_loss_disc_real_4: 0.23258165270090103 (+0.0046360790729522705) + | > avg_loss_disc_real_5: 0.2788407951593399 (+0.024175316095352173) + | > avg_loss_0: 2.4808558225631714 (-0.11187863349914551) + | > avg_loss_gen: 2.2810354232788086 (+0.13227713108062744) + | > avg_loss_kl: 2.2553101778030396 (+0.06604111194610596) + | > avg_loss_feat: 6.8942975997924805 (+0.5789535045623779) + | > avg_loss_mel: 18.482855796813965 (+0.9075698852539062) + | > avg_loss_duration: 1.632565677165985 (-0.010789811611175537) + | > avg_loss_1: 31.546064376831055 (+1.674051284790039) + + + > EPOCH: 60/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-18 23:59:40)  + + --> TIME: 2025-04-18 23:59:49 -- STEP: 23/142 -- GLOBAL_STEP: 387825 + | > loss_disc: 2.466646671295166 (2.4372410359589955) + | > loss_disc_real_0: 0.09478089958429337 (0.11523508183334184) + | > loss_disc_real_1: 0.1953212469816208 (0.20761665831441464) + | > loss_disc_real_2: 0.2212904691696167 (0.2312519019064696) + | > loss_disc_real_3: 0.2060883492231369 (0.2260956109865852) + | > loss_disc_real_4: 0.19768325984477997 (0.2208106718633486) + | > loss_disc_real_5: 0.21434356272220612 (0.23363442783770355) + | > loss_0: 2.466646671295166 (2.4372410359589955) + | > grad_norm_0: tensor(11.2379, device='cuda:0') (tensor(10.4718, device='cuda:0')) + | > loss_gen: 2.2162089347839355 (2.3683752806290337) + | > loss_kl: 1.9409732818603516 (1.8815775954205056) + | > loss_feat: 7.09419584274292 (7.131747266520625) + | > loss_mel: 16.596046447753906 (17.828458371369734) + | > loss_duration: 1.524782419204712 (1.474836634552997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.372207641601562 (30.684995070747707) + | > grad_norm_1: tensor(286.6960, device='cuda:0') (tensor(285.6298, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.3517 (0.3346346253934114) + | > loader_time: 0.0056 (0.00504215903904127) + + + --> TIME: 2025-04-18 23:59:58 -- STEP: 48/142 -- GLOBAL_STEP: 387850 + | > loss_disc: 2.5781631469726562 (2.4329838107029595) + | > loss_disc_real_0: 0.19556990265846252 (0.11874994340663154) + | > loss_disc_real_1: 0.24264077842235565 (0.2057600555320581) + | > loss_disc_real_2: 0.251459538936615 (0.22758173756301403) + | > loss_disc_real_3: 0.25957125425338745 (0.22564718096206585) + | > loss_disc_real_4: 0.27287957072257996 (0.22635802067816257) + | > loss_disc_real_5: 0.26776769757270813 (0.23137106467038393) + | > loss_0: 2.5781631469726562 (2.4329838107029595) + | > grad_norm_0: tensor(12.0586, device='cuda:0') (tensor(10.6284, device='cuda:0')) + | > loss_gen: 2.3748769760131836 (2.3767761091391244) + | > loss_kl: 1.8837753534317017 (1.9027330378691356) + | > loss_feat: 6.507340908050537 (7.147205839554469) + | > loss_mel: 17.413686752319336 (17.83073006073634) + | > loss_duration: 1.4770324230194092 (1.4848840981721878) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65671157836914 (30.742329200108845) + | > grad_norm_1: tensor(292.8016, device='cuda:0') (tensor(268.8035, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.3736 (0.3516871283451715) + | > loader_time: 0.0063 (0.005462969342867534) + + + --> TIME: 2025-04-19 00:00:08 -- STEP: 73/142 -- GLOBAL_STEP: 387875 + | > loss_disc: 2.426992416381836 (2.4447522000090722) + | > loss_disc_real_0: 0.12151684612035751 (0.1216714620998461) + | > loss_disc_real_1: 0.23903755843639374 (0.20497828701587573) + | > loss_disc_real_2: 0.18932217359542847 (0.2261524980198847) + | > loss_disc_real_3: 0.21210837364196777 (0.22669680865660105) + | > loss_disc_real_4: 0.25122177600860596 (0.22749051533333242) + | > loss_disc_real_5: 0.22921724617481232 (0.2347901412477232) + | > loss_0: 2.426992416381836 (2.4447522000090722) + | > grad_norm_0: tensor(6.8098, device='cuda:0') (tensor(10.6362, device='cuda:0')) + | > loss_gen: 2.350419759750366 (2.355705898102016) + | > loss_kl: 2.0919222831726074 (1.9235919420033285) + | > loss_feat: 7.364696979522705 (7.077168307892264) + | > loss_mel: 18.544967651367188 (17.80588907738254) + | > loss_duration: 1.5711777210235596 (1.4969273364707216) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.923185348510742 (30.659282605941982) + | > grad_norm_1: tensor(143.9375, device='cuda:0') (tensor(257.8725, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.4095 (0.3659500291902725) + | > loader_time: 0.0061 (0.00574926154254234) + + + --> TIME: 2025-04-19 00:00:19 -- STEP: 98/142 -- GLOBAL_STEP: 387900 + | > loss_disc: 2.451373815536499 (2.4535755770547043) + | > loss_disc_real_0: 0.1143173947930336 (0.12277796955741181) + | > loss_disc_real_1: 0.17813245952129364 (0.20508122991542427) + | > loss_disc_real_2: 0.20415832102298737 (0.22739627181875463) + | > loss_disc_real_3: 0.19105300307273865 (0.2264298028787788) + | > loss_disc_real_4: 0.1979549080133438 (0.22860831235136306) + | > loss_disc_real_5: 0.20021329820156097 (0.23401983903378856) + | > loss_0: 2.451373815536499 (2.4535755770547043) + | > grad_norm_0: tensor(7.7572, device='cuda:0') (tensor(10.5885, device='cuda:0')) + | > loss_gen: 2.240386724472046 (2.341996081021367) + | > loss_kl: 2.0451533794403076 (1.9380631811764775) + | > loss_feat: 7.432859420776367 (7.028806720461164) + | > loss_mel: 17.73586082458496 (17.79378883206114) + | > loss_duration: 1.4961965084075928 (1.5085168943113214) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.950458526611328 (30.61117176133759) + | > grad_norm_1: tensor(71.3489, device='cuda:0') (tensor(247.8129, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.4234 (0.37710461811143525) + | > loader_time: 0.0067 (0.005984836695145587) + + + --> TIME: 2025-04-19 00:00:30 -- STEP: 123/142 -- GLOBAL_STEP: 387925 + | > loss_disc: 2.3824920654296875 (2.456435984712305) + | > loss_disc_real_0: 0.08460203558206558 (0.12430442391130013) + | > loss_disc_real_1: 0.186444953083992 (0.2047865549481012) + | > loss_disc_real_2: 0.20793332159519196 (0.22663506865501404) + | > loss_disc_real_3: 0.19706818461418152 (0.22657080915400651) + | > loss_disc_real_4: 0.17826130986213684 (0.22904875014371018) + | > loss_disc_real_5: 0.18436823785305023 (0.2330302174498395) + | > loss_0: 2.3824920654296875 (2.456435984712305) + | > grad_norm_0: tensor(10.7554, device='cuda:0') (tensor(10.3967, device='cuda:0')) + | > loss_gen: 2.475639581680298 (2.335988870481165) + | > loss_kl: 1.9632606506347656 (1.949955908263602) + | > loss_feat: 7.32926082611084 (6.959857324274575) + | > loss_mel: 17.97587013244629 (17.76598616344172) + | > loss_duration: 1.5286202430725098 (1.5161237929894675) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.27264976501465 (30.52791207786498) + | > grad_norm_1: tensor(181.9222, device='cuda:0') (tensor(229.6204, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.45 (0.38955507433511377) + | > loader_time: 0.0067 (0.006203874339902304) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6178104877471924 (2.6178104877471924) + | > loss_disc_real_0: 0.24687635898590088 (0.24687635898590088) + | > loss_disc_real_1: 0.24822968244552612 (0.24822968244552612) + | > loss_disc_real_2: 0.2432790845632553 (0.2432790845632553) + | > loss_disc_real_3: 0.26518094539642334 (0.26518094539642334) + | > loss_disc_real_4: 0.26740920543670654 (0.26740920543670654) + | > loss_disc_real_5: 0.28404542803764343 (0.28404542803764343) + | > loss_0: 2.6178104877471924 (2.6178104877471924) + | > loss_gen: 2.4958598613739014 (2.4958598613739014) + | > loss_kl: 2.1947944164276123 (2.1947944164276123) + | > loss_feat: 5.380963325500488 (5.380963325500488) + | > loss_mel: 14.744341850280762 (14.744341850280762) + | > loss_duration: 1.6234036684036255 (1.6234036684036255) + | > loss_1: 26.439361572265625 (26.439361572265625) + + --> STEP: 1 + | > loss_disc: 2.5124781131744385 (2.5124781131744385) + | > loss_disc_real_0: 0.18239204585552216 (0.18239204585552216) + | > loss_disc_real_1: 0.2661760151386261 (0.2661760151386261) + | > loss_disc_real_2: 0.25202634930610657 (0.25202634930610657) + | > loss_disc_real_3: 0.27379873394966125 (0.27379873394966125) + | > loss_disc_real_4: 0.29319122433662415 (0.29319122433662415) + | > loss_disc_real_5: 0.22480368614196777 (0.22480368614196777) + | > loss_0: 2.5124781131744385 (2.5124781131744385) + | > loss_gen: 2.636575222015381 (2.636575222015381) + | > loss_kl: 2.520876407623291 (2.520876407623291) + | > loss_feat: 6.858415603637695 (6.858415603637695) + | > loss_mel: 19.196760177612305 (19.196760177612305) + | > loss_duration: 1.6463630199432373 (1.6463630199432373) + | > loss_1: 32.85898971557617 (32.85898971557617) + + --> STEP: 2 + | > loss_disc: 2.4530255794525146 (2.4827518463134766) + | > loss_disc_real_0: 0.18722116947174072 (0.18480660766363144) + | > loss_disc_real_1: 0.24244585633277893 (0.2543109357357025) + | > loss_disc_real_2: 0.209441676735878 (0.23073401302099228) + | > loss_disc_real_3: 0.25940728187561035 (0.2666030079126358) + | > loss_disc_real_4: 0.2059631645679474 (0.24957719445228577) + | > loss_disc_real_5: 0.23995742201805115 (0.23238055408000946) + | > loss_0: 2.4530255794525146 (2.4827518463134766) + | > loss_gen: 2.556089162826538 (2.5963321924209595) + | > loss_kl: 2.069692373275757 (2.295284390449524) + | > loss_feat: 8.149860382080078 (7.504137992858887) + | > loss_mel: 18.669458389282227 (18.933109283447266) + | > loss_duration: 1.5846424102783203 (1.6155027151107788) + | > loss_1: 33.02974319458008 (32.944366455078125) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003474712371826172 (-3.814697265625e-06) + | > avg_loss_disc: 2.4827518463134766 (+0.0018960237503051758) + | > avg_loss_disc_real_0: 0.18480660766363144 (+0.05858948826789856) + | > avg_loss_disc_real_1: 0.2543109357357025 (+0.04749958962202072) + | > avg_loss_disc_real_2: 0.23073401302099228 (+0.014776691794395447) + | > avg_loss_disc_real_3: 0.2666030079126358 (+0.02062537521123886) + | > avg_loss_disc_real_4: 0.24957719445228577 (+0.016995541751384735) + | > avg_loss_disc_real_5: 0.23238055408000946 (-0.046460241079330444) + | > avg_loss_0: 2.4827518463134766 (+0.0018960237503051758) + | > avg_loss_gen: 2.5963321924209595 (+0.3152967691421509) + | > avg_loss_kl: 2.295284390449524 (+0.039974212646484375) + | > avg_loss_feat: 7.504137992858887 (+0.6098403930664062) + | > avg_loss_mel: 18.933109283447266 (+0.4502534866333008) + | > avg_loss_duration: 1.6155027151107788 (-0.0170629620552063) + | > avg_loss_1: 32.944366455078125 (+1.3983020782470703) + + + > EPOCH: 61/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:00:40)  + + --> TIME: 2025-04-19 00:00:43 -- STEP: 6/142 -- GLOBAL_STEP: 387950 + | > loss_disc: 2.456411600112915 (2.4661146799723306) + | > loss_disc_real_0: 0.15291787683963776 (0.1266223800679048) + | > loss_disc_real_1: 0.23996248841285706 (0.2085031494498253) + | > loss_disc_real_2: 0.2235816866159439 (0.23258077849944434) + | > loss_disc_real_3: 0.2038252055644989 (0.2235537568728129) + | > loss_disc_real_4: 0.24214285612106323 (0.24086674551169077) + | > loss_disc_real_5: 0.21459460258483887 (0.24471223602692285) + | > loss_0: 2.456411600112915 (2.4661146799723306) + | > grad_norm_0: tensor(6.2653, device='cuda:0') (tensor(7.6936, device='cuda:0')) + | > loss_gen: 2.3621129989624023 (2.3655443588892617) + | > loss_kl: 1.7258358001708984 (1.8576401074727376) + | > loss_feat: 7.337047576904297 (7.201058387756348) + | > loss_mel: 18.131887435913086 (17.847660700480144) + | > loss_duration: 1.4797794818878174 (1.481783131758372) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.036664962768555 (30.753687222798664) + | > grad_norm_1: tensor(84.2660, device='cuda:0') (tensor(108.6245, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.3303 (0.323328177134196) + | > loader_time: 0.0042 (0.00440673033396403) + + + --> TIME: 2025-04-19 00:00:51 -- STEP: 31/142 -- GLOBAL_STEP: 387975 + | > loss_disc: 2.4038524627685547 (2.470482010995188) + | > loss_disc_real_0: 0.09139012545347214 (0.11853264128008197) + | > loss_disc_real_1: 0.20467883348464966 (0.20491950358113936) + | > loss_disc_real_2: 0.2482653558254242 (0.23004234029400733) + | > loss_disc_real_3: 0.23527438938617706 (0.2323073470784772) + | > loss_disc_real_4: 0.24566121399402618 (0.23479445615122396) + | > loss_disc_real_5: 0.26397788524627686 (0.23759410121748525) + | > loss_0: 2.4038524627685547 (2.470482010995188) + | > grad_norm_0: tensor(6.9075, device='cuda:0') (tensor(6.7442, device='cuda:0')) + | > loss_gen: 2.537144899368286 (2.3345394057612276) + | > loss_kl: 1.8872734308242798 (1.885293053042504) + | > loss_feat: 6.67860221862793 (7.048908387461016) + | > loss_mel: 18.229429244995117 (18.08492119081559) + | > loss_duration: 1.463844656944275 (1.4833478773793867) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.796295166015625 (30.83701010673277) + | > grad_norm_1: tensor(85.2460, device='cuda:0') (tensor(95.0953, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.3593 (0.33877305830678633) + | > loader_time: 0.0056 (0.005070040302891885) + + + --> TIME: 2025-04-19 00:01:01 -- STEP: 56/142 -- GLOBAL_STEP: 388000 + | > loss_disc: 2.404515027999878 (2.4607795306614455) + | > loss_disc_real_0: 0.09518406540155411 (0.12112061765843204) + | > loss_disc_real_1: 0.20546036958694458 (0.2054673180516277) + | > loss_disc_real_2: 0.22813428938388824 (0.22936571868402617) + | > loss_disc_real_3: 0.24372075498104095 (0.22825173128928458) + | > loss_disc_real_4: 0.2407158464193344 (0.23196581963981902) + | > loss_disc_real_5: 0.2265426516532898 (0.23438987747899123) + | > loss_0: 2.404515027999878 (2.4607795306614455) + | > grad_norm_0: tensor(3.8292, device='cuda:0') (tensor(6.8307, device='cuda:0')) + | > loss_gen: 2.5122692584991455 (2.349183235849653) + | > loss_kl: 1.859736442565918 (1.9139609251703535) + | > loss_feat: 7.23743200302124 (7.01171794959477) + | > loss_mel: 19.014955520629883 (18.02755675997053) + | > loss_duration: 1.533227801322937 (1.4917677385466437) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.15761947631836 (30.794186728341238) + | > grad_norm_1: tensor(62.1933, device='cuda:0') (tensor(124.1559, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.3865 (0.35449022906167166) + | > loader_time: 0.0057 (0.005605804068701608) + + + --> TIME: 2025-04-19 00:01:11 -- STEP: 81/142 -- GLOBAL_STEP: 388025 + | > loss_disc: 2.7088310718536377 (2.485336695188357) + | > loss_disc_real_0: 0.1272137314081192 (0.12706901330822779) + | > loss_disc_real_1: 0.23766714334487915 (0.20810758570830018) + | > loss_disc_real_2: 0.252297580242157 (0.22960040874687243) + | > loss_disc_real_3: 0.2320350855588913 (0.2306843876470754) + | > loss_disc_real_4: 0.2938913106918335 (0.23423267716978802) + | > loss_disc_real_5: 0.2739463746547699 (0.23558035971205912) + | > loss_0: 2.7088310718536377 (2.485336695188357) + | > grad_norm_0: tensor(8.1529, device='cuda:0') (tensor(7.5151, device='cuda:0')) + | > loss_gen: 2.2117486000061035 (2.337757852342393) + | > loss_kl: 1.8846122026443481 (1.9347013514718892) + | > loss_feat: 5.831217288970947 (6.932655428662712) + | > loss_mel: 15.937097549438477 (17.983226658385476) + | > loss_duration: 1.5276824235916138 (1.502135999408769) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.392358779907227 (30.690477277025764) + | > grad_norm_1: tensor(284.9496, device='cuda:0') (tensor(136.7717, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.4034 (0.36948004181002403) + | > loader_time: 0.0064 (0.005929723198031203) + + + --> TIME: 2025-04-19 00:01:22 -- STEP: 106/142 -- GLOBAL_STEP: 388050 + | > loss_disc: 2.483642101287842 (2.483319984292084) + | > loss_disc_real_0: 0.11568918079137802 (0.1284253375499316) + | > loss_disc_real_1: 0.2061440348625183 (0.20820824099036875) + | > loss_disc_real_2: 0.22218862175941467 (0.22821749057972207) + | > loss_disc_real_3: 0.22028200328350067 (0.22905171732857543) + | > loss_disc_real_4: 0.23026159405708313 (0.23336592260396705) + | > loss_disc_real_5: 0.2407979816198349 (0.23385386885899417) + | > loss_0: 2.483642101287842 (2.483319984292084) + | > grad_norm_0: tensor(12.6272, device='cuda:0') (tensor(8.5136, device='cuda:0')) + | > loss_gen: 2.240020513534546 (2.3278608614543694) + | > loss_kl: 1.9679312705993652 (1.9560204316984933) + | > loss_feat: 6.619781494140625 (6.893557894904658) + | > loss_mel: 16.981340408325195 (17.981213281739432) + | > loss_duration: 1.5338904857635498 (1.5129648604482977) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.34296417236328 (30.67161729200831) + | > grad_norm_1: tensor(333.5871, device='cuda:0') (tensor(170.0953, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.4204 (0.38087378807787625) + | > loader_time: 0.0071 (0.006130162275062419) + + + --> TIME: 2025-04-19 00:01:33 -- STEP: 131/142 -- GLOBAL_STEP: 388075 + | > loss_disc: 2.4811458587646484 (2.482595483765347) + | > loss_disc_real_0: 0.1879749298095703 (0.12965064222803552) + | > loss_disc_real_1: 0.22993452847003937 (0.2095118763337608) + | > loss_disc_real_2: 0.23984476923942566 (0.22757439702066756) + | > loss_disc_real_3: 0.24118675291538239 (0.22920656329347888) + | > loss_disc_real_4: 0.22021684050559998 (0.23230713378382092) + | > loss_disc_real_5: 0.1888052076101303 (0.23275619802129177) + | > loss_0: 2.4811458587646484 (2.482595483765347) + | > grad_norm_0: tensor(23.4097, device='cuda:0') (tensor(10.3940, device='cuda:0')) + | > loss_gen: 2.6104676723480225 (2.3384446515381785) + | > loss_kl: 2.060183048248291 (1.9665569367299554) + | > loss_feat: 7.384632587432861 (6.9113130969855625) + | > loss_mel: 18.614871978759766 (17.991623900318867) + | > loss_duration: 1.5536386966705322 (1.5188221039662837) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.22379684448242 (30.726760645859112) + | > grad_norm_1: tensor(118.4336, device='cuda:0') (tensor(206.7393, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.4606 (0.3949335604223586) + | > loader_time: 0.0079 (0.0063758296820953615) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.2988109588623047 (2.2988109588623047) + | > loss_disc_real_0: 0.12035111337900162 (0.12035111337900162) + | > loss_disc_real_1: 0.2183646559715271 (0.2183646559715271) + | > loss_disc_real_2: 0.2158842384815216 (0.2158842384815216) + | > loss_disc_real_3: 0.1976095736026764 (0.1976095736026764) + | > loss_disc_real_4: 0.16259901225566864 (0.16259901225566864) + | > loss_disc_real_5: 0.20942620933055878 (0.20942620933055878) + | > loss_0: 2.2988109588623047 (2.2988109588623047) + | > loss_gen: 2.3599512577056885 (2.3599512577056885) + | > loss_kl: 2.1076903343200684 (2.1076903343200684) + | > loss_feat: 7.103652000427246 (7.103652000427246) + | > loss_mel: 19.23761558532715 (19.23761558532715) + | > loss_duration: 1.6283493041992188 (1.6283493041992188) + | > loss_1: 32.437255859375 (32.437255859375) + + --> STEP: 1 + | > loss_disc: 2.3832144737243652 (2.3832144737243652) + | > loss_disc_real_0: 0.12420354038476944 (0.12420354038476944) + | > loss_disc_real_1: 0.21728837490081787 (0.21728837490081787) + | > loss_disc_real_2: 0.19073523581027985 (0.19073523581027985) + | > loss_disc_real_3: 0.21880973875522614 (0.21880973875522614) + | > loss_disc_real_4: 0.19860638678073883 (0.19860638678073883) + | > loss_disc_real_5: 0.20273199677467346 (0.20273199677467346) + | > loss_0: 2.3832144737243652 (2.3832144737243652) + | > loss_gen: 2.2118778228759766 (2.2118778228759766) + | > loss_kl: 2.5517632961273193 (2.5517632961273193) + | > loss_feat: 7.010997295379639 (7.010997295379639) + | > loss_mel: 19.2735595703125 (19.2735595703125) + | > loss_duration: 1.6702232360839844 (1.6702232360839844) + | > loss_1: 32.718421936035156 (32.718421936035156) + + --> STEP: 2 + | > loss_disc: 2.440065860748291 (2.411640167236328) + | > loss_disc_real_0: 0.13827070593833923 (0.13123712316155434) + | > loss_disc_real_1: 0.226482555270195 (0.22188546508550644) + | > loss_disc_real_2: 0.20018349587917328 (0.19545936584472656) + | > loss_disc_real_3: 0.20568276941776276 (0.21224625408649445) + | > loss_disc_real_4: 0.19930759072303772 (0.19895698875188828) + | > loss_disc_real_5: 0.19517773389816284 (0.19895486533641815) + | > loss_0: 2.440065860748291 (2.411640167236328) + | > loss_gen: 2.3140335083007812 (2.262955665588379) + | > loss_kl: 2.0201218128204346 (2.285942554473877) + | > loss_feat: 7.780918598175049 (7.395957946777344) + | > loss_mel: 17.92019271850586 (18.59687614440918) + | > loss_duration: 1.5916786193847656 (1.630950927734375) + | > loss_1: 31.62694549560547 (32.17268371582031) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038312673568725586 (+0.0003565549850463867) + | > avg_loss_disc: 2.411640167236328 (-0.07111167907714844) + | > avg_loss_disc_real_0: 0.13123712316155434 (-0.0535694845020771) + | > avg_loss_disc_real_1: 0.22188546508550644 (-0.032425470650196075) + | > avg_loss_disc_real_2: 0.19545936584472656 (-0.03527464717626572) + | > avg_loss_disc_real_3: 0.21224625408649445 (-0.05435675382614136) + | > avg_loss_disc_real_4: 0.19895698875188828 (-0.05062020570039749) + | > avg_loss_disc_real_5: 0.19895486533641815 (-0.03342568874359131) + | > avg_loss_0: 2.411640167236328 (-0.07111167907714844) + | > avg_loss_gen: 2.262955665588379 (-0.33337652683258057) + | > avg_loss_kl: 2.285942554473877 (-0.009341835975646973) + | > avg_loss_feat: 7.395957946777344 (-0.10818004608154297) + | > avg_loss_mel: 18.59687614440918 (-0.33623313903808594) + | > avg_loss_duration: 1.630950927734375 (+0.015448212623596191) + | > avg_loss_1: 32.17268371582031 (-0.7716827392578125) + + + > EPOCH: 62/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:01:40)  + + --> TIME: 2025-04-19 00:01:45 -- STEP: 14/142 -- GLOBAL_STEP: 388100 + | > loss_disc: 2.36684513092041 (2.3960621186665128) + | > loss_disc_real_0: 0.09061378985643387 (0.12177511251398496) + | > loss_disc_real_1: 0.1944630742073059 (0.20929785179240362) + | > loss_disc_real_2: 0.20382261276245117 (0.22036653757095337) + | > loss_disc_real_3: 0.18588876724243164 (0.2255798535687583) + | > loss_disc_real_4: 0.21941697597503662 (0.21912544859307154) + | > loss_disc_real_5: 0.1971762329339981 (0.22730545273848943) + | > loss_0: 2.36684513092041 (2.3960621186665128) + | > grad_norm_0: tensor(15.3487, device='cuda:0') (tensor(18.6308, device='cuda:0')) + | > loss_gen: 2.222322940826416 (2.4302446160997664) + | > loss_kl: 1.8980777263641357 (1.9549359934670585) + | > loss_feat: 7.628726005554199 (7.3667261600494385) + | > loss_mel: 19.319000244140625 (18.370013781956263) + | > loss_duration: 1.4795351028442383 (1.4782703263419015) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.54766082763672 (31.60019043513707) + | > grad_norm_1: tensor(454.7229, device='cuda:0') (tensor(401.9582, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.3307 (0.326960631779262) + | > loader_time: 0.005 (0.004580770220075335) + + + --> TIME: 2025-04-19 00:01:54 -- STEP: 39/142 -- GLOBAL_STEP: 388125 + | > loss_disc: 2.3556511402130127 (2.4312825447473765) + | > loss_disc_real_0: 0.07478728145360947 (0.11763599094672081) + | > loss_disc_real_1: 0.2063397914171219 (0.2083048319969422) + | > loss_disc_real_2: 0.22941593825817108 (0.22673968817943183) + | > loss_disc_real_3: 0.2519053518772125 (0.23001201336200422) + | > loss_disc_real_4: 0.2089628428220749 (0.22510582705338797) + | > loss_disc_real_5: 0.21742963790893555 (0.23073743971494529) + | > loss_0: 2.3556511402130127 (2.4312825447473765) + | > grad_norm_0: tensor(9.6126, device='cuda:0') (tensor(15.9605, device='cuda:0')) + | > loss_gen: 2.353288173675537 (2.3897475401560464) + | > loss_kl: 2.0500969886779785 (1.9410209839160626) + | > loss_feat: 7.1322197914123535 (7.151579477848151) + | > loss_mel: 17.493213653564453 (18.098586791600937) + | > loss_duration: 1.5125378370285034 (1.4858753161552625) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.54135513305664 (31.06680987431453) + | > grad_norm_1: tensor(280.0984, device='cuda:0') (tensor(361.0155, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.3563 (0.34119179309942776) + | > loader_time: 0.0061 (0.005046832255828075) + + + --> TIME: 2025-04-19 00:02:04 -- STEP: 64/142 -- GLOBAL_STEP: 388150 + | > loss_disc: 2.5769574642181396 (2.4493892081081867) + | > loss_disc_real_0: 0.14969472587108612 (0.12156412424519658) + | > loss_disc_real_1: 0.23084120452404022 (0.20748190209269524) + | > loss_disc_real_2: 0.24517510831356049 (0.22625638055615127) + | > loss_disc_real_3: 0.2567451596260071 (0.22923404909670353) + | > loss_disc_real_4: 0.2329111248254776 (0.22776496573351324) + | > loss_disc_real_5: 0.23177656531333923 (0.23266782658174634) + | > loss_0: 2.5769574642181396 (2.4493892081081867) + | > grad_norm_0: tensor(10.1054, device='cuda:0') (tensor(12.5519, device='cuda:0')) + | > loss_gen: 2.244401216506958 (2.364256523549557) + | > loss_kl: 2.0572967529296875 (1.9383422136306763) + | > loss_feat: 6.328872203826904 (7.028493769466877) + | > loss_mel: 17.836708068847656 (17.970377877354622) + | > loss_duration: 1.5517551898956299 (1.4967361129820347) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.01903533935547 (30.798206567764282) + | > grad_norm_1: tensor(200.6311, device='cuda:0') (tensor(273.9710, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.3859 (0.3538841828703879) + | > loader_time: 0.0063 (0.005449615418910979) + + + --> TIME: 2025-04-19 00:02:14 -- STEP: 89/142 -- GLOBAL_STEP: 388175 + | > loss_disc: 2.435622453689575 (2.4605304659082647) + | > loss_disc_real_0: 0.09576952457427979 (0.1231650137499477) + | > loss_disc_real_1: 0.26119232177734375 (0.20885752995362447) + | > loss_disc_real_2: 0.2301427125930786 (0.22695124685094598) + | > loss_disc_real_3: 0.21572303771972656 (0.22878806025124668) + | > loss_disc_real_4: 0.21611225605010986 (0.22801396100038893) + | > loss_disc_real_5: 0.26121240854263306 (0.23425846605488423) + | > loss_0: 2.435622453689575 (2.4605304659082647) + | > grad_norm_0: tensor(4.8314, device='cuda:0') (tensor(10.7804, device='cuda:0')) + | > loss_gen: 2.3069262504577637 (2.338656379935448) + | > loss_kl: 2.119107484817505 (1.9571324252010731) + | > loss_feat: 7.200585842132568 (6.957665041591346) + | > loss_mel: 17.78846549987793 (17.91793511422832) + | > loss_duration: 1.5048139095306396 (1.50784688987089) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.919898986816406 (30.67923582269904) + | > grad_norm_1: tensor(80.8763, device='cuda:0') (tensor(232.1478, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.4046 (0.3657686094219764) + | > loader_time: 0.007 (0.00585003113478757) + + + --> TIME: 2025-04-19 00:02:25 -- STEP: 114/142 -- GLOBAL_STEP: 388200 + | > loss_disc: 2.5501365661621094 (2.4637289737400256) + | > loss_disc_real_0: 0.1024685725569725 (0.12513182423355287) + | > loss_disc_real_1: 0.1912745237350464 (0.20747105711907673) + | > loss_disc_real_2: 0.211077481508255 (0.22674528182598583) + | > loss_disc_real_3: 0.20589731633663177 (0.22893643013217993) + | > loss_disc_real_4: 0.21377281844615936 (0.22908017339936473) + | > loss_disc_real_5: 0.20810887217521667 (0.2337466941068047) + | > loss_0: 2.5501365661621094 (2.4637289737400256) + | > grad_norm_0: tensor(5.7154, device='cuda:0') (tensor(9.8987, device='cuda:0')) + | > loss_gen: 2.088306188583374 (2.3354954175781786) + | > loss_kl: 1.9631884098052979 (1.962389021589045) + | > loss_feat: 6.226520538330078 (6.933854956375926) + | > loss_mel: 18.233430862426758 (17.94042909354493) + | > loss_duration: 1.5433099269866943 (1.5150584672626697) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05475616455078 (30.687226998178584) + | > grad_norm_1: tensor(50.7396, device='cuda:0') (tensor(213.6423, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.4256 (0.37718856962103586) + | > loader_time: 0.0069 (0.006116603550158049) + + + --> TIME: 2025-04-19 00:02:36 -- STEP: 139/142 -- GLOBAL_STEP: 388225 + | > loss_disc: 2.383604049682617 (2.4655248844366278) + | > loss_disc_real_0: 0.0959811881184578 (0.1261116707925316) + | > loss_disc_real_1: 0.2012866735458374 (0.2070732096330725) + | > loss_disc_real_2: 0.1904839128255844 (0.2269678978825645) + | > loss_disc_real_3: 0.21160806715488434 (0.2284933106719161) + | > loss_disc_real_4: 0.19512341916561127 (0.2293845637024735) + | > loss_disc_real_5: 0.1755026876926422 (0.23347905321086912) + | > loss_0: 2.383604049682617 (2.4655248844366278) + | > grad_norm_0: tensor(4.3930, device='cuda:0') (tensor(9.2890, device='cuda:0')) + | > loss_gen: 2.321296215057373 (2.3346212716411348) + | > loss_kl: 1.9741408824920654 (1.967263086236638) + | > loss_feat: 7.611932754516602 (6.953879726876458) + | > loss_mel: 18.764501571655273 (17.97663473911422) + | > loss_duration: 1.556147813796997 (1.5209135597558332) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.22801971435547 (30.753312433366297) + | > grad_norm_1: tensor(38.0307, device='cuda:0') (tensor(192.6953, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.4612 (0.39053951750556337) + | > loader_time: 0.0072 (0.006371269980780512) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.445397138595581 (2.445397138595581) + | > loss_disc_real_0: 0.11365451663732529 (0.11365451663732529) + | > loss_disc_real_1: 0.27811580896377563 (0.27811580896377563) + | > loss_disc_real_2: 0.27049776911735535 (0.27049776911735535) + | > loss_disc_real_3: 0.30265137553215027 (0.30265137553215027) + | > loss_disc_real_4: 0.22951717674732208 (0.22951717674732208) + | > loss_disc_real_5: 0.20643702149391174 (0.20643702149391174) + | > loss_0: 2.445397138595581 (2.445397138595581) + | > loss_gen: 2.50162672996521 (2.50162672996521) + | > loss_kl: 2.141282796859741 (2.141282796859741) + | > loss_feat: 7.0917067527771 (7.0917067527771) + | > loss_mel: 18.511323928833008 (18.511323928833008) + | > loss_duration: 1.6336109638214111 (1.6336109638214111) + | > loss_1: 31.87955093383789 (31.87955093383789) + + --> STEP: 1 + | > loss_disc: 2.4416823387145996 (2.4416823387145996) + | > loss_disc_real_0: 0.15376944839954376 (0.15376944839954376) + | > loss_disc_real_1: 0.23106445372104645 (0.23106445372104645) + | > loss_disc_real_2: 0.26905250549316406 (0.26905250549316406) + | > loss_disc_real_3: 0.2590015232563019 (0.2590015232563019) + | > loss_disc_real_4: 0.23458270728588104 (0.23458270728588104) + | > loss_disc_real_5: 0.21895644068717957 (0.21895644068717957) + | > loss_0: 2.4416823387145996 (2.4416823387145996) + | > loss_gen: 2.486178398132324 (2.486178398132324) + | > loss_kl: 2.395949602127075 (2.395949602127075) + | > loss_feat: 7.304386615753174 (7.304386615753174) + | > loss_mel: 17.770559310913086 (17.770559310913086) + | > loss_duration: 1.6581945419311523 (1.6581945419311523) + | > loss_1: 31.61526870727539 (31.61526870727539) + + --> STEP: 2 + | > loss_disc: 2.5663821697235107 (2.504032254219055) + | > loss_disc_real_0: 0.14251500368118286 (0.1481422260403633) + | > loss_disc_real_1: 0.22771844267845154 (0.229391448199749) + | > loss_disc_real_2: 0.26717692613601685 (0.26811471581459045) + | > loss_disc_real_3: 0.24115270376205444 (0.25007711350917816) + | > loss_disc_real_4: 0.2505781650543213 (0.24258043617010117) + | > loss_disc_real_5: 0.2389426976442337 (0.22894956916570663) + | > loss_0: 2.5663821697235107 (2.504032254219055) + | > loss_gen: 2.299588918685913 (2.3928836584091187) + | > loss_kl: 1.9598082304000854 (2.1778789162635803) + | > loss_feat: 7.222017765045166 (7.26320219039917) + | > loss_mel: 18.202335357666016 (17.98644733428955) + | > loss_duration: 1.5852328538894653 (1.6217136979103088) + | > loss_1: 31.268983840942383 (31.442126274108887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00348818302154541 (-0.00034308433532714844) + | > avg_loss_disc: 2.504032254219055 (+0.09239208698272705) + | > avg_loss_disc_real_0: 0.1481422260403633 (+0.016905102878808975) + | > avg_loss_disc_real_1: 0.229391448199749 (+0.007505983114242554) + | > avg_loss_disc_real_2: 0.26811471581459045 (+0.07265534996986389) + | > avg_loss_disc_real_3: 0.25007711350917816 (+0.037830859422683716) + | > avg_loss_disc_real_4: 0.24258043617010117 (+0.04362344741821289) + | > avg_loss_disc_real_5: 0.22894956916570663 (+0.029994703829288483) + | > avg_loss_0: 2.504032254219055 (+0.09239208698272705) + | > avg_loss_gen: 2.3928836584091187 (+0.12992799282073975) + | > avg_loss_kl: 2.1778789162635803 (-0.10806363821029663) + | > avg_loss_feat: 7.26320219039917 (-0.13275575637817383) + | > avg_loss_mel: 17.98644733428955 (-0.6104288101196289) + | > avg_loss_duration: 1.6217136979103088 (-0.009237229824066162) + | > avg_loss_1: 31.442126274108887 (-0.7305574417114258) + + + > EPOCH: 63/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:02:39)  + + --> TIME: 2025-04-19 00:02:47 -- STEP: 22/142 -- GLOBAL_STEP: 388250 + | > loss_disc: 2.4629108905792236 (2.4870629635724155) + | > loss_disc_real_0: 0.15409345924854279 (0.13751915063370357) + | > loss_disc_real_1: 0.20237509906291962 (0.20403312146663666) + | > loss_disc_real_2: 0.21553362905979156 (0.22375334934754806) + | > loss_disc_real_3: 0.2559300661087036 (0.23327589373696933) + | > loss_disc_real_4: 0.21749193966388702 (0.2310084653171626) + | > loss_disc_real_5: 0.22474405169487 (0.24028133804147894) + | > loss_0: 2.4629108905792236 (2.4870629635724155) + | > grad_norm_0: tensor(8.6882, device='cuda:0') (tensor(10.7878, device='cuda:0')) + | > loss_gen: 2.3195180892944336 (2.3546681729230015) + | > loss_kl: 1.913978934288025 (1.9020864313299006) + | > loss_feat: 7.420347213745117 (7.123194217681885) + | > loss_mel: 18.986772537231445 (18.22212366624312) + | > loss_duration: 1.4455018043518066 (1.4758395823565396) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.08612060546875 (31.077912243929777) + | > grad_norm_1: tensor(174.4857, device='cuda:0') (tensor(246.7212, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.3525 (0.33350975946946576) + | > loader_time: 0.0054 (0.005017063834450461) + + + --> TIME: 2025-04-19 00:02:57 -- STEP: 47/142 -- GLOBAL_STEP: 388275 + | > loss_disc: 2.5159480571746826 (2.469189963442214) + | > loss_disc_real_0: 0.11839920282363892 (0.13147222282404591) + | > loss_disc_real_1: 0.20216582715511322 (0.20396617689031235) + | > loss_disc_real_2: 0.2284553050994873 (0.2232913463673693) + | > loss_disc_real_3: 0.23463554680347443 (0.22984992411542446) + | > loss_disc_real_4: 0.21400116384029388 (0.22743394273392697) + | > loss_disc_real_5: 0.20757538080215454 (0.23382127189889867) + | > loss_0: 2.5159480571746826 (2.469189963442214) + | > grad_norm_0: tensor(9.4324, device='cuda:0') (tensor(9.7846, device='cuda:0')) + | > loss_gen: 2.314340591430664 (2.3427801233656855) + | > loss_kl: 1.8924123048782349 (1.907700777053833) + | > loss_feat: 6.6493425369262695 (7.022650201269921) + | > loss_mel: 17.1686954498291 (17.906503291840245) + | > loss_duration: 1.462397575378418 (1.4863671597014083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.487186431884766 (30.66600150250374) + | > grad_norm_1: tensor(182.0612, device='cuda:0') (tensor(209.5861, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.3726 (0.3516275223265304) + | > loader_time: 0.0058 (0.005434086982240068) + + + --> TIME: 2025-04-19 00:03:07 -- STEP: 72/142 -- GLOBAL_STEP: 388300 + | > loss_disc: 2.6047348976135254 (2.4785382350285854) + | > loss_disc_real_0: 0.13228456676006317 (0.1347117959004309) + | > loss_disc_real_1: 0.21473363041877747 (0.20577401667833328) + | > loss_disc_real_2: 0.23279903829097748 (0.22620567224091953) + | > loss_disc_real_3: 0.21393144130706787 (0.23024056396550602) + | > loss_disc_real_4: 0.24924354255199432 (0.22987827700045374) + | > loss_disc_real_5: 0.2410489171743393 (0.2345052104857233) + | > loss_0: 2.6047348976135254 (2.4785382350285854) + | > grad_norm_0: tensor(43.3488, device='cuda:0') (tensor(11.6862, device='cuda:0')) + | > loss_gen: 2.143240451812744 (2.3544584115346265) + | > loss_kl: 1.968569278717041 (1.9241779893636703) + | > loss_feat: 7.419637203216553 (6.991578724649218) + | > loss_mel: 18.484111785888672 (17.916886369387303) + | > loss_duration: 1.4901490211486816 (1.4967200060685475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.505706787109375 (30.68382149272495) + | > grad_norm_1: tensor(418.9798, device='cuda:0') (tensor(241.5204, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.4046 (0.36527979373931896) + | > loader_time: 0.0063 (0.005758017301559448) + + + --> TIME: 2025-04-19 00:03:17 -- STEP: 97/142 -- GLOBAL_STEP: 388325 + | > loss_disc: 2.639336109161377 (2.4791508310848935) + | > loss_disc_real_0: 0.10886120796203613 (0.13282936196966275) + | > loss_disc_real_1: 0.2655572295188904 (0.2067098287270241) + | > loss_disc_real_2: 0.23586894571781158 (0.2268140520324412) + | > loss_disc_real_3: 0.24475300312042236 (0.2287920161928098) + | > loss_disc_real_4: 0.26822102069854736 (0.22918263707578795) + | > loss_disc_real_5: 0.25465112924575806 (0.23272523821629199) + | > loss_0: 2.639336109161377 (2.4791508310848935) + | > grad_norm_0: tensor(24.4208, device='cuda:0') (tensor(13.8027, device='cuda:0')) + | > loss_gen: 2.148148536682129 (2.3694076366031283) + | > loss_kl: 2.0314245223999023 (1.934227461667405) + | > loss_feat: 5.57210636138916 (6.9889854696608085) + | > loss_mel: 15.706059455871582 (17.934620100198327) + | > loss_duration: 1.535238265991211 (1.5088950235819085) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.99297523498535 (30.736135581105025) + | > grad_norm_1: tensor(350.4926, device='cuda:0') (tensor(273.5117, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.4203 (0.37727998704025434) + | > loader_time: 0.0067 (0.005998837579156935) + + + --> TIME: 2025-04-19 00:03:28 -- STEP: 122/142 -- GLOBAL_STEP: 388350 + | > loss_disc: 2.620079755783081 (2.4830208410982255) + | > loss_disc_real_0: 0.1517314910888672 (0.13381815323087048) + | > loss_disc_real_1: 0.2513817548751831 (0.20729177042109065) + | > loss_disc_real_2: 0.24500617384910583 (0.22687016867223334) + | > loss_disc_real_3: 0.21923883259296417 (0.2292534420724775) + | > loss_disc_real_4: 0.26628389954566956 (0.22982653657921026) + | > loss_disc_real_5: 0.2731642425060272 (0.23389860499100606) + | > loss_0: 2.620079755783081 (2.4830208410982255) + | > grad_norm_0: tensor(4.9558, device='cuda:0') (tensor(13.8284, device='cuda:0')) + | > loss_gen: 2.242633819580078 (2.354851640638757) + | > loss_kl: 1.9689189195632935 (1.9446607173466293) + | > loss_feat: 6.570094585418701 (6.911865335996033) + | > loss_mel: 17.544448852539062 (17.842527498964397) + | > loss_duration: 1.5195095539093018 (1.5151400917866196) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.845605850219727 (30.569045098101505) + | > grad_norm_1: tensor(50.6432, device='cuda:0') (tensor(269.5287, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.4728 (0.38994469603554155) + | > loader_time: 0.0085 (0.006228267169389568) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.525084972381592 (2.525084972381592) + | > loss_disc_real_0: 0.17231498658657074 (0.17231498658657074) + | > loss_disc_real_1: 0.22236910462379456 (0.22236910462379456) + | > loss_disc_real_2: 0.22681564092636108 (0.22681564092636108) + | > loss_disc_real_3: 0.2606581449508667 (0.2606581449508667) + | > loss_disc_real_4: 0.25048011541366577 (0.25048011541366577) + | > loss_disc_real_5: 0.24717751145362854 (0.24717751145362854) + | > loss_0: 2.525084972381592 (2.525084972381592) + | > loss_gen: 2.3275060653686523 (2.3275060653686523) + | > loss_kl: 2.1425111293792725 (2.1425111293792725) + | > loss_feat: 6.59370231628418 (6.59370231628418) + | > loss_mel: 17.562522888183594 (17.562522888183594) + | > loss_duration: 1.612522840499878 (1.612522840499878) + | > loss_1: 30.238765716552734 (30.238765716552734) + + --> STEP: 1 + | > loss_disc: 2.472990036010742 (2.472990036010742) + | > loss_disc_real_0: 0.11165424436330795 (0.11165424436330795) + | > loss_disc_real_1: 0.1991765797138214 (0.1991765797138214) + | > loss_disc_real_2: 0.2650406062602997 (0.2650406062602997) + | > loss_disc_real_3: 0.24283674359321594 (0.24283674359321594) + | > loss_disc_real_4: 0.26253440976142883 (0.26253440976142883) + | > loss_disc_real_5: 0.21128317713737488 (0.21128317713737488) + | > loss_0: 2.472990036010742 (2.472990036010742) + | > loss_gen: 2.3169870376586914 (2.3169870376586914) + | > loss_kl: 2.4450485706329346 (2.4450485706329346) + | > loss_feat: 6.889432907104492 (6.889432907104492) + | > loss_mel: 19.45990753173828 (19.45990753173828) + | > loss_duration: 1.6444414854049683 (1.6444414854049683) + | > loss_1: 32.75581359863281 (32.75581359863281) + + --> STEP: 2 + | > loss_disc: 2.4899868965148926 (2.4814884662628174) + | > loss_disc_real_0: 0.15182259678840637 (0.13173842057585716) + | > loss_disc_real_1: 0.23790742456912994 (0.21854200214147568) + | > loss_disc_real_2: 0.19921241700649261 (0.23212651163339615) + | > loss_disc_real_3: 0.23892730474472046 (0.2408820241689682) + | > loss_disc_real_4: 0.22011014819145203 (0.24132227897644043) + | > loss_disc_real_5: 0.22510412335395813 (0.2181936502456665) + | > loss_0: 2.4899868965148926 (2.4814884662628174) + | > loss_gen: 2.2122397422790527 (2.264613389968872) + | > loss_kl: 2.1772539615631104 (2.3111512660980225) + | > loss_feat: 6.254547119140625 (6.571990013122559) + | > loss_mel: 16.603225708007812 (18.031566619873047) + | > loss_duration: 1.5949006080627441 (1.6196710467338562) + | > loss_1: 28.842166900634766 (30.79899024963379) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00327301025390625 (-0.00021517276763916016) + | > avg_loss_disc: 2.4814884662628174 (-0.022543787956237793) + | > avg_loss_disc_real_0: 0.13173842057585716 (-0.01640380546450615) + | > avg_loss_disc_real_1: 0.21854200214147568 (-0.010849446058273315) + | > avg_loss_disc_real_2: 0.23212651163339615 (-0.035988204181194305) + | > avg_loss_disc_real_3: 0.2408820241689682 (-0.009195089340209961) + | > avg_loss_disc_real_4: 0.24132227897644043 (-0.001258157193660736) + | > avg_loss_disc_real_5: 0.2181936502456665 (-0.01075591892004013) + | > avg_loss_0: 2.4814884662628174 (-0.022543787956237793) + | > avg_loss_gen: 2.264613389968872 (-0.12827026844024658) + | > avg_loss_kl: 2.3111512660980225 (+0.13327234983444214) + | > avg_loss_feat: 6.571990013122559 (-0.6912121772766113) + | > avg_loss_mel: 18.031566619873047 (+0.045119285583496094) + | > avg_loss_duration: 1.6196710467338562 (-0.0020426511764526367) + | > avg_loss_1: 30.79899024963379 (-0.6431360244750977) + + + > EPOCH: 64/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:03:39)  + + --> TIME: 2025-04-19 00:03:42 -- STEP: 5/142 -- GLOBAL_STEP: 388375 + | > loss_disc: 2.4802324771881104 (2.4736495494842528) + | > loss_disc_real_0: 0.18222235143184662 (0.12555502951145173) + | > loss_disc_real_1: 0.19270296394824982 (0.1964678704738617) + | > loss_disc_real_2: 0.21315374970436096 (0.245859757065773) + | > loss_disc_real_3: 0.23680393397808075 (0.21817429959774018) + | > loss_disc_real_4: 0.24489234387874603 (0.24358855187892914) + | > loss_disc_real_5: 0.23755356669425964 (0.2226064234972) + | > loss_0: 2.4802324771881104 (2.4736495494842528) + | > grad_norm_0: tensor(9.6923, device='cuda:0') (tensor(7.2975, device='cuda:0')) + | > loss_gen: 2.310358762741089 (2.3080230712890626) + | > loss_kl: 1.9156513214111328 (1.9390078783035278) + | > loss_feat: 6.520282745361328 (6.919904041290283) + | > loss_mel: 16.66480827331543 (17.87152099609375) + | > loss_duration: 1.4816482067108154 (1.480794858932495) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.89274787902832 (30.519249725341798) + | > grad_norm_1: tensor(125.8661, device='cuda:0') (tensor(131.9995, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.3281 (0.3274094581604004) + | > loader_time: 0.0041 (0.004186248779296875) + + + --> TIME: 2025-04-19 00:03:50 -- STEP: 30/142 -- GLOBAL_STEP: 388400 + | > loss_disc: 2.53436541557312 (2.479298973083496) + | > loss_disc_real_0: 0.11945851147174835 (0.11830957482258479) + | > loss_disc_real_1: 0.21716953814029694 (0.2097451274593671) + | > loss_disc_real_2: 0.21425758302211761 (0.23282912423213323) + | > loss_disc_real_3: 0.23241131007671356 (0.23350764016310374) + | > loss_disc_real_4: 0.22637224197387695 (0.23470116207997005) + | > loss_disc_real_5: 0.25830864906311035 (0.22977827489376068) + | > loss_0: 2.53436541557312 (2.479298973083496) + | > grad_norm_0: tensor(8.8754, device='cuda:0') (tensor(10.5797, device='cuda:0')) + | > loss_gen: 2.2415432929992676 (2.3409809112548827) + | > loss_kl: 1.8979414701461792 (1.9323927680651347) + | > loss_feat: 7.6769256591796875 (7.102181657155355) + | > loss_mel: 18.904003143310547 (17.901369539896645) + | > loss_duration: 1.486350655555725 (1.480725852648417) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.206764221191406 (30.7576504389445) + | > grad_norm_1: tensor(230.9135, device='cuda:0') (tensor(256.9577, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.3673 (0.34019282658894856) + | > loader_time: 0.0054 (0.005071496963500977) + + + --> TIME: 2025-04-19 00:04:00 -- STEP: 55/142 -- GLOBAL_STEP: 388425 + | > loss_disc: 2.4382545948028564 (2.4626580758528283) + | > loss_disc_real_0: 0.12850460410118103 (0.119608550992879) + | > loss_disc_real_1: 0.2693656384944916 (0.20824812027541073) + | > loss_disc_real_2: 0.20996184647083282 (0.22881475687026978) + | > loss_disc_real_3: 0.2177884578704834 (0.22995045428926295) + | > loss_disc_real_4: 0.21055234968662262 (0.23082488775253296) + | > loss_disc_real_5: 0.22139765322208405 (0.2320179429921237) + | > loss_0: 2.4382545948028564 (2.4626580758528283) + | > grad_norm_0: tensor(5.4411, device='cuda:0') (tensor(10.2943, device='cuda:0')) + | > loss_gen: 2.289877414703369 (2.3563037568872627) + | > loss_kl: 2.066483974456787 (1.945994689247825) + | > loss_feat: 6.880027770996094 (7.138899187608199) + | > loss_mel: 17.564308166503906 (18.083818539706137) + | > loss_duration: 1.549881935119629 (1.4855445059863004) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35057830810547 (31.010560365156692) + | > grad_norm_1: tensor(40.4090, device='cuda:0') (tensor(243.4809, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.384 (0.35466628941622647) + | > loader_time: 0.0063 (0.005483206835660067) + + + --> TIME: 2025-04-19 00:04:10 -- STEP: 80/142 -- GLOBAL_STEP: 388450 + | > loss_disc: 2.4498302936553955 (2.4661648303270343) + | > loss_disc_real_0: 0.10940628498792648 (0.12204035734757782) + | > loss_disc_real_1: 0.2622796595096588 (0.20836198832839728) + | > loss_disc_real_2: 0.2348853200674057 (0.22858330346643924) + | > loss_disc_real_3: 0.19924722611904144 (0.23089931178838013) + | > loss_disc_real_4: 0.19132472574710846 (0.23040022160857915) + | > loss_disc_real_5: 0.20803707838058472 (0.23075736481696368) + | > loss_0: 2.4498302936553955 (2.4661648303270343) + | > grad_norm_0: tensor(8.1437, device='cuda:0') (tensor(9.5540, device='cuda:0')) + | > loss_gen: 2.430623769760132 (2.353846997022629) + | > loss_kl: 1.988619327545166 (1.9606633648276328) + | > loss_feat: 7.01216459274292 (7.116941827535629) + | > loss_mel: 18.355072021484375 (18.048608732223506) + | > loss_duration: 1.5299708843231201 (1.4972240298986437) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.316450119018555 (30.977284717559815) + | > grad_norm_1: tensor(183.1724, device='cuda:0') (tensor(219.9621, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.4244 (0.3675945192575455) + | > loader_time: 0.0069 (0.005884453654289245) + + + --> TIME: 2025-04-19 00:04:21 -- STEP: 105/142 -- GLOBAL_STEP: 388475 + | > loss_disc: 2.454502582550049 (2.469165847415017) + | > loss_disc_real_0: 0.10374028980731964 (0.1244232904343378) + | > loss_disc_real_1: 0.18724583089351654 (0.20976203495547885) + | > loss_disc_real_2: 0.2105909287929535 (0.22698544987610408) + | > loss_disc_real_3: 0.1992022693157196 (0.2305160328036263) + | > loss_disc_real_4: 0.22550362348556519 (0.22985786823999313) + | > loss_disc_real_5: 0.21368536353111267 (0.23144623779115223) + | > loss_0: 2.454502582550049 (2.469165847415017) + | > grad_norm_0: tensor(20.6641, device='cuda:0') (tensor(9.9885, device='cuda:0')) + | > loss_gen: 2.2583608627319336 (2.33955283846174) + | > loss_kl: 1.9664312601089478 (1.9728501626423427) + | > loss_feat: 7.861141204833984 (7.040017763773601) + | > loss_mel: 18.29386329650879 (18.041023781186052) + | > loss_duration: 1.5573902130126953 (1.5081524599166143) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.93718910217285 (30.90159683227539) + | > grad_norm_1: tensor(479.6290, device='cuda:0') (tensor(222.6798, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.4207 (0.3797510442279634) + | > loader_time: 0.0072 (0.00615855852762858) + + + --> TIME: 2025-04-19 00:04:32 -- STEP: 130/142 -- GLOBAL_STEP: 388500 + | > loss_disc: 2.4466712474823 (2.474339543856108) + | > loss_disc_real_0: 0.13319556415081024 (0.1265825465321541) + | > loss_disc_real_1: 0.17241670191287994 (0.20930913044856145) + | > loss_disc_real_2: 0.23493079841136932 (0.22654446753171775) + | > loss_disc_real_3: 0.22684122622013092 (0.23003128480452759) + | > loss_disc_real_4: 0.2359330803155899 (0.229746009180179) + | > loss_disc_real_5: 0.2197377234697342 (0.23184650058929737) + | > loss_0: 2.4466712474823 (2.474339543856108) + | > grad_norm_0: tensor(7.2394, device='cuda:0') (tensor(10.0468, device='cuda:0')) + | > loss_gen: 2.346301555633545 (2.330172162789565) + | > loss_kl: 1.9470748901367188 (1.9700949283746572) + | > loss_feat: 6.464272499084473 (6.958246051348173) + | > loss_mel: 17.50059700012207 (17.905716521923353) + | > loss_duration: 1.5264068841934204 (1.5146880314900324) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.78465461730957 (30.678917518028847) + | > grad_norm_1: tensor(170.0481, device='cuda:0') (tensor(224.5918, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.4597 (0.391562085885268) + | > loader_time: 0.0076 (0.00640419079707219) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.686525344848633 (2.686525344848633) + | > loss_disc_real_0: 0.15417343378067017 (0.15417343378067017) + | > loss_disc_real_1: 0.25685879588127136 (0.25685879588127136) + | > loss_disc_real_2: 0.24557377398014069 (0.24557377398014069) + | > loss_disc_real_3: 0.28988757729530334 (0.28988757729530334) + | > loss_disc_real_4: 0.2804318368434906 (0.2804318368434906) + | > loss_disc_real_5: 0.2886473834514618 (0.2886473834514618) + | > loss_0: 2.686525344848633 (2.686525344848633) + | > loss_gen: 2.244826078414917 (2.244826078414917) + | > loss_kl: 1.980779767036438 (1.980779767036438) + | > loss_feat: 5.68782901763916 (5.68782901763916) + | > loss_mel: 17.19980812072754 (17.19980812072754) + | > loss_duration: 1.6158990859985352 (1.6158990859985352) + | > loss_1: 28.729141235351562 (28.729141235351562) + + --> STEP: 1 + | > loss_disc: 2.495100498199463 (2.495100498199463) + | > loss_disc_real_0: 0.12214427441358566 (0.12214427441358566) + | > loss_disc_real_1: 0.21073344349861145 (0.21073344349861145) + | > loss_disc_real_2: 0.23943570256233215 (0.23943570256233215) + | > loss_disc_real_3: 0.26111212372779846 (0.26111212372779846) + | > loss_disc_real_4: 0.24107208847999573 (0.24107208847999573) + | > loss_disc_real_5: 0.23427362740039825 (0.23427362740039825) + | > loss_0: 2.495100498199463 (2.495100498199463) + | > loss_gen: 2.2128138542175293 (2.2128138542175293) + | > loss_kl: 2.2101902961730957 (2.2101902961730957) + | > loss_feat: 6.438186168670654 (6.438186168670654) + | > loss_mel: 18.293054580688477 (18.293054580688477) + | > loss_duration: 1.6695835590362549 (1.6695835590362549) + | > loss_1: 30.823829650878906 (30.823829650878906) + + --> STEP: 2 + | > loss_disc: 2.444934844970703 (2.470017671585083) + | > loss_disc_real_0: 0.13047382235527039 (0.12630904838442802) + | > loss_disc_real_1: 0.21454490721225739 (0.21263917535543442) + | > loss_disc_real_2: 0.22280241549015045 (0.2311190590262413) + | > loss_disc_real_3: 0.21643158793449402 (0.23877185583114624) + | > loss_disc_real_4: 0.2581217586994171 (0.24959692358970642) + | > loss_disc_real_5: 0.2575087547302246 (0.24589119106531143) + | > loss_0: 2.444934844970703 (2.470017671585083) + | > loss_gen: 2.3508124351501465 (2.281813144683838) + | > loss_kl: 2.138028144836426 (2.1741092205047607) + | > loss_feat: 7.3995680809021 (6.918877124786377) + | > loss_mel: 19.458642959594727 (18.8758487701416) + | > loss_duration: 1.6037721633911133 (1.636677861213684) + | > loss_1: 32.95082473754883 (31.887327194213867) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003492593765258789 (+0.00021958351135253906) + | > avg_loss_disc: 2.470017671585083 (-0.011470794677734375) + | > avg_loss_disc_real_0: 0.12630904838442802 (-0.005429372191429138) + | > avg_loss_disc_real_1: 0.21263917535543442 (-0.00590282678604126) + | > avg_loss_disc_real_2: 0.2311190590262413 (-0.0010074526071548462) + | > avg_loss_disc_real_3: 0.23877185583114624 (-0.0021101683378219604) + | > avg_loss_disc_real_4: 0.24959692358970642 (+0.008274644613265991) + | > avg_loss_disc_real_5: 0.24589119106531143 (+0.027697540819644928) + | > avg_loss_0: 2.470017671585083 (-0.011470794677734375) + | > avg_loss_gen: 2.281813144683838 (+0.01719975471496582) + | > avg_loss_kl: 2.1741092205047607 (-0.13704204559326172) + | > avg_loss_feat: 6.918877124786377 (+0.34688711166381836) + | > avg_loss_mel: 18.8758487701416 (+0.8442821502685547) + | > avg_loss_duration: 1.636677861213684 (+0.01700681447982788) + | > avg_loss_1: 31.887327194213867 (+1.0883369445800781) + + + > EPOCH: 65/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:04:39)  + + --> TIME: 2025-04-19 00:04:44 -- STEP: 13/142 -- GLOBAL_STEP: 388525 + | > loss_disc: 2.5630173683166504 (2.4500638521634617) + | > loss_disc_real_0: 0.11542897671461105 (0.11973861089119545) + | > loss_disc_real_1: 0.21215753257274628 (0.20212086805930504) + | > loss_disc_real_2: 0.2600455582141876 (0.235990509390831) + | > loss_disc_real_3: 0.23389752209186554 (0.2347735476035338) + | > loss_disc_real_4: 0.23980724811553955 (0.23499462123100573) + | > loss_disc_real_5: 0.2167762964963913 (0.24257606611802027) + | > loss_0: 2.5630173683166504 (2.4500638521634617) + | > grad_norm_0: tensor(9.0680, device='cuda:0') (tensor(9.6180, device='cuda:0')) + | > loss_gen: 2.2256228923797607 (2.375192587192242) + | > loss_kl: 1.860641360282898 (1.8483312129974365) + | > loss_feat: 6.726043224334717 (7.227262240189773) + | > loss_mel: 17.44489288330078 (18.055899693415714) + | > loss_duration: 1.4901071786880493 (1.4719175467124352) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.74730682373047 (30.978603216317985) + | > grad_norm_1: tensor(130.6378, device='cuda:0') (tensor(247.3248, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.3304 (0.3287402849930983) + | > loader_time: 0.005 (0.004803180694580078) + + + --> TIME: 2025-04-19 00:04:53 -- STEP: 38/142 -- GLOBAL_STEP: 388550 + | > loss_disc: 2.4716217517852783 (2.428444975300839) + | > loss_disc_real_0: 0.14651653170585632 (0.12001987193760119) + | > loss_disc_real_1: 0.2222566306591034 (0.2006153032967919) + | > loss_disc_real_2: 0.23884539306163788 (0.22558910321248205) + | > loss_disc_real_3: 0.2706177234649658 (0.22868531432590986) + | > loss_disc_real_4: 0.27110299468040466 (0.2320616649169671) + | > loss_disc_real_5: 0.2159247100353241 (0.23422650836016) + | > loss_0: 2.4716217517852783 (2.428444975300839) + | > grad_norm_0: tensor(7.1805, device='cuda:0') (tensor(9.0889, device='cuda:0')) + | > loss_gen: 2.41831111907959 (2.372145044176202) + | > loss_kl: 1.9411226511001587 (1.8944824588926215) + | > loss_feat: 6.709862232208252 (7.131449435886584) + | > loss_mel: 17.704113006591797 (18.027559732135973) + | > loss_duration: 1.5092483758926392 (1.4818560826150995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.28265953063965 (30.907492788214434) + | > grad_norm_1: tensor(265.3716, device='cuda:0') (tensor(227.6846, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.351 (0.3418690279910439) + | > loader_time: 0.0058 (0.005332112312316896) + + + --> TIME: 2025-04-19 00:05:02 -- STEP: 63/142 -- GLOBAL_STEP: 388575 + | > loss_disc: 2.4953980445861816 (2.447422235731094) + | > loss_disc_real_0: 0.14082811772823334 (0.12345480907057958) + | > loss_disc_real_1: 0.21449315547943115 (0.20235352691203828) + | > loss_disc_real_2: 0.270714670419693 (0.22759061815246703) + | > loss_disc_real_3: 0.27267026901245117 (0.22904974908109696) + | > loss_disc_real_4: 0.2581738531589508 (0.23020789121824597) + | > loss_disc_real_5: 0.26122137904167175 (0.23315065886293138) + | > loss_0: 2.4953980445861816 (2.447422235731094) + | > grad_norm_0: tensor(15.9783, device='cuda:0') (tensor(11.0067, device='cuda:0')) + | > loss_gen: 2.4945054054260254 (2.3632565869225397) + | > loss_kl: 2.0932199954986572 (1.9120829521663605) + | > loss_feat: 6.58371639251709 (7.0769771242898605) + | > loss_mel: 18.08112907409668 (18.027549531724716) + | > loss_duration: 1.577353596687317 (1.4934890345921592) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.829923629760742 (30.873355259971014) + | > grad_norm_1: tensor(339.1924, device='cuda:0') (tensor(261.5452, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.3929 (0.3527127182672894) + | > loader_time: 0.0061 (0.005620044375222828) + + + --> TIME: 2025-04-19 00:05:13 -- STEP: 88/142 -- GLOBAL_STEP: 388600 + | > loss_disc: 2.715479850769043 (2.459358765320344) + | > loss_disc_real_0: 0.15048037469387054 (0.12530437933111732) + | > loss_disc_real_1: 0.21535496413707733 (0.2054276273331859) + | > loss_disc_real_2: 0.2324458360671997 (0.22665473073720932) + | > loss_disc_real_3: 0.23681221902370453 (0.22837236066433517) + | > loss_disc_real_4: 0.23046152293682098 (0.23021084408868442) + | > loss_disc_real_5: 0.2615901231765747 (0.23409732668237251) + | > loss_0: 2.715479850769043 (2.459358765320344) + | > grad_norm_0: tensor(7.9196, device='cuda:0') (tensor(12.8893, device='cuda:0')) + | > loss_gen: 2.261627435684204 (2.369167319752953) + | > loss_kl: 2.081441879272461 (1.9239294095472856) + | > loss_feat: 5.817598342895508 (7.094135235656392) + | > loss_mel: 16.516130447387695 (18.054092277180064) + | > loss_duration: 1.5910966396331787 (1.5048251003026962) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.267894744873047 (30.94614939256148) + | > grad_norm_1: tensor(198.1978, device='cuda:0') (tensor(294.1859, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.4008 (0.3650133961980993) + | > loader_time: 0.0074 (0.005970236929980194) + + + --> TIME: 2025-04-19 00:05:23 -- STEP: 113/142 -- GLOBAL_STEP: 388625 + | > loss_disc: 2.617966651916504 (2.466450651135065) + | > loss_disc_real_0: 0.14884768426418304 (0.12788284846902945) + | > loss_disc_real_1: 0.230959951877594 (0.2065271313211559) + | > loss_disc_real_2: 0.2504868805408478 (0.22654734077179325) + | > loss_disc_real_3: 0.2342127561569214 (0.22828617757928055) + | > loss_disc_real_4: 0.2546336352825165 (0.23132441120337596) + | > loss_disc_real_5: 0.24998345971107483 (0.23463244788942084) + | > loss_0: 2.617966651916504 (2.466450651135065) + | > grad_norm_0: tensor(10.0957, device='cuda:0') (tensor(13.9894, device='cuda:0')) + | > loss_gen: 2.139150381088257 (2.3658235073089595) + | > loss_kl: 1.845226764678955 (1.9334641695022583) + | > loss_feat: 5.1994476318359375 (7.016884162362698) + | > loss_mel: 14.824115753173828 (17.97876844996898) + | > loss_duration: 1.5429937839508057 (1.510916874472019) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.550933837890625 (30.80585721108766) + | > grad_norm_1: tensor(107.9591, device='cuda:0') (tensor(298.3738, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.4318 (0.37644919462963544) + | > loader_time: 0.0069 (0.006226925723320617) + + + --> TIME: 2025-04-19 00:05:35 -- STEP: 138/142 -- GLOBAL_STEP: 388650 + | > loss_disc: 2.567639112472534 (2.4656412394150444) + | > loss_disc_real_0: 0.14473803341388702 (0.12711587990971573) + | > loss_disc_real_1: 0.17901037633419037 (0.20632250628609589) + | > loss_disc_real_2: 0.23584820330142975 (0.22721130141745444) + | > loss_disc_real_3: 0.2566348612308502 (0.22883114639831625) + | > loss_disc_real_4: 0.2679046392440796 (0.23119842477034833) + | > loss_disc_real_5: 0.2578794062137604 (0.23337369358194046) + | > loss_0: 2.567639112472534 (2.4656412394150444) + | > grad_norm_0: tensor(6.1345, device='cuda:0') (tensor(14.2205, device='cuda:0')) + | > loss_gen: 2.150951862335205 (2.361025101896645) + | > loss_kl: 2.083864212036133 (1.9424352084380994) + | > loss_feat: 5.727889060974121 (7.004706420760224) + | > loss_mel: 17.99164581298828 (17.945612637893) + | > loss_duration: 1.5854904651641846 (1.5167391084242559) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.539840698242188 (30.77051852405935) + | > grad_norm_1: tensor(196.2843, device='cuda:0') (tensor(305.9631, device='cuda:0')) + | > current_lr_0: 0.00019838148297050769 + | > current_lr_1: 0.00019838148297050769 + | > step_time: 0.4685 (0.3902036780896394) + | > loader_time: 0.0076 (0.006500510201937912) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5010857582092285 (2.5010857582092285) + | > loss_disc_real_0: 0.1582280993461609 (0.1582280993461609) + | > loss_disc_real_1: 0.2505294978618622 (0.2505294978618622) + | > loss_disc_real_2: 0.2283877432346344 (0.2283877432346344) + | > loss_disc_real_3: 0.19642074406147003 (0.19642074406147003) + | > loss_disc_real_4: 0.2187953144311905 (0.2187953144311905) + | > loss_disc_real_5: 0.2173203080892563 (0.2173203080892563) + | > loss_0: 2.5010857582092285 (2.5010857582092285) + | > loss_gen: 2.2951486110687256 (2.2951486110687256) + | > loss_kl: 2.4588327407836914 (2.4588327407836914) + | > loss_feat: 6.785829544067383 (6.785829544067383) + | > loss_mel: 18.09117889404297 (18.09117889404297) + | > loss_duration: 1.626311182975769 (1.626311182975769) + | > loss_1: 31.257301330566406 (31.257301330566406) + + --> STEP: 1 + | > loss_disc: 2.5520615577697754 (2.5520615577697754) + | > loss_disc_real_0: 0.208214670419693 (0.208214670419693) + | > loss_disc_real_1: 0.2774295210838318 (0.2774295210838318) + | > loss_disc_real_2: 0.23477865755558014 (0.23477865755558014) + | > loss_disc_real_3: 0.21312810480594635 (0.21312810480594635) + | > loss_disc_real_4: 0.2203042209148407 (0.2203042209148407) + | > loss_disc_real_5: 0.2536275386810303 (0.2536275386810303) + | > loss_0: 2.5520615577697754 (2.5520615577697754) + | > loss_gen: 2.3574764728546143 (2.3574764728546143) + | > loss_kl: 2.2438526153564453 (2.2438526153564453) + | > loss_feat: 5.368035793304443 (5.368035793304443) + | > loss_mel: 17.122520446777344 (17.122520446777344) + | > loss_duration: 1.6478747129440308 (1.6478747129440308) + | > loss_1: 28.739761352539062 (28.739761352539062) + + --> STEP: 2 + | > loss_disc: 2.47383189201355 (2.5129467248916626) + | > loss_disc_real_0: 0.1507660448551178 (0.1794903576374054) + | > loss_disc_real_1: 0.2700929045677185 (0.27376121282577515) + | > loss_disc_real_2: 0.23024477064609528 (0.2325117141008377) + | > loss_disc_real_3: 0.19368325173854828 (0.20340567827224731) + | > loss_disc_real_4: 0.22424529492855072 (0.2222747579216957) + | > loss_disc_real_5: 0.2500569820404053 (0.2518422603607178) + | > loss_0: 2.47383189201355 (2.5129467248916626) + | > loss_gen: 2.478116512298584 (2.417796492576599) + | > loss_kl: 1.8474633693695068 (2.045657992362976) + | > loss_feat: 7.2013630867004395 (6.284699440002441) + | > loss_mel: 19.206689834594727 (18.164605140686035) + | > loss_duration: 1.5938516855239868 (1.6208631992340088) + | > loss_1: 32.327484130859375 (30.53362274169922) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003987789154052734 (+0.0004951953887939453) + | > avg_loss_disc: 2.5129467248916626 (+0.04292905330657959) + | > avg_loss_disc_real_0: 0.1794903576374054 (+0.05318130925297737) + | > avg_loss_disc_real_1: 0.27376121282577515 (+0.06112203747034073) + | > avg_loss_disc_real_2: 0.2325117141008377 (+0.001392655074596405) + | > avg_loss_disc_real_3: 0.20340567827224731 (-0.035366177558898926) + | > avg_loss_disc_real_4: 0.2222747579216957 (-0.02732216566801071) + | > avg_loss_disc_real_5: 0.2518422603607178 (+0.0059510692954063416) + | > avg_loss_0: 2.5129467248916626 (+0.04292905330657959) + | > avg_loss_gen: 2.417796492576599 (+0.13598334789276123) + | > avg_loss_kl: 2.045657992362976 (-0.12845122814178467) + | > avg_loss_feat: 6.284699440002441 (-0.6341776847839355) + | > avg_loss_mel: 18.164605140686035 (-0.7112436294555664) + | > avg_loss_duration: 1.6208631992340088 (-0.015814661979675293) + | > avg_loss_1: 30.53362274169922 (-1.3537044525146484) + + + > EPOCH: 66/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:05:38)  + + --> TIME: 2025-04-19 00:05:46 -- STEP: 21/142 -- GLOBAL_STEP: 388675 + | > loss_disc: 2.4250125885009766 (2.440726802462623) + | > loss_disc_real_0: 0.12342832237482071 (0.12012144603899547) + | > loss_disc_real_1: 0.21734534204006195 (0.20514274282114847) + | > loss_disc_real_2: 0.21024079620838165 (0.2266984780629476) + | > loss_disc_real_3: 0.22448262572288513 (0.23311837372325717) + | > loss_disc_real_4: 0.19698305428028107 (0.22857906775815146) + | > loss_disc_real_5: 0.2609054744243622 (0.23303885261217752) + | > loss_0: 2.4250125885009766 (2.440726802462623) + | > grad_norm_0: tensor(12.1100, device='cuda:0') (tensor(8.6003, device='cuda:0')) + | > loss_gen: 2.4631974697113037 (2.3342742125193277) + | > loss_kl: 1.9864516258239746 (1.8963223638988675) + | > loss_feat: 7.43242073059082 (6.981661592211042) + | > loss_mel: 19.520587921142578 (17.91036642165411) + | > loss_duration: 1.4939477443695068 (1.4813831420171828) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.8966064453125 (30.60400763012114) + | > grad_norm_1: tensor(289.6787, device='cuda:0') (tensor(200.2457, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.3413 (0.3400530701591855) + | > loader_time: 0.0046 (0.004865339824131557) + + + --> TIME: 2025-04-19 00:05:55 -- STEP: 46/142 -- GLOBAL_STEP: 388700 + | > loss_disc: 2.472790002822876 (2.418421584626903) + | > loss_disc_real_0: 0.12002821266651154 (0.11806676990312079) + | > loss_disc_real_1: 0.21685828268527985 (0.20406691179327344) + | > loss_disc_real_2: 0.21471630036830902 (0.22415527798559354) + | > loss_disc_real_3: 0.22126981616020203 (0.22653492766877878) + | > loss_disc_real_4: 0.23181480169296265 (0.22516630784324979) + | > loss_disc_real_5: 0.1997452974319458 (0.228122244062631) + | > loss_0: 2.472790002822876 (2.418421584626903) + | > grad_norm_0: tensor(5.3744, device='cuda:0') (tensor(8.8818, device='cuda:0')) + | > loss_gen: 2.462674379348755 (2.369056831235471) + | > loss_kl: 1.858567714691162 (1.9010889737502388) + | > loss_feat: 7.184902191162109 (7.142102231150088) + | > loss_mel: 17.654529571533203 (17.92674489643263) + | > loss_duration: 1.4722912311553955 (1.4880356011183367) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.632966995239258 (30.827028564784836) + | > grad_norm_1: tensor(119.7151, device='cuda:0') (tensor(212.8936, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.3675 (0.34875771273737366) + | > loader_time: 0.0059 (0.005363697591035261) + + + --> TIME: 2025-04-19 00:06:05 -- STEP: 71/142 -- GLOBAL_STEP: 388725 + | > loss_disc: 2.595994472503662 (2.4499232231731147) + | > loss_disc_real_0: 0.1863315850496292 (0.12283807352814875) + | > loss_disc_real_1: 0.1709136813879013 (0.20558075354972355) + | > loss_disc_real_2: 0.20758701860904694 (0.2258861520340745) + | > loss_disc_real_3: 0.2493702620267868 (0.2297358752136499) + | > loss_disc_real_4: 0.24411772191524506 (0.22938751166974994) + | > loss_disc_real_5: 0.21860194206237793 (0.23250212572829823) + | > loss_0: 2.595994472503662 (2.4499232231731147) + | > grad_norm_0: tensor(9.3018, device='cuda:0') (tensor(8.4351, device='cuda:0')) + | > loss_gen: 2.2113468647003174 (2.362942262434624) + | > loss_kl: 1.9967988729476929 (1.9211044462633804) + | > loss_feat: 6.3091206550598145 (7.114309962366669) + | > loss_mel: 16.799711227416992 (18.00853992515887) + | > loss_duration: 1.5261486768722534 (1.4981129555635049) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.84312629699707 (30.9050096592433) + | > grad_norm_1: tensor(309.3089, device='cuda:0') (tensor(193.7296, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.3909 (0.3593288609679316) + | > loader_time: 0.0065 (0.005663871765136718) + + + --> TIME: 2025-04-19 00:06:15 -- STEP: 96/142 -- GLOBAL_STEP: 388750 + | > loss_disc: 2.458075523376465 (2.4634265899658208) + | > loss_disc_real_0: 0.1439562290906906 (0.12533738176959255) + | > loss_disc_real_1: 0.205503910779953 (0.2060235763589541) + | > loss_disc_real_2: 0.20629358291625977 (0.22566035917649666) + | > loss_disc_real_3: 0.22047178447246552 (0.23003082939734062) + | > loss_disc_real_4: 0.22522850334644318 (0.2298669251613319) + | > loss_disc_real_5: 0.1995762437582016 (0.23335103426749507) + | > loss_0: 2.458075523376465 (2.4634265899658208) + | > grad_norm_0: tensor(17.9968, device='cuda:0') (tensor(9.2617, device='cuda:0')) + | > loss_gen: 2.250910758972168 (2.347173104683558) + | > loss_kl: 1.9587335586547852 (1.9287504330277443) + | > loss_feat: 6.94058895111084 (7.016343797246615) + | > loss_mel: 18.038118362426758 (17.995048582553874) + | > loss_duration: 1.5478408336639404 (1.5090047443906467) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.736190795898438 (30.79632079601288) + | > grad_norm_1: tensor(293.6820, device='cuda:0') (tensor(215.4976, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.4095 (0.37092279891173047) + | > loader_time: 0.0069 (0.005926502247651418) + + + --> TIME: 2025-04-19 00:06:26 -- STEP: 121/142 -- GLOBAL_STEP: 388775 + | > loss_disc: 2.4835093021392822 (2.4706547733180777) + | > loss_disc_real_0: 0.11213871091604233 (0.12603803440805308) + | > loss_disc_real_1: 0.21306081116199493 (0.20721802829710906) + | > loss_disc_real_2: 0.20507901906967163 (0.22665765204212882) + | > loss_disc_real_3: 0.24619974195957184 (0.23077707534486597) + | > loss_disc_real_4: 0.2323089838027954 (0.2305743529284296) + | > loss_disc_real_5: 0.23883609473705292 (0.2337708528623108) + | > loss_0: 2.4835093021392822 (2.4706547733180777) + | > grad_norm_0: tensor(8.8282, device='cuda:0') (tensor(9.7024, device='cuda:0')) + | > loss_gen: 2.368098735809326 (2.343225701781344) + | > loss_kl: 2.0730061531066895 (1.9343759885504226) + | > loss_feat: 6.588149070739746 (6.964127107100054) + | > loss_mel: 17.973041534423828 (17.9537589175642) + | > loss_duration: 1.5918452739715576 (1.5156521807032184) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.594141006469727 (30.71114004150895) + | > grad_norm_1: tensor(235.8448, device='cuda:0') (tensor(231.2014, device='cuda:0')) + | > current_lr_0: 0.00019835668528513637 + | > current_lr_1: 0.00019835668528513637 + | > step_time: 0.4535 (0.38279718801009754) + | > loader_time: 0.0079 (0.0062215367624582334) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.537700653076172 (2.537700653076172) + | > loss_disc_real_0: 0.1745315045118332 (0.1745315045118332) + | > loss_disc_real_1: 0.2214004099369049 (0.2214004099369049) + | > loss_disc_real_2: 0.23986271023750305 (0.23986271023750305) + | > loss_disc_real_3: 0.24069525301456451 (0.24069525301456451) + | > loss_disc_real_4: 0.22921471297740936 (0.22921471297740936) + | > loss_disc_real_5: 0.25366416573524475 (0.25366416573524475) + | > loss_0: 2.537700653076172 (2.537700653076172) + | > loss_gen: 2.354478359222412 (2.354478359222412) + | > loss_kl: 2.4780125617980957 (2.4780125617980957) + | > loss_feat: 6.847382545471191 (6.847382545471191) + | > loss_mel: 18.385265350341797 (18.385265350341797) + | > loss_duration: 1.6503422260284424 (1.6503422260284424) + | > loss_1: 31.715482711791992 (31.715482711791992) + + --> STEP: 1 + | > loss_disc: 2.4554529190063477 (2.4554529190063477) + | > loss_disc_real_0: 0.18111605942249298 (0.18111605942249298) + | > loss_disc_real_1: 0.24673302471637726 (0.24673302471637726) + | > loss_disc_real_2: 0.2349873036146164 (0.2349873036146164) + | > loss_disc_real_3: 0.2349812239408493 (0.2349812239408493) + | > loss_disc_real_4: 0.2333027571439743 (0.2333027571439743) + | > loss_disc_real_5: 0.26732590794563293 (0.26732590794563293) + | > loss_0: 2.4554529190063477 (2.4554529190063477) + | > loss_gen: 2.575094699859619 (2.575094699859619) + | > loss_kl: 2.4662628173828125 (2.4662628173828125) + | > loss_feat: 6.750588417053223 (6.750588417053223) + | > loss_mel: 18.682727813720703 (18.682727813720703) + | > loss_duration: 1.6479154825210571 (1.6479154825210571) + | > loss_1: 32.122589111328125 (32.122589111328125) + + --> STEP: 2 + | > loss_disc: 2.5293936729431152 (2.4924232959747314) + | > loss_disc_real_0: 0.2179499864578247 (0.19953302294015884) + | > loss_disc_real_1: 0.22618819773197174 (0.2364606112241745) + | > loss_disc_real_2: 0.24509373307228088 (0.24004051834344864) + | > loss_disc_real_3: 0.24330076575279236 (0.23914099484682083) + | > loss_disc_real_4: 0.2660194933414459 (0.2496611252427101) + | > loss_disc_real_5: 0.2672097086906433 (0.2672678083181381) + | > loss_0: 2.5293936729431152 (2.4924232959747314) + | > loss_gen: 2.594801902770996 (2.5849483013153076) + | > loss_kl: 1.7755147218704224 (2.1208887696266174) + | > loss_feat: 7.765019416809082 (7.257803916931152) + | > loss_mel: 18.830232620239258 (18.75648021697998) + | > loss_duration: 1.6072427034378052 (1.6275790929794312) + | > loss_1: 32.572811126708984 (32.347700119018555) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003328084945678711 (-0.0006597042083740234) + | > avg_loss_disc: 2.4924232959747314 (-0.020523428916931152) + | > avg_loss_disc_real_0: 0.19953302294015884 (+0.02004266530275345) + | > avg_loss_disc_real_1: 0.2364606112241745 (-0.03730060160160065) + | > avg_loss_disc_real_2: 0.24004051834344864 (+0.007528804242610931) + | > avg_loss_disc_real_3: 0.23914099484682083 (+0.03573531657457352) + | > avg_loss_disc_real_4: 0.2496611252427101 (+0.027386367321014404) + | > avg_loss_disc_real_5: 0.2672678083181381 (+0.015425547957420349) + | > avg_loss_0: 2.4924232959747314 (-0.020523428916931152) + | > avg_loss_gen: 2.5849483013153076 (+0.1671518087387085) + | > avg_loss_kl: 2.1208887696266174 (+0.07523077726364136) + | > avg_loss_feat: 7.257803916931152 (+0.9731044769287109) + | > avg_loss_mel: 18.75648021697998 (+0.5918750762939453) + | > avg_loss_duration: 1.6275790929794312 (+0.006715893745422363) + | > avg_loss_1: 32.347700119018555 (+1.814077377319336) + + + > EPOCH: 67/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:06:37)  + + --> TIME: 2025-04-19 00:06:39 -- STEP: 4/142 -- GLOBAL_STEP: 388800 + | > loss_disc: 2.4664342403411865 (2.4288196563720703) + | > loss_disc_real_0: 0.09467911720275879 (0.1033625602722168) + | > loss_disc_real_1: 0.18926377594470978 (0.2164704017341137) + | > loss_disc_real_2: 0.21291851997375488 (0.24648071452975273) + | > loss_disc_real_3: 0.2744527757167816 (0.2535957284271717) + | > loss_disc_real_4: 0.22441180050373077 (0.23347733914852142) + | > loss_disc_real_5: 0.24015891551971436 (0.25192447006702423) + | > loss_0: 2.4664342403411865 (2.4288196563720703) + | > grad_norm_0: tensor(9.0808, device='cuda:0') (tensor(11.1044, device='cuda:0')) + | > loss_gen: 2.458024501800537 (2.499582529067993) + | > loss_kl: 1.8887615203857422 (1.8786238431930542) + | > loss_feat: 7.230622291564941 (7.413397312164307) + | > loss_mel: 18.217432022094727 (18.31513261795044) + | > loss_duration: 1.4816381931304932 (1.4779821932315826) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.276477813720703 (31.584718227386475) + | > grad_norm_1: tensor(221.4461, device='cuda:0') (tensor(323.9726, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.3217 (0.32501018047332764) + | > loader_time: 0.0045 (0.004283487796783447) + + + --> TIME: 2025-04-19 00:06:48 -- STEP: 29/142 -- GLOBAL_STEP: 388825 + | > loss_disc: 2.356797695159912 (2.4517590670750056) + | > loss_disc_real_0: 0.14621983468532562 (0.11971520735272045) + | > loss_disc_real_1: 0.2249087542295456 (0.20886993819269642) + | > loss_disc_real_2: 0.259645015001297 (0.23254383072770876) + | > loss_disc_real_3: 0.23616847395896912 (0.23408427577594232) + | > loss_disc_real_4: 0.22111445665359497 (0.22715145606419135) + | > loss_disc_real_5: 0.20589600503444672 (0.22942614966425404) + | > loss_0: 2.356797695159912 (2.4517590670750056) + | > grad_norm_0: tensor(8.5518, device='cuda:0') (tensor(13.9255, device='cuda:0')) + | > loss_gen: 2.415933609008789 (2.3821571530966925) + | > loss_kl: 1.9753965139389038 (1.8786770393108498) + | > loss_feat: 7.568496227264404 (7.146371134396257) + | > loss_mel: 18.4569149017334 (18.04902103029448) + | > loss_duration: 1.4651203155517578 (1.478079364217561) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.88186264038086 (30.934305782975823) + | > grad_norm_1: tensor(399.9286, device='cuda:0') (tensor(337.0057, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.3481 (0.3359934461527857) + | > loader_time: 0.0052 (0.005101031270520442) + + + --> TIME: 2025-04-19 00:06:57 -- STEP: 54/142 -- GLOBAL_STEP: 388850 + | > loss_disc: 2.4932234287261963 (2.4529093680558383) + | > loss_disc_real_0: 0.12806575000286102 (0.11985723301768303) + | > loss_disc_real_1: 0.1860956847667694 (0.2091377209182139) + | > loss_disc_real_2: 0.22003501653671265 (0.22942953336018104) + | > loss_disc_real_3: 0.23676390945911407 (0.23136972773958137) + | > loss_disc_real_4: 0.2181052267551422 (0.22733138584428364) + | > loss_disc_real_5: 0.2208756059408188 (0.23144815448257658) + | > loss_0: 2.4932234287261963 (2.4529093680558383) + | > grad_norm_0: tensor(13.9759, device='cuda:0') (tensor(13.6144, device='cuda:0')) + | > loss_gen: 2.2576231956481934 (2.3752159763265546) + | > loss_kl: 1.9260601997375488 (1.8944050515139546) + | > loss_feat: 7.219850540161133 (7.148782226774427) + | > loss_mel: 17.482688903808594 (17.943314199094413) + | > loss_duration: 1.5208961963653564 (1.4852768822952551) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.407119750976562 (30.846994435345685) + | > grad_norm_1: tensor(417.4507, device='cuda:0') (tensor(349.5019, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.3702 (0.34784933814296015) + | > loader_time: 0.0059 (0.0055765046013726135) + + + --> TIME: 2025-04-19 00:07:07 -- STEP: 79/142 -- GLOBAL_STEP: 388875 + | > loss_disc: 2.504727363586426 (2.463309770897974) + | > loss_disc_real_0: 0.17390656471252441 (0.1235508450978919) + | > loss_disc_real_1: 0.2588130235671997 (0.2089408002322233) + | > loss_disc_real_2: 0.2694525420665741 (0.22981665443770494) + | > loss_disc_real_3: 0.2298726588487625 (0.23149448991576327) + | > loss_disc_real_4: 0.24567414820194244 (0.2284106921546067) + | > loss_disc_real_5: 0.21431922912597656 (0.23251938084258308) + | > loss_0: 2.504727363586426 (2.463309770897974) + | > grad_norm_0: tensor(18.1927, device='cuda:0') (tensor(14.2022, device='cuda:0')) + | > loss_gen: 2.5004870891571045 (2.373525764368758) + | > loss_kl: 1.9582738876342773 (1.916548896439468) + | > loss_feat: 7.092674255371094 (7.12774137303799) + | > loss_mel: 19.931657791137695 (17.950062643123566) + | > loss_duration: 1.5508869886398315 (1.4968777789345267) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.03397750854492 (30.864756354802772) + | > grad_norm_1: tensor(378.5252, device='cuda:0') (tensor(341.7863, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.3893 (0.3601711581024943) + | > loader_time: 0.0065 (0.0058596134185791016) + + + --> TIME: 2025-04-19 00:07:17 -- STEP: 104/142 -- GLOBAL_STEP: 388900 + | > loss_disc: 2.4982900619506836 (2.4723579677251664) + | > loss_disc_real_0: 0.1594552844762802 (0.1280556441141435) + | > loss_disc_real_1: 0.21120698750019073 (0.2094925303871815) + | > loss_disc_real_2: 0.22564834356307983 (0.22937727991778117) + | > loss_disc_real_3: 0.21791507303714752 (0.23132914605622107) + | > loss_disc_real_4: 0.21319739520549774 (0.22930135635229257) + | > loss_disc_real_5: 0.20762987434864044 (0.2309949601498934) + | > loss_0: 2.4982900619506836 (2.4723579677251664) + | > grad_norm_0: tensor(22.4348, device='cuda:0') (tensor(15.4534, device='cuda:0')) + | > loss_gen: 2.1467771530151367 (2.3621987585838022) + | > loss_kl: 2.1003291606903076 (1.9290861200827818) + | > loss_feat: 6.590818881988525 (7.048503729013296) + | > loss_mel: 16.781856536865234 (17.89162779771363) + | > loss_duration: 1.5546576976776123 (1.5076328687942946) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.1744384765625 (30.739049159563503) + | > grad_norm_1: tensor(322.3000, device='cuda:0') (tensor(342.1899, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.4144 (0.371518809061784) + | > loader_time: 0.0069 (0.006112518218847422) + + + --> TIME: 2025-04-19 00:07:29 -- STEP: 129/142 -- GLOBAL_STEP: 388925 + | > loss_disc: 2.400634527206421 (2.474571488624395) + | > loss_disc_real_0: 0.09208250045776367 (0.12786180667521418) + | > loss_disc_real_1: 0.1842663586139679 (0.20993663416814434) + | > loss_disc_real_2: 0.24838979542255402 (0.22871474274831224) + | > loss_disc_real_3: 0.24628400802612305 (0.2314342670431433) + | > loss_disc_real_4: 0.21409964561462402 (0.22936327894066655) + | > loss_disc_real_5: 0.22106724977493286 (0.23089733604312868) + | > loss_0: 2.400634527206421 (2.474571488624395) + | > grad_norm_0: tensor(29.3855, device='cuda:0') (tensor(16.3286, device='cuda:0')) + | > loss_gen: 2.3511719703674316 (2.361917917118516) + | > loss_kl: 2.0376710891723633 (1.9347264202990273) + | > loss_feat: 7.623456954956055 (7.019948671030444) + | > loss_mel: 18.939409255981445 (17.861228233159956) + | > loss_duration: 1.5286450386047363 (1.5137840879055882) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.48035430908203 (30.69160525004069) + | > grad_norm_1: tensor(752.8872, device='cuda:0') (tensor(347.7856, device='cuda:0')) + | > current_lr_0: 0.00019833189069947573 + | > current_lr_1: 0.00019833189069947573 + | > step_time: 0.4565 (0.384424558905668) + | > loader_time: 0.0074 (0.006360266559807829) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.2892541885375977 (2.2892541885375977) + | > loss_disc_real_0: 0.0562996044754982 (0.0562996044754982) + | > loss_disc_real_1: 0.19893823564052582 (0.19893823564052582) + | > loss_disc_real_2: 0.20230822265148163 (0.20230822265148163) + | > loss_disc_real_3: 0.22689537703990936 (0.22689537703990936) + | > loss_disc_real_4: 0.18637169897556305 (0.18637169897556305) + | > loss_disc_real_5: 0.2558963894844055 (0.2558963894844055) + | > loss_0: 2.2892541885375977 (2.2892541885375977) + | > loss_gen: 2.251227378845215 (2.251227378845215) + | > loss_kl: 2.2171337604522705 (2.2171337604522705) + | > loss_feat: 7.20914363861084 (7.20914363861084) + | > loss_mel: 19.320655822753906 (19.320655822753906) + | > loss_duration: 1.6297197341918945 (1.6297197341918945) + | > loss_1: 32.62788009643555 (32.62788009643555) + + --> STEP: 1 + | > loss_disc: 2.5979509353637695 (2.5979509353637695) + | > loss_disc_real_0: 0.14447109401226044 (0.14447109401226044) + | > loss_disc_real_1: 0.22716914117336273 (0.22716914117336273) + | > loss_disc_real_2: 0.23675702512264252 (0.23675702512264252) + | > loss_disc_real_3: 0.21537508070468903 (0.21537508070468903) + | > loss_disc_real_4: 0.2172231376171112 (0.2172231376171112) + | > loss_disc_real_5: 0.25110113620758057 (0.25110113620758057) + | > loss_0: 2.5979509353637695 (2.5979509353637695) + | > loss_gen: 1.9861074686050415 (1.9861074686050415) + | > loss_kl: 2.443286657333374 (2.443286657333374) + | > loss_feat: 6.08982515335083 (6.08982515335083) + | > loss_mel: 16.990571975708008 (16.990571975708008) + | > loss_duration: 1.6750850677490234 (1.6750850677490234) + | > loss_1: 29.18487548828125 (29.18487548828125) + + --> STEP: 2 + | > loss_disc: 2.5133023262023926 (2.555626630783081) + | > loss_disc_real_0: 0.08713863044977188 (0.11580486223101616) + | > loss_disc_real_1: 0.28271666169166565 (0.2549429014325142) + | > loss_disc_real_2: 0.20094412565231323 (0.21885057538747787) + | > loss_disc_real_3: 0.2140078991651535 (0.21469148993492126) + | > loss_disc_real_4: 0.21912449598312378 (0.2181738168001175) + | > loss_disc_real_5: 0.19877155125141144 (0.224936343729496) + | > loss_0: 2.5133023262023926 (2.555626630783081) + | > loss_gen: 2.0841476917266846 (2.035127580165863) + | > loss_kl: 2.142350673675537 (2.2928186655044556) + | > loss_feat: 6.768008232116699 (6.428916692733765) + | > loss_mel: 17.336538314819336 (17.163555145263672) + | > loss_duration: 1.5963819026947021 (1.6357334852218628) + | > loss_1: 29.927427291870117 (29.556151390075684) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003437042236328125 (+0.00010895729064941406) + | > avg_loss_disc: 2.555626630783081 (+0.06320333480834961) + | > avg_loss_disc_real_0: 0.11580486223101616 (-0.08372816070914268) + | > avg_loss_disc_real_1: 0.2549429014325142 (+0.01848229020833969) + | > avg_loss_disc_real_2: 0.21885057538747787 (-0.021189942955970764) + | > avg_loss_disc_real_3: 0.21469148993492126 (-0.024449504911899567) + | > avg_loss_disc_real_4: 0.2181738168001175 (-0.03148730844259262) + | > avg_loss_disc_real_5: 0.224936343729496 (-0.04233146458864212) + | > avg_loss_0: 2.555626630783081 (+0.06320333480834961) + | > avg_loss_gen: 2.035127580165863 (-0.5498207211494446) + | > avg_loss_kl: 2.2928186655044556 (+0.17192989587783813) + | > avg_loss_feat: 6.428916692733765 (-0.8288872241973877) + | > avg_loss_mel: 17.163555145263672 (-1.5929250717163086) + | > avg_loss_duration: 1.6357334852218628 (+0.00815439224243164) + | > avg_loss_1: 29.556151390075684 (-2.791548728942871) + + + > EPOCH: 68/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:07:36)  + + --> TIME: 2025-04-19 00:07:41 -- STEP: 12/142 -- GLOBAL_STEP: 388950 + | > loss_disc: 2.484016180038452 (2.436496456464132) + | > loss_disc_real_0: 0.11008644104003906 (0.12519063086559376) + | > loss_disc_real_1: 0.21157033741474152 (0.20362701267004013) + | > loss_disc_real_2: 0.23337756097316742 (0.227380300561587) + | > loss_disc_real_3: 0.25532785058021545 (0.22433321053783098) + | > loss_disc_real_4: 0.23726940155029297 (0.22232444708546004) + | > loss_disc_real_5: 0.23917463421821594 (0.22788472846150398) + | > loss_0: 2.484016180038452 (2.436496456464132) + | > grad_norm_0: tensor(11.8073, device='cuda:0') (tensor(12.2080, device='cuda:0')) + | > loss_gen: 2.4388132095336914 (2.373260577519735) + | > loss_kl: 1.9271669387817383 (1.9186948736508687) + | > loss_feat: 6.8865156173706055 (7.065888285636902) + | > loss_mel: 18.32863426208496 (17.67986861864726) + | > loss_duration: 1.4889891147613525 (1.46507262190183) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.070119857788086 (30.502785205841064) + | > grad_norm_1: tensor(304.6091, device='cuda:0') (tensor(278.3945, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.3545 (0.34272732337315875) + | > loader_time: 0.0049 (0.005005677541097005) + + + --> TIME: 2025-04-19 00:07:50 -- STEP: 37/142 -- GLOBAL_STEP: 388975 + | > loss_disc: 2.3483550548553467 (2.4458179667189315) + | > loss_disc_real_0: 0.10053694248199463 (0.12011971475707521) + | > loss_disc_real_1: 0.1812620311975479 (0.20489292410579887) + | > loss_disc_real_2: 0.23354287445545197 (0.22751555370317922) + | > loss_disc_real_3: 0.24638214707374573 (0.22740305154710203) + | > loss_disc_real_4: 0.24464209377765656 (0.22820603726683436) + | > loss_disc_real_5: 0.23699960112571716 (0.23323918677665093) + | > loss_0: 2.3483550548553467 (2.4458179667189315) + | > grad_norm_0: tensor(3.8707, device='cuda:0') (tensor(8.8816, device='cuda:0')) + | > loss_gen: 2.627166509628296 (2.3611816006737785) + | > loss_kl: 2.109914541244507 (1.919044861922393) + | > loss_feat: 7.587110996246338 (7.086743419234817) + | > loss_mel: 19.23699951171875 (17.780372078354294) + | > loss_duration: 1.5242927074432373 (1.4789176083899833) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.08548355102539 (30.62625941714725) + | > grad_norm_1: tensor(90.1806, device='cuda:0') (tensor(213.5162, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.3596 (0.3500657919290903) + | > loader_time: 0.0056 (0.005361808312905801) + + + --> TIME: 2025-04-19 00:08:00 -- STEP: 62/142 -- GLOBAL_STEP: 389000 + | > loss_disc: 2.457111120223999 (2.462928602772374) + | > loss_disc_real_0: 0.12240534275770187 (0.12465197562931048) + | > loss_disc_real_1: 0.2336461842060089 (0.20676442427981284) + | > loss_disc_real_2: 0.24084880948066711 (0.22820145949240653) + | > loss_disc_real_3: 0.21327200531959534 (0.2275466954996509) + | > loss_disc_real_4: 0.25215378403663635 (0.22833409689126477) + | > loss_disc_real_5: 0.2094610333442688 (0.23376248416400724) + | > loss_0: 2.457111120223999 (2.462928602772374) + | > grad_norm_0: tensor(17.6397, device='cuda:0') (tensor(8.6880, device='cuda:0')) + | > loss_gen: 2.3492612838745117 (2.348951274348844) + | > loss_kl: 1.8753689527511597 (1.9175622251725966) + | > loss_feat: 6.880192279815674 (6.99954350533024) + | > loss_mel: 18.0655574798584 (17.89841776509438) + | > loss_duration: 1.5435638427734375 (1.4879566815591627) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.713943481445312 (30.65243136498236) + | > grad_norm_1: tensor(110.2478, device='cuda:0') (tensor(177.6457, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.3943 (0.3617249688794536) + | > loader_time: 0.0066 (0.005744811027280746) + + + --> TIME: 2025-04-19 00:08:10 -- STEP: 87/142 -- GLOBAL_STEP: 389025 + | > loss_disc: 2.5668039321899414 (2.477191653744927) + | > loss_disc_real_0: 0.17425814270973206 (0.1288139892463712) + | > loss_disc_real_1: 0.22411222755908966 (0.20870177516306954) + | > loss_disc_real_2: 0.2115461230278015 (0.2291951621400899) + | > loss_disc_real_3: 0.21425554156303406 (0.23010896146297455) + | > loss_disc_real_4: 0.22478002309799194 (0.23081394550444065) + | > loss_disc_real_5: 0.261969655752182 (0.23192905797355476) + | > loss_0: 2.5668039321899414 (2.477191653744927) + | > grad_norm_0: tensor(13.1863, device='cuda:0') (tensor(9.0940, device='cuda:0')) + | > loss_gen: 2.2193260192871094 (2.3376148514363955) + | > loss_kl: 1.9813679456710815 (1.9286499146757454) + | > loss_feat: 6.795170783996582 (6.930336234213292) + | > loss_mel: 18.68270492553711 (17.921986119500527) + | > loss_duration: 1.5658819675445557 (1.499587361839996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.24445152282715 (30.618174541955707) + | > grad_norm_1: tensor(154.0107, device='cuda:0') (tensor(182.1362, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.4241 (0.37441888348809604) + | > loader_time: 0.007 (0.006028060255379513) + + + --> TIME: 2025-04-19 00:08:21 -- STEP: 112/142 -- GLOBAL_STEP: 389050 + | > loss_disc: 2.5582237243652344 (2.487547912767954) + | > loss_disc_real_0: 0.1040419489145279 (0.13071457146933038) + | > loss_disc_real_1: 0.24745506048202515 (0.20844804256090096) + | > loss_disc_real_2: 0.20142802596092224 (0.23000005220196076) + | > loss_disc_real_3: 0.22105193138122559 (0.2304307252966932) + | > loss_disc_real_4: 0.23332443833351135 (0.23185735608318023) + | > loss_disc_real_5: 0.22577473521232605 (0.23179907271904604) + | > loss_0: 2.5582237243652344 (2.487547912767954) + | > grad_norm_0: tensor(21.7054, device='cuda:0') (tensor(9.7121, device='cuda:0')) + | > loss_gen: 2.199244499206543 (2.3265803477593843) + | > loss_kl: 1.871983528137207 (1.9308390351278442) + | > loss_feat: 6.122730255126953 (6.881565817764828) + | > loss_mel: 17.01642417907715 (17.865405874592923) + | > loss_duration: 1.5016250610351562 (1.506706820002624) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.71200942993164 (30.51109799316951) + | > grad_norm_1: tensor(320.5172, device='cuda:0') (tensor(200.6069, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.4424 (0.3857344686985016) + | > loader_time: 0.0076 (0.006316495793206351) + + + --> TIME: 2025-04-19 00:08:33 -- STEP: 137/142 -- GLOBAL_STEP: 389075 + | > loss_disc: 2.4320719242095947 (2.4865934935799467) + | > loss_disc_real_0: 0.0873108059167862 (0.13103235890939294) + | > loss_disc_real_1: 0.20816999673843384 (0.20904367139739713) + | > loss_disc_real_2: 0.24601297080516815 (0.2295750091328238) + | > loss_disc_real_3: 0.24881260097026825 (0.23030051371476945) + | > loss_disc_real_4: 0.22504527866840363 (0.23140687218112668) + | > loss_disc_real_5: 0.21092379093170166 (0.23071499886738994) + | > loss_0: 2.4320719242095947 (2.4865934935799467) + | > grad_norm_0: tensor(12.3236, device='cuda:0') (tensor(10.8009, device='cuda:0')) + | > loss_gen: 2.4832942485809326 (2.3261732219779585) + | > loss_kl: 1.9511163234710693 (1.9336693669757703) + | > loss_feat: 6.644866943359375 (6.853469493615366) + | > loss_mel: 18.097087860107422 (17.82913480187855) + | > loss_duration: 1.5342501401901245 (1.5126987077894003) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.710615158081055 (30.455145661848306) + | > grad_norm_1: tensor(449.1098, device='cuda:0') (tensor(220.8178, device='cuda:0')) + | > current_lr_0: 0.0001983070992131383 + | > current_lr_1: 0.0001983070992131383 + | > step_time: 0.4653 (0.3980841027559155) + | > loader_time: 0.0079 (0.0065670100441814335) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3294522762298584 (2.3294522762298584) + | > loss_disc_real_0: 0.054976191371679306 (0.054976191371679306) + | > loss_disc_real_1: 0.18087652325630188 (0.18087652325630188) + | > loss_disc_real_2: 0.21655696630477905 (0.21655696630477905) + | > loss_disc_real_3: 0.19720493257045746 (0.19720493257045746) + | > loss_disc_real_4: 0.20274007320404053 (0.20274007320404053) + | > loss_disc_real_5: 0.22937622666358948 (0.22937622666358948) + | > loss_0: 2.3294522762298584 (2.3294522762298584) + | > loss_gen: 2.1490590572357178 (2.1490590572357178) + | > loss_kl: 2.4160358905792236 (2.4160358905792236) + | > loss_feat: 8.263574600219727 (8.263574600219727) + | > loss_mel: 19.368078231811523 (19.368078231811523) + | > loss_duration: 1.6334692239761353 (1.6334692239761353) + | > loss_1: 33.83021545410156 (33.83021545410156) + + --> STEP: 1 + | > loss_disc: 2.533487319946289 (2.533487319946289) + | > loss_disc_real_0: 0.13085559010505676 (0.13085559010505676) + | > loss_disc_real_1: 0.20423848927021027 (0.20423848927021027) + | > loss_disc_real_2: 0.22546818852424622 (0.22546818852424622) + | > loss_disc_real_3: 0.2156824916601181 (0.2156824916601181) + | > loss_disc_real_4: 0.21987920999526978 (0.21987920999526978) + | > loss_disc_real_5: 0.19935199618339539 (0.19935199618339539) + | > loss_0: 2.533487319946289 (2.533487319946289) + | > loss_gen: 1.9683101177215576 (1.9683101177215576) + | > loss_kl: 2.3249382972717285 (2.3249382972717285) + | > loss_feat: 5.007735252380371 (5.007735252380371) + | > loss_mel: 14.89276123046875 (14.89276123046875) + | > loss_duration: 1.6585386991500854 (1.6585386991500854) + | > loss_1: 25.852283477783203 (25.852283477783203) + + --> STEP: 2 + | > loss_disc: 2.375457525253296 (2.4544724225997925) + | > loss_disc_real_0: 0.10854773968458176 (0.11970166489481926) + | > loss_disc_real_1: 0.17021194100379944 (0.18722521513700485) + | > loss_disc_real_2: 0.1996704638004303 (0.21256932616233826) + | > loss_disc_real_3: 0.21355599164962769 (0.2146192416548729) + | > loss_disc_real_4: 0.2066078931093216 (0.21324355155229568) + | > loss_disc_real_5: 0.22221361100673676 (0.21078280359506607) + | > loss_0: 2.375457525253296 (2.4544724225997925) + | > loss_gen: 2.2719621658325195 (2.1201361417770386) + | > loss_kl: 2.0421080589294434 (2.183523178100586) + | > loss_feat: 7.386939525604248 (6.19733738899231) + | > loss_mel: 18.493541717529297 (16.693151473999023) + | > loss_duration: 1.58896803855896 (1.6237533688545227) + | > loss_1: 31.783517837524414 (28.81790065765381) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003158092498779297 (-0.0002789497375488281) + | > avg_loss_disc: 2.4544724225997925 (-0.10115420818328857) + | > avg_loss_disc_real_0: 0.11970166489481926 (+0.0038968026638031006) + | > avg_loss_disc_real_1: 0.18722521513700485 (-0.06771768629550934) + | > avg_loss_disc_real_2: 0.21256932616233826 (-0.006281249225139618) + | > avg_loss_disc_real_3: 0.2146192416548729 (-7.224828004837036e-05) + | > avg_loss_disc_real_4: 0.21324355155229568 (-0.004930265247821808) + | > avg_loss_disc_real_5: 0.21078280359506607 (-0.014153540134429932) + | > avg_loss_0: 2.4544724225997925 (-0.10115420818328857) + | > avg_loss_gen: 2.1201361417770386 (+0.08500856161117554) + | > avg_loss_kl: 2.183523178100586 (-0.10929548740386963) + | > avg_loss_feat: 6.19733738899231 (-0.23157930374145508) + | > avg_loss_mel: 16.693151473999023 (-0.47040367126464844) + | > avg_loss_duration: 1.6237533688545227 (-0.011980116367340088) + | > avg_loss_1: 28.81790065765381 (-0.738250732421875) + + + > EPOCH: 69/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:08:36)  + + --> TIME: 2025-04-19 00:08:44 -- STEP: 20/142 -- GLOBAL_STEP: 389100 + | > loss_disc: 2.4709243774414062 (2.395850157737732) + | > loss_disc_real_0: 0.16602598130702972 (0.11897035520523787) + | > loss_disc_real_1: 0.21241967380046844 (0.1966927908360958) + | > loss_disc_real_2: 0.19974827766418457 (0.2230169214308262) + | > loss_disc_real_3: 0.24069342017173767 (0.22697090804576875) + | > loss_disc_real_4: 0.22056111693382263 (0.2271064631640911) + | > loss_disc_real_5: 0.21069590747356415 (0.22394086197018623) + | > loss_0: 2.4709243774414062 (2.395850157737732) + | > grad_norm_0: tensor(27.6182, device='cuda:0') (tensor(19.1825, device='cuda:0')) + | > loss_gen: 2.6487348079681396 (2.501893937587738) + | > loss_kl: 1.89362370967865 (1.857861578464508) + | > loss_feat: 7.8668212890625 (7.443081593513488) + | > loss_mel: 19.107563018798828 (17.805550146102906) + | > loss_duration: 1.546036720275879 (1.4744269013404847) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.06277847290039 (31.082813835144044) + | > grad_norm_1: tensor(85.0561, device='cuda:0') (tensor(380.0028, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.3433 (0.33156237602233884) + | > loader_time: 0.0052 (0.004653012752532959) + + + --> TIME: 2025-04-19 00:08:53 -- STEP: 45/142 -- GLOBAL_STEP: 389125 + | > loss_disc: 2.43392014503479 (2.425632609261407) + | > loss_disc_real_0: 0.14744484424591064 (0.12009209427568647) + | > loss_disc_real_1: 0.19078266620635986 (0.20451859931151073) + | > loss_disc_real_2: 0.22307999432086945 (0.22417867283026377) + | > loss_disc_real_3: 0.2533840835094452 (0.22845396101474763) + | > loss_disc_real_4: 0.24106433987617493 (0.22830131815539467) + | > loss_disc_real_5: 0.2508127987384796 (0.22584428985913593) + | > loss_0: 2.43392014503479 (2.425632609261407) + | > grad_norm_0: tensor(7.1531, device='cuda:0') (tensor(17.6265, device='cuda:0')) + | > loss_gen: 2.402553081512451 (2.4325749450259737) + | > loss_kl: 1.968836784362793 (1.8793821652730307) + | > loss_feat: 6.502213001251221 (7.261615848541259) + | > loss_mel: 17.792625427246094 (17.86857329474554) + | > loss_duration: 1.5231215953826904 (1.4863423797819355) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.189350128173828 (30.928488540649415) + | > grad_norm_1: tensor(187.1055, device='cuda:0') (tensor(362.5012, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.3762 (0.3479956044091119) + | > loader_time: 0.0065 (0.005260335074530708) + + + --> TIME: 2025-04-19 00:09:03 -- STEP: 70/142 -- GLOBAL_STEP: 389150 + | > loss_disc: 2.5029656887054443 (2.428986780984061) + | > loss_disc_real_0: 0.16825763881206512 (0.12173845038882324) + | > loss_disc_real_1: 0.2030389904975891 (0.20436616539955138) + | > loss_disc_real_2: 0.24500609934329987 (0.22375932889325278) + | > loss_disc_real_3: 0.2546420097351074 (0.22907761931419374) + | > loss_disc_real_4: 0.20583657920360565 (0.22802457426275527) + | > loss_disc_real_5: 0.18075525760650635 (0.22645404253687176) + | > loss_0: 2.5029656887054443 (2.428986780984061) + | > grad_norm_0: tensor(21.3679, device='cuda:0') (tensor(14.5125, device='cuda:0')) + | > loss_gen: 2.2264485359191895 (2.416540452412196) + | > loss_kl: 1.940073847770691 (1.9043194583484104) + | > loss_feat: 7.136847496032715 (7.229688930511474) + | > loss_mel: 17.704328536987305 (17.820328358241483) + | > loss_duration: 1.5426406860351562 (1.495411198479789) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.550338745117188 (30.86628837585449) + | > grad_norm_1: tensor(37.9291, device='cuda:0') (tensor(272.0003, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.402 (0.36190565654209683) + | > loader_time: 0.0066 (0.005658224650791713) + + + --> TIME: 2025-04-19 00:09:13 -- STEP: 95/142 -- GLOBAL_STEP: 389175 + | > loss_disc: 2.5654420852661133 (2.45868145290174) + | > loss_disc_real_0: 0.199122354388237 (0.12625342360452602) + | > loss_disc_real_1: 0.24337126314640045 (0.2067885718847576) + | > loss_disc_real_2: 0.2324780523777008 (0.22644058826722596) + | > loss_disc_real_3: 0.22391125559806824 (0.22969121038913726) + | > loss_disc_real_4: 0.24077332019805908 (0.23105362575305136) + | > loss_disc_real_5: 0.2148626148700714 (0.22895856389873906) + | > loss_0: 2.5654420852661133 (2.45868145290174) + | > grad_norm_0: tensor(8.0966, device='cuda:0') (tensor(12.9008, device='cuda:0')) + | > loss_gen: 2.1939573287963867 (2.3781856963509003) + | > loss_kl: 1.9685217142105103 (1.914810626130355) + | > loss_feat: 6.249331474304199 (7.080392661847566) + | > loss_mel: 17.887651443481445 (17.82259651987176) + | > loss_duration: 1.5564711093902588 (1.5067521070179193) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.855932235717773 (30.702737607453997) + | > grad_norm_1: tensor(70.4852, device='cuda:0') (tensor(238.4531, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.413 (0.3742328719088906) + | > loader_time: 0.0067 (0.0059702622263055095) + + + --> TIME: 2025-04-19 00:09:24 -- STEP: 120/142 -- GLOBAL_STEP: 389200 + | > loss_disc: 2.4732680320739746 (2.459386825561524) + | > loss_disc_real_0: 0.09532927721738815 (0.12677759549890955) + | > loss_disc_real_1: 0.1881323605775833 (0.20706854946911335) + | > loss_disc_real_2: 0.19561976194381714 (0.2259161662310362) + | > loss_disc_real_3: 0.22128356993198395 (0.2292337030172348) + | > loss_disc_real_4: 0.20874075591564178 (0.23085036935905615) + | > loss_disc_real_5: 0.2663537263870239 (0.2284494770069917) + | > loss_0: 2.4732680320739746 (2.459386825561524) + | > grad_norm_0: tensor(8.2448, device='cuda:0') (tensor(12.1887, device='cuda:0')) + | > loss_gen: 2.217670202255249 (2.363685735066731) + | > loss_kl: 1.9814614057540894 (1.9170409639676411) + | > loss_feat: 6.74307107925415 (7.023766616980235) + | > loss_mel: 17.742027282714844 (17.801445976893106) + | > loss_duration: 1.546783208847046 (1.5127490540345514) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.231014251708984 (30.618688281377157) + | > grad_norm_1: tensor(44.0489, device='cuda:0') (tensor(224.6119, device='cuda:0')) + | > current_lr_0: 0.00019828231082573666 + | > current_lr_1: 0.00019828231082573666 + | > step_time: 0.451 (0.38651129603385925) + | > loader_time: 0.0079 (0.006256065766016642) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5412447452545166 (2.5412447452545166) + | > loss_disc_real_0: 0.22584418952465057 (0.22584418952465057) + | > loss_disc_real_1: 0.23469826579093933 (0.23469826579093933) + | > loss_disc_real_2: 0.21268773078918457 (0.21268773078918457) + | > loss_disc_real_3: 0.22279319167137146 (0.22279319167137146) + | > loss_disc_real_4: 0.20686960220336914 (0.20686960220336914) + | > loss_disc_real_5: 0.2456497997045517 (0.2456497997045517) + | > loss_0: 2.5412447452545166 (2.5412447452545166) + | > loss_gen: 2.4483981132507324 (2.4483981132507324) + | > loss_kl: 2.570563316345215 (2.570563316345215) + | > loss_feat: 7.001642227172852 (7.001642227172852) + | > loss_mel: 17.451866149902344 (17.451866149902344) + | > loss_duration: 1.62880277633667 (1.62880277633667) + | > loss_1: 31.101272583007812 (31.101272583007812) + + --> STEP: 1 + | > loss_disc: 2.4053542613983154 (2.4053542613983154) + | > loss_disc_real_0: 0.1738612949848175 (0.1738612949848175) + | > loss_disc_real_1: 0.19164642691612244 (0.19164642691612244) + | > loss_disc_real_2: 0.21274292469024658 (0.21274292469024658) + | > loss_disc_real_3: 0.1996126025915146 (0.1996126025915146) + | > loss_disc_real_4: 0.2075345516204834 (0.2075345516204834) + | > loss_disc_real_5: 0.2002897560596466 (0.2002897560596466) + | > loss_0: 2.4053542613983154 (2.4053542613983154) + | > loss_gen: 2.310474157333374 (2.310474157333374) + | > loss_kl: 2.518033266067505 (2.518033266067505) + | > loss_feat: 6.55832052230835 (6.55832052230835) + | > loss_mel: 17.057262420654297 (17.057262420654297) + | > loss_duration: 1.6373867988586426 (1.6373867988586426) + | > loss_1: 30.081478118896484 (30.081478118896484) + + --> STEP: 2 + | > loss_disc: 2.509744644165039 (2.4575494527816772) + | > loss_disc_real_0: 0.20006582140922546 (0.18696355819702148) + | > loss_disc_real_1: 0.19969959557056427 (0.19567301124334335) + | > loss_disc_real_2: 0.1920730471611023 (0.20240798592567444) + | > loss_disc_real_3: 0.1972598135471344 (0.1984362080693245) + | > loss_disc_real_4: 0.23323595523834229 (0.22038525342941284) + | > loss_disc_real_5: 0.22305627167224884 (0.21167301386594772) + | > loss_0: 2.509744644165039 (2.4575494527816772) + | > loss_gen: 2.430779218673706 (2.37062668800354) + | > loss_kl: 2.0571417808532715 (2.287587523460388) + | > loss_feat: 8.259544372558594 (7.408932447433472) + | > loss_mel: 19.597991943359375 (18.327627182006836) + | > loss_duration: 1.598668098449707 (1.6180274486541748) + | > loss_1: 33.94412612915039 (32.01280212402344) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004055619239807129 (+0.000897526741027832) + | > avg_loss_disc: 2.4575494527816772 (+0.0030770301818847656) + | > avg_loss_disc_real_0: 0.18696355819702148 (+0.06726189330220222) + | > avg_loss_disc_real_1: 0.19567301124334335 (+0.008447796106338501) + | > avg_loss_disc_real_2: 0.20240798592567444 (-0.010161340236663818) + | > avg_loss_disc_real_3: 0.1984362080693245 (-0.0161830335855484) + | > avg_loss_disc_real_4: 0.22038525342941284 (+0.007141701877117157) + | > avg_loss_disc_real_5: 0.21167301386594772 (+0.0008902102708816528) + | > avg_loss_0: 2.4575494527816772 (+0.0030770301818847656) + | > avg_loss_gen: 2.37062668800354 (+0.25049054622650146) + | > avg_loss_kl: 2.287587523460388 (+0.10406434535980225) + | > avg_loss_feat: 7.408932447433472 (+1.211595058441162) + | > avg_loss_mel: 18.327627182006836 (+1.6344757080078125) + | > avg_loss_duration: 1.6180274486541748 (-0.0057259202003479) + | > avg_loss_1: 32.01280212402344 (+3.194901466369629) + + + > EPOCH: 70/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:09:36)  + + --> TIME: 2025-04-19 00:09:38 -- STEP: 3/142 -- GLOBAL_STEP: 389225 + | > loss_disc: 2.4472193717956543 (2.3887928326924643) + | > loss_disc_real_0: 0.07110107690095901 (0.09165793905655543) + | > loss_disc_real_1: 0.1576928049325943 (0.22309542695681253) + | > loss_disc_real_2: 0.24776005744934082 (0.23129417995611826) + | > loss_disc_real_3: 0.24904970824718475 (0.2225750982761383) + | > loss_disc_real_4: 0.23107871413230896 (0.21781103809674582) + | > loss_disc_real_5: 0.22093577682971954 (0.24266120294729868) + | > loss_0: 2.4472193717956543 (2.3887928326924643) + | > grad_norm_0: tensor(10.2478, device='cuda:0') (tensor(12.1076, device='cuda:0')) + | > loss_gen: 2.4270241260528564 (2.4199179808298745) + | > loss_kl: 1.9314420223236084 (1.8495303392410278) + | > loss_feat: 7.089932918548584 (7.6097002029418945) + | > loss_mel: 18.03530502319336 (18.110022226969402) + | > loss_duration: 1.4976047277450562 (1.4830739895502727) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.981307983398438 (31.47224489847819) + | > grad_norm_1: tensor(72.8756, device='cuda:0') (tensor(250.2921, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.3226 (0.32492884000142414) + | > loader_time: 0.0042 (0.00432578722635905) + + + --> TIME: 2025-04-19 00:09:46 -- STEP: 28/142 -- GLOBAL_STEP: 389250 + | > loss_disc: 2.5225863456726074 (2.439303628035954) + | > loss_disc_real_0: 0.12523852288722992 (0.12125771466110434) + | > loss_disc_real_1: 0.21727196872234344 (0.20937537401914597) + | > loss_disc_real_2: 0.23821797966957092 (0.22825715690851212) + | > loss_disc_real_3: 0.20166625082492828 (0.22392779641917773) + | > loss_disc_real_4: 0.22262302041053772 (0.22770149899380548) + | > loss_disc_real_5: 0.2324865460395813 (0.23403023981622287) + | > loss_0: 2.5225863456726074 (2.439303628035954) + | > grad_norm_0: tensor(13.0591, device='cuda:0') (tensor(11.0508, device='cuda:0')) + | > loss_gen: 2.2233457565307617 (2.3853456463132594) + | > loss_kl: 1.9740972518920898 (1.8564203083515167) + | > loss_feat: 6.792146682739258 (7.207720330783299) + | > loss_mel: 17.220733642578125 (17.850472450256344) + | > loss_duration: 1.5166277885437012 (1.4785093367099762) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.726951599121094 (30.77846806389945) + | > grad_norm_1: tensor(233.2503, device='cuda:0') (tensor(259.2161, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.347 (0.33734430585588726) + | > loader_time: 0.0049 (0.005063687052045551) + + + --> TIME: 2025-04-19 00:09:56 -- STEP: 53/142 -- GLOBAL_STEP: 389275 + | > loss_disc: 2.4282166957855225 (2.4431003804476763) + | > loss_disc_real_0: 0.15154699981212616 (0.12135000077058684) + | > loss_disc_real_1: 0.2343505471944809 (0.20620818548607375) + | > loss_disc_real_2: 0.21822917461395264 (0.22733822015096555) + | > loss_disc_real_3: 0.19216260313987732 (0.22519292106043617) + | > loss_disc_real_4: 0.18878529965877533 (0.22791448713473553) + | > loss_disc_real_5: 0.17901161313056946 (0.23300451237075734) + | > loss_0: 2.4282166957855225 (2.4431003804476763) + | > grad_norm_0: tensor(8.5737, device='cuda:0') (tensor(11.6908, device='cuda:0')) + | > loss_gen: 2.343834400177002 (2.3840500903579436) + | > loss_kl: 1.812850832939148 (1.88248702715028) + | > loss_feat: 7.120267868041992 (7.198887834009135) + | > loss_mel: 16.641950607299805 (17.866610994878805) + | > loss_duration: 1.4805119037628174 (1.4846692220220026) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.399415969848633 (30.81670519990741) + | > grad_norm_1: tensor(380.3392, device='cuda:0') (tensor(279.4173, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.3805 (0.35020864684626735) + | > loader_time: 0.0058 (0.0054586113623853) + + + --> TIME: 2025-04-19 00:10:06 -- STEP: 78/142 -- GLOBAL_STEP: 389300 + | > loss_disc: 2.364452838897705 (2.441802513905061) + | > loss_disc_real_0: 0.10310616344213486 (0.12231089633244735) + | > loss_disc_real_1: 0.1838102489709854 (0.2053536839592151) + | > loss_disc_real_2: 0.2188698798418045 (0.22683869416897112) + | > loss_disc_real_3: 0.22262297570705414 (0.22623577809486634) + | > loss_disc_real_4: 0.2150391936302185 (0.22644224208899033) + | > loss_disc_real_5: 0.21389129757881165 (0.23198247853762063) + | > loss_0: 2.364452838897705 (2.441802513905061) + | > grad_norm_0: tensor(5.6589, device='cuda:0') (tensor(10.2783, device='cuda:0')) + | > loss_gen: 2.346930503845215 (2.371097265145718) + | > loss_kl: 1.9475075006484985 (1.9044768382341435) + | > loss_feat: 7.59414005279541 (7.140716943985376) + | > loss_mel: 18.614360809326172 (17.881703706888054) + | > loss_duration: 1.5397298336029053 (1.4975505669911702) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.04267120361328 (30.79554535792424) + | > grad_norm_1: tensor(118.7383, device='cuda:0') (tensor(238.5280, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.4017 (0.3616066498634142) + | > loader_time: 0.0063 (0.0057231707450671075) + + + --> TIME: 2025-04-19 00:10:16 -- STEP: 103/142 -- GLOBAL_STEP: 389325 + | > loss_disc: 2.493030548095703 (2.459309170547042) + | > loss_disc_real_0: 0.1768406480550766 (0.1254207698754894) + | > loss_disc_real_1: 0.1775680035352707 (0.20606234901159712) + | > loss_disc_real_2: 0.2138037085533142 (0.22808162925891506) + | > loss_disc_real_3: 0.22214087843894958 (0.22764328700824848) + | > loss_disc_real_4: 0.2070612758398056 (0.2288009686377442) + | > loss_disc_real_5: 0.2630542516708374 (0.2345612905557873) + | > loss_0: 2.493030548095703 (2.459309170547042) + | > grad_norm_0: tensor(5.2783, device='cuda:0') (tensor(9.7898, device='cuda:0')) + | > loss_gen: 2.2144880294799805 (2.3585929940047787) + | > loss_kl: 1.8914684057235718 (1.921803260312497) + | > loss_feat: 6.963292598724365 (7.05795597798616) + | > loss_mel: 19.26847267150879 (17.91135730558229) + | > loss_duration: 1.5700969696044922 (1.5083443065291469) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.907819747924805 (30.758053862932815) + | > grad_norm_1: tensor(41.1133, device='cuda:0') (tensor(210.8698, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.4127 (0.3716596098779474) + | > loader_time: 0.0064 (0.0059845054033890515) + + + --> TIME: 2025-04-19 00:10:27 -- STEP: 128/142 -- GLOBAL_STEP: 389350 + | > loss_disc: 2.655787944793701 (2.4597624260932216) + | > loss_disc_real_0: 0.17008183896541595 (0.1252143614110537) + | > loss_disc_real_1: 0.2230585813522339 (0.20705131825525314) + | > loss_disc_real_2: 0.23407824337482452 (0.2281332416459918) + | > loss_disc_real_3: 0.22441422939300537 (0.22710550262127072) + | > loss_disc_real_4: 0.24439844489097595 (0.22898162028286606) + | > loss_disc_real_5: 0.2647251784801483 (0.23375907342415303) + | > loss_0: 2.655787944793701 (2.4597624260932216) + | > grad_norm_0: tensor(12.3870, device='cuda:0') (tensor(10.4030, device='cuda:0')) + | > loss_gen: 2.2120351791381836 (2.35868150740862) + | > loss_kl: 2.03587007522583 (1.9377199271693826) + | > loss_feat: 6.684525489807129 (7.038658924400806) + | > loss_mel: 18.154340744018555 (17.953293919563286) + | > loss_duration: 1.5344181060791016 (1.5143873477354646) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.621191024780273 (30.80274160206318) + | > grad_norm_1: tensor(401.8316, device='cuda:0') (tensor(227.8133, device='cuda:0')) + | > current_lr_0: 0.00019825752553688343 + | > current_lr_1: 0.00019825752553688343 + | > step_time: 0.4375 (0.3829396534711122) + | > loader_time: 0.0075 (0.006229070946574211) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.480375289916992 (2.480375289916992) + | > loss_disc_real_0: 0.11934667825698853 (0.11934667825698853) + | > loss_disc_real_1: 0.18860016763210297 (0.18860016763210297) + | > loss_disc_real_2: 0.2356209009885788 (0.2356209009885788) + | > loss_disc_real_3: 0.28747931122779846 (0.28747931122779846) + | > loss_disc_real_4: 0.21343618631362915 (0.21343618631362915) + | > loss_disc_real_5: 0.26116204261779785 (0.26116204261779785) + | > loss_0: 2.480375289916992 (2.480375289916992) + | > loss_gen: 2.2179617881774902 (2.2179617881774902) + | > loss_kl: 2.1880745887756348 (2.1880745887756348) + | > loss_feat: 5.597228527069092 (5.597228527069092) + | > loss_mel: 17.167089462280273 (17.167089462280273) + | > loss_duration: 1.613192081451416 (1.613192081451416) + | > loss_1: 28.783546447753906 (28.783546447753906) + + --> STEP: 1 + | > loss_disc: 2.586320161819458 (2.586320161819458) + | > loss_disc_real_0: 0.15108726918697357 (0.15108726918697357) + | > loss_disc_real_1: 0.2220575511455536 (0.2220575511455536) + | > loss_disc_real_2: 0.25292837619781494 (0.25292837619781494) + | > loss_disc_real_3: 0.25994548201560974 (0.25994548201560974) + | > loss_disc_real_4: 0.23112565279006958 (0.23112565279006958) + | > loss_disc_real_5: 0.2298450767993927 (0.2298450767993927) + | > loss_0: 2.586320161819458 (2.586320161819458) + | > loss_gen: 2.1686949729919434 (2.1686949729919434) + | > loss_kl: 2.2337465286254883 (2.2337465286254883) + | > loss_feat: 5.9930315017700195 (5.9930315017700195) + | > loss_mel: 18.72160530090332 (18.72160530090332) + | > loss_duration: 1.6372805833816528 (1.6372805833816528) + | > loss_1: 30.75436019897461 (30.75436019897461) + + --> STEP: 2 + | > loss_disc: 2.6800825595855713 (2.6332013607025146) + | > loss_disc_real_0: 0.16474461555480957 (0.15791594237089157) + | > loss_disc_real_1: 0.25205960869789124 (0.2370585799217224) + | > loss_disc_real_2: 0.2867071330547333 (0.2698177546262741) + | > loss_disc_real_3: 0.2809012830257416 (0.27042338252067566) + | > loss_disc_real_4: 0.30910950899124146 (0.2701175808906555) + | > loss_disc_real_5: 0.25595492124557495 (0.24289999902248383) + | > loss_0: 2.6800825595855713 (2.6332013607025146) + | > loss_gen: 2.397003412246704 (2.2828491926193237) + | > loss_kl: 2.124676465988159 (2.1792114973068237) + | > loss_feat: 5.955416679382324 (5.974224090576172) + | > loss_mel: 17.819149017333984 (18.270377159118652) + | > loss_duration: 1.5965707302093506 (1.6169256567955017) + | > loss_1: 29.8928165435791 (30.323588371276855) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033473968505859375 (-0.0007082223892211914) + | > avg_loss_disc: 2.6332013607025146 (+0.1756519079208374) + | > avg_loss_disc_real_0: 0.15791594237089157 (-0.029047615826129913) + | > avg_loss_disc_real_1: 0.2370585799217224 (+0.04138556867837906) + | > avg_loss_disc_real_2: 0.2698177546262741 (+0.06740976870059967) + | > avg_loss_disc_real_3: 0.27042338252067566 (+0.07198717445135117) + | > avg_loss_disc_real_4: 0.2701175808906555 (+0.049732327461242676) + | > avg_loss_disc_real_5: 0.24289999902248383 (+0.031226985156536102) + | > avg_loss_0: 2.6332013607025146 (+0.1756519079208374) + | > avg_loss_gen: 2.2828491926193237 (-0.08777749538421631) + | > avg_loss_kl: 2.1792114973068237 (-0.10837602615356445) + | > avg_loss_feat: 5.974224090576172 (-1.4347083568572998) + | > avg_loss_mel: 18.270377159118652 (-0.057250022888183594) + | > avg_loss_duration: 1.6169256567955017 (-0.0011017918586730957) + | > avg_loss_1: 30.323588371276855 (-1.689213752746582) + + + > EPOCH: 71/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:10:35)  + + --> TIME: 2025-04-19 00:10:39 -- STEP: 11/142 -- GLOBAL_STEP: 389375 + | > loss_disc: 2.3464431762695312 (2.4053251960060815) + | > loss_disc_real_0: 0.116093710064888 (0.11155587570233778) + | > loss_disc_real_1: 0.2270684689283371 (0.20919466696002267) + | > loss_disc_real_2: 0.23425151407718658 (0.2237327830357985) + | > loss_disc_real_3: 0.26791852712631226 (0.23161927542903207) + | > loss_disc_real_4: 0.20297257602214813 (0.2249529930678281) + | > loss_disc_real_5: 0.22246867418289185 (0.2309207022190094) + | > loss_0: 2.3464431762695312 (2.4053251960060815) + | > grad_norm_0: tensor(3.0193, device='cuda:0') (tensor(13.2856, device='cuda:0')) + | > loss_gen: 2.444993734359741 (2.4664126092737373) + | > loss_kl: 1.9772441387176514 (1.9106154658577659) + | > loss_feat: 6.738309383392334 (7.422448678450151) + | > loss_mel: 17.13601303100586 (18.008671847256746) + | > loss_duration: 1.4512399435043335 (1.4603167447176846) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.747800827026367 (31.268465735695578) + | > grad_norm_1: tensor(206.9698, device='cuda:0') (tensor(317.8870, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.3299 (0.32772896506569604) + | > loader_time: 0.0052 (0.004614526575261896) + + + --> TIME: 2025-04-19 00:10:48 -- STEP: 36/142 -- GLOBAL_STEP: 389400 + | > loss_disc: 2.2854831218719482 (2.432970517211491) + | > loss_disc_real_0: 0.10633514076471329 (0.11681582426859273) + | > loss_disc_real_1: 0.18100200593471527 (0.20674411124653286) + | > loss_disc_real_2: 0.2207382321357727 (0.22560704499483109) + | > loss_disc_real_3: 0.21270783245563507 (0.23070403850740856) + | > loss_disc_real_4: 0.21060685813426971 (0.2251187769903077) + | > loss_disc_real_5: 0.24033810198307037 (0.23180287165774238) + | > loss_0: 2.2854831218719482 (2.432970517211491) + | > grad_norm_0: tensor(13.3814, device='cuda:0') (tensor(15.0871, device='cuda:0')) + | > loss_gen: 2.5772085189819336 (2.4104244179195833) + | > loss_kl: 2.062628746032715 (1.9141988125112321) + | > loss_feat: 7.959729194641113 (7.263257543245952) + | > loss_mel: 18.556840896606445 (17.923144155078468) + | > loss_duration: 1.4677659273147583 (1.473724325497945) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.62417221069336 (30.98474915822347) + | > grad_norm_1: tensor(596.5579, device='cuda:0') (tensor(347.4813, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.3545 (0.3397056195471022) + | > loader_time: 0.006 (0.005195074611239963) + + + --> TIME: 2025-04-19 00:10:57 -- STEP: 61/142 -- GLOBAL_STEP: 389425 + | > loss_disc: 2.5105466842651367 (2.464434170332112) + | > loss_disc_real_0: 0.0890972763299942 (0.11925410748016639) + | > loss_disc_real_1: 0.16872693598270416 (0.20842860858948503) + | > loss_disc_real_2: 0.2176894247531891 (0.2287383797715922) + | > loss_disc_real_3: 0.1988859325647354 (0.23219930295084343) + | > loss_disc_real_4: 0.2359636127948761 (0.231793076532786) + | > loss_disc_real_5: 0.21293258666992188 (0.23265393444749174) + | > loss_0: 2.5105466842651367 (2.464434170332112) + | > grad_norm_0: tensor(24.4757, device='cuda:0') (tensor(14.7076, device='cuda:0')) + | > loss_gen: 2.4197301864624023 (2.3851024830927625) + | > loss_kl: 1.9828877449035645 (1.9227544421055278) + | > loss_feat: 7.473262310028076 (7.14449740237877) + | > loss_mel: 18.88988494873047 (17.901337748668233) + | > loss_duration: 1.5044821500778198 (1.4864259723757136) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.27024841308594 (30.84011790791496) + | > grad_norm_1: tensor(563.4371, device='cuda:0') (tensor(336.0282, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.3848 (0.350836241831545) + | > loader_time: 0.0063 (0.005535911341182521) + + + --> TIME: 2025-04-19 00:11:07 -- STEP: 86/142 -- GLOBAL_STEP: 389450 + | > loss_disc: 2.6272494792938232 (2.4689298424609873) + | > loss_disc_real_0: 0.0763467326760292 (0.12357081654806469) + | > loss_disc_real_1: 0.22948653995990753 (0.20828440646792568) + | > loss_disc_real_2: 0.23759913444519043 (0.22875033578900403) + | > loss_disc_real_3: 0.21037301421165466 (0.22988146075675653) + | > loss_disc_real_4: 0.2269340604543686 (0.23126874188351076) + | > loss_disc_real_5: 0.20907875895500183 (0.23226147023744362) + | > loss_0: 2.6272494792938232 (2.4689298424609873) + | > grad_norm_0: tensor(17.7617, device='cuda:0') (tensor(16.6778, device='cuda:0')) + | > loss_gen: 2.102475881576538 (2.374780846196552) + | > loss_kl: 1.9934111833572388 (1.931084656438162) + | > loss_feat: 6.39730167388916 (7.0446811942167065) + | > loss_mel: 17.306697845458984 (17.894819913908496) + | > loss_duration: 1.5541307926177979 (1.498341296994409) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.35401725769043 (30.743707767752714) + | > grad_norm_1: tensor(204.0639, device='cuda:0') (tensor(345.1070, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.398 (0.36251027362291205) + | > loader_time: 0.0079 (0.005838624266690986) + + + --> TIME: 2025-04-19 00:11:18 -- STEP: 111/142 -- GLOBAL_STEP: 389475 + | > loss_disc: 2.461665630340576 (2.4766279126072783) + | > loss_disc_real_0: 0.1475210040807724 (0.12597130198736445) + | > loss_disc_real_1: 0.2079218178987503 (0.20862043185814008) + | > loss_disc_real_2: 0.23378126323223114 (0.22783160370749397) + | > loss_disc_real_3: 0.24467076361179352 (0.23010020964854472) + | > loss_disc_real_4: 0.2679067552089691 (0.23219672087076548) + | > loss_disc_real_5: 0.21950164437294006 (0.23296745349695017) + | > loss_0: 2.461665630340576 (2.4766279126072783) + | > grad_norm_0: tensor(12.7549, device='cuda:0') (tensor(16.2147, device='cuda:0')) + | > loss_gen: 2.3226442337036133 (2.3574100120647543) + | > loss_kl: 2.0496838092803955 (1.944758474289834) + | > loss_feat: 6.213908672332764 (6.932788956272709) + | > loss_mel: 18.613798141479492 (17.830527116586495) + | > loss_duration: 1.5671968460083008 (1.5064709208033107) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.767230987548828 (30.571955311405766) + | > grad_norm_1: tensor(274.3074, device='cuda:0') (tensor(332.4890, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.424 (0.373123854130238) + | > loader_time: 0.0069 (0.006071717889459284) + + + --> TIME: 2025-04-19 00:11:29 -- STEP: 136/142 -- GLOBAL_STEP: 389500 + | > loss_disc: 2.3650317192077637 (2.474068866056554) + | > loss_disc_real_0: 0.11284501850605011 (0.1265499745769535) + | > loss_disc_real_1: 0.19167137145996094 (0.20751640102004304) + | > loss_disc_real_2: 0.2025289088487625 (0.2268680317656082) + | > loss_disc_real_3: 0.21231234073638916 (0.22959456461317398) + | > loss_disc_real_4: 0.20892348885536194 (0.2310145702213049) + | > loss_disc_real_5: 0.18156808614730835 (0.23277181715649717) + | > loss_0: 2.3650317192077637 (2.474068866056554) + | > grad_norm_0: tensor(3.8978, device='cuda:0') (tensor(14.6846, device='cuda:0')) + | > loss_gen: 2.447960615158081 (2.3485420170952302) + | > loss_kl: 1.9967265129089355 (1.9452815984978395) + | > loss_feat: 7.250308513641357 (6.923902753521414) + | > loss_mel: 17.470544815063477 (17.81955623626709) + | > loss_duration: 1.5476253032684326 (1.5126011792351217) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.713163375854492 (30.549883688197415) + | > grad_norm_1: tensor(175.8105, device='cuda:0') (tensor(301.6619, device='cuda:0')) + | > current_lr_0: 0.0001982327433461913 + | > current_lr_1: 0.0001982327433461913 + | > step_time: 0.4484 (0.38502233519273643) + | > loader_time: 0.0078 (0.006307033931507784) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.209260940551758 (2.209260940551758) + | > loss_disc_real_0: 0.11522839218378067 (0.11522839218378067) + | > loss_disc_real_1: 0.1964069902896881 (0.1964069902896881) + | > loss_disc_real_2: 0.23960871994495392 (0.23960871994495392) + | > loss_disc_real_3: 0.2381027340888977 (0.2381027340888977) + | > loss_disc_real_4: 0.21397776901721954 (0.21397776901721954) + | > loss_disc_real_5: 0.2515289783477783 (0.2515289783477783) + | > loss_0: 2.209260940551758 (2.209260940551758) + | > loss_gen: 2.6383118629455566 (2.6383118629455566) + | > loss_kl: 2.127911329269409 (2.127911329269409) + | > loss_feat: 7.538409233093262 (7.538409233093262) + | > loss_mel: 19.464921951293945 (19.464921951293945) + | > loss_duration: 1.618720531463623 (1.618720531463623) + | > loss_1: 33.388275146484375 (33.388275146484375) + + --> STEP: 1 + | > loss_disc: 2.595458507537842 (2.595458507537842) + | > loss_disc_real_0: 0.2734543979167938 (0.2734543979167938) + | > loss_disc_real_1: 0.21640321612358093 (0.21640321612358093) + | > loss_disc_real_2: 0.22201243042945862 (0.22201243042945862) + | > loss_disc_real_3: 0.2556895911693573 (0.2556895911693573) + | > loss_disc_real_4: 0.24994361400604248 (0.24994361400604248) + | > loss_disc_real_5: 0.2752968668937683 (0.2752968668937683) + | > loss_0: 2.595458507537842 (2.595458507537842) + | > loss_gen: 2.3837194442749023 (2.3837194442749023) + | > loss_kl: 2.4233005046844482 (2.4233005046844482) + | > loss_feat: 5.8229780197143555 (5.8229780197143555) + | > loss_mel: 16.660205841064453 (16.660205841064453) + | > loss_duration: 1.6607065200805664 (1.6607065200805664) + | > loss_1: 28.950908660888672 (28.950908660888672) + + --> STEP: 2 + | > loss_disc: 2.510869026184082 (2.553163766860962) + | > loss_disc_real_0: 0.23503346741199493 (0.2542439326643944) + | > loss_disc_real_1: 0.2435757964849472 (0.22998950630426407) + | > loss_disc_real_2: 0.261761873960495 (0.2418871521949768) + | > loss_disc_real_3: 0.2464461773633957 (0.2510678842663765) + | > loss_disc_real_4: 0.2435217797756195 (0.246732696890831) + | > loss_disc_real_5: 0.22043560445308685 (0.24786623567342758) + | > loss_0: 2.510869026184082 (2.553163766860962) + | > loss_gen: 2.5627334117889404 (2.4732264280319214) + | > loss_kl: 2.2259037494659424 (2.3246021270751953) + | > loss_feat: 6.965471267700195 (6.394224643707275) + | > loss_mel: 17.078453063964844 (16.86932945251465) + | > loss_duration: 1.613236665725708 (1.6369715929031372) + | > loss_1: 30.445798873901367 (29.69835376739502) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034885406494140625 (+0.000141143798828125) + | > avg_loss_disc: 2.553163766860962 (-0.08003759384155273) + | > avg_loss_disc_real_0: 0.2542439326643944 (+0.09632799029350281) + | > avg_loss_disc_real_1: 0.22998950630426407 (-0.0070690736174583435) + | > avg_loss_disc_real_2: 0.2418871521949768 (-0.027930602431297302) + | > avg_loss_disc_real_3: 0.2510678842663765 (-0.019355498254299164) + | > avg_loss_disc_real_4: 0.246732696890831 (-0.023384883999824524) + | > avg_loss_disc_real_5: 0.24786623567342758 (+0.004966236650943756) + | > avg_loss_0: 2.553163766860962 (-0.08003759384155273) + | > avg_loss_gen: 2.4732264280319214 (+0.19037723541259766) + | > avg_loss_kl: 2.3246021270751953 (+0.14539062976837158) + | > avg_loss_feat: 6.394224643707275 (+0.4200005531311035) + | > avg_loss_mel: 16.86932945251465 (-1.401047706604004) + | > avg_loss_duration: 1.6369715929031372 (+0.020045936107635498) + | > avg_loss_1: 29.69835376739502 (-0.6252346038818359) + + + > EPOCH: 72/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:11:33)  + + --> TIME: 2025-04-19 00:11:40 -- STEP: 19/142 -- GLOBAL_STEP: 389525 + | > loss_disc: 2.4648537635803223 (2.4889706686923376) + | > loss_disc_real_0: 0.08463986217975616 (0.13709219937261782) + | > loss_disc_real_1: 0.18530239164829254 (0.20470170206145236) + | > loss_disc_real_2: 0.23305919766426086 (0.23167923014414937) + | > loss_disc_real_3: 0.2200549691915512 (0.2351862333322826) + | > loss_disc_real_4: 0.23042981326580048 (0.2343525329702779) + | > loss_disc_real_5: 0.266580194234848 (0.2296407301174967) + | > loss_0: 2.4648537635803223 (2.4889706686923376) + | > grad_norm_0: tensor(6.9642, device='cuda:0') (tensor(10.2093, device='cuda:0')) + | > loss_gen: 2.2123067378997803 (2.3277118331507634) + | > loss_kl: 1.939829707145691 (1.8934329246219836) + | > loss_feat: 6.967087745666504 (6.987394734432823) + | > loss_mel: 17.63423728942871 (17.753806867097552) + | > loss_duration: 1.471724510192871 (1.467265016154239) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.22518539428711 (30.429611206054688) + | > grad_norm_1: tensor(74.9358, device='cuda:0') (tensor(147.2491, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.3383 (0.33066956620467336) + | > loader_time: 0.0054 (0.004996237001921001) + + + --> TIME: 2025-04-19 00:11:49 -- STEP: 44/142 -- GLOBAL_STEP: 389550 + | > loss_disc: 2.4635744094848633 (2.4598916769027714) + | > loss_disc_real_0: 0.15376171469688416 (0.13527360947971995) + | > loss_disc_real_1: 0.2262524515390396 (0.20554399287158792) + | > loss_disc_real_2: 0.20915715396404266 (0.22748825360428204) + | > loss_disc_real_3: 0.2534814476966858 (0.2295970026065003) + | > loss_disc_real_4: 0.2601325809955597 (0.22843242267316038) + | > loss_disc_real_5: 0.2202441394329071 (0.22868626903403888) + | > loss_0: 2.4635744094848633 (2.4598916769027714) + | > grad_norm_0: tensor(5.6253, device='cuda:0') (tensor(8.4124, device='cuda:0')) + | > loss_gen: 2.274113416671753 (2.3481037129055373) + | > loss_kl: 1.8312642574310303 (1.8959621732885188) + | > loss_feat: 7.107801914215088 (7.064178065820173) + | > loss_mel: 18.24447250366211 (17.829846208745785) + | > loss_duration: 1.5167274475097656 (1.4805596606297926) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.97437858581543 (30.61864961277355) + | > grad_norm_1: tensor(211.6290, device='cuda:0') (tensor(138.6125, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.3648 (0.34404164552688593) + | > loader_time: 0.0057 (0.005338099869814786) + + + --> TIME: 2025-04-19 00:11:59 -- STEP: 69/142 -- GLOBAL_STEP: 389575 + | > loss_disc: 2.533137798309326 (2.4646493455638057) + | > loss_disc_real_0: 0.12187769263982773 (0.1330682693616203) + | > loss_disc_real_1: 0.23353800177574158 (0.20502102396626404) + | > loss_disc_real_2: 0.27695178985595703 (0.2281224760024444) + | > loss_disc_real_3: 0.2596170902252197 (0.22882509771464526) + | > loss_disc_real_4: 0.2382993847131729 (0.2280410474193269) + | > loss_disc_real_5: 0.2660665214061737 (0.22975931478583295) + | > loss_0: 2.533137798309326 (2.4646493455638057) + | > grad_norm_0: tensor(6.8040, device='cuda:0') (tensor(8.7589, device='cuda:0')) + | > loss_gen: 2.449791193008423 (2.3475532186204107) + | > loss_kl: 2.0447278022766113 (1.9090420422346697) + | > loss_feat: 6.846843242645264 (6.98944379972375) + | > loss_mel: 17.02519989013672 (17.83516762222069) + | > loss_duration: 1.5677108764648438 (1.4909137124600618) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.93427276611328 (30.5721202518629) + | > grad_norm_1: tensor(45.2945, device='cuda:0') (tensor(157.0321, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.3992 (0.35636347964190057) + | > loader_time: 0.0064 (0.005584817001785057) + + + --> TIME: 2025-04-19 00:12:09 -- STEP: 94/142 -- GLOBAL_STEP: 389600 + | > loss_disc: 2.443080186843872 (2.4640072837789) + | > loss_disc_real_0: 0.09909337013959885 (0.13305313060892382) + | > loss_disc_real_1: 0.22101731598377228 (0.20530680313389352) + | > loss_disc_real_2: 0.20743593573570251 (0.2276949204028921) + | > loss_disc_real_3: 0.21360883116722107 (0.22888735602510737) + | > loss_disc_real_4: 0.21841782331466675 (0.22842001280886062) + | > loss_disc_real_5: 0.21595941483974457 (0.22933703803635658) + | > loss_0: 2.443080186843872 (2.4640072837789) + | > grad_norm_0: tensor(10.6224, device='cuda:0') (tensor(9.6443, device='cuda:0')) + | > loss_gen: 2.3573718070983887 (2.349152587829752) + | > loss_kl: 1.9391896724700928 (1.9222903340420825) + | > loss_feat: 7.158843040466309 (6.97436635037686) + | > loss_mel: 19.251745223999023 (17.83735898200502) + | > loss_duration: 1.538370132446289 (1.5025018872098719) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.245521545410156 (30.58567000449972) + | > grad_norm_1: tensor(226.8210, device='cuda:0') (tensor(191.0614, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.4032 (0.3677027555222207) + | > loader_time: 0.0067 (0.0058636995072060445) + + + --> TIME: 2025-04-19 00:12:20 -- STEP: 119/142 -- GLOBAL_STEP: 389625 + | > loss_disc: 2.5886106491088867 (2.4669076594985824) + | > loss_disc_real_0: 0.15541133284568787 (0.13215012278626936) + | > loss_disc_real_1: 0.20651428401470184 (0.20668098605981394) + | > loss_disc_real_2: 0.22731737792491913 (0.226416399123288) + | > loss_disc_real_3: 0.2248890995979309 (0.22911714167655015) + | > loss_disc_real_4: 0.263303279876709 (0.22808284594231293) + | > loss_disc_real_5: 0.21255677938461304 (0.23027101835282912) + | > loss_0: 2.5886106491088867 (2.4669076594985824) + | > grad_norm_0: tensor(6.2087, device='cuda:0') (tensor(9.4335, device='cuda:0')) + | > loss_gen: 2.239997386932373 (2.3457705453664315) + | > loss_kl: 1.9528547525405884 (1.923909629092497) + | > loss_feat: 6.195404052734375 (6.97442412977459) + | > loss_mel: 18.265483856201172 (17.861507912643816) + | > loss_duration: 1.5909428596496582 (1.5090989866176574) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.24468231201172 (30.61471108829274) + | > grad_norm_1: tensor(87.1250, device='cuda:0') (tensor(193.0136, device='cuda:0')) + | > current_lr_0: 0.00019820796425327303 + | > current_lr_1: 0.00019820796425327303 + | > step_time: 0.4266 (0.37947894545162425) + | > loader_time: 0.0079 (0.006212903671905774) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3922977447509766 (2.3922977447509766) + | > loss_disc_real_0: 0.13100512325763702 (0.13100512325763702) + | > loss_disc_real_1: 0.23809711635112762 (0.23809711635112762) + | > loss_disc_real_2: 0.2308027744293213 (0.2308027744293213) + | > loss_disc_real_3: 0.2209126204252243 (0.2209126204252243) + | > loss_disc_real_4: 0.24580205976963043 (0.24580205976963043) + | > loss_disc_real_5: 0.2554323375225067 (0.2554323375225067) + | > loss_0: 2.3922977447509766 (2.3922977447509766) + | > loss_gen: 2.5836031436920166 (2.5836031436920166) + | > loss_kl: 2.291606903076172 (2.291606903076172) + | > loss_feat: 7.3360419273376465 (7.3360419273376465) + | > loss_mel: 20.048023223876953 (20.048023223876953) + | > loss_duration: 1.6437360048294067 (1.6437360048294067) + | > loss_1: 33.90301513671875 (33.90301513671875) + + --> STEP: 1 + | > loss_disc: 2.505557060241699 (2.505557060241699) + | > loss_disc_real_0: 0.11660601198673248 (0.11660601198673248) + | > loss_disc_real_1: 0.2229829877614975 (0.2229829877614975) + | > loss_disc_real_2: 0.22232945263385773 (0.22232945263385773) + | > loss_disc_real_3: 0.24319736659526825 (0.24319736659526825) + | > loss_disc_real_4: 0.2626209557056427 (0.2626209557056427) + | > loss_disc_real_5: 0.26139530539512634 (0.26139530539512634) + | > loss_0: 2.505557060241699 (2.505557060241699) + | > loss_gen: 2.325209617614746 (2.325209617614746) + | > loss_kl: 2.5401172637939453 (2.5401172637939453) + | > loss_feat: 7.0221662521362305 (7.0221662521362305) + | > loss_mel: 18.6723690032959 (18.6723690032959) + | > loss_duration: 1.6704901456832886 (1.6704901456832886) + | > loss_1: 32.23035430908203 (32.23035430908203) + + --> STEP: 2 + | > loss_disc: 2.596163272857666 (2.5508601665496826) + | > loss_disc_real_0: 0.1660967916250229 (0.14135140180587769) + | > loss_disc_real_1: 0.21012260019779205 (0.21655279397964478) + | > loss_disc_real_2: 0.25705111026763916 (0.23969028145074844) + | > loss_disc_real_3: 0.23325875401496887 (0.23822806030511856) + | > loss_disc_real_4: 0.22709256410598755 (0.24485675990581512) + | > loss_disc_real_5: 0.2505475580692291 (0.25597143173217773) + | > loss_0: 2.596163272857666 (2.5508601665496826) + | > loss_gen: 2.171760320663452 (2.248484969139099) + | > loss_kl: 2.00681209564209 (2.2734646797180176) + | > loss_feat: 6.6589179039001465 (6.8405420780181885) + | > loss_mel: 16.91836166381836 (17.79536533355713) + | > loss_duration: 1.6220083236694336 (1.646249234676361) + | > loss_1: 29.37786102294922 (30.804107666015625) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032879114151000977 (-0.00020062923431396484) + | > avg_loss_disc: 2.5508601665496826 (-0.002303600311279297) + | > avg_loss_disc_real_0: 0.14135140180587769 (-0.1128925308585167) + | > avg_loss_disc_real_1: 0.21655279397964478 (-0.013436712324619293) + | > avg_loss_disc_real_2: 0.23969028145074844 (-0.002196870744228363) + | > avg_loss_disc_real_3: 0.23822806030511856 (-0.012839823961257935) + | > avg_loss_disc_real_4: 0.24485675990581512 (-0.0018759369850158691) + | > avg_loss_disc_real_5: 0.25597143173217773 (+0.008105196058750153) + | > avg_loss_0: 2.5508601665496826 (-0.002303600311279297) + | > avg_loss_gen: 2.248484969139099 (-0.22474145889282227) + | > avg_loss_kl: 2.2734646797180176 (-0.051137447357177734) + | > avg_loss_feat: 6.8405420780181885 (+0.4463174343109131) + | > avg_loss_mel: 17.79536533355713 (+0.9260358810424805) + | > avg_loss_duration: 1.646249234676361 (+0.009277641773223877) + | > avg_loss_1: 30.804107666015625 (+1.1057538986206055) + + + > EPOCH: 73/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:12:32)  + + --> TIME: 2025-04-19 00:12:34 -- STEP: 2/142 -- GLOBAL_STEP: 389650 + | > loss_disc: 2.4904863834381104 (2.4854499101638794) + | > loss_disc_real_0: 0.14742295444011688 (0.12597419694066048) + | > loss_disc_real_1: 0.18994145095348358 (0.19190829247236252) + | > loss_disc_real_2: 0.2125701904296875 (0.2158687338232994) + | > loss_disc_real_3: 0.2384333461523056 (0.23475729674100876) + | > loss_disc_real_4: 0.24512343108654022 (0.240196093916893) + | > loss_disc_real_5: 0.21308091282844543 (0.20402242243289948) + | > loss_0: 2.4904863834381104 (2.4854499101638794) + | > grad_norm_0: tensor(8.1354, device='cuda:0') (tensor(7.3626, device='cuda:0')) + | > loss_gen: 2.398782253265381 (2.3646955490112305) + | > loss_kl: 1.9724637269973755 (1.9286311864852905) + | > loss_feat: 6.690906524658203 (6.900357961654663) + | > loss_mel: 17.572351455688477 (17.921966552734375) + | > loss_duration: 1.4866806268692017 (1.46481055021286) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.121185302734375 (30.580461502075195) + | > grad_norm_1: tensor(170.8289, device='cuda:0') (tensor(174.3729, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.3227 (0.33316659927368164) + | > loader_time: 0.005 (0.004507660865783691) + + + --> TIME: 2025-04-19 00:12:42 -- STEP: 27/142 -- GLOBAL_STEP: 389675 + | > loss_disc: 2.452893018722534 (2.453866826163398) + | > loss_disc_real_0: 0.06376473605632782 (0.11954231836177685) + | > loss_disc_real_1: 0.20902886986732483 (0.2026647439709416) + | > loss_disc_real_2: 0.19179560244083405 (0.2255229585700565) + | > loss_disc_real_3: 0.20131519436836243 (0.2280858799263283) + | > loss_disc_real_4: 0.22030816972255707 (0.2277385515195352) + | > loss_disc_real_5: 0.2198609560728073 (0.23560821071819024) + | > loss_0: 2.452893018722534 (2.453866826163398) + | > grad_norm_0: tensor(17.0340, device='cuda:0') (tensor(11.2875, device='cuda:0')) + | > loss_gen: 2.5279810428619385 (2.3770553359278925) + | > loss_kl: 2.0145881175994873 (1.9056410833641335) + | > loss_feat: 7.857332229614258 (7.053882757822673) + | > loss_mel: 18.440616607666016 (18.03930699383771) + | > loss_duration: 1.4924533367156982 (1.470838330410145) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.33297348022461 (30.846724262943976) + | > grad_norm_1: tensor(434.1346, device='cuda:0') (tensor(295.0938, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.3492 (0.33617670447738085) + | > loader_time: 0.0056 (0.005065926799067744) + + + --> TIME: 2025-04-19 00:12:51 -- STEP: 52/142 -- GLOBAL_STEP: 389700 + | > loss_disc: 2.4888033866882324 (2.446468701729408) + | > loss_disc_real_0: 0.21101699769496918 (0.12894123921600675) + | > loss_disc_real_1: 0.20203928649425507 (0.20481963083148003) + | > loss_disc_real_2: 0.23097451031208038 (0.22393243587934053) + | > loss_disc_real_3: 0.25320059061050415 (0.22730105542219603) + | > loss_disc_real_4: 0.2479073703289032 (0.22581069830518502) + | > loss_disc_real_5: 0.2408401221036911 (0.22985426078622156) + | > loss_0: 2.4888033866882324 (2.446468701729408) + | > grad_norm_0: tensor(40.4441, device='cuda:0') (tensor(15.3171, device='cuda:0')) + | > loss_gen: 2.3094639778137207 (2.40356022119522) + | > loss_kl: 2.0388131141662598 (1.9191775436584766) + | > loss_feat: 6.3573832511901855 (7.115324047895578) + | > loss_mel: 17.261993408203125 (18.035946075732895) + | > loss_duration: 1.5440223217010498 (1.47938838830361) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.511674880981445 (30.953396320343018) + | > grad_norm_1: tensor(384.1939, device='cuda:0') (tensor(306.6582, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.363 (0.34699365267386806) + | > loader_time: 0.0061 (0.0054310101729172925) + + + --> TIME: 2025-04-19 00:13:01 -- STEP: 77/142 -- GLOBAL_STEP: 389725 + | > loss_disc: 2.400412082672119 (2.4548639105511945) + | > loss_disc_real_0: 0.09283297508955002 (0.12904760865615567) + | > loss_disc_real_1: 0.15771861374378204 (0.2053889109716787) + | > loss_disc_real_2: 0.24071769416332245 (0.2254430450402297) + | > loss_disc_real_3: 0.23790989816188812 (0.2273941742522376) + | > loss_disc_real_4: 0.22134236991405487 (0.2273149656784999) + | > loss_disc_real_5: 0.185731902718544 (0.22994418016501836) + | > loss_0: 2.400412082672119 (2.4548639105511945) + | > grad_norm_0: tensor(22.5937, device='cuda:0') (tensor(17.3142, device='cuda:0')) + | > loss_gen: 2.416668653488159 (2.4112100384452115) + | > loss_kl: 1.845452904701233 (1.9302144035116418) + | > loss_feat: 8.093891143798828 (7.148529411910416) + | > loss_mel: 19.006092071533203 (18.022744562718774) + | > loss_duration: 1.5013928413391113 (1.491905147379095) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.86349868774414 (31.00460364601829) + | > grad_norm_1: tensor(588.2383, device='cuda:0') (tensor(332.2331, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.3914 (0.35926544201838506) + | > loader_time: 0.0069 (0.005775575513963575) + + + --> TIME: 2025-04-19 00:13:12 -- STEP: 102/142 -- GLOBAL_STEP: 389750 + | > loss_disc: 2.5284032821655273 (2.4639126366259996) + | > loss_disc_real_0: 0.14083276689052582 (0.1282000639201964) + | > loss_disc_real_1: 0.20750127732753754 (0.20599637691881142) + | > loss_disc_real_2: 0.25435349345207214 (0.22712084928563997) + | > loss_disc_real_3: 0.2364712953567505 (0.2282955642132198) + | > loss_disc_real_4: 0.30091139674186707 (0.23057565443656025) + | > loss_disc_real_5: 0.2540978789329529 (0.23111238520519406) + | > loss_0: 2.5284032821655273 (2.4639126366259996) + | > grad_norm_0: tensor(5.6169, device='cuda:0') (tensor(15.9449, device='cuda:0')) + | > loss_gen: 2.343374252319336 (2.3873892671921664) + | > loss_kl: 1.9858369827270508 (1.9441476511020286) + | > loss_feat: 6.460043430328369 (7.074637740266089) + | > loss_mel: 17.949909210205078 (17.989958688324574) + | > loss_duration: 1.5176944732666016 (1.503527256787992) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.256858825683594 (30.899660690158022) + | > grad_norm_1: tensor(122.4884, device='cuda:0') (tensor(325.1864, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.4292 (0.3706677474227606) + | > loader_time: 0.0069 (0.006039252468183928) + + + --> TIME: 2025-04-19 00:13:23 -- STEP: 127/142 -- GLOBAL_STEP: 389775 + | > loss_disc: 2.3786566257476807 (2.4666188870827974) + | > loss_disc_real_0: 0.14234542846679688 (0.1294700842145391) + | > loss_disc_real_1: 0.17354042828083038 (0.20681161099062192) + | > loss_disc_real_2: 0.19719143211841583 (0.22665685145404396) + | > loss_disc_real_3: 0.2094269096851349 (0.22828173696056125) + | > loss_disc_real_4: 0.21871966123580933 (0.2295693502886089) + | > loss_disc_real_5: 0.23809348046779633 (0.23075916060781854) + | > loss_0: 2.3786566257476807 (2.4666188870827974) + | > grad_norm_0: tensor(5.8360, device='cuda:0') (tensor(15.3540, device='cuda:0')) + | > loss_gen: 2.3691165447235107 (2.373530530554102) + | > loss_kl: 1.9341950416564941 (1.946791893853916) + | > loss_feat: 7.507170677185059 (7.003655242168997) + | > loss_mel: 18.386985778808594 (17.914207548607052) + | > loss_duration: 1.5449254512786865 (1.5110568671714601) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.742393493652344 (30.749242166834552) + | > grad_norm_1: tensor(84.0501, device='cuda:0') (tensor(310.7144, device='cuda:0')) + | > current_lr_0: 0.00019818318825774137 + | > current_lr_1: 0.00019818318825774137 + | > step_time: 0.4535 (0.38385895105797474) + | > loader_time: 0.0074 (0.006335168372927688) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.59511661529541 (2.59511661529541) + | > loss_disc_real_0: 0.19572404026985168 (0.19572404026985168) + | > loss_disc_real_1: 0.20946279168128967 (0.20946279168128967) + | > loss_disc_real_2: 0.25519344210624695 (0.25519344210624695) + | > loss_disc_real_3: 0.24738740921020508 (0.24738740921020508) + | > loss_disc_real_4: 0.24546349048614502 (0.24546349048614502) + | > loss_disc_real_5: 0.33726051449775696 (0.33726051449775696) + | > loss_0: 2.59511661529541 (2.59511661529541) + | > loss_gen: 2.4691123962402344 (2.4691123962402344) + | > loss_kl: 2.353424072265625 (2.353424072265625) + | > loss_feat: 6.487667560577393 (6.487667560577393) + | > loss_mel: 19.13692283630371 (19.13692283630371) + | > loss_duration: 1.6719313859939575 (1.6719313859939575) + | > loss_1: 32.119056701660156 (32.119056701660156) + + --> STEP: 1 + | > loss_disc: 2.3738224506378174 (2.3738224506378174) + | > loss_disc_real_0: 0.16156668961048126 (0.16156668961048126) + | > loss_disc_real_1: 0.22894978523254395 (0.22894978523254395) + | > loss_disc_real_2: 0.2563036382198334 (0.2563036382198334) + | > loss_disc_real_3: 0.19193731248378754 (0.19193731248378754) + | > loss_disc_real_4: 0.21194033324718475 (0.21194033324718475) + | > loss_disc_real_5: 0.25710222125053406 (0.25710222125053406) + | > loss_0: 2.3738224506378174 (2.3738224506378174) + | > loss_gen: 2.5181472301483154 (2.5181472301483154) + | > loss_kl: 2.249575614929199 (2.249575614929199) + | > loss_feat: 7.824413776397705 (7.824413776397705) + | > loss_mel: 19.898025512695312 (19.898025512695312) + | > loss_duration: 1.6311366558074951 (1.6311366558074951) + | > loss_1: 34.121299743652344 (34.121299743652344) + + --> STEP: 2 + | > loss_disc: 2.405977725982666 (2.3899000883102417) + | > loss_disc_real_0: 0.1483086496591568 (0.15493766963481903) + | > loss_disc_real_1: 0.21799089014530182 (0.22347033768892288) + | > loss_disc_real_2: 0.2585297226905823 (0.2574166804552078) + | > loss_disc_real_3: 0.21011044085025787 (0.2010238766670227) + | > loss_disc_real_4: 0.17906618118286133 (0.19550325721502304) + | > loss_disc_real_5: 0.24342644214630127 (0.25026433169841766) + | > loss_0: 2.405977725982666 (2.3899000883102417) + | > loss_gen: 2.408585786819458 (2.4633665084838867) + | > loss_kl: 1.8931607007980347 (2.071368157863617) + | > loss_feat: 8.475208282470703 (8.149811029434204) + | > loss_mel: 18.885061264038086 (19.3915433883667) + | > loss_duration: 1.623122215270996 (1.6271294355392456) + | > loss_1: 33.28513717651367 (33.70321846008301) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003422975540161133 (+0.00013506412506103516) + | > avg_loss_disc: 2.3899000883102417 (-0.16096007823944092) + | > avg_loss_disc_real_0: 0.15493766963481903 (+0.013586267828941345) + | > avg_loss_disc_real_1: 0.22347033768892288 (+0.006917543709278107) + | > avg_loss_disc_real_2: 0.2574166804552078 (+0.01772639900445938) + | > avg_loss_disc_real_3: 0.2010238766670227 (-0.037204183638095856) + | > avg_loss_disc_real_4: 0.19550325721502304 (-0.049353502690792084) + | > avg_loss_disc_real_5: 0.25026433169841766 (-0.005707100033760071) + | > avg_loss_0: 2.3899000883102417 (-0.16096007823944092) + | > avg_loss_gen: 2.4633665084838867 (+0.2148815393447876) + | > avg_loss_kl: 2.071368157863617 (-0.20209652185440063) + | > avg_loss_feat: 8.149811029434204 (+1.3092689514160156) + | > avg_loss_mel: 19.3915433883667 (+1.5961780548095703) + | > avg_loss_duration: 1.6271294355392456 (-0.01911979913711548) + | > avg_loss_1: 33.70321846008301 (+2.899110794067383) + + + > EPOCH: 74/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:13:31)  + + --> TIME: 2025-04-19 00:13:35 -- STEP: 10/142 -- GLOBAL_STEP: 389800 + | > loss_disc: 2.3590784072875977 (2.4597419261932374) + | > loss_disc_real_0: 0.1194528341293335 (0.12443567365407944) + | > loss_disc_real_1: 0.20607295632362366 (0.20484413355588912) + | > loss_disc_real_2: 0.22182908654212952 (0.22812538892030715) + | > loss_disc_real_3: 0.2297370433807373 (0.23188774585723876) + | > loss_disc_real_4: 0.22054129838943481 (0.23236992061138154) + | > loss_disc_real_5: 0.23964916169643402 (0.23446893692016602) + | > loss_0: 2.3590784072875977 (2.4597419261932374) + | > grad_norm_0: tensor(10.7422, device='cuda:0') (tensor(8.0933, device='cuda:0')) + | > loss_gen: 2.5546693801879883 (2.3546964168548583) + | > loss_kl: 1.9285370111465454 (1.8264553546905518) + | > loss_feat: 7.427361965179443 (6.974710941314697) + | > loss_mel: 18.276710510253906 (17.795943450927734) + | > loss_duration: 1.4368129968643188 (1.4691308259963989) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.62409210205078 (30.42093734741211) + | > grad_norm_1: tensor(231.3516, device='cuda:0') (tensor(185.3208, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.3333 (0.32643072605133056) + | > loader_time: 0.0053 (0.004579973220825195) + + + --> TIME: 2025-04-19 00:13:44 -- STEP: 35/142 -- GLOBAL_STEP: 389825 + | > loss_disc: 2.4137792587280273 (2.4200814519609724) + | > loss_disc_real_0: 0.14782218635082245 (0.12121311213288988) + | > loss_disc_real_1: 0.17409150302410126 (0.20477530189922877) + | > loss_disc_real_2: 0.2048303186893463 (0.22104403291429792) + | > loss_disc_real_3: 0.19285765290260315 (0.22877127485615867) + | > loss_disc_real_4: 0.1931021362543106 (0.22452079355716706) + | > loss_disc_real_5: 0.19135618209838867 (0.22979446479252408) + | > loss_0: 2.4137792587280273 (2.4200814519609724) + | > grad_norm_0: tensor(11.7352, device='cuda:0') (tensor(7.6192, device='cuda:0')) + | > loss_gen: 2.3953781127929688 (2.3661543028695244) + | > loss_kl: 1.7381259202957153 (1.8641925062452043) + | > loss_feat: 7.453769207000732 (7.0980853080749515) + | > loss_mel: 18.549182891845703 (17.824697821480886) + | > loss_duration: 1.482773780822754 (1.477205160685948) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.61922836303711 (30.630335290091377) + | > grad_norm_1: tensor(148.5087, device='cuda:0') (tensor(164.4505, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.3571 (0.33979134559631347) + | > loader_time: 0.0061 (0.005191864286150251) + + + --> TIME: 2025-04-19 00:13:54 -- STEP: 60/142 -- GLOBAL_STEP: 389850 + | > loss_disc: 2.5220248699188232 (2.4484856804211934) + | > loss_disc_real_0: 0.09366612881422043 (0.12691672096649806) + | > loss_disc_real_1: 0.21199467778205872 (0.20740663756926855) + | > loss_disc_real_2: 0.19544382393360138 (0.22232874209682146) + | > loss_disc_real_3: 0.1862645000219345 (0.23017583961288135) + | > loss_disc_real_4: 0.22271105647087097 (0.22586572269598643) + | > loss_disc_real_5: 0.24197545647621155 (0.23169618820150692) + | > loss_0: 2.5220248699188232 (2.4484856804211934) + | > grad_norm_0: tensor(8.7532, device='cuda:0') (tensor(7.8080, device='cuda:0')) + | > loss_gen: 2.2538866996765137 (2.3411208192507424) + | > loss_kl: 1.9026517868041992 (1.8821998059749603) + | > loss_feat: 7.107544898986816 (6.959049956003825) + | > loss_mel: 18.34261131286621 (17.741253821055093) + | > loss_duration: 1.527873158454895 (1.4868966917196906) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.13456916809082 (30.41052131652832) + | > grad_norm_1: tensor(222.6848, device='cuda:0') (tensor(171.0748, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.3762 (0.3504891633987427) + | > loader_time: 0.0063 (0.005531438191731771) + + + --> TIME: 2025-04-19 00:14:04 -- STEP: 85/142 -- GLOBAL_STEP: 389875 + | > loss_disc: 2.464120626449585 (2.456102043039658) + | > loss_disc_real_0: 0.09965147078037262 (0.1272309182321324) + | > loss_disc_real_1: 0.21033437550067902 (0.2085640774053686) + | > loss_disc_real_2: 0.22271515429019928 (0.2248291280339746) + | > loss_disc_real_3: 0.22071532905101776 (0.22880416389773875) + | > loss_disc_real_4: 0.21994753181934357 (0.227631032817504) + | > loss_disc_real_5: 0.2695794105529785 (0.23044549927991978) + | > loss_0: 2.464120626449585 (2.456102043039658) + | > grad_norm_0: tensor(16.9441, device='cuda:0') (tensor(8.9032, device='cuda:0')) + | > loss_gen: 2.3349266052246094 (2.3411684456993553) + | > loss_kl: 1.8597549200057983 (1.8991698082755595) + | > loss_feat: 7.051499366760254 (6.957666643928079) + | > loss_mel: 18.197025299072266 (17.876504740995514) + | > loss_duration: 1.5258219242095947 (1.4992636287913605) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.96902847290039 (30.573773440192728) + | > grad_norm_1: tensor(419.1727, device='cuda:0') (tensor(195.8073, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.3942 (0.36289510165943817) + | > loader_time: 0.0065 (0.005846203074735755) + + + --> TIME: 2025-04-19 00:14:14 -- STEP: 110/142 -- GLOBAL_STEP: 389900 + | > loss_disc: 2.462782382965088 (2.4589025714180686) + | > loss_disc_real_0: 0.11536122858524323 (0.128229390897534) + | > loss_disc_real_1: 0.17010414600372314 (0.20743122683330015) + | > loss_disc_real_2: 0.18674398958683014 (0.2244861428033222) + | > loss_disc_real_3: 0.21861927211284637 (0.2291349473324689) + | > loss_disc_real_4: 0.2233450710773468 (0.22824619981375607) + | > loss_disc_real_5: 0.2317180335521698 (0.23103609085083007) + | > loss_0: 2.462782382965088 (2.4589025714180686) + | > grad_norm_0: tensor(14.0125, device='cuda:0') (tensor(9.4715, device='cuda:0')) + | > loss_gen: 2.347548484802246 (2.3370053919878866) + | > loss_kl: 1.9112142324447632 (1.9132812489162792) + | > loss_feat: 6.958865165710449 (6.923886195096102) + | > loss_mel: 17.307395935058594 (17.854408420215947) + | > loss_duration: 1.520663857460022 (1.5079206607558513) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.04568862915039 (30.53650210987438) + | > grad_norm_1: tensor(278.9147, device='cuda:0') (tensor(216.5320, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.4338 (0.37334480285644533) + | > loader_time: 0.007 (0.006114515391263096) + + + --> TIME: 2025-04-19 00:14:25 -- STEP: 135/142 -- GLOBAL_STEP: 389925 + | > loss_disc: 2.4831719398498535 (2.459663989808824) + | > loss_disc_real_0: 0.12076612561941147 (0.12864136166042742) + | > loss_disc_real_1: 0.22448842227458954 (0.20740409714204294) + | > loss_disc_real_2: 0.21312442421913147 (0.2246197885937161) + | > loss_disc_real_3: 0.21266594529151917 (0.22925734277124757) + | > loss_disc_real_4: 0.22473591566085815 (0.2284208212737684) + | > loss_disc_real_5: 0.21682021021842957 (0.22997681818626545) + | > loss_0: 2.4831719398498535 (2.459663989808824) + | > grad_norm_0: tensor(7.5642, device='cuda:0') (tensor(10.3380, device='cuda:0')) + | > loss_gen: 2.294229507446289 (2.3430735958947073) + | > loss_kl: 1.8927236795425415 (1.9202404066368386) + | > loss_feat: 7.096482753753662 (6.965373805717186) + | > loss_mel: 17.455339431762695 (17.873657954180675) + | > loss_duration: 1.4962410926818848 (1.5131924752835877) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.235015869140625 (30.615538434629087) + | > grad_norm_1: tensor(175.4264, device='cuda:0') (tensor(230.3186, device='cuda:0')) + | > current_lr_0: 0.00019815841535920914 + | > current_lr_1: 0.00019815841535920914 + | > step_time: 0.4565 (0.3857020484076606) + | > loader_time: 0.0076 (0.006374572824548793) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6023130416870117 (2.6023130416870117) + | > loss_disc_real_0: 0.1038835272192955 (0.1038835272192955) + | > loss_disc_real_1: 0.2629801034927368 (0.2629801034927368) + | > loss_disc_real_2: 0.224742129445076 (0.224742129445076) + | > loss_disc_real_3: 0.2707042992115021 (0.2707042992115021) + | > loss_disc_real_4: 0.22984009981155396 (0.22984009981155396) + | > loss_disc_real_5: 0.25445374846458435 (0.25445374846458435) + | > loss_0: 2.6023130416870117 (2.6023130416870117) + | > loss_gen: 2.237015724182129 (2.237015724182129) + | > loss_kl: 2.4578516483306885 (2.4578516483306885) + | > loss_feat: 6.800503253936768 (6.800503253936768) + | > loss_mel: 18.393556594848633 (18.393556594848633) + | > loss_duration: 1.647276520729065 (1.647276520729065) + | > loss_1: 31.536203384399414 (31.536203384399414) + + --> STEP: 1 + | > loss_disc: 2.5485219955444336 (2.5485219955444336) + | > loss_disc_real_0: 0.09817681461572647 (0.09817681461572647) + | > loss_disc_real_1: 0.2531876564025879 (0.2531876564025879) + | > loss_disc_real_2: 0.24226674437522888 (0.24226674437522888) + | > loss_disc_real_3: 0.22209812700748444 (0.22209812700748444) + | > loss_disc_real_4: 0.23975412547588348 (0.23975412547588348) + | > loss_disc_real_5: 0.23694275319576263 (0.23694275319576263) + | > loss_0: 2.5485219955444336 (2.5485219955444336) + | > loss_gen: 2.1377129554748535 (2.1377129554748535) + | > loss_kl: 2.406149387359619 (2.406149387359619) + | > loss_feat: 6.994079113006592 (6.994079113006592) + | > loss_mel: 19.897258758544922 (19.897258758544922) + | > loss_duration: 1.6506236791610718 (1.6506236791610718) + | > loss_1: 33.08582305908203 (33.08582305908203) + + --> STEP: 2 + | > loss_disc: 2.389378309249878 (2.4689501523971558) + | > loss_disc_real_0: 0.08831489086151123 (0.09324585273861885) + | > loss_disc_real_1: 0.21552476286888123 (0.23435620963573456) + | > loss_disc_real_2: 0.22587929666042328 (0.23407302051782608) + | > loss_disc_real_3: 0.2038145661354065 (0.21295634657144547) + | > loss_disc_real_4: 0.24401837587356567 (0.24188625067472458) + | > loss_disc_real_5: 0.24356074631214142 (0.24025174975395203) + | > loss_0: 2.389378309249878 (2.4689501523971558) + | > loss_gen: 2.317901849746704 (2.227807402610779) + | > loss_kl: 1.9962356090545654 (2.2011924982070923) + | > loss_feat: 7.0103230476379395 (7.002201080322266) + | > loss_mel: 15.360054969787598 (17.62865686416626) + | > loss_duration: 1.59315025806427 (1.621886968612671) + | > loss_1: 28.277666091918945 (30.68174457550049) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036510229110717773 (+0.00022804737091064453) + | > avg_loss_disc: 2.4689501523971558 (+0.07905006408691406) + | > avg_loss_disc_real_0: 0.09324585273861885 (-0.06169181689620018) + | > avg_loss_disc_real_1: 0.23435620963573456 (+0.010885871946811676) + | > avg_loss_disc_real_2: 0.23407302051782608 (-0.023343659937381744) + | > avg_loss_disc_real_3: 0.21295634657144547 (+0.01193246990442276) + | > avg_loss_disc_real_4: 0.24188625067472458 (+0.04638299345970154) + | > avg_loss_disc_real_5: 0.24025174975395203 (-0.010012581944465637) + | > avg_loss_0: 2.4689501523971558 (+0.07905006408691406) + | > avg_loss_gen: 2.227807402610779 (-0.2355591058731079) + | > avg_loss_kl: 2.2011924982070923 (+0.12982434034347534) + | > avg_loss_feat: 7.002201080322266 (-1.1476099491119385) + | > avg_loss_mel: 17.62865686416626 (-1.7628865242004395) + | > avg_loss_duration: 1.621886968612671 (-0.005242466926574707) + | > avg_loss_1: 30.68174457550049 (-3.0214738845825195) + + + > EPOCH: 75/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:14:30)  + + --> TIME: 2025-04-19 00:14:37 -- STEP: 18/142 -- GLOBAL_STEP: 389950 + | > loss_disc: 2.326176881790161 (2.395902527703179) + | > loss_disc_real_0: 0.12396323680877686 (0.12013666124807464) + | > loss_disc_real_1: 0.2091134786605835 (0.2070637047290802) + | > loss_disc_real_2: 0.24043530225753784 (0.22376252048545414) + | > loss_disc_real_3: 0.20494288206100464 (0.22012307743231455) + | > loss_disc_real_4: 0.20653654634952545 (0.22178710997104645) + | > loss_disc_real_5: 0.2166108638048172 (0.2165371005733808) + | > loss_0: 2.326176881790161 (2.395902527703179) + | > grad_norm_0: tensor(14.4732, device='cuda:0') (tensor(9.2773, device='cuda:0')) + | > loss_gen: 2.559356212615967 (2.3996002939012318) + | > loss_kl: 1.7735252380371094 (1.8824071023199294) + | > loss_feat: 7.498772144317627 (7.277306874593099) + | > loss_mel: 17.838762283325195 (18.113433519999187) + | > loss_duration: 1.4759008884429932 (1.461774594253964) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.146316528320312 (31.134522755940754) + | > grad_norm_1: tensor(290.9034, device='cuda:0') (tensor(179.8409, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.3348 (0.3302495612038506) + | > loader_time: 0.0054 (0.0050586726930406355) + + + --> TIME: 2025-04-19 00:14:46 -- STEP: 43/142 -- GLOBAL_STEP: 389975 + | > loss_disc: 2.5349984169006348 (2.446777438008508) + | > loss_disc_real_0: 0.12209808826446533 (0.12384946141825166) + | > loss_disc_real_1: 0.21919961273670197 (0.20479926570903423) + | > loss_disc_real_2: 0.23521041870117188 (0.22778534958528918) + | > loss_disc_real_3: 0.24171486496925354 (0.22775084854558456) + | > loss_disc_real_4: 0.24091215431690216 (0.22790812718313794) + | > loss_disc_real_5: 0.25319862365722656 (0.22645179372887278) + | > loss_0: 2.5349984169006348 (2.446777438008508) + | > grad_norm_0: tensor(10.1997, device='cuda:0') (tensor(10.6819, device='cuda:0')) + | > loss_gen: 2.2012832164764404 (2.3689525848211246) + | > loss_kl: 1.8409960269927979 (1.8982014323389806) + | > loss_feat: 6.619626998901367 (7.0899479999098665) + | > loss_mel: 18.155790328979492 (17.953587310258733) + | > loss_duration: 1.5070348978042603 (1.475907372873883) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.324729919433594 (30.78659696357195) + | > grad_norm_1: tensor(343.5542, device='cuda:0') (tensor(217.4419, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.378 (0.34429729816525473) + | > loader_time: 0.0059 (0.005367434302041697) + + + --> TIME: 2025-04-19 00:14:55 -- STEP: 68/142 -- GLOBAL_STEP: 390000 + | > loss_disc: 2.4265122413635254 (2.464203084216398) + | > loss_disc_real_0: 0.13253675401210785 (0.1256794593132594) + | > loss_disc_real_1: 0.1858709752559662 (0.20695775352856693) + | > loss_disc_real_2: 0.20974920690059662 (0.2271800242802676) + | > loss_disc_real_3: 0.20173639059066772 (0.2276397080105894) + | > loss_disc_real_4: 0.2018464207649231 (0.2278871014714241) + | > loss_disc_real_5: 0.21656857430934906 (0.22819306504200487) + | > loss_0: 2.4265122413635254 (2.464203084216398) + | > grad_norm_0: tensor(8.3574, device='cuda:0') (tensor(11.1418, device='cuda:0')) + | > loss_gen: 2.5163540840148926 (2.3536323834868034) + | > loss_kl: 2.015728712081909 (1.912641975809546) + | > loss_feat: 7.56240701675415 (6.992187044199775) + | > loss_mel: 17.320161819458008 (17.807532857446102) + | > loss_duration: 1.495782732963562 (1.4880540546248942) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.910432815551758 (30.55404836991254) + | > grad_norm_1: tensor(287.1744, device='cuda:0') (tensor(232.4209, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.388 (0.35633967203252453) + | > loader_time: 0.0064 (0.0057109699529760026) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_390000.pth + + --> TIME: 2025-04-19 00:15:07 -- STEP: 93/142 -- GLOBAL_STEP: 390025 + | > loss_disc: 2.575870990753174 (2.4709018404765803) + | > loss_disc_real_0: 0.21832449734210968 (0.12661028573269484) + | > loss_disc_real_1: 0.17221292853355408 (0.20832037957765723) + | > loss_disc_real_2: 0.21000975370407104 (0.22722203180354128) + | > loss_disc_real_3: 0.2512328624725342 (0.22993006661374082) + | > loss_disc_real_4: 0.2536395788192749 (0.22940369190708285) + | > loss_disc_real_5: 0.24705247581005096 (0.23184665797218199) + | > loss_0: 2.575870990753174 (2.4709018404765803) + | > grad_norm_0: tensor(24.3334, device='cuda:0') (tensor(11.9864, device='cuda:0')) + | > loss_gen: 2.2077603340148926 (2.3541657053014298) + | > loss_kl: 2.1114869117736816 (1.9285538914383098) + | > loss_feat: 5.7407732009887695 (6.967297595034363) + | > loss_mel: 16.94379425048828 (17.821043516999925) + | > loss_duration: 1.516188383102417 (1.5010057995396275) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.520002365112305 (30.57206654292281) + | > grad_norm_1: tensor(209.7493, device='cuda:0') (tensor(253.6662, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.4076 (0.36744990400088745) + | > loader_time: 0.0115 (0.005955601251253519) + + + --> TIME: 2025-04-19 00:15:18 -- STEP: 118/142 -- GLOBAL_STEP: 390050 + | > loss_disc: 2.445967197418213 (2.469543699490823) + | > loss_disc_real_0: 0.10028743743896484 (0.12573796226564105) + | > loss_disc_real_1: 0.22111837565898895 (0.2083485255302009) + | > loss_disc_real_2: 0.22563418745994568 (0.22754801279407436) + | > loss_disc_real_3: 0.2086086869239807 (0.22904660085500297) + | > loss_disc_real_4: 0.20991022884845734 (0.22864591544967586) + | > loss_disc_real_5: 0.21155595779418945 (0.23252951416928888) + | > loss_0: 2.445967197418213 (2.469543699490823) + | > grad_norm_0: tensor(25.6444, device='cuda:0') (tensor(13.4001, device='cuda:0')) + | > loss_gen: 2.4560792446136475 (2.3584767984131645) + | > loss_kl: 2.048313617706299 (1.9379698502815377) + | > loss_feat: 7.0746073722839355 (6.974945925049862) + | > loss_mel: 17.626659393310547 (17.842647261538733) + | > loss_duration: 1.521836280822754 (1.508528000217373) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.727497100830078 (30.62256785570565) + | > grad_norm_1: tensor(549.3906, device='cuda:0') (tensor(283.5137, device='cuda:0')) + | > current_lr_0: 0.00019813364555728923 + | > current_lr_1: 0.00019813364555728923 + | > step_time: 0.4421 (0.3784033080278818) + | > loader_time: 0.0076 (0.006191138493812691) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.446336269378662 (2.446336269378662) + | > loss_disc_real_0: 0.12585389614105225 (0.12585389614105225) + | > loss_disc_real_1: 0.19384773075580597 (0.19384773075580597) + | > loss_disc_real_2: 0.24650529026985168 (0.24650529026985168) + | > loss_disc_real_3: 0.24417221546173096 (0.24417221546173096) + | > loss_disc_real_4: 0.2236432433128357 (0.2236432433128357) + | > loss_disc_real_5: 0.23788709938526154 (0.23788709938526154) + | > loss_0: 2.446336269378662 (2.446336269378662) + | > loss_gen: 2.2648532390594482 (2.2648532390594482) + | > loss_kl: 2.1195461750030518 (2.1195461750030518) + | > loss_feat: 7.030242443084717 (7.030242443084717) + | > loss_mel: 18.927898406982422 (18.927898406982422) + | > loss_duration: 1.6179245710372925 (1.6179245710372925) + | > loss_1: 31.96046257019043 (31.96046257019043) + + --> STEP: 1 + | > loss_disc: 2.464573383331299 (2.464573383331299) + | > loss_disc_real_0: 0.12762822210788727 (0.12762822210788727) + | > loss_disc_real_1: 0.263944149017334 (0.263944149017334) + | > loss_disc_real_2: 0.25440478324890137 (0.25440478324890137) + | > loss_disc_real_3: 0.27275311946868896 (0.27275311946868896) + | > loss_disc_real_4: 0.23946358263492584 (0.23946358263492584) + | > loss_disc_real_5: 0.20120200514793396 (0.20120200514793396) + | > loss_0: 2.464573383331299 (2.464573383331299) + | > loss_gen: 2.3687636852264404 (2.3687636852264404) + | > loss_kl: 2.6013736724853516 (2.6013736724853516) + | > loss_feat: 5.883274555206299 (5.883274555206299) + | > loss_mel: 16.2392635345459 (16.2392635345459) + | > loss_duration: 1.6657216548919678 (1.6657216548919678) + | > loss_1: 28.75839614868164 (28.75839614868164) + + --> STEP: 2 + | > loss_disc: 2.3887343406677246 (2.4266538619995117) + | > loss_disc_real_0: 0.13316389918327332 (0.1303960606455803) + | > loss_disc_real_1: 0.28831079602241516 (0.2761274725198746) + | > loss_disc_real_2: 0.2582312524318695 (0.25631801784038544) + | > loss_disc_real_3: 0.23977436125278473 (0.25626374036073685) + | > loss_disc_real_4: 0.251374751329422 (0.24541916698217392) + | > loss_disc_real_5: 0.21846656501293182 (0.2098342850804329) + | > loss_0: 2.3887343406677246 (2.4266538619995117) + | > loss_gen: 2.6413750648498535 (2.505069375038147) + | > loss_kl: 2.142803192138672 (2.3720884323120117) + | > loss_feat: 7.258335590362549 (6.570805072784424) + | > loss_mel: 18.279155731201172 (17.259209632873535) + | > loss_duration: 1.5824072360992432 (1.6240644454956055) + | > loss_1: 31.904075622558594 (30.331235885620117) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034265518188476562 (-0.0002244710922241211) + | > avg_loss_disc: 2.4266538619995117 (-0.04229629039764404) + | > avg_loss_disc_real_0: 0.1303960606455803 (+0.03715020790696144) + | > avg_loss_disc_real_1: 0.2761274725198746 (+0.041771262884140015) + | > avg_loss_disc_real_2: 0.25631801784038544 (+0.022244997322559357) + | > avg_loss_disc_real_3: 0.25626374036073685 (+0.04330739378929138) + | > avg_loss_disc_real_4: 0.24541916698217392 (+0.003532916307449341) + | > avg_loss_disc_real_5: 0.2098342850804329 (-0.030417464673519135) + | > avg_loss_0: 2.4266538619995117 (-0.04229629039764404) + | > avg_loss_gen: 2.505069375038147 (+0.27726197242736816) + | > avg_loss_kl: 2.3720884323120117 (+0.17089593410491943) + | > avg_loss_feat: 6.570805072784424 (-0.4313960075378418) + | > avg_loss_mel: 17.259209632873535 (-0.3694472312927246) + | > avg_loss_duration: 1.6240644454956055 (+0.0021774768829345703) + | > avg_loss_1: 30.331235885620117 (-0.3505086898803711) + + + > EPOCH: 76/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:15:30)  + + --> TIME: 2025-04-19 00:15:31 -- STEP: 1/142 -- GLOBAL_STEP: 390075 + | > loss_disc: 2.4126508235931396 (2.4126508235931396) + | > loss_disc_real_0: 0.15419505536556244 (0.15419505536556244) + | > loss_disc_real_1: 0.2338779717683792 (0.2338779717683792) + | > loss_disc_real_2: 0.24487127363681793 (0.24487127363681793) + | > loss_disc_real_3: 0.24287191033363342 (0.24287191033363342) + | > loss_disc_real_4: 0.24489164352416992 (0.24489164352416992) + | > loss_disc_real_5: 0.1883019059896469 (0.1883019059896469) + | > loss_0: 2.4126508235931396 (2.4126508235931396) + | > grad_norm_0: tensor(31.4426, device='cuda:0') (tensor(31.4426, device='cuda:0')) + | > loss_gen: 2.420144557952881 (2.420144557952881) + | > loss_kl: 1.8465566635131836 (1.8465566635131836) + | > loss_feat: 7.411402702331543 (7.411402702331543) + | > loss_mel: 17.80491828918457 (17.80491828918457) + | > loss_duration: 1.4397714138031006 (1.4397714138031006) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.922794342041016 (30.922794342041016) + | > grad_norm_1: tensor(320.7898, device='cuda:0') (tensor(320.7898, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.3271 (0.3270847797393799) + | > loader_time: 0.0042 (0.004214286804199219) + + + --> TIME: 2025-04-19 00:15:40 -- STEP: 26/142 -- GLOBAL_STEP: 390100 + | > loss_disc: 2.380399227142334 (2.418678081952608) + | > loss_disc_real_0: 0.09796904772520065 (0.11199933032576855) + | > loss_disc_real_1: 0.19916850328445435 (0.2017732709646225) + | > loss_disc_real_2: 0.19535300135612488 (0.22487541345449594) + | > loss_disc_real_3: 0.20357142388820648 (0.22725660353899002) + | > loss_disc_real_4: 0.19251976907253265 (0.22584524693397376) + | > loss_disc_real_5: 0.17906109988689423 (0.22480299782294494) + | > loss_0: 2.380399227142334 (2.418678081952608) + | > grad_norm_0: tensor(13.3274, device='cuda:0') (tensor(12.8548, device='cuda:0')) + | > loss_gen: 2.372236728668213 (2.3622760681005626) + | > loss_kl: 1.908036708831787 (1.8827142944702735) + | > loss_feat: 7.271984577178955 (7.1221717320955715) + | > loss_mel: 17.328495025634766 (17.717069002298214) + | > loss_duration: 1.4819635152816772 (1.4735113153090844) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.362716674804688 (30.5577424122737) + | > grad_norm_1: tensor(443.9652, device='cuda:0') (tensor(294.1135, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.3502 (0.33428431474245507) + | > loader_time: 0.0053 (0.005135939671443058) + + + --> TIME: 2025-04-19 00:15:49 -- STEP: 51/142 -- GLOBAL_STEP: 390125 + | > loss_disc: 2.462118148803711 (2.4347280053531417) + | > loss_disc_real_0: 0.11692158132791519 (0.11951822992049012) + | > loss_disc_real_1: 0.15729038417339325 (0.20259085852725833) + | > loss_disc_real_2: 0.21259337663650513 (0.22606206787567512) + | > loss_disc_real_3: 0.24828706681728363 (0.22932582772245594) + | > loss_disc_real_4: 0.23350553214550018 (0.22767037328551798) + | > loss_disc_real_5: 0.2555617094039917 (0.22930537456390904) + | > loss_0: 2.462118148803711 (2.4347280053531417) + | > grad_norm_0: tensor(14.6251, device='cuda:0') (tensor(11.6677, device='cuda:0')) + | > loss_gen: 2.262119770050049 (2.354805581709918) + | > loss_kl: 1.960705041885376 (1.8829128531848682) + | > loss_feat: 7.071725368499756 (7.071667830149333) + | > loss_mel: 17.219602584838867 (17.711869333304612) + | > loss_duration: 1.4986588954925537 (1.4814649189219762) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.0128116607666 (30.50272055233226) + | > grad_norm_1: tensor(390.4248, device='cuda:0') (tensor(280.7599, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.3725 (0.3460501970029345) + | > loader_time: 0.0061 (0.005483435649497837) + + + --> TIME: 2025-04-19 00:15:59 -- STEP: 76/142 -- GLOBAL_STEP: 390150 + | > loss_disc: 2.4353392124176025 (2.437005090086084) + | > loss_disc_real_0: 0.10243115574121475 (0.12228837432829957) + | > loss_disc_real_1: 0.22109070420265198 (0.20277438959793037) + | > loss_disc_real_2: 0.25857412815093994 (0.22403052135517723) + | > loss_disc_real_3: 0.2396538108587265 (0.2285999530240109) + | > loss_disc_real_4: 0.24236856400966644 (0.22660400169460396) + | > loss_disc_real_5: 0.22919154167175293 (0.22976826013703094) + | > loss_0: 2.4353392124176025 (2.437005090086084) + | > grad_norm_0: tensor(9.1312, device='cuda:0') (tensor(10.8273, device='cuda:0')) + | > loss_gen: 2.3859148025512695 (2.352485267739547) + | > loss_kl: 1.9594507217407227 (1.9019556108273958) + | > loss_feat: 7.35075569152832 (7.07441406500967) + | > loss_mel: 19.439401626586914 (17.831768362145674) + | > loss_duration: 1.5223557949066162 (1.492655909375141) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.657875061035156 (30.65327925431101) + | > grad_norm_1: tensor(151.4068, device='cuda:0') (tensor(246.6514, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.3916 (0.3578727057105616) + | > loader_time: 0.0064 (0.005821576243952701) + + + --> TIME: 2025-04-19 00:16:09 -- STEP: 101/142 -- GLOBAL_STEP: 390175 + | > loss_disc: 2.386439323425293 (2.457773961643181) + | > loss_disc_real_0: 0.14374053478240967 (0.12519836278245) + | > loss_disc_real_1: 0.22910954058170319 (0.2048276053501828) + | > loss_disc_real_2: 0.18775594234466553 (0.2253856200097811) + | > loss_disc_real_3: 0.2063537985086441 (0.228248397902687) + | > loss_disc_real_4: 0.17995654046535492 (0.2280674259556402) + | > loss_disc_real_5: 0.24691633880138397 (0.2305371123965424) + | > loss_0: 2.386439323425293 (2.457773961643181) + | > grad_norm_0: tensor(5.9859, device='cuda:0') (tensor(10.9051, device='cuda:0')) + | > loss_gen: 2.4197824001312256 (2.339475202088309) + | > loss_kl: 2.0200090408325195 (1.9186830756687883) + | > loss_feat: 7.329019546508789 (7.0190073334344545) + | > loss_mel: 19.302745819091797 (17.884003138778233) + | > loss_duration: 1.547087550163269 (1.5029617677820797) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.61864471435547 (30.66413059801158) + | > grad_norm_1: tensor(163.0916, device='cuda:0') (tensor(240.0336, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.4168 (0.3690886639132358) + | > loader_time: 0.0069 (0.006128301714906599) + + + --> TIME: 2025-04-19 00:16:20 -- STEP: 126/142 -- GLOBAL_STEP: 390200 + | > loss_disc: 2.363863468170166 (2.4622783225680154) + | > loss_disc_real_0: 0.1582430750131607 (0.1260105331265737) + | > loss_disc_real_1: 0.20327448844909668 (0.2064053921235932) + | > loss_disc_real_2: 0.2054532766342163 (0.22648602381112085) + | > loss_disc_real_3: 0.24169543385505676 (0.2292150952040203) + | > loss_disc_real_4: 0.2170795351266861 (0.2286492341331073) + | > loss_disc_real_5: 0.2184067666530609 (0.23021423308149216) + | > loss_0: 2.363863468170166 (2.4622783225680154) + | > grad_norm_0: tensor(8.1456, device='cuda:0') (tensor(10.5697, device='cuda:0')) + | > loss_gen: 2.446406364440918 (2.3337026910176353) + | > loss_kl: 1.9095460176467896 (1.9257690849758329) + | > loss_feat: 6.910402774810791 (6.955572480247135) + | > loss_mel: 17.32028579711914 (17.881500107901438) + | > loss_duration: 1.5293232202529907 (1.5090764306840447) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.115966796875 (30.605620883759997) + | > grad_norm_1: tensor(202.3599, device='cuda:0') (tensor(236.1144, device='cuda:0')) + | > current_lr_0: 0.00019810887885159456 + | > current_lr_1: 0.00019810887885159456 + | > step_time: 0.4385 (0.38060373351687476) + | > loader_time: 0.0072 (0.00634817850022089) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.477304220199585 (2.477304220199585) + | > loss_disc_real_0: 0.12106602638959885 (0.12106602638959885) + | > loss_disc_real_1: 0.20465905964374542 (0.20465905964374542) + | > loss_disc_real_2: 0.20098207890987396 (0.20098207890987396) + | > loss_disc_real_3: 0.23846659064292908 (0.23846659064292908) + | > loss_disc_real_4: 0.19675114750862122 (0.19675114750862122) + | > loss_disc_real_5: 0.23267827928066254 (0.23267827928066254) + | > loss_0: 2.477304220199585 (2.477304220199585) + | > loss_gen: 2.0676798820495605 (2.0676798820495605) + | > loss_kl: 2.380542516708374 (2.380542516708374) + | > loss_feat: 7.113861560821533 (7.113861560821533) + | > loss_mel: 17.6373233795166 (17.6373233795166) + | > loss_duration: 1.6307493448257446 (1.6307493448257446) + | > loss_1: 30.830156326293945 (30.830156326293945) + + --> STEP: 1 + | > loss_disc: 2.3566389083862305 (2.3566389083862305) + | > loss_disc_real_0: 0.07846630364656448 (0.07846630364656448) + | > loss_disc_real_1: 0.2135850042104721 (0.2135850042104721) + | > loss_disc_real_2: 0.19682319462299347 (0.19682319462299347) + | > loss_disc_real_3: 0.2404797226190567 (0.2404797226190567) + | > loss_disc_real_4: 0.21389231085777283 (0.21389231085777283) + | > loss_disc_real_5: 0.24638894200325012 (0.24638894200325012) + | > loss_0: 2.3566389083862305 (2.3566389083862305) + | > loss_gen: 2.35823917388916 (2.35823917388916) + | > loss_kl: 2.532228946685791 (2.532228946685791) + | > loss_feat: 7.896737575531006 (7.896737575531006) + | > loss_mel: 19.001359939575195 (19.001359939575195) + | > loss_duration: 1.6506516933441162 (1.6506516933441162) + | > loss_1: 33.43921661376953 (33.43921661376953) + + --> STEP: 2 + | > loss_disc: 2.5125608444213867 (2.4345998764038086) + | > loss_disc_real_0: 0.13617075979709625 (0.10731853172183037) + | > loss_disc_real_1: 0.22358521819114685 (0.21858511120080948) + | > loss_disc_real_2: 0.2144397795200348 (0.20563148707151413) + | > loss_disc_real_3: 0.22306017577648163 (0.23176994919776917) + | > loss_disc_real_4: 0.2122138887643814 (0.21305309981107712) + | > loss_disc_real_5: 0.2838246822357178 (0.26510681211948395) + | > loss_0: 2.5125608444213867 (2.4345998764038086) + | > loss_gen: 2.157101631164551 (2.2576704025268555) + | > loss_kl: 2.113280773162842 (2.3227548599243164) + | > loss_feat: 5.994162082672119 (6.9454498291015625) + | > loss_mel: 15.985304832458496 (17.493332386016846) + | > loss_duration: 1.6142289638519287 (1.6324403285980225) + | > loss_1: 27.864076614379883 (30.651646614074707) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035883188247680664 (+0.00016176700592041016) + | > avg_loss_disc: 2.4345998764038086 (+0.007946014404296875) + | > avg_loss_disc_real_0: 0.10731853172183037 (-0.023077528923749924) + | > avg_loss_disc_real_1: 0.21858511120080948 (-0.057542361319065094) + | > avg_loss_disc_real_2: 0.20563148707151413 (-0.05068653076887131) + | > avg_loss_disc_real_3: 0.23176994919776917 (-0.024493791162967682) + | > avg_loss_disc_real_4: 0.21305309981107712 (-0.0323660671710968) + | > avg_loss_disc_real_5: 0.26510681211948395 (+0.055272527039051056) + | > avg_loss_0: 2.4345998764038086 (+0.007946014404296875) + | > avg_loss_gen: 2.2576704025268555 (-0.2473989725112915) + | > avg_loss_kl: 2.3227548599243164 (-0.04933357238769531) + | > avg_loss_feat: 6.9454498291015625 (+0.37464475631713867) + | > avg_loss_mel: 17.493332386016846 (+0.23412275314331055) + | > avg_loss_duration: 1.6324403285980225 (+0.008375883102416992) + | > avg_loss_1: 30.651646614074707 (+0.32041072845458984) + + + > EPOCH: 77/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:16:29)  + + --> TIME: 2025-04-19 00:16:32 -- STEP: 9/142 -- GLOBAL_STEP: 390225 + | > loss_disc: 2.4501261711120605 (2.4498232205708823) + | > loss_disc_real_0: 0.11261706799268723 (0.1115535373489062) + | > loss_disc_real_1: 0.17959599196910858 (0.20296663377020094) + | > loss_disc_real_2: 0.23143504559993744 (0.2324517915646235) + | > loss_disc_real_3: 0.23937523365020752 (0.23883042567306095) + | > loss_disc_real_4: 0.25682705640792847 (0.23304761780632866) + | > loss_disc_real_5: 0.22578009963035583 (0.23735026684072283) + | > loss_0: 2.4501261711120605 (2.4498232205708823) + | > grad_norm_0: tensor(16.7751, device='cuda:0') (tensor(9.9601, device='cuda:0')) + | > loss_gen: 2.408618927001953 (2.4161880811055503) + | > loss_kl: 1.817077398300171 (1.8586200343237982) + | > loss_feat: 7.373508453369141 (7.258716106414795) + | > loss_mel: 18.98462677001953 (18.100639555189346) + | > loss_duration: 1.442901372909546 (1.4642379548814561) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.0267333984375 (31.09840159946018) + | > grad_norm_1: tensor(250.3879, device='cuda:0') (tensor(259.0022, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.3247 (0.3249477810329861) + | > loader_time: 0.0048 (0.004438612196180556) + + + --> TIME: 2025-04-19 00:16:41 -- STEP: 34/142 -- GLOBAL_STEP: 390250 + | > loss_disc: 2.508078098297119 (2.4366219183977913) + | > loss_disc_real_0: 0.17329533398151398 (0.11985376849770546) + | > loss_disc_real_1: 0.19347356259822845 (0.20560182148919387) + | > loss_disc_real_2: 0.2323600947856903 (0.22488255448201122) + | > loss_disc_real_3: 0.2037004977464676 (0.2294126166140332) + | > loss_disc_real_4: 0.1974923014640808 (0.22605084715520635) + | > loss_disc_real_5: 0.19844762980937958 (0.23276999636608012) + | > loss_0: 2.508078098297119 (2.4366219183977913) + | > grad_norm_0: tensor(15.0723, device='cuda:0') (tensor(12.0358, device='cuda:0')) + | > loss_gen: 2.3587188720703125 (2.388662576675415) + | > loss_kl: 1.8657289743423462 (1.8863591201165144) + | > loss_feat: 7.412563323974609 (7.005281756905949) + | > loss_mel: 17.59869956970215 (17.563014759736898) + | > loss_duration: 1.4989972114562988 (1.472996694200179) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.73470687866211 (30.316314865561093) + | > grad_norm_1: tensor(449.1865, device='cuda:0') (tensor(281.6078, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.3552 (0.3409761891645543) + | > loader_time: 0.0059 (0.005268714007209329) + + + --> TIME: 2025-04-19 00:16:51 -- STEP: 59/142 -- GLOBAL_STEP: 390275 + | > loss_disc: 2.347430944442749 (2.457150859347844) + | > loss_disc_real_0: 0.15706460177898407 (0.1244428141389863) + | > loss_disc_real_1: 0.17169331014156342 (0.20864663412005213) + | > loss_disc_real_2: 0.20262038707733154 (0.2268256850666919) + | > loss_disc_real_3: 0.20542356371879578 (0.2290649876251059) + | > loss_disc_real_4: 0.20113389194011688 (0.22649324798988082) + | > loss_disc_real_5: 0.1971919983625412 (0.23039798084962165) + | > loss_0: 2.347430944442749 (2.457150859347844) + | > grad_norm_0: tensor(4.4536, device='cuda:0') (tensor(11.6472, device='cuda:0')) + | > loss_gen: 2.4130258560180664 (2.361434633448971) + | > loss_kl: 1.899452567100525 (1.9001825522568265) + | > loss_feat: 7.33572244644165 (6.955771882655257) + | > loss_mel: 17.1512508392334 (17.65516371646169) + | > loss_duration: 1.4985454082489014 (1.483907309629149) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.29799461364746 (30.35646011869786) + | > grad_norm_1: tensor(286.5079, device='cuda:0') (tensor(274.2465, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.3744 (0.35151060961060615) + | > loader_time: 0.0062 (0.0056537571599928) + + + --> TIME: 2025-04-19 00:17:01 -- STEP: 84/142 -- GLOBAL_STEP: 390300 + | > loss_disc: 2.539062261581421 (2.464932640393575) + | > loss_disc_real_0: 0.19964008033275604 (0.12816599987092475) + | > loss_disc_real_1: 0.22268211841583252 (0.20915957983760608) + | > loss_disc_real_2: 0.23010483384132385 (0.22597521401586987) + | > loss_disc_real_3: 0.22839881479740143 (0.2289039803047975) + | > loss_disc_real_4: 0.20715510845184326 (0.2272006424055213) + | > loss_disc_real_5: 0.22537586092948914 (0.23104151923741614) + | > loss_0: 2.539062261581421 (2.464932640393575) + | > grad_norm_0: tensor(32.4940, device='cuda:0') (tensor(12.1324, device='cuda:0')) + | > loss_gen: 2.350860357284546 (2.360779413155146) + | > loss_kl: 1.922224998474121 (1.9212915429047175) + | > loss_feat: 6.881709575653076 (6.975696484247844) + | > loss_mel: 17.88144874572754 (17.731785660698293) + | > loss_duration: 1.5553308725357056 (1.4964315124920435) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.59157371520996 (30.485984643300373) + | > grad_norm_1: tensor(171.5324, device='cuda:0') (tensor(273.8613, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.3944 (0.3636210873013451) + | > loader_time: 0.0069 (0.0059666321391151035) + + + --> TIME: 2025-04-19 00:17:11 -- STEP: 109/142 -- GLOBAL_STEP: 390325 + | > loss_disc: 2.575380325317383 (2.4808251988997165) + | > loss_disc_real_0: 0.12798653542995453 (0.1300133445189086) + | > loss_disc_real_1: 0.22459281980991364 (0.21124490781114735) + | > loss_disc_real_2: 0.25501561164855957 (0.22860578645806795) + | > loss_disc_real_3: 0.26860311627388 (0.23043594857968322) + | > loss_disc_real_4: 0.23551015555858612 (0.2294291177200615) + | > loss_disc_real_5: 0.2832483947277069 (0.23177899939751406) + | > loss_0: 2.575380325317383 (2.4808251988997165) + | > grad_norm_0: tensor(24.0756, device='cuda:0') (tensor(13.8594, device='cuda:0')) + | > loss_gen: 2.3149490356445312 (2.359849811693942) + | > loss_kl: 1.983155608177185 (1.9265249202010828) + | > loss_feat: 6.46660041809082 (6.932030642798187) + | > loss_mel: 18.129682540893555 (17.75996733149257) + | > loss_duration: 1.5419902801513672 (1.5050098119525734) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.436376571655273 (30.483382636253985) + | > grad_norm_1: tensor(377.4396, device='cuda:0') (tensor(292.3356, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.4227 (0.373803661503923) + | > loader_time: 0.0071 (0.006195022425520309) + + + --> TIME: 2025-04-19 00:17:22 -- STEP: 134/142 -- GLOBAL_STEP: 390350 + | > loss_disc: 2.4867115020751953 (2.4829369907948515) + | > loss_disc_real_0: 0.12307341396808624 (0.1309189750020628) + | > loss_disc_real_1: 0.20322532951831818 (0.20995963798530065) + | > loss_disc_real_2: 0.20842835307121277 (0.22779060599964057) + | > loss_disc_real_3: 0.2128102332353592 (0.23005776440919334) + | > loss_disc_real_4: 0.22965194284915924 (0.22988397772632427) + | > loss_disc_real_5: 0.2530509829521179 (0.23064961484563884) + | > loss_0: 2.4867115020751953 (2.4829369907948515) + | > grad_norm_0: tensor(13.5588, device='cuda:0') (tensor(13.9460, device='cuda:0')) + | > loss_gen: 2.2748570442199707 (2.346656555560096) + | > loss_kl: 2.076270341873169 (1.9311692091956068) + | > loss_feat: 6.732442855834961 (6.876939727299249) + | > loss_mel: 17.848556518554688 (17.69072650795552) + | > loss_duration: 1.5594134330749512 (1.5111417868244117) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.491539001464844 (30.356633826867856) + | > grad_norm_1: tensor(349.4209, device='cuda:0') (tensor(293.6507, device='cuda:0')) + | > current_lr_0: 0.0001980841152417381 + | > current_lr_1: 0.0001980841152417381 + | > step_time: 0.4435 (0.38608879473671986) + | > loader_time: 0.0076 (0.006449767013094317) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.414011001586914 (2.414011001586914) + | > loss_disc_real_0: 0.1521776169538498 (0.1521776169538498) + | > loss_disc_real_1: 0.20708198845386505 (0.20708198845386505) + | > loss_disc_real_2: 0.22656786441802979 (0.22656786441802979) + | > loss_disc_real_3: 0.22718720138072968 (0.22718720138072968) + | > loss_disc_real_4: 0.22934654355049133 (0.22934654355049133) + | > loss_disc_real_5: 0.26379433274269104 (0.26379433274269104) + | > loss_0: 2.414011001586914 (2.414011001586914) + | > loss_gen: 2.360684871673584 (2.360684871673584) + | > loss_kl: 2.2145447731018066 (2.2145447731018066) + | > loss_feat: 6.698416233062744 (6.698416233062744) + | > loss_mel: 17.73611068725586 (17.73611068725586) + | > loss_duration: 1.6365151405334473 (1.6365151405334473) + | > loss_1: 30.646270751953125 (30.646270751953125) + + --> STEP: 1 + | > loss_disc: 2.4926705360412598 (2.4926705360412598) + | > loss_disc_real_0: 0.1773890256881714 (0.1773890256881714) + | > loss_disc_real_1: 0.24734674394130707 (0.24734674394130707) + | > loss_disc_real_2: 0.24140767753124237 (0.24140767753124237) + | > loss_disc_real_3: 0.20591814815998077 (0.20591814815998077) + | > loss_disc_real_4: 0.22062551975250244 (0.22062551975250244) + | > loss_disc_real_5: 0.2419794201850891 (0.2419794201850891) + | > loss_0: 2.4926705360412598 (2.4926705360412598) + | > loss_gen: 2.270580530166626 (2.270580530166626) + | > loss_kl: 2.4882378578186035 (2.4882378578186035) + | > loss_feat: 6.400643825531006 (6.400643825531006) + | > loss_mel: 16.895626068115234 (16.895626068115234) + | > loss_duration: 1.68497896194458 (1.68497896194458) + | > loss_1: 29.740066528320312 (29.740066528320312) + + --> STEP: 2 + | > loss_disc: 2.312305212020874 (2.402487874031067) + | > loss_disc_real_0: 0.15543235838413239 (0.16641069203615189) + | > loss_disc_real_1: 0.20950190722942352 (0.2284243255853653) + | > loss_disc_real_2: 0.2200460284948349 (0.23072685301303864) + | > loss_disc_real_3: 0.15591686964035034 (0.18091750890016556) + | > loss_disc_real_4: 0.16789646446704865 (0.19426099210977554) + | > loss_disc_real_5: 0.190652534365654 (0.21631597727537155) + | > loss_0: 2.312305212020874 (2.402487874031067) + | > loss_gen: 2.492420196533203 (2.3815003633499146) + | > loss_kl: 2.1243460178375244 (2.306291937828064) + | > loss_feat: 8.333349227905273 (7.36699652671814) + | > loss_mel: 18.14185905456543 (17.518742561340332) + | > loss_duration: 1.6082475185394287 (1.6466132402420044) + | > loss_1: 32.70022201538086 (31.220144271850586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003876924514770508 (+0.0002886056900024414) + | > avg_loss_disc: 2.402487874031067 (-0.0321120023727417) + | > avg_loss_disc_real_0: 0.16641069203615189 (+0.05909216031432152) + | > avg_loss_disc_real_1: 0.2284243255853653 (+0.009839214384555817) + | > avg_loss_disc_real_2: 0.23072685301303864 (+0.025095365941524506) + | > avg_loss_disc_real_3: 0.18091750890016556 (-0.05085244029760361) + | > avg_loss_disc_real_4: 0.19426099210977554 (-0.018792107701301575) + | > avg_loss_disc_real_5: 0.21631597727537155 (-0.048790834844112396) + | > avg_loss_0: 2.402487874031067 (-0.0321120023727417) + | > avg_loss_gen: 2.3815003633499146 (+0.12382996082305908) + | > avg_loss_kl: 2.306291937828064 (-0.01646292209625244) + | > avg_loss_feat: 7.36699652671814 (+0.42154669761657715) + | > avg_loss_mel: 17.518742561340332 (+0.025410175323486328) + | > avg_loss_duration: 1.6466132402420044 (+0.014172911643981934) + | > avg_loss_1: 31.220144271850586 (+0.5684976577758789) + + + > EPOCH: 78/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:17:27)  + + --> TIME: 2025-04-19 00:17:34 -- STEP: 17/142 -- GLOBAL_STEP: 390375 + | > loss_disc: 2.4890198707580566 (2.453730386846206) + | > loss_disc_real_0: 0.1259913295507431 (0.1225278377532959) + | > loss_disc_real_1: 0.217655748128891 (0.20640061795711517) + | > loss_disc_real_2: 0.22176238894462585 (0.2237102433162577) + | > loss_disc_real_3: 0.23453593254089355 (0.2299492183853598) + | > loss_disc_real_4: 0.2128535360097885 (0.2286121336852803) + | > loss_disc_real_5: 0.25281497836112976 (0.2282524792587056) + | > loss_0: 2.4890198707580566 (2.453730386846206) + | > grad_norm_0: tensor(5.1224, device='cuda:0') (tensor(10.8667, device='cuda:0')) + | > loss_gen: 2.3954007625579834 (2.3746789623709286) + | > loss_kl: 1.774451732635498 (1.8922917281880098) + | > loss_feat: 7.651953220367432 (7.302476013407988) + | > loss_mel: 18.471729278564453 (18.077461803660672) + | > loss_duration: 1.4487757682800293 (1.457423441550311) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.742311477661133 (31.1043321946088) + | > grad_norm_1: tensor(135.2637, device='cuda:0') (tensor(205.9216, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.3347 (0.32875240550321694) + | > loader_time: 0.0053 (0.00497132189133588) + + + --> TIME: 2025-04-19 00:17:43 -- STEP: 42/142 -- GLOBAL_STEP: 390400 + | > loss_disc: 2.3614087104797363 (2.477564351899283) + | > loss_disc_real_0: 0.0985272154211998 (0.12303424733025688) + | > loss_disc_real_1: 0.16786901652812958 (0.20708116605168297) + | > loss_disc_real_2: 0.20479536056518555 (0.22904675773211888) + | > loss_disc_real_3: 0.2364431619644165 (0.23207669527757735) + | > loss_disc_real_4: 0.24538934230804443 (0.23103736411957515) + | > loss_disc_real_5: 0.23253804445266724 (0.23568462083737055) + | > loss_0: 2.3614087104797363 (2.477564351899283) + | > grad_norm_0: tensor(7.7285, device='cuda:0') (tensor(9.3692, device='cuda:0')) + | > loss_gen: 2.330146551132202 (2.33628701596033) + | > loss_kl: 1.9151711463928223 (1.9060136051405043) + | > loss_feat: 7.354251861572266 (7.093168939862933) + | > loss_mel: 18.357189178466797 (17.912444750467937) + | > loss_duration: 1.520809531211853 (1.4736367009934925) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.477567672729492 (30.721551123119536) + | > grad_norm_1: tensor(179.6579, device='cuda:0') (tensor(219.8749, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.3607 (0.3415858121145339) + | > loader_time: 0.0059 (0.00542951197851272) + + + --> TIME: 2025-04-19 00:17:52 -- STEP: 67/142 -- GLOBAL_STEP: 390425 + | > loss_disc: 2.5662119388580322 (2.475203635087653) + | > loss_disc_real_0: 0.12156277894973755 (0.12435585215910157) + | > loss_disc_real_1: 0.2248416244983673 (0.2064092090325569) + | > loss_disc_real_2: 0.2082972675561905 (0.22722645793388138) + | > loss_disc_real_3: 0.2285032719373703 (0.23109638579745792) + | > loss_disc_real_4: 0.19423209130764008 (0.23064351504418387) + | > loss_disc_real_5: 0.2398499697446823 (0.23435337730308078) + | > loss_0: 2.5662119388580322 (2.475203635087653) + | > grad_norm_0: tensor(8.9348, device='cuda:0') (tensor(9.1905, device='cuda:0')) + | > loss_gen: 2.361666202545166 (2.341792284552731) + | > loss_kl: 1.8162111043930054 (1.9152069447645501) + | > loss_feat: 7.10669469833374 (7.047446151277912) + | > loss_mel: 18.530288696289062 (17.87066898061268) + | > loss_duration: 1.4501982927322388 (1.4859022756121052) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.265058517456055 (30.661016606572847) + | > grad_norm_1: tensor(273.9025, device='cuda:0') (tensor(218.4479, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.3819 (0.35378007746454493) + | > loader_time: 0.0062 (0.00574081691343393) + + + --> TIME: 2025-04-19 00:18:03 -- STEP: 92/142 -- GLOBAL_STEP: 390450 + | > loss_disc: 2.335391044616699 (2.476902508217354) + | > loss_disc_real_0: 0.07596997171640396 (0.126814184791368) + | > loss_disc_real_1: 0.21787729859352112 (0.20746255163913188) + | > loss_disc_real_2: 0.2550007402896881 (0.22877246578750404) + | > loss_disc_real_3: 0.25557222962379456 (0.2313640480456145) + | > loss_disc_real_4: 0.2270634025335312 (0.22880343444969342) + | > loss_disc_real_5: 0.2553323805332184 (0.23243994460157727) + | > loss_0: 2.335391044616699 (2.476902508217354) + | > grad_norm_0: tensor(24.1621, device='cuda:0') (tensor(10.3367, device='cuda:0')) + | > loss_gen: 2.5378353595733643 (2.332993564398393) + | > loss_kl: 2.0049972534179688 (1.9334332735642143) + | > loss_feat: 6.883822441101074 (6.947095233461131) + | > loss_mel: 17.165769577026367 (17.812219319136247) + | > loss_duration: 1.4898788928985596 (1.4990621960681418) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.082304000854492 (30.52480355553005) + | > grad_norm_1: tensor(441.4866, device='cuda:0') (tensor(246.5814, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.4258 (0.3669166202130526) + | > loader_time: 0.007 (0.0060276026311128035) + + + --> TIME: 2025-04-19 00:18:14 -- STEP: 117/142 -- GLOBAL_STEP: 390475 + | > loss_disc: 2.42995548248291 (2.4838807970030685) + | > loss_disc_real_0: 0.08860516548156738 (0.1282319840457705) + | > loss_disc_real_1: 0.19744925200939178 (0.2079369778561796) + | > loss_disc_real_2: 0.2409433275461197 (0.22841126159725025) + | > loss_disc_real_3: 0.23575380444526672 (0.23128167546202993) + | > loss_disc_real_4: 0.20905843377113342 (0.22881800573096317) + | > loss_disc_real_5: 0.22116272151470184 (0.23373116756606305) + | > loss_0: 2.42995548248291 (2.4838807970030685) + | > grad_norm_0: tensor(12.0923, device='cuda:0') (tensor(10.0017, device='cuda:0')) + | > loss_gen: 2.355560779571533 (2.329295900132921) + | > loss_kl: 1.8753013610839844 (1.9467777726996658) + | > loss_feat: 7.184214115142822 (6.939940036871494) + | > loss_mel: 16.40987205505371 (17.783142309922447) + | > loss_duration: 1.5030168294906616 (1.5050643325870872) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.327964782714844 (30.50422031043941) + | > grad_norm_1: tensor(186.7813, device='cuda:0') (tensor(239.8120, device='cuda:0')) + | > current_lr_0: 0.00019805935472733287 + | > current_lr_1: 0.00019805935472733287 + | > step_time: 0.4483 (0.3806408164847611) + | > loader_time: 0.0069 (0.006338243810539572) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4810941219329834 (2.4810941219329834) + | > loss_disc_real_0: 0.17988573014736176 (0.17988573014736176) + | > loss_disc_real_1: 0.20516757667064667 (0.20516757667064667) + | > loss_disc_real_2: 0.21436068415641785 (0.21436068415641785) + | > loss_disc_real_3: 0.24788838624954224 (0.24788838624954224) + | > loss_disc_real_4: 0.24008570611476898 (0.24008570611476898) + | > loss_disc_real_5: 0.2549557685852051 (0.2549557685852051) + | > loss_0: 2.4810941219329834 (2.4810941219329834) + | > loss_gen: 2.384455680847168 (2.384455680847168) + | > loss_kl: 2.348902702331543 (2.348902702331543) + | > loss_feat: 6.9821648597717285 (6.9821648597717285) + | > loss_mel: 19.750539779663086 (19.750539779663086) + | > loss_duration: 1.6550383567810059 (1.6550383567810059) + | > loss_1: 33.12110137939453 (33.12110137939453) + + --> STEP: 1 + | > loss_disc: 2.452399253845215 (2.452399253845215) + | > loss_disc_real_0: 0.18722866475582123 (0.18722866475582123) + | > loss_disc_real_1: 0.18920819461345673 (0.18920819461345673) + | > loss_disc_real_2: 0.2135717123746872 (0.2135717123746872) + | > loss_disc_real_3: 0.21418385207653046 (0.21418385207653046) + | > loss_disc_real_4: 0.22389622032642365 (0.22389622032642365) + | > loss_disc_real_5: 0.2327762097120285 (0.2327762097120285) + | > loss_0: 2.452399253845215 (2.452399253845215) + | > loss_gen: 2.254817485809326 (2.254817485809326) + | > loss_kl: 2.431520700454712 (2.431520700454712) + | > loss_feat: 5.903848648071289 (5.903848648071289) + | > loss_mel: 17.886737823486328 (17.886737823486328) + | > loss_duration: 1.6681066751480103 (1.6681066751480103) + | > loss_1: 30.145030975341797 (30.145030975341797) + + --> STEP: 2 + | > loss_disc: 2.52346134185791 (2.4879302978515625) + | > loss_disc_real_0: 0.22843100130558014 (0.20782983303070068) + | > loss_disc_real_1: 0.20660685002803802 (0.19790752232074738) + | > loss_disc_real_2: 0.25146540999412537 (0.23251856118440628) + | > loss_disc_real_3: 0.2120470404624939 (0.21311544626951218) + | > loss_disc_real_4: 0.23768919706344604 (0.23079270869493484) + | > loss_disc_real_5: 0.18808086216449738 (0.21042853593826294) + | > loss_0: 2.52346134185791 (2.4879302978515625) + | > loss_gen: 2.3927693367004395 (2.323793411254883) + | > loss_kl: 2.3243885040283203 (2.377954602241516) + | > loss_feat: 7.12844705581665 (6.51614785194397) + | > loss_mel: 18.343082427978516 (18.114910125732422) + | > loss_duration: 1.6285052299499512 (1.6483059525489807) + | > loss_1: 31.81719207763672 (30.981111526489258) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034415721893310547 (-0.0004353523254394531) + | > avg_loss_disc: 2.4879302978515625 (+0.0854424238204956) + | > avg_loss_disc_real_0: 0.20782983303070068 (+0.0414191409945488) + | > avg_loss_disc_real_1: 0.19790752232074738 (-0.03051680326461792) + | > avg_loss_disc_real_2: 0.23251856118440628 (+0.0017917081713676453) + | > avg_loss_disc_real_3: 0.21311544626951218 (+0.03219793736934662) + | > avg_loss_disc_real_4: 0.23079270869493484 (+0.0365317165851593) + | > avg_loss_disc_real_5: 0.21042853593826294 (-0.005887441337108612) + | > avg_loss_0: 2.4879302978515625 (+0.0854424238204956) + | > avg_loss_gen: 2.323793411254883 (-0.05770695209503174) + | > avg_loss_kl: 2.377954602241516 (+0.07166266441345215) + | > avg_loss_feat: 6.51614785194397 (-0.8508486747741699) + | > avg_loss_mel: 18.114910125732422 (+0.5961675643920898) + | > avg_loss_duration: 1.6483059525489807 (+0.0016927123069763184) + | > avg_loss_1: 30.981111526489258 (-0.23903274536132812) + + + > EPOCH: 79/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:18:27)  + + --> TIME: 2025-04-19 00:18:27 -- STEP: 0/142 -- GLOBAL_STEP: 390500 + | > loss_disc: 2.544219493865967 (2.544219493865967) + | > loss_disc_real_0: 0.21429280936717987 (0.21429280936717987) + | > loss_disc_real_1: 0.23697714507579803 (0.23697714507579803) + | > loss_disc_real_2: 0.2373262345790863 (0.2373262345790863) + | > loss_disc_real_3: 0.22929705679416656 (0.22929705679416656) + | > loss_disc_real_4: 0.24116116762161255 (0.24116116762161255) + | > loss_disc_real_5: 0.22715045511722565 (0.22715045511722565) + | > loss_0: 2.544219493865967 (2.544219493865967) + | > grad_norm_0: tensor(32.0093, device='cuda:0') (tensor(32.0093, device='cuda:0')) + | > loss_gen: 2.3400070667266846 (2.3400070667266846) + | > loss_kl: 1.7283780574798584 (1.7283780574798584) + | > loss_feat: 6.429980278015137 (6.429980278015137) + | > loss_mel: 16.129491806030273 (16.129491806030273) + | > loss_duration: 1.4785373210906982 (1.4785373210906982) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.106393814086914 (28.106393814086914) + | > grad_norm_1: tensor(134.8905, device='cuda:0') (tensor(134.8905, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.3545 (0.3545246124267578) + | > loader_time: 0.3138 (0.3138267993927002) + + + --> TIME: 2025-04-19 00:18:36 -- STEP: 25/142 -- GLOBAL_STEP: 390525 + | > loss_disc: 2.4204394817352295 (2.4705297470092775) + | > loss_disc_real_0: 0.12580391764640808 (0.12120420932769775) + | > loss_disc_real_1: 0.20605146884918213 (0.203926157951355) + | > loss_disc_real_2: 0.24969862401485443 (0.22727982580661774) + | > loss_disc_real_3: 0.20518486201763153 (0.22833628416061402) + | > loss_disc_real_4: 0.23527145385742188 (0.2350138795375824) + | > loss_disc_real_5: 0.20984160900115967 (0.24116009414196016) + | > loss_0: 2.4204394817352295 (2.4705297470092775) + | > grad_norm_0: tensor(4.1711, device='cuda:0') (tensor(8.4979, device='cuda:0')) + | > loss_gen: 2.385000228881836 (2.3567351055145265) + | > loss_kl: 1.80293607711792 (1.8948343515396118) + | > loss_feat: 6.633902549743652 (7.122222805023194) + | > loss_mel: 16.624906539916992 (17.98193817138672) + | > loss_duration: 1.5056171417236328 (1.4663405084609986) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.952362060546875 (30.82207084655762) + | > grad_norm_1: tensor(171.6178, device='cuda:0') (tensor(179.6783, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.3451 (0.3342575550079346) + | > loader_time: 0.0056 (0.005021486282348633) + + + --> TIME: 2025-04-19 00:18:45 -- STEP: 50/142 -- GLOBAL_STEP: 390550 + | > loss_disc: 2.4614975452423096 (2.471204748153686) + | > loss_disc_real_0: 0.1509981006383896 (0.12363612726330757) + | > loss_disc_real_1: 0.16525118052959442 (0.20592088997364044) + | > loss_disc_real_2: 0.2172553688287735 (0.22714690178632735) + | > loss_disc_real_3: 0.2404826432466507 (0.22940546542406082) + | > loss_disc_real_4: 0.26617005467414856 (0.23377667725086212) + | > loss_disc_real_5: 0.25638413429260254 (0.2356218045949936) + | > loss_0: 2.4614975452423096 (2.471204748153686) + | > grad_norm_0: tensor(4.2763, device='cuda:0') (tensor(9.3923, device='cuda:0')) + | > loss_gen: 2.29697847366333 (2.3445317220687865) + | > loss_kl: 1.9609085321426392 (1.9163632249832154) + | > loss_feat: 6.667850017547607 (6.943521223068237) + | > loss_mel: 17.009872436523438 (17.812855587005618) + | > loss_duration: 1.5208224058151245 (1.475629780292511) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.456430435180664 (30.492901458740235) + | > grad_norm_1: tensor(229.2518, device='cuda:0') (tensor(212.8397, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.3711 (0.347162127494812) + | > loader_time: 0.006 (0.005454511642456055) + + + --> TIME: 2025-04-19 00:18:55 -- STEP: 75/142 -- GLOBAL_STEP: 390575 + | > loss_disc: 2.466265916824341 (2.4710596974690757) + | > loss_disc_real_0: 0.14359579980373383 (0.12707127799590434) + | > loss_disc_real_1: 0.1890089213848114 (0.20705075482527413) + | > loss_disc_real_2: 0.23636570572853088 (0.22736788928508758) + | > loss_disc_real_3: 0.24630771577358246 (0.22804790377616882) + | > loss_disc_real_4: 0.21103587746620178 (0.23102761149406434) + | > loss_disc_real_5: 0.21720395982265472 (0.23498829682668051) + | > loss_0: 2.466265916824341 (2.4710596974690757) + | > grad_norm_0: tensor(7.3503, device='cuda:0') (tensor(10.8296, device='cuda:0')) + | > loss_gen: 2.3814079761505127 (2.343770618438721) + | > loss_kl: 2.060086965560913 (1.9362484471003214) + | > loss_feat: 7.486081123352051 (6.939207611083984) + | > loss_mel: 17.75322151184082 (17.83210046132406) + | > loss_duration: 1.483925223350525 (1.4870960474014283) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.164722442626953 (30.538423156738283) + | > grad_norm_1: tensor(350.5880, device='cuda:0') (tensor(250.6523, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.3974 (0.3594770717620849) + | > loader_time: 0.0065 (0.005707047780354818) + + + --> TIME: 2025-04-19 00:19:05 -- STEP: 100/142 -- GLOBAL_STEP: 390600 + | > loss_disc: 2.4805312156677246 (2.4711170411109937) + | > loss_disc_real_0: 0.13021989166736603 (0.12743560470640666) + | > loss_disc_real_1: 0.1876557171344757 (0.20617834493517875) + | > loss_disc_real_2: 0.23980079591274261 (0.22670275807380677) + | > loss_disc_real_3: 0.1971266120672226 (0.22833392575383185) + | > loss_disc_real_4: 0.2060299515724182 (0.23109223201870918) + | > loss_disc_real_5: 0.22337064146995544 (0.23504038974642755) + | > loss_0: 2.4805312156677246 (2.4711170411109937) + | > grad_norm_0: tensor(8.8446, device='cuda:0') (tensor(12.0810, device='cuda:0')) + | > loss_gen: 2.4320456981658936 (2.347918574810028) + | > loss_kl: 2.0896599292755127 (1.9457439041137696) + | > loss_feat: 7.229337215423584 (6.920530242919922) + | > loss_mel: 18.074068069458008 (17.814304933547973) + | > loss_duration: 1.5284181833267212 (1.4989176273345948) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.35352897644043 (30.52741527557373) + | > grad_norm_1: tensor(151.0100, device='cuda:0') (tensor(266.9630, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.4353 (0.3697387123107909) + | > loader_time: 0.0073 (0.0059992241859436034) + + + --> TIME: 2025-04-19 00:19:16 -- STEP: 125/142 -- GLOBAL_STEP: 390625 + | > loss_disc: 2.4508235454559326 (2.469482681274415) + | > loss_disc_real_0: 0.1667562574148178 (0.13034063023328782) + | > loss_disc_real_1: 0.21343158185482025 (0.2063572441339493) + | > loss_disc_real_2: 0.23302969336509705 (0.2263629183769226) + | > loss_disc_real_3: 0.26213523745536804 (0.22882989835739137) + | > loss_disc_real_4: 0.26137232780456543 (0.22988129317760467) + | > loss_disc_real_5: 0.22409118711948395 (0.23382632195949554) + | > loss_0: 2.4508235454559326 (2.469482681274415) + | > grad_norm_0: tensor(29.1307, device='cuda:0') (tensor(13.4546, device='cuda:0')) + | > loss_gen: 2.290898084640503 (2.3493793277740487) + | > loss_kl: 2.099844455718994 (1.948040452003479) + | > loss_feat: 6.570016860961914 (6.91252193069458) + | > loss_mel: 17.102373123168945 (17.74586558532715) + | > loss_duration: 1.581982135772705 (1.5063607702255248) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.645112991333008 (30.462167999267578) + | > grad_norm_1: tensor(287.7012, device='cuda:0') (tensor(280.3112, device='cuda:0')) + | > current_lr_0: 0.00019803459730799195 + | > current_lr_1: 0.00019803459730799195 + | > step_time: 0.4361 (0.38117138290405267) + | > loader_time: 0.0079 (0.006227886199951172) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.625670909881592 (2.625670909881592) + | > loss_disc_real_0: 0.2220732718706131 (0.2220732718706131) + | > loss_disc_real_1: 0.2556490898132324 (0.2556490898132324) + | > loss_disc_real_2: 0.2711835503578186 (0.2711835503578186) + | > loss_disc_real_3: 0.26086270809173584 (0.26086270809173584) + | > loss_disc_real_4: 0.26763716340065 (0.26763716340065) + | > loss_disc_real_5: 0.256511390209198 (0.256511390209198) + | > loss_0: 2.625670909881592 (2.625670909881592) + | > loss_gen: 2.4276535511016846 (2.4276535511016846) + | > loss_kl: 2.032031774520874 (2.032031774520874) + | > loss_feat: 5.844070911407471 (5.844070911407471) + | > loss_mel: 17.193077087402344 (17.193077087402344) + | > loss_duration: 1.6583750247955322 (1.6583750247955322) + | > loss_1: 29.155208587646484 (29.155208587646484) + + --> STEP: 1 + | > loss_disc: 2.516136884689331 (2.516136884689331) + | > loss_disc_real_0: 0.19571611285209656 (0.19571611285209656) + | > loss_disc_real_1: 0.19318397343158722 (0.19318397343158722) + | > loss_disc_real_2: 0.26578426361083984 (0.26578426361083984) + | > loss_disc_real_3: 0.2611640393733978 (0.2611640393733978) + | > loss_disc_real_4: 0.27351289987564087 (0.27351289987564087) + | > loss_disc_real_5: 0.25129640102386475 (0.25129640102386475) + | > loss_0: 2.516136884689331 (2.516136884689331) + | > loss_gen: 2.505906343460083 (2.505906343460083) + | > loss_kl: 2.4454407691955566 (2.4454407691955566) + | > loss_feat: 6.777402877807617 (6.777402877807617) + | > loss_mel: 17.85320281982422 (17.85320281982422) + | > loss_duration: 1.6318182945251465 (1.6318182945251465) + | > loss_1: 31.21377182006836 (31.21377182006836) + + --> STEP: 2 + | > loss_disc: 2.3191404342651367 (2.417638659477234) + | > loss_disc_real_0: 0.19326362013816833 (0.19448986649513245) + | > loss_disc_real_1: 0.1705724000930786 (0.18187818676233292) + | > loss_disc_real_2: 0.2124093770980835 (0.23909682035446167) + | > loss_disc_real_3: 0.23228760063648224 (0.24672582000494003) + | > loss_disc_real_4: 0.29235032200813293 (0.2829316109418869) + | > loss_disc_real_5: 0.22967122495174408 (0.2404838129878044) + | > loss_0: 2.3191404342651367 (2.417638659477234) + | > loss_gen: 2.61140775680542 (2.5586570501327515) + | > loss_kl: 2.1485557556152344 (2.2969982624053955) + | > loss_feat: 7.767784595489502 (7.27259373664856) + | > loss_mel: 18.826095581054688 (18.339649200439453) + | > loss_duration: 1.591588020324707 (1.6117031574249268) + | > loss_1: 32.945430755615234 (32.0796012878418) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036041736602783203 (+0.00016260147094726562) + | > avg_loss_disc: 2.417638659477234 (-0.07029163837432861) + | > avg_loss_disc_real_0: 0.19448986649513245 (-0.013339966535568237) + | > avg_loss_disc_real_1: 0.18187818676233292 (-0.01602933555841446) + | > avg_loss_disc_real_2: 0.23909682035446167 (+0.006578259170055389) + | > avg_loss_disc_real_3: 0.24672582000494003 (+0.033610373735427856) + | > avg_loss_disc_real_4: 0.2829316109418869 (+0.05213890224695206) + | > avg_loss_disc_real_5: 0.2404838129878044 (+0.030055277049541473) + | > avg_loss_0: 2.417638659477234 (-0.07029163837432861) + | > avg_loss_gen: 2.5586570501327515 (+0.23486363887786865) + | > avg_loss_kl: 2.2969982624053955 (-0.0809563398361206) + | > avg_loss_feat: 7.27259373664856 (+0.7564458847045898) + | > avg_loss_mel: 18.339649200439453 (+0.22473907470703125) + | > avg_loss_duration: 1.6117031574249268 (-0.036602795124053955) + | > avg_loss_1: 32.0796012878418 (+1.098489761352539) + + + > EPOCH: 80/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:19:25)  + + --> TIME: 2025-04-19 00:19:29 -- STEP: 8/142 -- GLOBAL_STEP: 390650 + | > loss_disc: 2.3994498252868652 (2.395942062139511) + | > loss_disc_real_0: 0.11887311935424805 (0.11885362025350332) + | > loss_disc_real_1: 0.15978041291236877 (0.20315727032721043) + | > loss_disc_real_2: 0.23596349358558655 (0.22121259197592735) + | > loss_disc_real_3: 0.2454805225133896 (0.2224374171346426) + | > loss_disc_real_4: 0.21794800460338593 (0.2210848331451416) + | > loss_disc_real_5: 0.2581552565097809 (0.22857479751110077) + | > loss_0: 2.3994498252868652 (2.395942062139511) + | > grad_norm_0: tensor(5.1544, device='cuda:0') (tensor(5.7558, device='cuda:0')) + | > loss_gen: 2.2233636379241943 (2.3471288681030273) + | > loss_kl: 1.8894374370574951 (1.8475249707698822) + | > loss_feat: 6.730485916137695 (7.309893906116486) + | > loss_mel: 18.08139419555664 (18.238725900650024) + | > loss_duration: 1.4504942893981934 (1.4691470116376877) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.37517547607422 (31.21242046356201) + | > grad_norm_1: tensor(65.1442, device='cuda:0') (tensor(129.5084, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.3254 (0.3242618143558502) + | > loader_time: 0.0051 (0.004441112279891968) + + + --> TIME: 2025-04-19 00:19:37 -- STEP: 33/142 -- GLOBAL_STEP: 390675 + | > loss_disc: 2.510899543762207 (2.4387459104711358) + | > loss_disc_real_0: 0.11917517334222794 (0.11690238027861624) + | > loss_disc_real_1: 0.23823031783103943 (0.20526453626878333) + | > loss_disc_real_2: 0.2150820791721344 (0.22443889397563357) + | > loss_disc_real_3: 0.2578704357147217 (0.2299992965929436) + | > loss_disc_real_4: 0.23725956678390503 (0.2271822690963745) + | > loss_disc_real_5: 0.24306036531925201 (0.23309198067043768) + | > loss_0: 2.510899543762207 (2.4387459104711358) + | > grad_norm_0: tensor(15.6661, device='cuda:0') (tensor(7.3625, device='cuda:0')) + | > loss_gen: 2.313357353210449 (2.3547845898252544) + | > loss_kl: 1.949418067932129 (1.9022134072852857) + | > loss_feat: 7.303586483001709 (7.171652317047119) + | > loss_mel: 18.18788719177246 (17.989190246119644) + | > loss_duration: 1.4784440994262695 (1.4735631003524319) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.23269271850586 (30.891403602831293) + | > grad_norm_1: tensor(392.4023, device='cuda:0') (tensor(129.0002, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.3561 (0.3367760614915328) + | > loader_time: 0.0055 (0.005129792473532937) + + + --> TIME: 2025-04-19 00:19:47 -- STEP: 58/142 -- GLOBAL_STEP: 390700 + | > loss_disc: 2.502650260925293 (2.45584383504144) + | > loss_disc_real_0: 0.1476123332977295 (0.12055137740640805) + | > loss_disc_real_1: 0.22816506028175354 (0.20723770327609162) + | > loss_disc_real_2: 0.1808580756187439 (0.2260892578753932) + | > loss_disc_real_3: 0.2344094216823578 (0.22825413325737262) + | > loss_disc_real_4: 0.22880633175373077 (0.22825957988870554) + | > loss_disc_real_5: 0.2721560001373291 (0.23291953627405496) + | > loss_0: 2.502650260925293 (2.45584383504144) + | > grad_norm_0: tensor(6.2064, device='cuda:0') (tensor(7.8801, device='cuda:0')) + | > loss_gen: 2.319810390472412 (2.3463999896213927) + | > loss_kl: 1.986938238143921 (1.899596734293576) + | > loss_feat: 6.950484752655029 (7.103842570863921) + | > loss_mel: 19.617904663085938 (18.093275481256953) + | > loss_duration: 1.50296950340271 (1.4838334465848988) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.378108978271484 (30.92694821851007) + | > grad_norm_1: tensor(111.9204, device='cuda:0') (tensor(156.0757, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.3747 (0.34832086645323657) + | > loader_time: 0.0063 (0.00552986819168617) + + + --> TIME: 2025-04-19 00:19:57 -- STEP: 83/142 -- GLOBAL_STEP: 390725 + | > loss_disc: 2.4657089710235596 (2.4687954603907585) + | > loss_disc_real_0: 0.1743905246257782 (0.12609813434172826) + | > loss_disc_real_1: 0.22215989232063293 (0.20804853575775423) + | > loss_disc_real_2: 0.17972607910633087 (0.22649179093809013) + | > loss_disc_real_3: 0.22810421884059906 (0.22818989979933543) + | > loss_disc_real_4: 0.22035004198551178 (0.22809089253465814) + | > loss_disc_real_5: 0.19692668318748474 (0.23087731793702368) + | > loss_0: 2.4657089710235596 (2.4687954603907585) + | > grad_norm_0: tensor(11.7657, device='cuda:0') (tensor(7.6188, device='cuda:0')) + | > loss_gen: 2.376718282699585 (2.3302969300603293) + | > loss_kl: 2.0599606037139893 (1.906749897692577) + | > loss_feat: 7.056313991546631 (7.013705937259169) + | > loss_mel: 18.853111267089844 (18.040699246418047) + | > loss_duration: 1.5889582633972168 (1.494005540767348) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.9350643157959 (30.785457565123778) + | > grad_norm_1: tensor(126.3475, device='cuda:0') (tensor(141.7712, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.4134 (0.36226031291915706) + | > loader_time: 0.0068 (0.005842157157070664) + + + --> TIME: 2025-04-19 00:20:08 -- STEP: 108/142 -- GLOBAL_STEP: 390750 + | > loss_disc: 2.3879430294036865 (2.4767664207352538) + | > loss_disc_real_0: 0.13370144367218018 (0.1288407080013443) + | > loss_disc_real_1: 0.2074895203113556 (0.2084244034356541) + | > loss_disc_real_2: 0.2569909989833832 (0.2261463842458195) + | > loss_disc_real_3: 0.25400564074516296 (0.22839660018130584) + | > loss_disc_real_4: 0.2329251915216446 (0.2294691061532056) + | > loss_disc_real_5: 0.24988780915737152 (0.23347977548837662) + | > loss_0: 2.3879430294036865 (2.4767664207352538) + | > grad_norm_0: tensor(9.1742, device='cuda:0') (tensor(7.7953, device='cuda:0')) + | > loss_gen: 2.4673268795013428 (2.3185268861276125) + | > loss_kl: 2.00618839263916 (1.9198090621718653) + | > loss_feat: 6.750796318054199 (6.937895227361609) + | > loss_mel: 18.232513427734375 (17.992220110363434) + | > loss_duration: 1.5343847274780273 (1.5036253896024492) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.9912109375 (30.672076719778556) + | > grad_norm_1: tensor(172.8337, device='cuda:0') (tensor(144.3699, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.4347 (0.3754882083998786) + | > loader_time: 0.0072 (0.006141130571012144) + + + --> TIME: 2025-04-19 00:20:19 -- STEP: 133/142 -- GLOBAL_STEP: 390775 + | > loss_disc: 2.565835475921631 (2.4783594841347605) + | > loss_disc_real_0: 0.179839089512825 (0.1299175644727578) + | > loss_disc_real_1: 0.21359677612781525 (0.20876507875614597) + | > loss_disc_real_2: 0.25332432985305786 (0.22687957598302597) + | > loss_disc_real_3: 0.22487109899520874 (0.22857107804682023) + | > loss_disc_real_4: 0.22009776532649994 (0.2287904915953041) + | > loss_disc_real_5: 0.21819369494915009 (0.23322402187307975) + | > loss_0: 2.565835475921631 (2.4783594841347605) + | > grad_norm_0: tensor(8.7935, device='cuda:0') (tensor(8.1704, device='cuda:0')) + | > loss_gen: 2.1492919921875 (2.3186534257759717) + | > loss_kl: 1.9688324928283691 (1.9311245769486391) + | > loss_feat: 5.64268159866333 (6.906335378948011) + | > loss_mel: 15.95422077178955 (17.923990808931514) + | > loss_duration: 1.6005990505218506 (1.5096773339393443) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.31562614440918 (30.58978153888444) + | > grad_norm_1: tensor(150.6954, device='cuda:0') (tensor(157.3548, device='cuda:0')) + | > current_lr_0: 0.00019800984298332845 + | > current_lr_1: 0.00019800984298332845 + | > step_time: 0.462 (0.3894416837763965) + | > loader_time: 0.0079 (0.006418199467479735) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4212069511413574 (2.4212069511413574) + | > loss_disc_real_0: 0.09985467046499252 (0.09985467046499252) + | > loss_disc_real_1: 0.19915850460529327 (0.19915850460529327) + | > loss_disc_real_2: 0.2673874497413635 (0.2673874497413635) + | > loss_disc_real_3: 0.2309877723455429 (0.2309877723455429) + | > loss_disc_real_4: 0.22503094375133514 (0.22503094375133514) + | > loss_disc_real_5: 0.23267167806625366 (0.23267167806625366) + | > loss_0: 2.4212069511413574 (2.4212069511413574) + | > loss_gen: 2.3438241481781006 (2.3438241481781006) + | > loss_kl: 2.1597206592559814 (2.1597206592559814) + | > loss_feat: 7.2402024269104 (7.2402024269104) + | > loss_mel: 18.529985427856445 (18.529985427856445) + | > loss_duration: 1.6418555974960327 (1.6418555974960327) + | > loss_1: 31.91558837890625 (31.91558837890625) + + --> STEP: 1 + | > loss_disc: 2.460482597351074 (2.460482597351074) + | > loss_disc_real_0: 0.10767453163862228 (0.10767453163862228) + | > loss_disc_real_1: 0.20151180028915405 (0.20151180028915405) + | > loss_disc_real_2: 0.28697049617767334 (0.28697049617767334) + | > loss_disc_real_3: 0.2655509114265442 (0.2655509114265442) + | > loss_disc_real_4: 0.23336994647979736 (0.23336994647979736) + | > loss_disc_real_5: 0.24710716307163239 (0.24710716307163239) + | > loss_0: 2.460482597351074 (2.460482597351074) + | > loss_gen: 2.3458261489868164 (2.3458261489868164) + | > loss_kl: 2.480992078781128 (2.480992078781128) + | > loss_feat: 5.849161624908447 (5.849161624908447) + | > loss_mel: 18.211666107177734 (18.211666107177734) + | > loss_duration: 1.6751137971878052 (1.6751137971878052) + | > loss_1: 30.562759399414062 (30.562759399414062) + + --> STEP: 2 + | > loss_disc: 2.4841251373291016 (2.472303867340088) + | > loss_disc_real_0: 0.12442837655544281 (0.11605145409703255) + | > loss_disc_real_1: 0.2222219556570053 (0.21186687797307968) + | > loss_disc_real_2: 0.25710639357566833 (0.27203844487667084) + | > loss_disc_real_3: 0.2624121308326721 (0.26398152112960815) + | > loss_disc_real_4: 0.26826173067092896 (0.25081583857536316) + | > loss_disc_real_5: 0.2457473874092102 (0.2464272752404213) + | > loss_0: 2.4841251373291016 (2.472303867340088) + | > loss_gen: 2.3401126861572266 (2.3429694175720215) + | > loss_kl: 2.1007392406463623 (2.290865659713745) + | > loss_feat: 6.675312042236328 (6.262236833572388) + | > loss_mel: 16.547576904296875 (17.379621505737305) + | > loss_duration: 1.5976730585098267 (1.636393427848816) + | > loss_1: 29.26141357421875 (29.912086486816406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003981351852416992 (+0.0003771781921386719) + | > avg_loss_disc: 2.472303867340088 (+0.054665207862854004) + | > avg_loss_disc_real_0: 0.11605145409703255 (-0.0784384123980999) + | > avg_loss_disc_real_1: 0.21186687797307968 (+0.029988691210746765) + | > avg_loss_disc_real_2: 0.27203844487667084 (+0.03294162452220917) + | > avg_loss_disc_real_3: 0.26398152112960815 (+0.01725570112466812) + | > avg_loss_disc_real_4: 0.25081583857536316 (-0.03211577236652374) + | > avg_loss_disc_real_5: 0.2464272752404213 (+0.005943462252616882) + | > avg_loss_0: 2.472303867340088 (+0.054665207862854004) + | > avg_loss_gen: 2.3429694175720215 (-0.21568763256072998) + | > avg_loss_kl: 2.290865659713745 (-0.006132602691650391) + | > avg_loss_feat: 6.262236833572388 (-1.0103569030761719) + | > avg_loss_mel: 17.379621505737305 (-0.9600276947021484) + | > avg_loss_duration: 1.636393427848816 (+0.02469027042388916) + | > avg_loss_1: 29.912086486816406 (-2.1675148010253906) + + + > EPOCH: 81/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:20:25)  + + --> TIME: 2025-04-19 00:20:31 -- STEP: 16/142 -- GLOBAL_STEP: 390800 + | > loss_disc: 2.4304776191711426 (2.4385991245508194) + | > loss_disc_real_0: 0.11308087408542633 (0.11586898285895586) + | > loss_disc_real_1: 0.20223106443881989 (0.19936255272477865) + | > loss_disc_real_2: 0.2584896385669708 (0.2240582499653101) + | > loss_disc_real_3: 0.23030973970890045 (0.2268393812701106) + | > loss_disc_real_4: 0.2723385691642761 (0.2322323126718402) + | > loss_disc_real_5: 0.23979641497135162 (0.22924305219203234) + | > loss_0: 2.4304776191711426 (2.4385991245508194) + | > grad_norm_0: tensor(15.7804, device='cuda:0') (tensor(14.3237, device='cuda:0')) + | > loss_gen: 2.6510074138641357 (2.4155451208353047) + | > loss_kl: 1.913769245147705 (1.8753615692257881) + | > loss_feat: 7.83265495300293 (7.231087028980255) + | > loss_mel: 19.2637882232666 (17.997206568717957) + | > loss_duration: 1.4522582292556763 (1.4603227600455284) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.11347579956055 (30.979523062705994) + | > grad_norm_1: tensor(179.7997, device='cuda:0') (tensor(310.2621, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.3452 (0.3294845074415207) + | > loader_time: 0.0059 (0.005007162690162659) + + + --> TIME: 2025-04-19 00:20:40 -- STEP: 41/142 -- GLOBAL_STEP: 390825 + | > loss_disc: 2.465681552886963 (2.432654037708189) + | > loss_disc_real_0: 0.1272919923067093 (0.11699793978435237) + | > loss_disc_real_1: 0.1862894892692566 (0.19991135015720274) + | > loss_disc_real_2: 0.2343711107969284 (0.22199309635453107) + | > loss_disc_real_3: 0.2164769321680069 (0.22619525179630373) + | > loss_disc_real_4: 0.23906052112579346 (0.23077331810462764) + | > loss_disc_real_5: 0.2311393767595291 (0.23429704657415065) + | > loss_0: 2.465681552886963 (2.432654037708189) + | > grad_norm_0: tensor(10.7434, device='cuda:0') (tensor(15.5571, device='cuda:0')) + | > loss_gen: 2.4750747680664062 (2.403158129715338) + | > loss_kl: 1.8143864870071411 (1.9073792201716726) + | > loss_feat: 7.1660685539245605 (7.086309840039509) + | > loss_mel: 17.47550392150879 (17.91245797785317) + | > loss_duration: 1.4512410163879395 (1.4726500016887014) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.382274627685547 (30.781955067704363) + | > grad_norm_1: tensor(566.3479, device='cuda:0') (tensor(343.6288, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.3607 (0.34227634057766054) + | > loader_time: 0.0057 (0.005387719084576863) + + + --> TIME: 2025-04-19 00:20:49 -- STEP: 66/142 -- GLOBAL_STEP: 390850 + | > loss_disc: 2.465763807296753 (2.438004555124225) + | > loss_disc_real_0: 0.11042582988739014 (0.11785067403406808) + | > loss_disc_real_1: 0.2404387891292572 (0.20508578758348117) + | > loss_disc_real_2: 0.23927472531795502 (0.22260925873662485) + | > loss_disc_real_3: 0.2048170417547226 (0.22754714005824292) + | > loss_disc_real_4: 0.20348097383975983 (0.22770997046521216) + | > loss_disc_real_5: 0.19516851007938385 (0.23314686932347037) + | > loss_0: 2.465763807296753 (2.438004555124225) + | > grad_norm_0: tensor(14.8260, device='cuda:0') (tensor(15.4054, device='cuda:0')) + | > loss_gen: 2.35140323638916 (2.3947221979950406) + | > loss_kl: 1.9374449253082275 (1.9205720280155991) + | > loss_feat: 7.177034378051758 (7.1304449384862725) + | > loss_mel: 17.868555068969727 (18.033714713472307) + | > loss_duration: 1.4829950332641602 (1.486676340753382) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.817432403564453 (30.966130285552055) + | > grad_norm_1: tensor(544.4977, device='cuda:0') (tensor(344.0488, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.384 (0.3545045563668916) + | > loader_time: 0.006 (0.005681157112121582) + + + --> TIME: 2025-04-19 00:20:59 -- STEP: 91/142 -- GLOBAL_STEP: 390875 + | > loss_disc: 2.41886568069458 (2.450517303341037) + | > loss_disc_real_0: 0.10169842094182968 (0.1201487072630898) + | > loss_disc_real_1: 0.20256808400154114 (0.20409072509833745) + | > loss_disc_real_2: 0.24254728853702545 (0.22472498950722453) + | > loss_disc_real_3: 0.2022796869277954 (0.2271075278193086) + | > loss_disc_real_4: 0.2253725379705429 (0.22863438004975792) + | > loss_disc_real_5: 0.1982031613588333 (0.2350863529430641) + | > loss_0: 2.41886568069458 (2.450517303341037) + | > grad_norm_0: tensor(12.4039, device='cuda:0') (tensor(16.0156, device='cuda:0')) + | > loss_gen: 2.499055862426758 (2.398685104244357) + | > loss_kl: 2.037534475326538 (1.9356088022609332) + | > loss_feat: 7.709296703338623 (7.083061291621282) + | > loss_mel: 18.313844680786133 (18.03300215123773) + | > loss_duration: 1.5310871601104736 (1.4990652865105936) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.0908203125 (30.949422647664836) + | > grad_norm_1: tensor(538.6533, device='cuda:0') (tensor(344.7136, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.4009 (0.3654300170940357) + | > loader_time: 0.007 (0.005907011556101369) + + + --> TIME: 2025-04-19 00:21:10 -- STEP: 116/142 -- GLOBAL_STEP: 390900 + | > loss_disc: 2.5288901329040527 (2.462910216430137) + | > loss_disc_real_0: 0.22280637919902802 (0.12235921357983146) + | > loss_disc_real_1: 0.20124931633472443 (0.20613171070300298) + | > loss_disc_real_2: 0.2426009625196457 (0.22621235174351725) + | > loss_disc_real_3: 0.22785699367523193 (0.22806414550748363) + | > loss_disc_real_4: 0.25019583106040955 (0.22992100412475652) + | > loss_disc_real_5: 0.2430570274591446 (0.23524020381014923) + | > loss_0: 2.5288901329040527 (2.462910216430137) + | > grad_norm_0: tensor(25.2233, device='cuda:0') (tensor(16.9887, device='cuda:0')) + | > loss_gen: 2.461702823638916 (2.3874357486593296) + | > loss_kl: 1.9040844440460205 (1.9426191048375492) + | > loss_feat: 6.728201866149902 (7.023004478421704) + | > loss_mel: 17.04949378967285 (17.976905206154125) + | > loss_duration: 1.4929113388061523 (1.505161022317821) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.636394500732422 (30.835125528532885) + | > grad_norm_1: tensor(419.7760, device='cuda:0') (tensor(355.2765, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.4298 (0.37640629554616994) + | > loader_time: 0.0089 (0.0061804467234118235) + + + --> TIME: 2025-04-19 00:21:21 -- STEP: 141/142 -- GLOBAL_STEP: 390925 + | > loss_disc: 2.4037694931030273 (2.4623586032407494) + | > loss_disc_real_0: 0.12463714182376862 (0.12217603280400553) + | > loss_disc_real_1: 0.1900172084569931 (0.2058290106819031) + | > loss_disc_real_2: 0.2073977142572403 (0.22567026391096992) + | > loss_disc_real_3: 0.20512063801288605 (0.2277416613296414) + | > loss_disc_real_4: 0.15155349671840668 (0.22867312372153534) + | > loss_disc_real_5: 0.19421076774597168 (0.23472449257441447) + | > loss_0: 2.4037694931030273 (2.4623586032407494) + | > grad_norm_0: tensor(9.5787, device='cuda:0') (tensor(16.4382, device='cuda:0')) + | > loss_gen: 2.6187901496887207 (2.374605440924352) + | > loss_kl: 1.775591492652893 (1.951562247377761) + | > loss_feat: 8.06195068359375 (6.979724082540958) + | > loss_mel: 16.798789978027344 (17.889947580107552) + | > loss_duration: 1.4006361961364746 (1.5102216478780655) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.655759811401367 (30.7060609776923) + | > grad_norm_1: tensor(672.8529, device='cuda:0') (tensor(347.4920, device='cuda:0')) + | > current_lr_0: 0.00019798509175295552 + | > current_lr_1: 0.00019798509175295552 + | > step_time: 0.1993 (0.3865644678156426) + | > loader_time: 0.003 (0.006386958115489771) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4589576721191406 (2.4589576721191406) + | > loss_disc_real_0: 0.1394406408071518 (0.1394406408071518) + | > loss_disc_real_1: 0.24081456661224365 (0.24081456661224365) + | > loss_disc_real_2: 0.24927139282226562 (0.24927139282226562) + | > loss_disc_real_3: 0.21091988682746887 (0.21091988682746887) + | > loss_disc_real_4: 0.22266237437725067 (0.22266237437725067) + | > loss_disc_real_5: 0.21476610004901886 (0.21476610004901886) + | > loss_0: 2.4589576721191406 (2.4589576721191406) + | > loss_gen: 2.372821569442749 (2.372821569442749) + | > loss_kl: 2.5065195560455322 (2.5065195560455322) + | > loss_feat: 7.1676154136657715 (7.1676154136657715) + | > loss_mel: 19.778488159179688 (19.778488159179688) + | > loss_duration: 1.6175825595855713 (1.6175825595855713) + | > loss_1: 33.44302749633789 (33.44302749633789) + + --> STEP: 1 + | > loss_disc: 2.6008853912353516 (2.6008853912353516) + | > loss_disc_real_0: 0.13827888667583466 (0.13827888667583466) + | > loss_disc_real_1: 0.29005178809165955 (0.29005178809165955) + | > loss_disc_real_2: 0.27288591861724854 (0.27288591861724854) + | > loss_disc_real_3: 0.2484673708677292 (0.2484673708677292) + | > loss_disc_real_4: 0.24372145533561707 (0.24372145533561707) + | > loss_disc_real_5: 0.25262317061424255 (0.25262317061424255) + | > loss_0: 2.6008853912353516 (2.6008853912353516) + | > loss_gen: 2.5043301582336426 (2.5043301582336426) + | > loss_kl: 2.492314100265503 (2.492314100265503) + | > loss_feat: 6.183503150939941 (6.183503150939941) + | > loss_mel: 16.592304229736328 (16.592304229736328) + | > loss_duration: 1.6456842422485352 (1.6456842422485352) + | > loss_1: 29.418136596679688 (29.418136596679688) + + --> STEP: 2 + | > loss_disc: 2.4411470890045166 (2.521016240119934) + | > loss_disc_real_0: 0.10360654443502426 (0.12094271555542946) + | > loss_disc_real_1: 0.29011544585227966 (0.2900836169719696) + | > loss_disc_real_2: 0.23719292879104614 (0.25503942370414734) + | > loss_disc_real_3: 0.1941056102514267 (0.22128649055957794) + | > loss_disc_real_4: 0.22064830362796783 (0.23218487948179245) + | > loss_disc_real_5: 0.2099333256483078 (0.23127824813127518) + | > loss_0: 2.4411470890045166 (2.521016240119934) + | > loss_gen: 2.409149408340454 (2.4567397832870483) + | > loss_kl: 2.0419533252716064 (2.2671337127685547) + | > loss_feat: 6.5010294914245605 (6.342266321182251) + | > loss_mel: 15.249347686767578 (15.920825958251953) + | > loss_duration: 1.6114921569824219 (1.6285881996154785) + | > loss_1: 27.812973022460938 (28.615554809570312) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003444075584411621 (-0.0005372762680053711) + | > avg_loss_disc: 2.521016240119934 (+0.04871237277984619) + | > avg_loss_disc_real_0: 0.12094271555542946 (+0.004891261458396912) + | > avg_loss_disc_real_1: 0.2900836169719696 (+0.07821673899888992) + | > avg_loss_disc_real_2: 0.25503942370414734 (-0.0169990211725235) + | > avg_loss_disc_real_3: 0.22128649055957794 (-0.04269503057003021) + | > avg_loss_disc_real_4: 0.23218487948179245 (-0.01863095909357071) + | > avg_loss_disc_real_5: 0.23127824813127518 (-0.015149027109146118) + | > avg_loss_0: 2.521016240119934 (+0.04871237277984619) + | > avg_loss_gen: 2.4567397832870483 (+0.11377036571502686) + | > avg_loss_kl: 2.2671337127685547 (-0.02373194694519043) + | > avg_loss_feat: 6.342266321182251 (+0.08002948760986328) + | > avg_loss_mel: 15.920825958251953 (-1.4587955474853516) + | > avg_loss_duration: 1.6285881996154785 (-0.007805228233337402) + | > avg_loss_1: 28.615554809570312 (-1.2965316772460938) + + + > EPOCH: 82/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:21:23)  + + --> TIME: 2025-04-19 00:21:32 -- STEP: 24/142 -- GLOBAL_STEP: 390950 + | > loss_disc: 2.4557301998138428 (2.453358550866445) + | > loss_disc_real_0: 0.12636087834835052 (0.11345531356831391) + | > loss_disc_real_1: 0.23773571848869324 (0.20908904634416103) + | > loss_disc_real_2: 0.22888311743736267 (0.2302139134456714) + | > loss_disc_real_3: 0.23814594745635986 (0.23156882201631865) + | > loss_disc_real_4: 0.22728680074214935 (0.23374339751899242) + | > loss_disc_real_5: 0.2219824194908142 (0.23697554816802344) + | > loss_0: 2.4557301998138428 (2.453358550866445) + | > grad_norm_0: tensor(10.2541, device='cuda:0') (tensor(11.2389, device='cuda:0')) + | > loss_gen: 2.3178257942199707 (2.3611740867296858) + | > loss_kl: 1.9368568658828735 (1.9110345244407654) + | > loss_feat: 7.099208831787109 (7.1217682758967085) + | > loss_mel: 17.999727249145508 (17.887327233950298) + | > loss_duration: 1.5060720443725586 (1.465942457318306) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.859691619873047 (30.747246583302815) + | > grad_norm_1: tensor(201.3878, device='cuda:0') (tensor(304.2882, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.34 (0.33233941594759625) + | > loader_time: 0.0054 (0.005137910445531209) + + + --> TIME: 2025-04-19 00:21:41 -- STEP: 49/142 -- GLOBAL_STEP: 390975 + | > loss_disc: 2.5420613288879395 (2.4516932477756415) + | > loss_disc_real_0: 0.21563558280467987 (0.12074713272099592) + | > loss_disc_real_1: 0.27936917543411255 (0.20871350260413424) + | > loss_disc_real_2: 0.23443050682544708 (0.22493804656729408) + | > loss_disc_real_3: 0.23051254451274872 (0.22890725062817943) + | > loss_disc_real_4: 0.25193434953689575 (0.22863102263333845) + | > loss_disc_real_5: 0.21908079087734222 (0.235068573027241) + | > loss_0: 2.5420613288879395 (2.4516932477756415) + | > grad_norm_0: tensor(8.4847, device='cuda:0') (tensor(10.6597, device='cuda:0')) + | > loss_gen: 2.3275299072265625 (2.3640313099841683) + | > loss_kl: 1.8767542839050293 (1.9202284520986128) + | > loss_feat: 6.839320659637451 (7.108106243367097) + | > loss_mel: 17.7803897857666 (17.979224127166127) + | > loss_duration: 1.481506109237671 (1.4761019896487801) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.305500030517578 (30.847692217145646) + | > grad_norm_1: tensor(43.9988, device='cuda:0') (tensor(276.8636, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.3681 (0.3447785085561324) + | > loader_time: 0.0059 (0.0055042334965297165) + + + --> TIME: 2025-04-19 00:21:51 -- STEP: 74/142 -- GLOBAL_STEP: 391000 + | > loss_disc: 2.4634087085723877 (2.470680562225547) + | > loss_disc_real_0: 0.13036179542541504 (0.12439919585311734) + | > loss_disc_real_1: 0.18526266515254974 (0.20705521630274282) + | > loss_disc_real_2: 0.18386058509349823 (0.22600494405707797) + | > loss_disc_real_3: 0.21899296343326569 (0.23084703087806702) + | > loss_disc_real_4: 0.221830353140831 (0.22937631445962028) + | > loss_disc_real_5: 0.2850084900856018 (0.23634999668275988) + | > loss_0: 2.4634087085723877 (2.470680562225547) + | > grad_norm_0: tensor(15.7346, device='cuda:0') (tensor(10.2319, device='cuda:0')) + | > loss_gen: 2.3469626903533936 (2.340189709856704) + | > loss_kl: 1.9010177850723267 (1.9418665122341465) + | > loss_feat: 7.2578911781311035 (7.027410152796152) + | > loss_mel: 18.736896514892578 (17.908612135294323) + | > loss_duration: 1.5081173181533813 (1.4892813740549862) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.750885009765625 (30.7073599325644) + | > grad_norm_1: tensor(213.4904, device='cuda:0') (tensor(240.5774, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.3891 (0.35752048041369466) + | > loader_time: 0.0067 (0.005782926404798354) + + + --> TIME: 2025-04-19 00:22:01 -- STEP: 99/142 -- GLOBAL_STEP: 391025 + | > loss_disc: 2.52908992767334 (2.466820107565985) + | > loss_disc_real_0: 0.12624439597129822 (0.12626706853960495) + | > loss_disc_real_1: 0.20905902981758118 (0.20778305196400845) + | > loss_disc_real_2: 0.26264703273773193 (0.2259361224644112) + | > loss_disc_real_3: 0.25959187746047974 (0.23006636732154423) + | > loss_disc_real_4: 0.2771378457546234 (0.22855577399634353) + | > loss_disc_real_5: 0.2536948025226593 (0.23467842015353116) + | > loss_0: 2.52908992767334 (2.466820107565985) + | > grad_norm_0: tensor(7.5101, device='cuda:0') (tensor(10.0167, device='cuda:0')) + | > loss_gen: 2.360647201538086 (2.332124431928) + | > loss_kl: 1.975313663482666 (1.9513632528709643) + | > loss_feat: 5.526032447814941 (6.961375405090024) + | > loss_mel: 16.134967803955078 (17.793285629966046) + | > loss_duration: 1.524436593055725 (1.4991429863554058) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.521398544311523 (30.53729184468587) + | > grad_norm_1: tensor(236.6849, device='cuda:0') (tensor(226.0644, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.4038 (0.3711596691247189) + | > loader_time: 0.0068 (0.006008810467190213) + + + --> TIME: 2025-04-19 00:22:12 -- STEP: 124/142 -- GLOBAL_STEP: 391050 + | > loss_disc: 2.4468462467193604 (2.4699066396682485) + | > loss_disc_real_0: 0.14455406367778778 (0.12758670951570233) + | > loss_disc_real_1: 0.20768871903419495 (0.20690476413696043) + | > loss_disc_real_2: 0.24092502892017365 (0.22622854370743997) + | > loss_disc_real_3: 0.22597885131835938 (0.23026837564764485) + | > loss_disc_real_4: 0.20274882018566132 (0.22885581658732507) + | > loss_disc_real_5: 0.213997945189476 (0.23392521902438132) + | > loss_0: 2.4468462467193604 (2.4699066396682485) + | > grad_norm_0: tensor(8.6473, device='cuda:0') (tensor(10.0738, device='cuda:0')) + | > loss_gen: 2.3039093017578125 (2.332196888423736) + | > loss_kl: 2.019883394241333 (1.9603206313425494) + | > loss_feat: 7.229525089263916 (6.9626884537358436) + | > loss_mel: 18.87287712097168 (17.849582556755315) + | > loss_duration: 1.5670740604400635 (1.5058553411114601) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.993268966674805 (30.61064398673273) + | > grad_norm_1: tensor(359.0784, device='cuda:0') (tensor(225.5943, device='cuda:0')) + | > current_lr_0: 0.0001979603436164864 + | > current_lr_1: 0.0001979603436164864 + | > step_time: 0.445 (0.38263438786229786) + | > loader_time: 0.0076 (0.006274375223344372) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.555227041244507 (2.555227041244507) + | > loss_disc_real_0: 0.1353374719619751 (0.1353374719619751) + | > loss_disc_real_1: 0.1992795169353485 (0.1992795169353485) + | > loss_disc_real_2: 0.2488228678703308 (0.2488228678703308) + | > loss_disc_real_3: 0.3017776906490326 (0.3017776906490326) + | > loss_disc_real_4: 0.2111859917640686 (0.2111859917640686) + | > loss_disc_real_5: 0.24112752079963684 (0.24112752079963684) + | > loss_0: 2.555227041244507 (2.555227041244507) + | > loss_gen: 2.1974070072174072 (2.1974070072174072) + | > loss_kl: 2.1996304988861084 (2.1996304988861084) + | > loss_feat: 6.402410507202148 (6.402410507202148) + | > loss_mel: 17.302669525146484 (17.302669525146484) + | > loss_duration: 1.6359044313430786 (1.6359044313430786) + | > loss_1: 29.738021850585938 (29.738021850585938) + + --> STEP: 1 + | > loss_disc: 2.5444936752319336 (2.5444936752319336) + | > loss_disc_real_0: 0.20572130382061005 (0.20572130382061005) + | > loss_disc_real_1: 0.19345422089099884 (0.19345422089099884) + | > loss_disc_real_2: 0.24080513417720795 (0.24080513417720795) + | > loss_disc_real_3: 0.2606855630874634 (0.2606855630874634) + | > loss_disc_real_4: 0.23460565507411957 (0.23460565507411957) + | > loss_disc_real_5: 0.20474809408187866 (0.20474809408187866) + | > loss_0: 2.5444936752319336 (2.5444936752319336) + | > loss_gen: 2.2581989765167236 (2.2581989765167236) + | > loss_kl: 2.4425296783447266 (2.4425296783447266) + | > loss_feat: 6.063767433166504 (6.063767433166504) + | > loss_mel: 18.304096221923828 (18.304096221923828) + | > loss_duration: 1.668513298034668 (1.668513298034668) + | > loss_1: 30.737106323242188 (30.737106323242188) + + --> STEP: 2 + | > loss_disc: 2.489095449447632 (2.5167945623397827) + | > loss_disc_real_0: 0.1397533416748047 (0.17273732274770737) + | > loss_disc_real_1: 0.19502268731594086 (0.19423845410346985) + | > loss_disc_real_2: 0.25089478492736816 (0.24584995955228806) + | > loss_disc_real_3: 0.2910597324371338 (0.2758726477622986) + | > loss_disc_real_4: 0.25307390093803406 (0.2438397780060768) + | > loss_disc_real_5: 0.21551786363124847 (0.21013297885656357) + | > loss_0: 2.489095449447632 (2.5167945623397827) + | > loss_gen: 2.29168701171875 (2.274942994117737) + | > loss_kl: 1.9383816719055176 (2.190455675125122) + | > loss_feat: 6.563819885253906 (6.313793659210205) + | > loss_mel: 17.859256744384766 (18.081676483154297) + | > loss_duration: 1.6043434143066406 (1.6364283561706543) + | > loss_1: 30.257488250732422 (30.497297286987305) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003494739532470703 (+5.066394805908203e-05) + | > avg_loss_disc: 2.5167945623397827 (-0.004221677780151367) + | > avg_loss_disc_real_0: 0.17273732274770737 (+0.05179460719227791) + | > avg_loss_disc_real_1: 0.19423845410346985 (-0.09584516286849976) + | > avg_loss_disc_real_2: 0.24584995955228806 (-0.009189464151859283) + | > avg_loss_disc_real_3: 0.2758726477622986 (+0.05458615720272064) + | > avg_loss_disc_real_4: 0.2438397780060768 (+0.011654898524284363) + | > avg_loss_disc_real_5: 0.21013297885656357 (-0.02114526927471161) + | > avg_loss_0: 2.5167945623397827 (-0.004221677780151367) + | > avg_loss_gen: 2.274942994117737 (-0.18179678916931152) + | > avg_loss_kl: 2.190455675125122 (-0.07667803764343262) + | > avg_loss_feat: 6.313793659210205 (-0.0284726619720459) + | > avg_loss_mel: 18.081676483154297 (+2.1608505249023438) + | > avg_loss_duration: 1.6364283561706543 (+0.007840156555175781) + | > avg_loss_1: 30.497297286987305 (+1.8817424774169922) + + + > EPOCH: 83/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:22:22)  + + --> TIME: 2025-04-19 00:22:25 -- STEP: 7/142 -- GLOBAL_STEP: 391075 + | > loss_disc: 2.4688401222229004 (2.4264841420309886) + | > loss_disc_real_0: 0.14842677116394043 (0.14050822066409246) + | > loss_disc_real_1: 0.22104084491729736 (0.2043148683650153) + | > loss_disc_real_2: 0.21168354153633118 (0.2251734584569931) + | > loss_disc_real_3: 0.22700002789497375 (0.23104606994560786) + | > loss_disc_real_4: 0.2267863154411316 (0.22527042882783072) + | > loss_disc_real_5: 0.2194787561893463 (0.2434420543057578) + | > loss_0: 2.4688401222229004 (2.4264841420309886) + | > grad_norm_0: tensor(17.3908, device='cuda:0') (tensor(10.2791, device='cuda:0')) + | > loss_gen: 2.2821481227874756 (2.452059950147356) + | > loss_kl: 1.8021345138549805 (1.85871388231005) + | > loss_feat: 7.240523338317871 (7.436514173235212) + | > loss_mel: 17.859886169433594 (18.152610506330216) + | > loss_duration: 1.4588638544082642 (1.465746317590986) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.643556594848633 (31.365644999912806) + | > grad_norm_1: tensor(188.6333, device='cuda:0') (tensor(212.6741, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.3523 (0.33899218695504324) + | > loader_time: 0.0045 (0.004749093736921038) + + + --> TIME: 2025-04-19 00:22:34 -- STEP: 32/142 -- GLOBAL_STEP: 391100 + | > loss_disc: 2.5109190940856934 (2.430427365005016) + | > loss_disc_real_0: 0.12662366032600403 (0.12118328898213804) + | > loss_disc_real_1: 0.18786518275737762 (0.20421292027458549) + | > loss_disc_real_2: 0.21297557651996613 (0.22323221946135163) + | > loss_disc_real_3: 0.2141086906194687 (0.225086132530123) + | > loss_disc_real_4: 0.2674029767513275 (0.22666773293167353) + | > loss_disc_real_5: 0.2253195196390152 (0.23119653994217515) + | > loss_0: 2.5109190940856934 (2.430427365005016) + | > grad_norm_0: tensor(11.7284, device='cuda:0') (tensor(11.9154, device='cuda:0')) + | > loss_gen: 2.395571708679199 (2.4256249666213994) + | > loss_kl: 2.053412675857544 (1.901735845953226) + | > loss_feat: 6.962825298309326 (7.2697369158267975) + | > loss_mel: 17.17681121826172 (17.973310470581062) + | > loss_duration: 1.4662964344024658 (1.4690482206642628) + | > amp_scaler: 256.0 (216.0) + | > loss_1: 30.054920196533203 (31.039456844329834) + | > grad_norm_1: tensor(454.6393, device='cuda:0') (tensor(275.9708, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.3541 (0.3401378467679024) + | > loader_time: 0.0057 (0.005289681255817413) + + + --> TIME: 2025-04-19 00:22:43 -- STEP: 57/142 -- GLOBAL_STEP: 391125 + | > loss_disc: 2.6463067531585693 (2.4414573677799156) + | > loss_disc_real_0: 0.20228008925914764 (0.1226803951880388) + | > loss_disc_real_1: 0.208876833319664 (0.2037171870470047) + | > loss_disc_real_2: 0.24374696612358093 (0.22349321005637185) + | > loss_disc_real_3: 0.24750502407550812 (0.22739328078010626) + | > loss_disc_real_4: 0.23738281428813934 (0.22870297076409324) + | > loss_disc_real_5: 0.266414612531662 (0.23216614022589566) + | > loss_0: 2.6463067531585693 (2.4414573677799156) + | > grad_norm_0: tensor(31.0882, device='cuda:0') (tensor(13.7366, device='cuda:0')) + | > loss_gen: 2.322144031524658 (2.4081834115480123) + | > loss_kl: 1.9421758651733398 (1.9108730868289345) + | > loss_feat: 7.1136908531188965 (7.241763031273558) + | > loss_mel: 18.54256248474121 (18.053470511185488) + | > loss_duration: 1.5320682525634766 (1.4789964504409254) + | > amp_scaler: 256.0 (233.5438596491228) + | > loss_1: 31.452640533447266 (31.09328678198028) + | > grad_norm_1: tensor(104.2910, device='cuda:0') (tensor(308.9719, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.3741 (0.3529056122428493) + | > loader_time: 0.0062 (0.005649073082104064) + + + --> TIME: 2025-04-19 00:22:53 -- STEP: 82/142 -- GLOBAL_STEP: 391150 + | > loss_disc: 2.5277023315429688 (2.451301740437019) + | > loss_disc_real_0: 0.14661747217178345 (0.12372696853992415) + | > loss_disc_real_1: 0.24331364035606384 (0.20530373457728363) + | > loss_disc_real_2: 0.27803996205329895 (0.22520446886376635) + | > loss_disc_real_3: 0.25622430443763733 (0.22794390724199573) + | > loss_disc_real_4: 0.26402825117111206 (0.2302314713597297) + | > loss_disc_real_5: 0.2516271471977234 (0.23181501957701472) + | > loss_0: 2.5277023315429688 (2.451301740437019) + | > grad_norm_0: tensor(23.6644, device='cuda:0') (tensor(15.2194, device='cuda:0')) + | > loss_gen: 2.262190580368042 (2.40228947488273) + | > loss_kl: 2.0996737480163574 (1.9241478661211526) + | > loss_feat: 6.293527603149414 (7.146043980993876) + | > loss_mel: 17.046524047851562 (17.97475230984571) + | > loss_duration: 1.5810062885284424 (1.4909958563199857) + | > amp_scaler: 256.0 (240.390243902439) + | > loss_1: 29.282922744750977 (30.938229746934844) + | > grad_norm_1: tensor(467.5013, device='cuda:0') (tensor(317.2139, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.393 (0.3642472726542775) + | > loader_time: 0.0068 (0.005971117717463796) + + + --> TIME: 2025-04-19 00:23:04 -- STEP: 107/142 -- GLOBAL_STEP: 391175 + | > loss_disc: 2.590289354324341 (2.4574825162085414) + | > loss_disc_real_0: 0.0971391424536705 (0.12444432706476372) + | > loss_disc_real_1: 0.22490911185741425 (0.20560758345037977) + | > loss_disc_real_2: 0.2589515149593353 (0.2250929814632808) + | > loss_disc_real_3: 0.23592232167720795 (0.2286806412946398) + | > loss_disc_real_4: 0.23073242604732513 (0.23056504074658185) + | > loss_disc_real_5: 0.2471330761909485 (0.2318579076885063) + | > loss_0: 2.590289354324341 (2.4574825162085414) + | > grad_norm_0: tensor(10.7150, device='cuda:0') (tensor(15.2826, device='cuda:0')) + | > loss_gen: 2.3593828678131104 (2.3880126988776365) + | > loss_kl: 1.8827804327011108 (1.9360342805630693) + | > loss_feat: 7.7497968673706055 (7.112978204388485) + | > loss_mel: 19.097469329833984 (18.00932930563097) + | > loss_duration: 1.509265422821045 (1.5005856329035538) + | > amp_scaler: 256.0 (244.0373831775701) + | > loss_1: 32.59869384765625 (30.94694023488838) + | > grad_norm_1: tensor(91.1722, device='cuda:0') (tensor(318.5044, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.4161 (0.37414650605103666) + | > loader_time: 0.0076 (0.006224868453551676) + + + --> TIME: 2025-04-19 00:23:15 -- STEP: 132/142 -- GLOBAL_STEP: 391200 + | > loss_disc: 2.453645944595337 (2.4622445016196277) + | > loss_disc_real_0: 0.12756679952144623 (0.12753254406605707) + | > loss_disc_real_1: 0.21828022599220276 (0.20582925900816917) + | > loss_disc_real_2: 0.23984113335609436 (0.22534776185498093) + | > loss_disc_real_3: 0.19443540275096893 (0.2288670161682548) + | > loss_disc_real_4: 0.20580773055553436 (0.2299053400303378) + | > loss_disc_real_5: 0.24755743145942688 (0.23277286485289084) + | > loss_0: 2.453645944595337 (2.4622445016196277) + | > grad_norm_0: tensor(4.7654, device='cuda:0') (tensor(14.3079, device='cuda:0')) + | > loss_gen: 2.3449130058288574 (2.374592772035887) + | > loss_kl: 2.1228621006011963 (1.936053131565903) + | > loss_feat: 6.796839237213135 (7.022541425444863) + | > loss_mel: 17.97696304321289 (17.923814983078923) + | > loss_duration: 1.515748381614685 (1.5065667593117917) + | > amp_scaler: 256.0 (246.3030303030303) + | > loss_1: 30.757326126098633 (30.7635691093676) + | > grad_norm_1: tensor(116.1934, device='cuda:0') (tensor(294.2460, device='cuda:0')) + | > current_lr_0: 0.00019793559857353432 + | > current_lr_1: 0.00019793559857353432 + | > step_time: 0.4423 (0.38616934870228614) + | > loader_time: 0.0074 (0.006474361275181625) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4140262603759766 (2.4140262603759766) + | > loss_disc_real_0: 0.17408226430416107 (0.17408226430416107) + | > loss_disc_real_1: 0.21165473759174347 (0.21165473759174347) + | > loss_disc_real_2: 0.1840689331293106 (0.1840689331293106) + | > loss_disc_real_3: 0.17915281653404236 (0.17915281653404236) + | > loss_disc_real_4: 0.1994021236896515 (0.1994021236896515) + | > loss_disc_real_5: 0.234406515955925 (0.234406515955925) + | > loss_0: 2.4140262603759766 (2.4140262603759766) + | > loss_gen: 2.352038860321045 (2.352038860321045) + | > loss_kl: 2.12958025932312 (2.12958025932312) + | > loss_feat: 6.574989318847656 (6.574989318847656) + | > loss_mel: 18.91768455505371 (18.91768455505371) + | > loss_duration: 1.643237590789795 (1.643237590789795) + | > loss_1: 31.617530822753906 (31.617530822753906) + + --> STEP: 1 + | > loss_disc: 2.3779940605163574 (2.3779940605163574) + | > loss_disc_real_0: 0.22021380066871643 (0.22021380066871643) + | > loss_disc_real_1: 0.23235949873924255 (0.23235949873924255) + | > loss_disc_real_2: 0.18424300849437714 (0.18424300849437714) + | > loss_disc_real_3: 0.1910904049873352 (0.1910904049873352) + | > loss_disc_real_4: 0.18559284508228302 (0.18559284508228302) + | > loss_disc_real_5: 0.2359016090631485 (0.2359016090631485) + | > loss_0: 2.3779940605163574 (2.3779940605163574) + | > loss_gen: 2.5142393112182617 (2.5142393112182617) + | > loss_kl: 2.384061098098755 (2.384061098098755) + | > loss_feat: 6.822709083557129 (6.822709083557129) + | > loss_mel: 18.25099754333496 (18.25099754333496) + | > loss_duration: 1.6752614974975586 (1.6752614974975586) + | > loss_1: 31.64727020263672 (31.64727020263672) + + --> STEP: 2 + | > loss_disc: 2.566807270050049 (2.472400665283203) + | > loss_disc_real_0: 0.2275218963623047 (0.22386784851551056) + | > loss_disc_real_1: 0.23644128441810608 (0.23440039157867432) + | > loss_disc_real_2: 0.24513690173625946 (0.2146899551153183) + | > loss_disc_real_3: 0.20912787318229675 (0.20010913908481598) + | > loss_disc_real_4: 0.250188410282135 (0.21789062768220901) + | > loss_disc_real_5: 0.2750881314277649 (0.2554948702454567) + | > loss_0: 2.566807270050049 (2.472400665283203) + | > loss_gen: 2.554218292236328 (2.534228801727295) + | > loss_kl: 2.2424583435058594 (2.313259720802307) + | > loss_feat: 6.999917030334473 (6.911313056945801) + | > loss_mel: 17.63922882080078 (17.94511318206787) + | > loss_duration: 1.6267223358154297 (1.6509919166564941) + | > loss_1: 31.062543869018555 (31.354907035827637) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00358736515045166 (+9.262561798095703e-05) + | > avg_loss_disc: 2.472400665283203 (-0.04439389705657959) + | > avg_loss_disc_real_0: 0.22386784851551056 (+0.05113052576780319) + | > avg_loss_disc_real_1: 0.23440039157867432 (+0.04016193747520447) + | > avg_loss_disc_real_2: 0.2146899551153183 (-0.031160004436969757) + | > avg_loss_disc_real_3: 0.20010913908481598 (-0.0757635086774826) + | > avg_loss_disc_real_4: 0.21789062768220901 (-0.025949150323867798) + | > avg_loss_disc_real_5: 0.2554948702454567 (+0.04536189138889313) + | > avg_loss_0: 2.472400665283203 (-0.04439389705657959) + | > avg_loss_gen: 2.534228801727295 (+0.2592858076095581) + | > avg_loss_kl: 2.313259720802307 (+0.12280404567718506) + | > avg_loss_feat: 6.911313056945801 (+0.5975193977355957) + | > avg_loss_mel: 17.94511318206787 (-0.13656330108642578) + | > avg_loss_duration: 1.6509919166564941 (+0.014563560485839844) + | > avg_loss_1: 31.354907035827637 (+0.857609748840332) + + + > EPOCH: 84/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:23:21)  + + --> TIME: 2025-04-19 00:23:27 -- STEP: 15/142 -- GLOBAL_STEP: 391225 + | > loss_disc: 2.4462215900421143 (2.444779888788859) + | > loss_disc_real_0: 0.12770767509937286 (0.13387141178051631) + | > loss_disc_real_1: 0.16884011030197144 (0.2070049931605657) + | > loss_disc_real_2: 0.23531046509742737 (0.2232294281323751) + | > loss_disc_real_3: 0.2224925309419632 (0.23217817147572836) + | > loss_disc_real_4: 0.19151945412158966 (0.22763600250085195) + | > loss_disc_real_5: 0.19874753057956696 (0.2309359182914098) + | > loss_0: 2.4462215900421143 (2.444779888788859) + | > grad_norm_0: tensor(18.2165, device='cuda:0') (tensor(9.2229, device='cuda:0')) + | > loss_gen: 2.268601417541504 (2.3574046611785895) + | > loss_kl: 1.9297536611557007 (1.9078305880228679) + | > loss_feat: 7.843726634979248 (7.230329640706381) + | > loss_mel: 18.470558166503906 (18.117471694946286) + | > loss_duration: 1.4754955768585205 (1.4624826828638713) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.988134384155273 (31.075519307454428) + | > grad_norm_1: tensor(437.9722, device='cuda:0') (tensor(171.2421, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.3347 (0.32839535077412924) + | > loader_time: 0.0056 (0.004928859074910482) + + + --> TIME: 2025-04-19 00:23:36 -- STEP: 40/142 -- GLOBAL_STEP: 391250 + | > loss_disc: 2.529454231262207 (2.455488336086273) + | > loss_disc_real_0: 0.17356149852275848 (0.13291646856814626) + | > loss_disc_real_1: 0.17072784900665283 (0.2055184155702591) + | > loss_disc_real_2: 0.22467896342277527 (0.22464493624866008) + | > loss_disc_real_3: 0.2214309275150299 (0.22877914160490037) + | > loss_disc_real_4: 0.23128831386566162 (0.22819082103669644) + | > loss_disc_real_5: 0.23661449551582336 (0.23448426462709904) + | > loss_0: 2.529454231262207 (2.455488336086273) + | > grad_norm_0: tensor(17.6408, device='cuda:0') (tensor(10.0374, device='cuda:0')) + | > loss_gen: 2.1597678661346436 (2.369737833738328) + | > loss_kl: 1.9076703786849976 (1.8944726884365082) + | > loss_feat: 6.527654647827148 (7.071148860454559) + | > loss_mel: 16.642024993896484 (17.845179939270015) + | > loss_duration: 1.4555147886276245 (1.473525995016098) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.6926326751709 (30.65406541824341) + | > grad_norm_1: tensor(349.0880, device='cuda:0') (tensor(203.4587, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.3603 (0.3410998821258545) + | > loader_time: 0.0059 (0.005401086807250976) + + + --> TIME: 2025-04-19 00:23:45 -- STEP: 65/142 -- GLOBAL_STEP: 391275 + | > loss_disc: 2.4000496864318848 (2.4543063273796664) + | > loss_disc_real_0: 0.10966377705335617 (0.1285237471644696) + | > loss_disc_real_1: 0.1767081618309021 (0.2063103980742968) + | > loss_disc_real_2: 0.21810179948806763 (0.22620260348686805) + | > loss_disc_real_3: 0.22233261168003082 (0.22942757675280936) + | > loss_disc_real_4: 0.23655815422534943 (0.22996679980021256) + | > loss_disc_real_5: 0.2373027503490448 (0.23216011959772845) + | > loss_0: 2.4000496864318848 (2.4543063273796664) + | > grad_norm_0: tensor(9.5559, device='cuda:0') (tensor(9.9548, device='cuda:0')) + | > loss_gen: 2.3652420043945312 (2.3601078290205746) + | > loss_kl: 1.9015812873840332 (1.9056238192778368) + | > loss_feat: 7.336505889892578 (7.093983320089487) + | > loss_mel: 18.307350158691406 (17.93821581326999) + | > loss_duration: 1.4950882196426392 (1.4860106413181011) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.4057674407959 (30.783941533015323) + | > grad_norm_1: tensor(364.7893, device='cuda:0') (tensor(220.0191, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.4026 (0.35320592660170336) + | > loader_time: 0.0066 (0.005733570685753455) + + + --> TIME: 2025-04-19 00:23:55 -- STEP: 90/142 -- GLOBAL_STEP: 391300 + | > loss_disc: 2.451070785522461 (2.4637539916568323) + | > loss_disc_real_0: 0.17021211981773376 (0.12916655316948902) + | > loss_disc_real_1: 0.21043139696121216 (0.20812947005033494) + | > loss_disc_real_2: 0.21541881561279297 (0.22535762124591405) + | > loss_disc_real_3: 0.2159392088651657 (0.22980934083461763) + | > loss_disc_real_4: 0.23835378885269165 (0.2303766848312484) + | > loss_disc_real_5: 0.21230074763298035 (0.23230646004279454) + | > loss_0: 2.451070785522461 (2.4637539916568323) + | > grad_norm_0: tensor(11.0289, device='cuda:0') (tensor(9.6762, device='cuda:0')) + | > loss_gen: 2.408682107925415 (2.3498827695846565) + | > loss_kl: 1.89425528049469 (1.9240484277407328) + | > loss_feat: 7.0909223556518555 (7.032060766220093) + | > loss_mel: 17.679019927978516 (17.870697424146865) + | > loss_duration: 1.5332592725753784 (1.4978388547897339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.606138229370117 (30.67452841864692) + | > grad_norm_1: tensor(341.0027, device='cuda:0') (tensor(212.4235, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.4018 (0.36440649032592776) + | > loader_time: 0.0068 (0.005990886688232423) + + + --> TIME: 2025-04-19 00:24:06 -- STEP: 115/142 -- GLOBAL_STEP: 391325 + | > loss_disc: 2.315049648284912 (2.465841917369677) + | > loss_disc_real_0: 0.16645430028438568 (0.13018148690462114) + | > loss_disc_real_1: 0.17809045314788818 (0.20758336354856904) + | > loss_disc_real_2: 0.2052590399980545 (0.22591614438139873) + | > loss_disc_real_3: 0.18473558127880096 (0.2281968089549438) + | > loss_disc_real_4: 0.15876984596252441 (0.23045488259066707) + | > loss_disc_real_5: 0.1874772012233734 (0.2308146349761797) + | > loss_0: 2.315049648284912 (2.465841917369677) + | > grad_norm_0: tensor(5.4214, device='cuda:0') (tensor(9.9413, device='cuda:0')) + | > loss_gen: 2.316023826599121 (2.3420262564783503) + | > loss_kl: 2.0055959224700928 (1.9303039965422257) + | > loss_feat: 7.600958824157715 (7.016211153113324) + | > loss_mel: 18.68731689453125 (17.89792929939602) + | > loss_duration: 1.5371071100234985 (1.503444106682487) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.147003173828125 (30.689914885811184) + | > grad_norm_1: tensor(49.7900, device='cuda:0') (tensor(210.5922, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.4249 (0.3750732463339101) + | > loader_time: 0.0074 (0.0062028449514637835) + + + --> TIME: 2025-04-19 00:24:17 -- STEP: 140/142 -- GLOBAL_STEP: 391350 + | > loss_disc: 2.458479881286621 (2.467953601905278) + | > loss_disc_real_0: 0.13577623665332794 (0.1314045189214604) + | > loss_disc_real_1: 0.20508867502212524 (0.20772169402667454) + | > loss_disc_real_2: 0.2423565536737442 (0.22707112452813558) + | > loss_disc_real_3: 0.23504097759723663 (0.22866563435111728) + | > loss_disc_real_4: 0.2393985241651535 (0.2306826148714338) + | > loss_disc_real_5: 0.23003007471561432 (0.2314769022166729) + | > loss_0: 2.458479881286621 (2.467953601905278) + | > grad_norm_0: tensor(6.4843, device='cuda:0') (tensor(9.9734, device='cuda:0')) + | > loss_gen: 2.407932996749878 (2.3442754030227655) + | > loss_kl: 1.7934757471084595 (1.9292599575860159) + | > loss_feat: 7.860130310058594 (6.993495631217956) + | > loss_mel: 18.4719181060791 (17.86395867211478) + | > loss_duration: 1.5599664449691772 (1.5097115193094526) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.093421936035156 (30.640701293945312) + | > grad_norm_1: tensor(248.2278, device='cuda:0') (tensor(207.4217, device='cuda:0')) + | > current_lr_0: 0.00019791085662371262 + | > current_lr_1: 0.00019791085662371262 + | > step_time: 0.4589 (0.3880015407289778) + | > loader_time: 0.0082 (0.0064803481101989744) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4200692176818848 (2.4200692176818848) + | > loss_disc_real_0: 0.12233219295740128 (0.12233219295740128) + | > loss_disc_real_1: 0.21237896382808685 (0.21237896382808685) + | > loss_disc_real_2: 0.24294354021549225 (0.24294354021549225) + | > loss_disc_real_3: 0.25885334610939026 (0.25885334610939026) + | > loss_disc_real_4: 0.19650277495384216 (0.19650277495384216) + | > loss_disc_real_5: 0.2553249001502991 (0.2553249001502991) + | > loss_0: 2.4200692176818848 (2.4200692176818848) + | > loss_gen: 2.4023609161376953 (2.4023609161376953) + | > loss_kl: 2.2587473392486572 (2.2587473392486572) + | > loss_feat: 7.440920352935791 (7.440920352935791) + | > loss_mel: 20.005159378051758 (20.005159378051758) + | > loss_duration: 1.6388845443725586 (1.6388845443725586) + | > loss_1: 33.746070861816406 (33.746070861816406) + + --> STEP: 1 + | > loss_disc: 2.444154739379883 (2.444154739379883) + | > loss_disc_real_0: 0.1645113229751587 (0.1645113229751587) + | > loss_disc_real_1: 0.21880410611629486 (0.21880410611629486) + | > loss_disc_real_2: 0.21561332046985626 (0.21561332046985626) + | > loss_disc_real_3: 0.22538988292217255 (0.22538988292217255) + | > loss_disc_real_4: 0.22867701947689056 (0.22867701947689056) + | > loss_disc_real_5: 0.23839333653450012 (0.23839333653450012) + | > loss_0: 2.444154739379883 (2.444154739379883) + | > loss_gen: 2.31473445892334 (2.31473445892334) + | > loss_kl: 2.4449124336242676 (2.4449124336242676) + | > loss_feat: 6.806879997253418 (6.806879997253418) + | > loss_mel: 18.694293975830078 (18.694293975830078) + | > loss_duration: 1.6732631921768188 (1.6732631921768188) + | > loss_1: 31.934085845947266 (31.934085845947266) + + --> STEP: 2 + | > loss_disc: 2.3071277141571045 (2.3756412267684937) + | > loss_disc_real_0: 0.09993626177310944 (0.13222379237413406) + | > loss_disc_real_1: 0.20206193625926971 (0.2104330211877823) + | > loss_disc_real_2: 0.24077638983726501 (0.22819485515356064) + | > loss_disc_real_3: 0.2392762452363968 (0.23233306407928467) + | > loss_disc_real_4: 0.1857101172208786 (0.20719356834888458) + | > loss_disc_real_5: 0.2942691445350647 (0.2663312405347824) + | > loss_0: 2.3071277141571045 (2.3756412267684937) + | > loss_gen: 2.577810525894165 (2.4462724924087524) + | > loss_kl: 2.152341604232788 (2.298627018928528) + | > loss_feat: 7.639872074127197 (7.223376035690308) + | > loss_mel: 19.374479293823242 (19.03438663482666) + | > loss_duration: 1.6112613677978516 (1.6422622799873352) + | > loss_1: 33.35576629638672 (32.64492607116699) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032711029052734375 (-0.00031626224517822266) + | > avg_loss_disc: 2.3756412267684937 (-0.09675943851470947) + | > avg_loss_disc_real_0: 0.13222379237413406 (-0.0916440561413765) + | > avg_loss_disc_real_1: 0.2104330211877823 (-0.02396737039089203) + | > avg_loss_disc_real_2: 0.22819485515356064 (+0.01350490003824234) + | > avg_loss_disc_real_3: 0.23233306407928467 (+0.03222392499446869) + | > avg_loss_disc_real_4: 0.20719356834888458 (-0.010697059333324432) + | > avg_loss_disc_real_5: 0.2663312405347824 (+0.010836370289325714) + | > avg_loss_0: 2.3756412267684937 (-0.09675943851470947) + | > avg_loss_gen: 2.4462724924087524 (-0.08795630931854248) + | > avg_loss_kl: 2.298627018928528 (-0.014632701873779297) + | > avg_loss_feat: 7.223376035690308 (+0.31206297874450684) + | > avg_loss_mel: 19.03438663482666 (+1.089273452758789) + | > avg_loss_duration: 1.6422622799873352 (-0.008729636669158936) + | > avg_loss_1: 32.64492607116699 (+1.2900190353393555) + + + > EPOCH: 85/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:24:20)  + + --> TIME: 2025-04-19 00:24:28 -- STEP: 23/142 -- GLOBAL_STEP: 391375 + | > loss_disc: 2.5416555404663086 (2.464955132940541) + | > loss_disc_real_0: 0.09978237003087997 (0.1181635710856189) + | > loss_disc_real_1: 0.22135798633098602 (0.212531469438387) + | > loss_disc_real_2: 0.2371310144662857 (0.22812591233979101) + | > loss_disc_real_3: 0.2563009560108185 (0.2277628429557966) + | > loss_disc_real_4: 0.2515847980976105 (0.23011086881160736) + | > loss_disc_real_5: 0.23701225221157074 (0.22644916500734247) + | > loss_0: 2.5416555404663086 (2.464955132940541) + | > grad_norm_0: tensor(12.9392, device='cuda:0') (tensor(9.3517, device='cuda:0')) + | > loss_gen: 2.2826220989227295 (2.3670183161030645) + | > loss_kl: 1.9059187173843384 (1.8644258768662163) + | > loss_feat: 6.778028964996338 (7.108371796815292) + | > loss_mel: 16.02836799621582 (17.942913055419922) + | > loss_duration: 1.4999916553497314 (1.4648416767949644) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.49492835998535 (30.747571032980215) + | > grad_norm_1: tensor(282.6700, device='cuda:0') (tensor(231.0138, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.352 (0.33370481366696564) + | > loader_time: 0.0056 (0.005192093227220618) + + + --> TIME: 2025-04-19 00:24:38 -- STEP: 48/142 -- GLOBAL_STEP: 391400 + | > loss_disc: 2.4491405487060547 (2.4698594411214194) + | > loss_disc_real_0: 0.07684069126844406 (0.12448617303743963) + | > loss_disc_real_1: 0.20439022779464722 (0.20874783707161745) + | > loss_disc_real_2: 0.2511584162712097 (0.2256666962057352) + | > loss_disc_real_3: 0.2612646520137787 (0.22761867133279642) + | > loss_disc_real_4: 0.2451283484697342 (0.23210642331590256) + | > loss_disc_real_5: 0.2378360778093338 (0.22988655045628548) + | > loss_0: 2.4491405487060547 (2.4698594411214194) + | > grad_norm_0: tensor(9.7246, device='cuda:0') (tensor(10.5797, device='cuda:0')) + | > loss_gen: 2.4289824962615967 (2.361533164978027) + | > loss_kl: 1.8684513568878174 (1.8973488633831341) + | > loss_feat: 7.094516277313232 (7.0894900262355804) + | > loss_mel: 18.581836700439453 (17.9903777440389) + | > loss_duration: 1.4630377292633057 (1.4744151781002683) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.43682289123535 (30.813165148099262) + | > grad_norm_1: tensor(239.1379, device='cuda:0') (tensor(244.5953, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.3638 (0.3522612551848094) + | > loader_time: 0.0058 (0.005498453974723816) + + + --> TIME: 2025-04-19 00:24:47 -- STEP: 73/142 -- GLOBAL_STEP: 391425 + | > loss_disc: 2.483328104019165 (2.467988994023571) + | > loss_disc_real_0: 0.1600186824798584 (0.12490274563227619) + | > loss_disc_real_1: 0.19805589318275452 (0.20753280491861575) + | > loss_disc_real_2: 0.21777333319187164 (0.22581614513103276) + | > loss_disc_real_3: 0.21707352995872498 (0.22609166474375006) + | > loss_disc_real_4: 0.18915893137454987 (0.2306994326310615) + | > loss_disc_real_5: 0.2121608704328537 (0.23075030172524386) + | > loss_0: 2.483328104019165 (2.467988994023571) + | > grad_norm_0: tensor(32.6596, device='cuda:0') (tensor(12.0848, device='cuda:0')) + | > loss_gen: 2.4345157146453857 (2.3632490896198846) + | > loss_kl: 2.0785396099090576 (1.9185797221040073) + | > loss_feat: 7.581640720367432 (7.03210743159464) + | > loss_mel: 18.453580856323242 (17.98349343913875) + | > loss_duration: 1.5602878332138062 (1.4871658103106773) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.10856628417969 (30.784595515630016) + | > grad_norm_1: tensor(314.0446, device='cuda:0') (tensor(264.7387, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.3929 (0.36245293813209006) + | > loader_time: 0.0065 (0.005743029999406371) + + + --> TIME: 2025-04-19 00:24:58 -- STEP: 98/142 -- GLOBAL_STEP: 391450 + | > loss_disc: 2.573982000350952 (2.473575073845533) + | > loss_disc_real_0: 0.22127209603786469 (0.12948402640770892) + | > loss_disc_real_1: 0.23204341530799866 (0.2085534650452283) + | > loss_disc_real_2: 0.21187171339988708 (0.22642765346230292) + | > loss_disc_real_3: 0.2553848624229431 (0.2272661449957867) + | > loss_disc_real_4: 0.2539385259151459 (0.23006952295497973) + | > loss_disc_real_5: 0.22963298857212067 (0.2298623568245343) + | > loss_0: 2.573982000350952 (2.473575073845533) + | > grad_norm_0: tensor(41.8627, device='cuda:0') (tensor(14.2515, device='cuda:0')) + | > loss_gen: 2.4089274406433105 (2.3684806848058884) + | > loss_kl: 1.9531962871551514 (1.9245572138805778) + | > loss_feat: 7.249497890472412 (6.99208817676622) + | > loss_mel: 18.52873420715332 (17.90803857725493) + | > loss_duration: 1.4844744205474854 (1.4979563902835462) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.62483024597168 (30.691121101379395) + | > grad_norm_1: tensor(523.6831, device='cuda:0') (tensor(283.5918, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.4186 (0.3726261756858047) + | > loader_time: 0.0067 (0.006029053610198354) + + + --> TIME: 2025-04-19 00:25:09 -- STEP: 123/142 -- GLOBAL_STEP: 391475 + | > loss_disc: 2.5375940799713135 (2.4803675519741657) + | > loss_disc_real_0: 0.1449650675058365 (0.1336004946956305) + | > loss_disc_real_1: 0.21965450048446655 (0.2078023389345262) + | > loss_disc_real_2: 0.21458977460861206 (0.22596275503557872) + | > loss_disc_real_3: 0.21590636670589447 (0.22744149466355643) + | > loss_disc_real_4: 0.2296752631664276 (0.23041083185168787) + | > loss_disc_real_5: 0.224175825715065 (0.23087578151768784) + | > loss_0: 2.5375940799713135 (2.4803675519741657) + | > grad_norm_0: tensor(29.4869, device='cuda:0') (tensor(16.2177, device='cuda:0')) + | > loss_gen: 2.322650909423828 (2.364366554632418) + | > loss_kl: 1.884608268737793 (1.929872585505974) + | > loss_feat: 6.562191963195801 (6.969454253592143) + | > loss_mel: 16.708982467651367 (17.891713964260685) + | > loss_duration: 1.5117766857147217 (1.5050345785249544) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.990209579467773 (30.660441918101736) + | > grad_norm_1: tensor(290.9177, device='cuda:0') (tensor(295.3150, device='cuda:0')) + | > current_lr_0: 0.00019788611776663464 + | > current_lr_1: 0.00019788611776663464 + | > step_time: 0.4445 (0.3835133381975376) + | > loader_time: 0.0078 (0.006316136538497803) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4184818267822266 (2.4184818267822266) + | > loss_disc_real_0: 0.15757811069488525 (0.15757811069488525) + | > loss_disc_real_1: 0.20841766893863678 (0.20841766893863678) + | > loss_disc_real_2: 0.21582314372062683 (0.21582314372062683) + | > loss_disc_real_3: 0.22349423170089722 (0.22349423170089722) + | > loss_disc_real_4: 0.25747257471084595 (0.25747257471084595) + | > loss_disc_real_5: 0.21030358970165253 (0.21030358970165253) + | > loss_0: 2.4184818267822266 (2.4184818267822266) + | > loss_gen: 2.677645206451416 (2.677645206451416) + | > loss_kl: 2.1587469577789307 (2.1587469577789307) + | > loss_feat: 7.800553321838379 (7.800553321838379) + | > loss_mel: 19.539812088012695 (19.539812088012695) + | > loss_duration: 1.631897211074829 (1.631897211074829) + | > loss_1: 33.80865478515625 (33.80865478515625) + + --> STEP: 1 + | > loss_disc: 2.573437452316284 (2.573437452316284) + | > loss_disc_real_0: 0.17843806743621826 (0.17843806743621826) + | > loss_disc_real_1: 0.26824912428855896 (0.26824912428855896) + | > loss_disc_real_2: 0.25658082962036133 (0.25658082962036133) + | > loss_disc_real_3: 0.21478423476219177 (0.21478423476219177) + | > loss_disc_real_4: 0.2812981605529785 (0.2812981605529785) + | > loss_disc_real_5: 0.21231399476528168 (0.21231399476528168) + | > loss_0: 2.573437452316284 (2.573437452316284) + | > loss_gen: 2.553213119506836 (2.553213119506836) + | > loss_kl: 2.27178692817688 (2.27178692817688) + | > loss_feat: 7.047128677368164 (7.047128677368164) + | > loss_mel: 18.804065704345703 (18.804065704345703) + | > loss_duration: 1.6602680683135986 (1.6602680683135986) + | > loss_1: 32.336463928222656 (32.336463928222656) + + --> STEP: 2 + | > loss_disc: 2.513136863708496 (2.54328715801239) + | > loss_disc_real_0: 0.24691623449325562 (0.21267715096473694) + | > loss_disc_real_1: 0.21372178196907043 (0.2409854531288147) + | > loss_disc_real_2: 0.21642647683620453 (0.23650365322828293) + | > loss_disc_real_3: 0.2361154705286026 (0.2254498526453972) + | > loss_disc_real_4: 0.29627668857574463 (0.2887874245643616) + | > loss_disc_real_5: 0.25110962986946106 (0.23171181231737137) + | > loss_0: 2.513136863708496 (2.54328715801239) + | > loss_gen: 2.681239128112793 (2.6172261238098145) + | > loss_kl: 2.225959539413452 (2.248873233795166) + | > loss_feat: 7.883415222167969 (7.465271949768066) + | > loss_mel: 19.5058536529541 (19.154959678649902) + | > loss_duration: 1.6083259582519531 (1.6342970132827759) + | > loss_1: 33.90479278564453 (33.120628356933594) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003447294235229492 (+0.0001761913299560547) + | > avg_loss_disc: 2.54328715801239 (+0.16764593124389648) + | > avg_loss_disc_real_0: 0.21267715096473694 (+0.08045335859060287) + | > avg_loss_disc_real_1: 0.2409854531288147 (+0.03055243194103241) + | > avg_loss_disc_real_2: 0.23650365322828293 (+0.00830879807472229) + | > avg_loss_disc_real_3: 0.2254498526453972 (-0.006883211433887482) + | > avg_loss_disc_real_4: 0.2887874245643616 (+0.08159385621547699) + | > avg_loss_disc_real_5: 0.23171181231737137 (-0.03461942821741104) + | > avg_loss_0: 2.54328715801239 (+0.16764593124389648) + | > avg_loss_gen: 2.6172261238098145 (+0.170953631401062) + | > avg_loss_kl: 2.248873233795166 (-0.049753785133361816) + | > avg_loss_feat: 7.465271949768066 (+0.2418959140777588) + | > avg_loss_mel: 19.154959678649902 (+0.12057304382324219) + | > avg_loss_duration: 1.6342970132827759 (-0.007965266704559326) + | > avg_loss_1: 33.120628356933594 (+0.47570228576660156) + + + > EPOCH: 86/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:25:19)  + + --> TIME: 2025-04-19 00:25:21 -- STEP: 6/142 -- GLOBAL_STEP: 391500 + | > loss_disc: 2.503072738647461 (2.4298136234283447) + | > loss_disc_real_0: 0.12689527869224548 (0.10988462716341019) + | > loss_disc_real_1: 0.2890233099460602 (0.22000636905431747) + | > loss_disc_real_2: 0.23538538813591003 (0.23548568785190582) + | > loss_disc_real_3: 0.2601753771305084 (0.2314586564898491) + | > loss_disc_real_4: 0.23646120727062225 (0.22764556606610617) + | > loss_disc_real_5: 0.2541721761226654 (0.2262774258852005) + | > loss_0: 2.503072738647461 (2.4298136234283447) + | > grad_norm_0: tensor(7.3951, device='cuda:0') (tensor(9.4289, device='cuda:0')) + | > loss_gen: 2.3508567810058594 (2.365691860516866) + | > loss_kl: 1.907753586769104 (1.8439966440200806) + | > loss_feat: 6.787445545196533 (7.174986441930135) + | > loss_mel: 17.188228607177734 (18.124216715494793) + | > loss_duration: 1.4759031534194946 (1.4734835028648376) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.710186004638672 (30.982375462849934) + | > grad_norm_1: tensor(64.8728, device='cuda:0') (tensor(218.6582, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.3317 (0.32555309931437176) + | > loader_time: 0.0047 (0.004758119583129883) + + + --> TIME: 2025-04-19 00:25:30 -- STEP: 31/142 -- GLOBAL_STEP: 391525 + | > loss_disc: 2.5712900161743164 (2.4494014093952794) + | > loss_disc_real_0: 0.1252429336309433 (0.11579168660025443) + | > loss_disc_real_1: 0.2013666033744812 (0.20634030526684177) + | > loss_disc_real_2: 0.21850258111953735 (0.22886059889870305) + | > loss_disc_real_3: 0.25065740942955017 (0.22867555435626738) + | > loss_disc_real_4: 0.2625639736652374 (0.22938559324510635) + | > loss_disc_real_5: 0.24737010896205902 (0.2304284567794492) + | > loss_0: 2.5712900161743164 (2.4494014093952794) + | > grad_norm_0: tensor(6.0157, device='cuda:0') (tensor(9.3650, device='cuda:0')) + | > loss_gen: 2.18940806388855 (2.3272013818064043) + | > loss_kl: 1.7531309127807617 (1.8616624839844242) + | > loss_feat: 6.130429267883301 (6.9962697336750646) + | > loss_mel: 15.317849159240723 (17.815756397862593) + | > loss_duration: 1.4621367454528809 (1.4717158156056558) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.85295295715332 (30.472605489915416) + | > grad_norm_1: tensor(57.5803, device='cuda:0') (tensor(172.9314, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.3529 (0.33678210166192823) + | > loader_time: 0.0059 (0.005313273399106917) + + + --> TIME: 2025-04-19 00:25:39 -- STEP: 56/142 -- GLOBAL_STEP: 391550 + | > loss_disc: 2.4584550857543945 (2.450087764433453) + | > loss_disc_real_0: 0.1209656223654747 (0.12059882110250848) + | > loss_disc_real_1: 0.2415277361869812 (0.20614677614399365) + | > loss_disc_real_2: 0.20897822082042694 (0.225830005215747) + | > loss_disc_real_3: 0.2269555926322937 (0.22837048955261707) + | > loss_disc_real_4: 0.22855614125728607 (0.22888214300785745) + | > loss_disc_real_5: 0.25911104679107666 (0.23045328631997108) + | > loss_0: 2.4584550857543945 (2.450087764433453) + | > grad_norm_0: tensor(10.0872, device='cuda:0') (tensor(8.2007, device='cuda:0')) + | > loss_gen: 2.3940210342407227 (2.327615580388478) + | > loss_kl: 1.9969509840011597 (1.887472152709961) + | > loss_feat: 6.443528652191162 (7.018111935683659) + | > loss_mel: 18.039737701416016 (17.88057592936925) + | > loss_duration: 1.5027251243591309 (1.4800184028489247) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.376962661743164 (30.593793664659774) + | > grad_norm_1: tensor(69.1147, device='cuda:0') (tensor(142.2979, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.3873 (0.3491862842014858) + | > loader_time: 0.0064 (0.005604271377835955) + + + --> TIME: 2025-04-19 00:25:49 -- STEP: 81/142 -- GLOBAL_STEP: 391575 + | > loss_disc: 2.4150567054748535 (2.4578921971497723) + | > loss_disc_real_0: 0.16663102805614471 (0.12469431656746217) + | > loss_disc_real_1: 0.24012596905231476 (0.2075633184041506) + | > loss_disc_real_2: 0.21271109580993652 (0.22478243432663106) + | > loss_disc_real_3: 0.18211476504802704 (0.22681259483466915) + | > loss_disc_real_4: 0.2071024626493454 (0.22872867812345057) + | > loss_disc_real_5: 0.2293945848941803 (0.23070614187069882) + | > loss_0: 2.4150567054748535 (2.4578921971497723) + | > grad_norm_0: tensor(9.4945, device='cuda:0') (tensor(8.1679, device='cuda:0')) + | > loss_gen: 2.471000909805298 (2.330904569154905) + | > loss_kl: 1.9136583805084229 (1.908223271369934) + | > loss_feat: 7.202188491821289 (7.005366166432698) + | > loss_mel: 17.710430145263672 (17.92155345869653) + | > loss_duration: 1.5000391006469727 (1.4901739756266275) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.797317504882812 (30.656221272032937) + | > grad_norm_1: tensor(226.9619, device='cuda:0') (tensor(144.2025, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.392 (0.3615621460808648) + | > loader_time: 0.007 (0.0058893745328173226) + + + --> TIME: 2025-04-19 00:26:00 -- STEP: 106/142 -- GLOBAL_STEP: 391600 + | > loss_disc: 2.5668420791625977 (2.465132411920801) + | > loss_disc_real_0: 0.12486868351697922 (0.1268021763073948) + | > loss_disc_real_1: 0.21839065849781036 (0.20772814652267493) + | > loss_disc_real_2: 0.22769217193126678 (0.2263572634953373) + | > loss_disc_real_3: 0.24430988729000092 (0.22771331330515304) + | > loss_disc_real_4: 0.21358861029148102 (0.2285313589392968) + | > loss_disc_real_5: 0.2432710975408554 (0.23014937481790218) + | > loss_0: 2.5668420791625977 (2.465132411920801) + | > grad_norm_0: tensor(5.5375, device='cuda:0') (tensor(7.7168, device='cuda:0')) + | > loss_gen: 2.2545864582061768 (2.324651630419606) + | > loss_kl: 1.9575636386871338 (1.9202724942621194) + | > loss_feat: 6.716963768005371 (6.955149025287268) + | > loss_mel: 17.37757682800293 (17.940221822486738) + | > loss_duration: 1.5373775959014893 (1.5012962953099664) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.844070434570312 (30.64159118004565) + | > grad_norm_1: tensor(100.1277, device='cuda:0') (tensor(132.2296, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.4087 (0.3718139023151038) + | > loader_time: 0.007 (0.006141453419091567) + + + --> TIME: 2025-04-19 00:26:11 -- STEP: 131/142 -- GLOBAL_STEP: 391625 + | > loss_disc: 2.6002745628356934 (2.468255583566565) + | > loss_disc_real_0: 0.16578832268714905 (0.12937494839420757) + | > loss_disc_real_1: 0.24624569714069366 (0.20855617693817344) + | > loss_disc_real_2: 0.2570205628871918 (0.22602941539451366) + | > loss_disc_real_3: 0.25251203775405884 (0.2286500793149453) + | > loss_disc_real_4: 0.25064757466316223 (0.22874018163171433) + | > loss_disc_real_5: 0.2415243536233902 (0.2302335057094807) + | > loss_0: 2.6002745628356934 (2.468255583566565) + | > grad_norm_0: tensor(5.9626, device='cuda:0') (tensor(7.4451, device='cuda:0')) + | > loss_gen: 2.1928348541259766 (2.320645312316546) + | > loss_kl: 1.9764975309371948 (1.923381362252563) + | > loss_feat: 6.248139381408691 (6.923221282376588) + | > loss_mel: 16.16237449645996 (17.89473453550849) + | > loss_duration: 1.5308046340942383 (1.506757705266239) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.110652923583984 (30.568740087611076) + | > grad_norm_1: tensor(130.2834, device='cuda:0') (tensor(120.9890, device='cuda:0')) + | > current_lr_0: 0.0001978613820019138 + | > current_lr_1: 0.0001978613820019138 + | > step_time: 0.4529 (0.38405064044107917) + | > loader_time: 0.0074 (0.00641728721502173) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3823118209838867 (2.3823118209838867) + | > loss_disc_real_0: 0.1444210410118103 (0.1444210410118103) + | > loss_disc_real_1: 0.26347437500953674 (0.26347437500953674) + | > loss_disc_real_2: 0.2487192302942276 (0.2487192302942276) + | > loss_disc_real_3: 0.23527221381664276 (0.23527221381664276) + | > loss_disc_real_4: 0.1977514922618866 (0.1977514922618866) + | > loss_disc_real_5: 0.22670938074588776 (0.22670938074588776) + | > loss_0: 2.3823118209838867 (2.3823118209838867) + | > loss_gen: 2.4546327590942383 (2.4546327590942383) + | > loss_kl: 2.328714609146118 (2.328714609146118) + | > loss_feat: 6.447871685028076 (6.447871685028076) + | > loss_mel: 18.622377395629883 (18.622377395629883) + | > loss_duration: 1.6362416744232178 (1.6362416744232178) + | > loss_1: 31.489837646484375 (31.489837646484375) + + --> STEP: 1 + | > loss_disc: 2.5852854251861572 (2.5852854251861572) + | > loss_disc_real_0: 0.18739821016788483 (0.18739821016788483) + | > loss_disc_real_1: 0.2922191023826599 (0.2922191023826599) + | > loss_disc_real_2: 0.20419546961784363 (0.20419546961784363) + | > loss_disc_real_3: 0.20803749561309814 (0.20803749561309814) + | > loss_disc_real_4: 0.1942087858915329 (0.1942087858915329) + | > loss_disc_real_5: 0.23742836713790894 (0.23742836713790894) + | > loss_0: 2.5852854251861572 (2.5852854251861572) + | > loss_gen: 2.2035818099975586 (2.2035818099975586) + | > loss_kl: 2.24414324760437 (2.24414324760437) + | > loss_feat: 5.9409708976745605 (5.9409708976745605) + | > loss_mel: 17.25399398803711 (17.25399398803711) + | > loss_duration: 1.650083303451538 (1.650083303451538) + | > loss_1: 29.29277229309082 (29.29277229309082) + + --> STEP: 2 + | > loss_disc: 2.505171775817871 (2.545228600502014) + | > loss_disc_real_0: 0.22522519528865814 (0.20631170272827148) + | > loss_disc_real_1: 0.27997511625289917 (0.28609710931777954) + | > loss_disc_real_2: 0.23120862245559692 (0.21770204603672028) + | > loss_disc_real_3: 0.20396220684051514 (0.20599985122680664) + | > loss_disc_real_4: 0.20761625468730927 (0.20091252028942108) + | > loss_disc_real_5: 0.21875612437725067 (0.2280922457575798) + | > loss_0: 2.505171775817871 (2.545228600502014) + | > loss_gen: 2.3430447578430176 (2.273313283920288) + | > loss_kl: 2.112877368927002 (2.178510308265686) + | > loss_feat: 5.807161808013916 (5.874066352844238) + | > loss_mel: 16.72176742553711 (16.98788070678711) + | > loss_duration: 1.6125741004943848 (1.6313287019729614) + | > loss_1: 28.59742546081543 (28.945098876953125) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003267049789428711 (-0.00018024444580078125) + | > avg_loss_disc: 2.545228600502014 (+0.0019414424896240234) + | > avg_loss_disc_real_0: 0.20631170272827148 (-0.006365448236465454) + | > avg_loss_disc_real_1: 0.28609710931777954 (+0.045111656188964844) + | > avg_loss_disc_real_2: 0.21770204603672028 (-0.018801607191562653) + | > avg_loss_disc_real_3: 0.20599985122680664 (-0.019450001418590546) + | > avg_loss_disc_real_4: 0.20091252028942108 (-0.08787490427494049) + | > avg_loss_disc_real_5: 0.2280922457575798 (-0.003619566559791565) + | > avg_loss_0: 2.545228600502014 (+0.0019414424896240234) + | > avg_loss_gen: 2.273313283920288 (-0.34391283988952637) + | > avg_loss_kl: 2.178510308265686 (-0.07036292552947998) + | > avg_loss_feat: 5.874066352844238 (-1.5912055969238281) + | > avg_loss_mel: 16.98788070678711 (-2.167078971862793) + | > avg_loss_duration: 1.6313287019729614 (-0.002968311309814453) + | > avg_loss_1: 28.945098876953125 (-4.175529479980469) + + + > EPOCH: 87/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:26:17)  + + --> TIME: 2025-04-19 00:26:23 -- STEP: 14/142 -- GLOBAL_STEP: 391650 + | > loss_disc: 2.363985061645508 (2.3902029991149902) + | > loss_disc_real_0: 0.09865756332874298 (0.11611726081797055) + | > loss_disc_real_1: 0.201488196849823 (0.20477629133633204) + | > loss_disc_real_2: 0.20424775779247284 (0.21623779301132476) + | > loss_disc_real_3: 0.22299440205097198 (0.2197523351226534) + | > loss_disc_real_4: 0.2050905078649521 (0.22218656220606395) + | > loss_disc_real_5: 0.21134497225284576 (0.22438068900789535) + | > loss_0: 2.363985061645508 (2.3902029991149902) + | > grad_norm_0: tensor(11.2540, device='cuda:0') (tensor(9.7623, device='cuda:0')) + | > loss_gen: 2.499264717102051 (2.4142199754714966) + | > loss_kl: 1.893768310546875 (1.8616679395948137) + | > loss_feat: 7.403454303741455 (7.271915095193045) + | > loss_mel: 18.49326515197754 (18.17776380266462) + | > loss_duration: 1.4493122100830078 (1.4617483104978288) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.739065170288086 (31.187314987182617) + | > grad_norm_1: tensor(308.9353, device='cuda:0') (tensor(223.2882, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.3335 (0.32806267057146343) + | > loader_time: 0.005 (0.0047539472579956055) + + + --> TIME: 2025-04-19 00:26:32 -- STEP: 39/142 -- GLOBAL_STEP: 391675 + | > loss_disc: 2.5488193035125732 (2.455622746394231) + | > loss_disc_real_0: 0.12272772938013077 (0.12041605703341655) + | > loss_disc_real_1: 0.21266217529773712 (0.20959714131477553) + | > loss_disc_real_2: 0.24239656329154968 (0.224673446172323) + | > loss_disc_real_3: 0.2693743109703064 (0.2268832620137777) + | > loss_disc_real_4: 0.23441597819328308 (0.2296957499705828) + | > loss_disc_real_5: 0.2693653404712677 (0.2337955435117086) + | > loss_0: 2.5488193035125732 (2.455622746394231) + | > grad_norm_0: tensor(5.3842, device='cuda:0') (tensor(9.3120, device='cuda:0')) + | > loss_gen: 2.321164608001709 (2.3864312722132754) + | > loss_kl: 1.9293969869613647 (1.8602312467037103) + | > loss_feat: 6.827113151550293 (7.232623772743421) + | > loss_mel: 18.780309677124023 (18.13556020687788) + | > loss_duration: 1.5013737678527832 (1.4719113783958633) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.359357833862305 (31.086757708818485) + | > grad_norm_1: tensor(153.8453, device='cuda:0') (tensor(228.4523, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.359 (0.34116652073004305) + | > loader_time: 0.0056 (0.005234143672845303) + + + --> TIME: 2025-04-19 00:26:41 -- STEP: 64/142 -- GLOBAL_STEP: 391700 + | > loss_disc: 2.48659610748291 (2.455664973706007) + | > loss_disc_real_0: 0.17414169013500214 (0.12525598192587495) + | > loss_disc_real_1: 0.18730053305625916 (0.20681183133274317) + | > loss_disc_real_2: 0.2519577741622925 (0.2263016861397773) + | > loss_disc_real_3: 0.2290111631155014 (0.22636489057913423) + | > loss_disc_real_4: 0.22342322766780853 (0.22788840276189148) + | > loss_disc_real_5: 0.21835529804229736 (0.2327877739444375) + | > loss_0: 2.48659610748291 (2.455664973706007) + | > grad_norm_0: tensor(8.4223, device='cuda:0') (tensor(9.5622, device='cuda:0')) + | > loss_gen: 2.425863742828369 (2.374897424131632) + | > loss_kl: 1.8622822761535645 (1.890934919938445) + | > loss_feat: 6.710115432739258 (7.135272093117237) + | > loss_mel: 17.357677459716797 (18.08951959013939) + | > loss_duration: 1.52536940574646 (1.4838897362351418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.88130760192871 (30.974513679742813) + | > grad_norm_1: tensor(129.9534, device='cuda:0') (tensor(230.8566, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.3866 (0.35395918413996696) + | > loader_time: 0.0063 (0.005650866776704789) + + + --> TIME: 2025-04-19 00:26:51 -- STEP: 89/142 -- GLOBAL_STEP: 391725 + | > loss_disc: 2.4804365634918213 (2.4642726785681206) + | > loss_disc_real_0: 0.11125009506940842 (0.12608085212747708) + | > loss_disc_real_1: 0.19256596267223358 (0.2075613114271271) + | > loss_disc_real_2: 0.23786526918411255 (0.22722874046041724) + | > loss_disc_real_3: 0.22919264435768127 (0.2293685060203745) + | > loss_disc_real_4: 0.25116482377052307 (0.22935813297046703) + | > loss_disc_real_5: 0.2598271667957306 (0.2323794775129704) + | > loss_0: 2.4804365634918213 (2.4642726785681206) + | > grad_norm_0: tensor(8.3382, device='cuda:0') (tensor(10.8426, device='cuda:0')) + | > loss_gen: 2.3164992332458496 (2.362640980924114) + | > loss_kl: 1.9883285760879517 (1.9077794659003784) + | > loss_feat: 6.573238372802734 (7.0613819240184315) + | > loss_mel: 16.664470672607422 (18.042338553439368) + | > loss_duration: 1.4903912544250488 (1.4951973122157405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.032928466796875 (30.869338282038658) + | > grad_norm_1: tensor(283.5468, device='cuda:0') (tensor(252.3657, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.4124 (0.36650662743643436) + | > loader_time: 0.0069 (0.005903169010462387) + + + --> TIME: 2025-04-19 00:27:02 -- STEP: 114/142 -- GLOBAL_STEP: 391750 + | > loss_disc: 2.4254202842712402 (2.465244048520137) + | > loss_disc_real_0: 0.11801517009735107 (0.1271897607989479) + | > loss_disc_real_1: 0.21037089824676514 (0.20774922180071212) + | > loss_disc_real_2: 0.2034505009651184 (0.22684371026984432) + | > loss_disc_real_3: 0.21553151309490204 (0.22882384427806787) + | > loss_disc_real_4: 0.2075631320476532 (0.2293089399473709) + | > loss_disc_real_5: 0.2208847999572754 (0.23118256934379278) + | > loss_0: 2.4254202842712402 (2.465244048520137) + | > grad_norm_0: tensor(5.0281, device='cuda:0') (tensor(10.8110, device='cuda:0')) + | > loss_gen: 2.252786636352539 (2.3575304893025177) + | > loss_kl: 1.964383602142334 (1.9138812025388081) + | > loss_feat: 6.731106758117676 (7.035164226565445) + | > loss_mel: 17.834287643432617 (17.9694040449042) + | > loss_duration: 1.5223004817962646 (1.5018656536152486) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.30486488342285 (30.77784558346397) + | > grad_norm_1: tensor(56.9809, device='cuda:0') (tensor(249.8015, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.4287 (0.37886747561003037) + | > loader_time: 0.0077 (0.006256080510323509) + + + --> TIME: 2025-04-19 00:27:14 -- STEP: 139/142 -- GLOBAL_STEP: 391775 + | > loss_disc: 2.5449373722076416 (2.474431555905787) + | > loss_disc_real_0: 0.09788721054792404 (0.12867762277023398) + | > loss_disc_real_1: 0.19236984848976135 (0.20801025039429288) + | > loss_disc_real_2: 0.23374925553798676 (0.22756620802038866) + | > loss_disc_real_3: 0.24346618354320526 (0.229839487791919) + | > loss_disc_real_4: 0.22930507361888885 (0.23051583595413097) + | > loss_disc_real_5: 0.25778791308403015 (0.2318381684289562) + | > loss_0: 2.5449373722076416 (2.474431555905787) + | > grad_norm_0: tensor(10.1069, device='cuda:0') (tensor(11.1973, device='cuda:0')) + | > loss_gen: 2.2834224700927734 (2.346631113573801) + | > loss_kl: 1.9297267198562622 (1.918954861249855) + | > loss_feat: 5.935277462005615 (6.936955781291715) + | > loss_mel: 17.739587783813477 (17.88303410235068) + | > loss_duration: 1.533574104309082 (1.5078507233009062) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.421588897705078 (30.593426505438714) + | > grad_norm_1: tensor(61.8615, device='cuda:0') (tensor(251.8214, device='cuda:0')) + | > current_lr_0: 0.00019783664932916355 + | > current_lr_1: 0.00019783664932916355 + | > step_time: 0.4577 (0.3913690289147467) + | > loader_time: 0.0079 (0.006509784314272215) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.796909809112549 (2.796909809112549) + | > loss_disc_real_0: 0.2572607100009918 (0.2572607100009918) + | > loss_disc_real_1: 0.23991462588310242 (0.23991462588310242) + | > loss_disc_real_2: 0.2440517246723175 (0.2440517246723175) + | > loss_disc_real_3: 0.23403945565223694 (0.23403945565223694) + | > loss_disc_real_4: 0.2621581554412842 (0.2621581554412842) + | > loss_disc_real_5: 0.27459949254989624 (0.27459949254989624) + | > loss_0: 2.796909809112549 (2.796909809112549) + | > loss_gen: 2.258497953414917 (2.258497953414917) + | > loss_kl: 2.4241018295288086 (2.4241018295288086) + | > loss_feat: 6.294677257537842 (6.294677257537842) + | > loss_mel: 18.55183982849121 (18.55183982849121) + | > loss_duration: 1.643568754196167 (1.643568754196167) + | > loss_1: 31.172683715820312 (31.172683715820312) + + --> STEP: 1 + | > loss_disc: 2.4927263259887695 (2.4927263259887695) + | > loss_disc_real_0: 0.2725558876991272 (0.2725558876991272) + | > loss_disc_real_1: 0.20562222599983215 (0.20562222599983215) + | > loss_disc_real_2: 0.2136298567056656 (0.2136298567056656) + | > loss_disc_real_3: 0.2268984317779541 (0.2268984317779541) + | > loss_disc_real_4: 0.26736515760421753 (0.26736515760421753) + | > loss_disc_real_5: 0.21140386164188385 (0.21140386164188385) + | > loss_0: 2.4927263259887695 (2.4927263259887695) + | > loss_gen: 2.499858856201172 (2.499858856201172) + | > loss_kl: 2.4355621337890625 (2.4355621337890625) + | > loss_feat: 5.731895923614502 (5.731895923614502) + | > loss_mel: 17.12834930419922 (17.12834930419922) + | > loss_duration: 1.654984474182129 (1.654984474182129) + | > loss_1: 29.45064926147461 (29.45064926147461) + + --> STEP: 2 + | > loss_disc: 2.519568920135498 (2.506147623062134) + | > loss_disc_real_0: 0.2542213201522827 (0.26338860392570496) + | > loss_disc_real_1: 0.176632359623909 (0.19112729281187057) + | > loss_disc_real_2: 0.22478824853897095 (0.21920905262231827) + | > loss_disc_real_3: 0.20892959833145142 (0.21791401505470276) + | > loss_disc_real_4: 0.2358681559562683 (0.2516166567802429) + | > loss_disc_real_5: 0.20057743787765503 (0.20599064975976944) + | > loss_0: 2.519568920135498 (2.506147623062134) + | > loss_gen: 2.564957618713379 (2.5324082374572754) + | > loss_kl: 2.0309066772460938 (2.233234405517578) + | > loss_feat: 7.781264305114746 (6.756580114364624) + | > loss_mel: 19.347780227661133 (18.238064765930176) + | > loss_duration: 1.5949046611785889 (1.6249445676803589) + | > loss_1: 33.3198127746582 (31.385231018066406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003801703453063965 (+0.0005346536636352539) + | > avg_loss_disc: 2.506147623062134 (-0.03908097743988037) + | > avg_loss_disc_real_0: 0.26338860392570496 (+0.05707690119743347) + | > avg_loss_disc_real_1: 0.19112729281187057 (-0.09496981650590897) + | > avg_loss_disc_real_2: 0.21920905262231827 (+0.001507006585597992) + | > avg_loss_disc_real_3: 0.21791401505470276 (+0.011914163827896118) + | > avg_loss_disc_real_4: 0.2516166567802429 (+0.05070413649082184) + | > avg_loss_disc_real_5: 0.20599064975976944 (-0.022101595997810364) + | > avg_loss_0: 2.506147623062134 (-0.03908097743988037) + | > avg_loss_gen: 2.5324082374572754 (+0.2590949535369873) + | > avg_loss_kl: 2.233234405517578 (+0.05472409725189209) + | > avg_loss_feat: 6.756580114364624 (+0.8825137615203857) + | > avg_loss_mel: 18.238064765930176 (+1.2501840591430664) + | > avg_loss_duration: 1.6249445676803589 (-0.006384134292602539) + | > avg_loss_1: 31.385231018066406 (+2.4401321411132812) + + + > EPOCH: 88/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:27:17)  + + --> TIME: 2025-04-19 00:27:25 -- STEP: 22/142 -- GLOBAL_STEP: 391800 + | > loss_disc: 2.325711250305176 (2.3941858898509634) + | > loss_disc_real_0: 0.11186134070158005 (0.1134176982397383) + | > loss_disc_real_1: 0.23411700129508972 (0.20363922078501095) + | > loss_disc_real_2: 0.23760151863098145 (0.22025803950699893) + | > loss_disc_real_3: 0.2510870099067688 (0.2251922677863728) + | > loss_disc_real_4: 0.22052815556526184 (0.2254918867891485) + | > loss_disc_real_5: 0.25795942544937134 (0.22957941280169922) + | > loss_0: 2.325711250305176 (2.3941858898509634) + | > grad_norm_0: tensor(4.9187, device='cuda:0') (tensor(10.2237, device='cuda:0')) + | > loss_gen: 2.428703784942627 (2.37540066242218) + | > loss_kl: 1.9453474283218384 (1.8818331577561118) + | > loss_feat: 7.034180641174316 (7.205583810806274) + | > loss_mel: 18.516550064086914 (17.87165815179998) + | > loss_duration: 1.4260389804840088 (1.4609282396056436) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.350820541381836 (30.795404347506437) + | > grad_norm_1: tensor(84.3991, device='cuda:0') (tensor(259.6611, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.3439 (0.33202576637268066) + | > loader_time: 0.0055 (0.0049761750481345434) + + + --> TIME: 2025-04-19 00:27:34 -- STEP: 47/142 -- GLOBAL_STEP: 391825 + | > loss_disc: 2.486706256866455 (2.4202466213956786) + | > loss_disc_real_0: 0.11187707632780075 (0.11689989950428618) + | > loss_disc_real_1: 0.21523135900497437 (0.2061858595685756) + | > loss_disc_real_2: 0.2469499260187149 (0.22299706840768774) + | > loss_disc_real_3: 0.2596568465232849 (0.2231037695991232) + | > loss_disc_real_4: 0.2522646486759186 (0.22635186637969726) + | > loss_disc_real_5: 0.23873740434646606 (0.22702086098650667) + | > loss_0: 2.486706256866455 (2.4202466213956786) + | > grad_norm_0: tensor(7.4728, device='cuda:0') (tensor(10.9604, device='cuda:0')) + | > loss_gen: 2.552896738052368 (2.3780613605012286) + | > loss_kl: 1.8061513900756836 (1.8799060583114624) + | > loss_feat: 7.230599880218506 (7.133367102196876) + | > loss_mel: 17.34638214111328 (17.776589657397974) + | > loss_duration: 1.4484572410583496 (1.4720921034508563) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.38448715209961 (30.640016596368017) + | > grad_norm_1: tensor(386.6561, device='cuda:0') (tensor(280.7026, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.3628 (0.34424980650556847) + | > loader_time: 0.0061 (0.005360121422625603) + + + --> TIME: 2025-04-19 00:27:44 -- STEP: 72/142 -- GLOBAL_STEP: 391850 + | > loss_disc: 2.53240966796875 (2.445674690935346) + | > loss_disc_real_0: 0.13076651096343994 (0.1200741808861494) + | > loss_disc_real_1: 0.2136414647102356 (0.20848551123506492) + | > loss_disc_real_2: 0.214987114071846 (0.2244630911284023) + | > loss_disc_real_3: 0.2230614423751831 (0.22596139916115338) + | > loss_disc_real_4: 0.22609657049179077 (0.22789508601029715) + | > loss_disc_real_5: 0.2534409165382385 (0.23041847058468395) + | > loss_0: 2.53240966796875 (2.445674690935346) + | > grad_norm_0: tensor(8.1443, device='cuda:0') (tensor(11.0903, device='cuda:0')) + | > loss_gen: 2.237699031829834 (2.352610306607352) + | > loss_kl: 1.9541559219360352 (1.891718899210294) + | > loss_feat: 6.911826133728027 (7.032551778687371) + | > loss_mel: 17.6108455657959 (17.711461345354714) + | > loss_duration: 1.4678473472595215 (1.4837355001105206) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.182374954223633 (30.47207808494568) + | > grad_norm_1: tensor(122.0336, device='cuda:0') (tensor(266.9372, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.399 (0.35728489028082955) + | > loader_time: 0.0069 (0.005724612209531996) + + + --> TIME: 2025-04-19 00:27:54 -- STEP: 97/142 -- GLOBAL_STEP: 391875 + | > loss_disc: 2.5330424308776855 (2.4608350414590747) + | > loss_disc_real_0: 0.22281260788440704 (0.12474285949444033) + | > loss_disc_real_1: 0.1997506022453308 (0.20803568827122756) + | > loss_disc_real_2: 0.2349209189414978 (0.22497764490928845) + | > loss_disc_real_3: 0.24732619524002075 (0.227743878653369) + | > loss_disc_real_4: 0.24737092852592468 (0.22913109196215561) + | > loss_disc_real_5: 0.26959410309791565 (0.23100990118439665) + | > loss_0: 2.5330424308776855 (2.4608350414590747) + | > grad_norm_0: tensor(7.0978, device='cuda:0') (tensor(10.2290, device='cuda:0')) + | > loss_gen: 2.378807783126831 (2.335806168231768) + | > loss_kl: 2.0116348266601562 (1.9102142410180003) + | > loss_feat: 7.306944847106934 (6.932359572538395) + | > loss_mel: 17.83536148071289 (17.672396885980035) + | > loss_duration: 1.5120158195495605 (1.495869324379361) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.04476547241211 (30.346646377720784) + | > grad_norm_1: tensor(108.4094, device='cuda:0') (tensor(243.2922, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.4129 (0.3691514300316878) + | > loader_time: 0.0077 (0.006023652774771464) + + + --> TIME: 2025-04-19 00:28:05 -- STEP: 122/142 -- GLOBAL_STEP: 391900 + | > loss_disc: 2.5835068225860596 (2.470523130698282) + | > loss_disc_real_0: 0.11553360521793365 (0.12610898435604376) + | > loss_disc_real_1: 0.21369262039661407 (0.20831283475043344) + | > loss_disc_real_2: 0.23216424882411957 (0.22559467031330357) + | > loss_disc_real_3: 0.21558625996112823 (0.2286550159581372) + | > loss_disc_real_4: 0.21964100003242493 (0.23010392254981837) + | > loss_disc_real_5: 0.2486184984445572 (0.23199339479696554) + | > loss_0: 2.5835068225860596 (2.470523130698282) + | > grad_norm_0: tensor(8.0989, device='cuda:0') (tensor(10.3169, device='cuda:0')) + | > loss_gen: 2.279480218887329 (2.333721704170353) + | > loss_kl: 1.8736047744750977 (1.9166811978230711) + | > loss_feat: 6.566617965698242 (6.913050886060371) + | > loss_mel: 16.713756561279297 (17.683003644474216) + | > loss_duration: 1.5046656131744385 (1.5029690871473222) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.938125610351562 (30.34942669164939) + | > grad_norm_1: tensor(329.7155, device='cuda:0') (tensor(237.9222, device='cuda:0')) + | > current_lr_0: 0.0001978119197479974 + | > current_lr_1: 0.0001978119197479974 + | > step_time: 0.456 (0.3816133170831398) + | > loader_time: 0.0079 (0.00636484974720439) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.49269700050354 (2.49269700050354) + | > loss_disc_real_0: 0.14751288294792175 (0.14751288294792175) + | > loss_disc_real_1: 0.2663968801498413 (0.2663968801498413) + | > loss_disc_real_2: 0.2949211597442627 (0.2949211597442627) + | > loss_disc_real_3: 0.26096877455711365 (0.26096877455711365) + | > loss_disc_real_4: 0.25492146611213684 (0.25492146611213684) + | > loss_disc_real_5: 0.2465449422597885 (0.2465449422597885) + | > loss_0: 2.49269700050354 (2.49269700050354) + | > loss_gen: 2.6448962688446045 (2.6448962688446045) + | > loss_kl: 2.335679054260254 (2.335679054260254) + | > loss_feat: 7.8975934982299805 (7.8975934982299805) + | > loss_mel: 19.297943115234375 (19.297943115234375) + | > loss_duration: 1.6415002346038818 (1.6415002346038818) + | > loss_1: 33.81761169433594 (33.81761169433594) + + --> STEP: 1 + | > loss_disc: 2.514125347137451 (2.514125347137451) + | > loss_disc_real_0: 0.1430756002664566 (0.1430756002664566) + | > loss_disc_real_1: 0.2536700963973999 (0.2536700963973999) + | > loss_disc_real_2: 0.2672956883907318 (0.2672956883907318) + | > loss_disc_real_3: 0.2514994442462921 (0.2514994442462921) + | > loss_disc_real_4: 0.25828322768211365 (0.25828322768211365) + | > loss_disc_real_5: 0.2302210032939911 (0.2302210032939911) + | > loss_0: 2.514125347137451 (2.514125347137451) + | > loss_gen: 2.313889265060425 (2.313889265060425) + | > loss_kl: 2.4333817958831787 (2.4333817958831787) + | > loss_feat: 5.776101589202881 (5.776101589202881) + | > loss_mel: 17.151508331298828 (17.151508331298828) + | > loss_duration: 1.6670875549316406 (1.6670875549316406) + | > loss_1: 29.341968536376953 (29.341968536376953) + + --> STEP: 2 + | > loss_disc: 2.5160021781921387 (2.515063762664795) + | > loss_disc_real_0: 0.1452169269323349 (0.14414626359939575) + | > loss_disc_real_1: 0.2547033131122589 (0.2541867047548294) + | > loss_disc_real_2: 0.24461916089057922 (0.2559574246406555) + | > loss_disc_real_3: 0.20194044709205627 (0.2267199456691742) + | > loss_disc_real_4: 0.21785704791545868 (0.23807013779878616) + | > loss_disc_real_5: 0.24685858190059662 (0.23853979259729385) + | > loss_0: 2.5160021781921387 (2.515063762664795) + | > loss_gen: 2.1392343044281006 (2.2265617847442627) + | > loss_kl: 2.0908780097961426 (2.2621299028396606) + | > loss_feat: 5.060573577880859 (5.41833758354187) + | > loss_mel: 13.974666595458984 (15.563087463378906) + | > loss_duration: 1.6057589054107666 (1.6364232301712036) + | > loss_1: 24.871110916137695 (27.106539726257324) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003398895263671875 (-0.00040280818939208984) + | > avg_loss_disc: 2.515063762664795 (+0.008916139602661133) + | > avg_loss_disc_real_0: 0.14414626359939575 (-0.1192423403263092) + | > avg_loss_disc_real_1: 0.2541867047548294 (+0.06305941194295883) + | > avg_loss_disc_real_2: 0.2559574246406555 (+0.03674837201833725) + | > avg_loss_disc_real_3: 0.2267199456691742 (+0.008805930614471436) + | > avg_loss_disc_real_4: 0.23807013779878616 (-0.013546518981456757) + | > avg_loss_disc_real_5: 0.23853979259729385 (+0.032549142837524414) + | > avg_loss_0: 2.515063762664795 (+0.008916139602661133) + | > avg_loss_gen: 2.2265617847442627 (-0.3058464527130127) + | > avg_loss_kl: 2.2621299028396606 (+0.02889549732208252) + | > avg_loss_feat: 5.41833758354187 (-1.338242530822754) + | > avg_loss_mel: 15.563087463378906 (-2.6749773025512695) + | > avg_loss_duration: 1.6364232301712036 (+0.011478662490844727) + | > avg_loss_1: 27.106539726257324 (-4.278691291809082) + + + > EPOCH: 89/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:28:16)  + + --> TIME: 2025-04-19 00:28:18 -- STEP: 5/142 -- GLOBAL_STEP: 391925 + | > loss_disc: 2.4779436588287354 (2.4038226127624513) + | > loss_disc_real_0: 0.13611029088497162 (0.1291466012597084) + | > loss_disc_real_1: 0.22683565318584442 (0.19886821806430816) + | > loss_disc_real_2: 0.21926306188106537 (0.22080285251140594) + | > loss_disc_real_3: 0.23682278394699097 (0.218727245926857) + | > loss_disc_real_4: 0.23787161707878113 (0.23046714067459106) + | > loss_disc_real_5: 0.24679265916347504 (0.2326493054628372) + | > loss_0: 2.4779436588287354 (2.4038226127624513) + | > grad_norm_0: tensor(13.3802, device='cuda:0') (tensor(10.2837, device='cuda:0')) + | > loss_gen: 2.429786443710327 (2.3358238697052003) + | > loss_kl: 1.7462705373764038 (1.86340012550354) + | > loss_feat: 6.993096351623535 (6.96983699798584) + | > loss_mel: 17.571224212646484 (17.97703742980957) + | > loss_duration: 1.4697383642196655 (1.4696332693099976) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.210115432739258 (30.61573181152344) + | > grad_norm_1: tensor(325.5042, device='cuda:0') (tensor(232.1100, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.3249 (0.3226041793823242) + | > loader_time: 0.0046 (0.004639196395874024) + + + --> TIME: 2025-04-19 00:28:27 -- STEP: 30/142 -- GLOBAL_STEP: 391950 + | > loss_disc: 2.3558340072631836 (2.4420029560724896) + | > loss_disc_real_0: 0.10034482926130295 (0.12257494976123177) + | > loss_disc_real_1: 0.2062993198633194 (0.20646827965974807) + | > loss_disc_real_2: 0.21790753304958344 (0.22648709466060002) + | > loss_disc_real_3: 0.23099301755428314 (0.2278933435678482) + | > loss_disc_real_4: 0.19421043992042542 (0.2261326476931572) + | > loss_disc_real_5: 0.25854530930519104 (0.2315021057923635) + | > loss_0: 2.3558340072631836 (2.4420029560724896) + | > grad_norm_0: tensor(15.0244, device='cuda:0') (tensor(11.1337, device='cuda:0')) + | > loss_gen: 2.5140488147735596 (2.3787089188893638) + | > loss_kl: 1.8245131969451904 (1.8771014134089152) + | > loss_feat: 7.718523025512695 (7.208015521367391) + | > loss_mel: 18.132904052734375 (18.11863479614258) + | > loss_duration: 1.4804232120513916 (1.4691746989885965) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.670413970947266 (31.051635424296062) + | > grad_norm_1: tensor(400.7302, device='cuda:0') (tensor(267.9043, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.3495 (0.33921267986297604) + | > loader_time: 0.0056 (0.0052239418029785155) + + + --> TIME: 2025-04-19 00:28:36 -- STEP: 55/142 -- GLOBAL_STEP: 391975 + | > loss_disc: 2.382636547088623 (2.4515002814206204) + | > loss_disc_real_0: 0.11279342323541641 (0.12624333798885343) + | > loss_disc_real_1: 0.19246236979961395 (0.20505270930853756) + | > loss_disc_real_2: 0.2300102859735489 (0.22595136301083998) + | > loss_disc_real_3: 0.22727178037166595 (0.2267822501334277) + | > loss_disc_real_4: 0.20369477570056915 (0.22835652177984064) + | > loss_disc_real_5: 0.22554950416088104 (0.22889432338151064) + | > loss_0: 2.382636547088623 (2.4515002814206204) + | > grad_norm_0: tensor(15.1480, device='cuda:0') (tensor(11.5604, device='cuda:0')) + | > loss_gen: 2.3507044315338135 (2.3703076752749355) + | > loss_kl: 1.9489418268203735 (1.8851966879584572) + | > loss_feat: 7.2331862449646 (7.217937824942849) + | > loss_mel: 17.941556930541992 (18.039444247159086) + | > loss_duration: 1.535449504852295 (1.4759577014229515) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.009838104248047 (30.988844229958275) + | > grad_norm_1: tensor(323.7055, device='cuda:0') (tensor(270.9037, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.3733 (0.34942700212652034) + | > loader_time: 0.0061 (0.005550276149402964) + + + --> TIME: 2025-04-19 00:28:46 -- STEP: 80/142 -- GLOBAL_STEP: 392000 + | > loss_disc: 2.4073970317840576 (2.461593022942543) + | > loss_disc_real_0: 0.09122860431671143 (0.12733144052326678) + | > loss_disc_real_1: 0.2180127501487732 (0.20706182774156331) + | > loss_disc_real_2: 0.2248254418373108 (0.22765230294317007) + | > loss_disc_real_3: 0.2121315747499466 (0.2278263485059142) + | > loss_disc_real_4: 0.21147984266281128 (0.2277984105050564) + | > loss_disc_real_5: 0.19228187203407288 (0.23103646226227284) + | > loss_0: 2.4073970317840576 (2.461593022942543) + | > grad_norm_0: tensor(4.6600, device='cuda:0') (tensor(11.3637, device='cuda:0')) + | > loss_gen: 2.328374147415161 (2.359562283754349) + | > loss_kl: 1.723087191581726 (1.9003730580210685) + | > loss_feat: 7.399499416351318 (7.116882091760635) + | > loss_mel: 19.47879981994629 (17.964763450622556) + | > loss_duration: 1.533820390701294 (1.4872403889894485) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.46358108520508 (30.82882125377655) + | > grad_norm_1: tensor(76.8831, device='cuda:0') (tensor(262.9301, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.4072 (0.3617800861597062) + | > loader_time: 0.0069 (0.0058185309171676644) + + + --> TIME: 2025-04-19 00:28:56 -- STEP: 105/142 -- GLOBAL_STEP: 392025 + | > loss_disc: 2.4965641498565674 (2.466340970993041) + | > loss_disc_real_0: 0.1130591556429863 (0.127651641198567) + | > loss_disc_real_1: 0.20009200274944305 (0.2075002316917692) + | > loss_disc_real_2: 0.2278747707605362 (0.22729957742350443) + | > loss_disc_real_3: 0.21446655690670013 (0.22851172770772663) + | > loss_disc_real_4: 0.2270301878452301 (0.2291883728333882) + | > loss_disc_real_5: 0.22956909239292145 (0.23207535715330216) + | > loss_0: 2.4965641498565674 (2.466340970993041) + | > grad_norm_0: tensor(9.0083, device='cuda:0') (tensor(11.6496, device='cuda:0')) + | > loss_gen: 2.269334316253662 (2.3474364734831314) + | > loss_kl: 2.1381027698516846 (1.9081619625999815) + | > loss_feat: 6.981562614440918 (7.048428635370164) + | > loss_mel: 17.28221893310547 (17.916674495878667) + | > loss_duration: 1.523043155670166 (1.4980606158574423) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.19426155090332 (30.718762188866023) + | > grad_norm_1: tensor(208.0778, device='cuda:0') (tensor(265.9198, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.4177 (0.37184022040594206) + | > loader_time: 0.007 (0.006086705979846773) + + + --> TIME: 2025-04-19 00:29:07 -- STEP: 130/142 -- GLOBAL_STEP: 392050 + | > loss_disc: 2.450963258743286 (2.467188147398141) + | > loss_disc_real_0: 0.10053791105747223 (0.12861121004590625) + | > loss_disc_real_1: 0.20160184800624847 (0.20751916961028025) + | > loss_disc_real_2: 0.1866339147090912 (0.22672594338655472) + | > loss_disc_real_3: 0.1901058852672577 (0.22915367231919215) + | > loss_disc_real_4: 0.16931867599487305 (0.22900609637682254) + | > loss_disc_real_5: 0.1863422691822052 (0.23005036952403876) + | > loss_0: 2.450963258743286 (2.467188147398141) + | > grad_norm_0: tensor(19.2189, device='cuda:0') (tensor(11.9765, device='cuda:0')) + | > loss_gen: 2.3383169174194336 (2.3408175230026256) + | > loss_kl: 1.977124571800232 (1.9085298327299265) + | > loss_feat: 7.274772644042969 (6.983846448018) + | > loss_mel: 17.883106231689453 (17.83948753063496) + | > loss_duration: 1.50555419921875 (1.504024847654196) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.97887420654297 (30.576706196711612) + | > grad_norm_1: tensor(475.5005, device='cuda:0') (tensor(266.8246, device='cuda:0')) + | > current_lr_0: 0.0001977871932580289 + | > current_lr_1: 0.0001977871932580289 + | > step_time: 0.4541 (0.384194196187533) + | > loader_time: 0.0076 (0.00636903872856727) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.503052234649658 (2.503052234649658) + | > loss_disc_real_0: 0.17851221561431885 (0.17851221561431885) + | > loss_disc_real_1: 0.20069198310375214 (0.20069198310375214) + | > loss_disc_real_2: 0.2934315800666809 (0.2934315800666809) + | > loss_disc_real_3: 0.24404168128967285 (0.24404168128967285) + | > loss_disc_real_4: 0.23475369811058044 (0.23475369811058044) + | > loss_disc_real_5: 0.21250039339065552 (0.21250039339065552) + | > loss_0: 2.503052234649658 (2.503052234649658) + | > loss_gen: 2.5024359226226807 (2.5024359226226807) + | > loss_kl: 2.118647575378418 (2.118647575378418) + | > loss_feat: 8.004989624023438 (8.004989624023438) + | > loss_mel: 19.79142951965332 (19.79142951965332) + | > loss_duration: 1.6372755765914917 (1.6372755765914917) + | > loss_1: 34.054779052734375 (34.054779052734375) + + --> STEP: 1 + | > loss_disc: 2.4858791828155518 (2.4858791828155518) + | > loss_disc_real_0: 0.09801235049962997 (0.09801235049962997) + | > loss_disc_real_1: 0.20789463818073273 (0.20789463818073273) + | > loss_disc_real_2: 0.2527901530265808 (0.2527901530265808) + | > loss_disc_real_3: 0.24372252821922302 (0.24372252821922302) + | > loss_disc_real_4: 0.22741718590259552 (0.22741718590259552) + | > loss_disc_real_5: 0.23658408224582672 (0.23658408224582672) + | > loss_0: 2.4858791828155518 (2.4858791828155518) + | > loss_gen: 2.232022762298584 (2.232022762298584) + | > loss_kl: 2.57279372215271 (2.57279372215271) + | > loss_feat: 7.063761234283447 (7.063761234283447) + | > loss_mel: 18.2100887298584 (18.2100887298584) + | > loss_duration: 1.648512363433838 (1.648512363433838) + | > loss_1: 31.7271785736084 (31.7271785736084) + + --> STEP: 2 + | > loss_disc: 2.5097086429595947 (2.4977939128875732) + | > loss_disc_real_0: 0.14350509643554688 (0.12075872346758842) + | > loss_disc_real_1: 0.2473781555891037 (0.2276363968849182) + | > loss_disc_real_2: 0.23083361983299255 (0.24181188642978668) + | > loss_disc_real_3: 0.22635279595851898 (0.235037662088871) + | > loss_disc_real_4: 0.22339914739131927 (0.2254081666469574) + | > loss_disc_real_5: 0.21670101583003998 (0.22664254903793335) + | > loss_0: 2.5097086429595947 (2.4977939128875732) + | > loss_gen: 2.261953353881836 (2.24698805809021) + | > loss_kl: 2.074169874191284 (2.323481798171997) + | > loss_feat: 6.672205924987793 (6.86798357963562) + | > loss_mel: 17.61785316467285 (17.913970947265625) + | > loss_duration: 1.6103744506835938 (1.6294434070587158) + | > loss_1: 30.236557006835938 (30.981867790222168) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00336456298828125 (-3.4332275390625e-05) + | > avg_loss_disc: 2.4977939128875732 (-0.01726984977722168) + | > avg_loss_disc_real_0: 0.12075872346758842 (-0.023387540131807327) + | > avg_loss_disc_real_1: 0.2276363968849182 (-0.026550307869911194) + | > avg_loss_disc_real_2: 0.24181188642978668 (-0.014145538210868835) + | > avg_loss_disc_real_3: 0.235037662088871 (+0.008317716419696808) + | > avg_loss_disc_real_4: 0.2254081666469574 (-0.012661971151828766) + | > avg_loss_disc_real_5: 0.22664254903793335 (-0.011897243559360504) + | > avg_loss_0: 2.4977939128875732 (-0.01726984977722168) + | > avg_loss_gen: 2.24698805809021 (+0.020426273345947266) + | > avg_loss_kl: 2.323481798171997 (+0.061351895332336426) + | > avg_loss_feat: 6.86798357963562 (+1.44964599609375) + | > avg_loss_mel: 17.913970947265625 (+2.3508834838867188) + | > avg_loss_duration: 1.6294434070587158 (-0.006979823112487793) + | > avg_loss_1: 30.981867790222168 (+3.8753280639648438) + + + > EPOCH: 90/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:29:14)  + + --> TIME: 2025-04-19 00:29:19 -- STEP: 13/142 -- GLOBAL_STEP: 392075 + | > loss_disc: 2.4987001419067383 (2.4175866567171536) + | > loss_disc_real_0: 0.11748220771551132 (0.1237131145138007) + | > loss_disc_real_1: 0.19200661778450012 (0.19832966190118057) + | > loss_disc_real_2: 0.22514976561069489 (0.22626463954265302) + | > loss_disc_real_3: 0.22226737439632416 (0.22385676090533918) + | > loss_disc_real_4: 0.21948735415935516 (0.22609072350538695) + | > loss_disc_real_5: 0.20599839091300964 (0.23690949838895065) + | > loss_0: 2.4987001419067383 (2.4175866567171536) + | > grad_norm_0: tensor(6.9942, device='cuda:0') (tensor(11.6537, device='cuda:0')) + | > loss_gen: 2.2227277755737305 (2.3945290675530067) + | > loss_kl: 1.9107030630111694 (1.8310381540885339) + | > loss_feat: 7.03600549697876 (7.172911167144775) + | > loss_mel: 18.27921485900879 (17.99404687147874) + | > loss_duration: 1.4744170904159546 (1.4642261266708374) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.923067092895508 (30.856751075157753) + | > grad_norm_1: tensor(147.5090, device='cuda:0') (tensor(271.7661, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.3303 (0.3276075766636775) + | > loader_time: 0.0052 (0.004748032643244817) + + + --> TIME: 2025-04-19 00:29:28 -- STEP: 38/142 -- GLOBAL_STEP: 392100 + | > loss_disc: 2.3769335746765137 (2.462138897494266) + | > loss_disc_real_0: 0.08581588417291641 (0.12538961164261164) + | > loss_disc_real_1: 0.1839766651391983 (0.20622918911670385) + | > loss_disc_real_2: 0.18767817318439484 (0.22783870681336052) + | > loss_disc_real_3: 0.21894632279872894 (0.22934401623512568) + | > loss_disc_real_4: 0.17337630689144135 (0.22816708370258934) + | > loss_disc_real_5: 0.1893073469400406 (0.23111094573610708) + | > loss_0: 2.3769335746765137 (2.462138897494266) + | > grad_norm_0: tensor(7.1918, device='cuda:0') (tensor(12.3639, device='cuda:0')) + | > loss_gen: 2.369377613067627 (2.3767402987731123) + | > loss_kl: 1.9476635456085205 (1.8887270500785427) + | > loss_feat: 7.250814914703369 (7.114464935503508) + | > loss_mel: 18.509963989257812 (17.95102842230547) + | > loss_duration: 1.4986488819122314 (1.474906259461453) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.57646942138672 (30.8058668437757) + | > grad_norm_1: tensor(180.8498, device='cuda:0') (tensor(284.8517, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.3514 (0.33971747599150004) + | > loader_time: 0.0058 (0.005319087128890188) + + + --> TIME: 2025-04-19 00:29:38 -- STEP: 63/142 -- GLOBAL_STEP: 392125 + | > loss_disc: 2.5150210857391357 (2.4544650448693166) + | > loss_disc_real_0: 0.11151635646820068 (0.12622999967563717) + | > loss_disc_real_1: 0.2488185316324234 (0.20720934229237692) + | > loss_disc_real_2: 0.2523549497127533 (0.22653931071833958) + | > loss_disc_real_3: 0.2620994448661804 (0.22843246445769355) + | > loss_disc_real_4: 0.24462105333805084 (0.2283243631559705) + | > loss_disc_real_5: 0.23907387256622314 (0.22750652120226905) + | > loss_0: 2.5150210857391357 (2.4544650448693166) + | > grad_norm_0: tensor(7.2078, device='cuda:0') (tensor(11.1756, device='cuda:0')) + | > loss_gen: 2.426818370819092 (2.3737750621069047) + | > loss_kl: 2.01584529876709 (1.8926788114366078) + | > loss_feat: 5.941618919372559 (7.074572472345261) + | > loss_mel: 18.021930694580078 (17.92993227640789) + | > loss_duration: 1.556915283203125 (1.4855623926435197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9631290435791 (30.756520801120335) + | > grad_norm_1: tensor(67.3081, device='cuda:0') (tensor(255.3353, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.3896 (0.3513260493202815) + | > loader_time: 0.006 (0.005616434036739289) + + + --> TIME: 2025-04-19 00:29:48 -- STEP: 88/142 -- GLOBAL_STEP: 392150 + | > loss_disc: 2.4459145069122314 (2.4685365232554344) + | > loss_disc_real_0: 0.13712655007839203 (0.12968830154700706) + | > loss_disc_real_1: 0.1916796714067459 (0.2095066381787712) + | > loss_disc_real_2: 0.20939409732818604 (0.22709205153990875) + | > loss_disc_real_3: 0.21166157722473145 (0.22947520237754693) + | > loss_disc_real_4: 0.2413550317287445 (0.2304158698428761) + | > loss_disc_real_5: 0.20458273589611053 (0.22982252287593755) + | > loss_0: 2.4459145069122314 (2.4685365232554344) + | > grad_norm_0: tensor(7.7852, device='cuda:0') (tensor(11.1930, device='cuda:0')) + | > loss_gen: 2.3727049827575684 (2.3658868372440343) + | > loss_kl: 2.0697989463806152 (1.9049513651566072) + | > loss_feat: 7.125360012054443 (7.0222019444812425) + | > loss_mel: 18.152053833007812 (17.918195160952497) + | > loss_duration: 1.5876048803329468 (1.4977554380893707) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.307523727416992 (30.70899055220864) + | > grad_norm_1: tensor(338.5533, device='cuda:0') (tensor(239.9827, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.4092 (0.364919673312794) + | > loader_time: 0.0068 (0.005944181572307242) + + + --> TIME: 2025-04-19 00:29:59 -- STEP: 113/142 -- GLOBAL_STEP: 392175 + | > loss_disc: 2.388231039047241 (2.467833917752831) + | > loss_disc_real_0: 0.14332501590251923 (0.13013055581039012) + | > loss_disc_real_1: 0.16849935054779053 (0.2085055466246816) + | > loss_disc_real_2: 0.23584488034248352 (0.22702358794950805) + | > loss_disc_real_3: 0.22980166971683502 (0.22921642120433064) + | > loss_disc_real_4: 0.23425866663455963 (0.22971085254597454) + | > loss_disc_real_5: 0.22667893767356873 (0.23011414139671663) + | > loss_0: 2.388231039047241 (2.467833917752831) + | > grad_norm_0: tensor(7.3740, device='cuda:0') (tensor(12.3877, device='cuda:0')) + | > loss_gen: 2.5656943321228027 (2.371612941269326) + | > loss_kl: 1.8817157745361328 (1.9159083229250613) + | > loss_feat: 6.725514888763428 (7.023094527489316) + | > loss_mel: 16.841089248657227 (17.946601057474602) + | > loss_duration: 1.5236769914627075 (1.5035780773753613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.53769302368164 (30.76079468178538) + | > grad_norm_1: tensor(213.9050, device='cuda:0') (tensor(262.7762, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.4259 (0.37955741755730277) + | > loader_time: 0.0068 (0.006209232110892779) + + + --> TIME: 2025-04-19 00:30:10 -- STEP: 138/142 -- GLOBAL_STEP: 392200 + | > loss_disc: 2.5965144634246826 (2.4718282153640967) + | > loss_disc_real_0: 0.08531052619218826 (0.13068266242634544) + | > loss_disc_real_1: 0.20344778895378113 (0.20826701873886413) + | > loss_disc_real_2: 0.2553671896457672 (0.22802600782850516) + | > loss_disc_real_3: 0.2670181095600128 (0.2298264985067257) + | > loss_disc_real_4: 0.21359610557556152 (0.22943992308084515) + | > loss_disc_real_5: 0.23005864024162292 (0.23041652736888416) + | > loss_0: 2.5965144634246826 (2.4718282153640967) + | > grad_norm_0: tensor(10.3125, device='cuda:0') (tensor(13.6408, device='cuda:0')) + | > loss_gen: 2.2432706356048584 (2.3687551833581235) + | > loss_kl: 1.9930473566055298 (1.9260416264119355) + | > loss_feat: 4.890174865722656 (6.981295347213745) + | > loss_mel: 14.812834739685059 (17.86634217828945) + | > loss_duration: 1.5847084522247314 (1.5095883391905522) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.524036407470703 (30.652022403219473) + | > grad_norm_1: tensor(283.4425, device='cuda:0') (tensor(277.5960, device='cuda:0')) + | > current_lr_0: 0.00019776246985887165 + | > current_lr_1: 0.00019776246985887165 + | > step_time: 0.4579 (0.39128906484963233) + | > loader_time: 0.008 (0.0064504509386808994) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4146952629089355 (2.4146952629089355) + | > loss_disc_real_0: 0.1435975730419159 (0.1435975730419159) + | > loss_disc_real_1: 0.22905118763446808 (0.22905118763446808) + | > loss_disc_real_2: 0.2858249545097351 (0.2858249545097351) + | > loss_disc_real_3: 0.23404094576835632 (0.23404094576835632) + | > loss_disc_real_4: 0.23990172147750854 (0.23990172147750854) + | > loss_disc_real_5: 0.2563936710357666 (0.2563936710357666) + | > loss_0: 2.4146952629089355 (2.4146952629089355) + | > loss_gen: 2.52417254447937 (2.52417254447937) + | > loss_kl: 2.2288355827331543 (2.2288355827331543) + | > loss_feat: 7.43220853805542 (7.43220853805542) + | > loss_mel: 17.589502334594727 (17.589502334594727) + | > loss_duration: 1.6232891082763672 (1.6232891082763672) + | > loss_1: 31.398006439208984 (31.398006439208984) + + --> STEP: 1 + | > loss_disc: 2.532550573348999 (2.532550573348999) + | > loss_disc_real_0: 0.13410793244838715 (0.13410793244838715) + | > loss_disc_real_1: 0.27269870042800903 (0.27269870042800903) + | > loss_disc_real_2: 0.2764090895652771 (0.2764090895652771) + | > loss_disc_real_3: 0.25796252489089966 (0.25796252489089966) + | > loss_disc_real_4: 0.2615587115287781 (0.2615587115287781) + | > loss_disc_real_5: 0.2660181224346161 (0.2660181224346161) + | > loss_0: 2.532550573348999 (2.532550573348999) + | > loss_gen: 2.498014211654663 (2.498014211654663) + | > loss_kl: 2.4758801460266113 (2.4758801460266113) + | > loss_feat: 6.314375400543213 (6.314375400543213) + | > loss_mel: 19.46480941772461 (19.46480941772461) + | > loss_duration: 1.6584277153015137 (1.6584277153015137) + | > loss_1: 32.41150665283203 (32.41150665283203) + + --> STEP: 2 + | > loss_disc: 2.595280647277832 (2.5639156103134155) + | > loss_disc_real_0: 0.1217605322599411 (0.12793423235416412) + | > loss_disc_real_1: 0.23701944947242737 (0.2548590749502182) + | > loss_disc_real_2: 0.2722504138946533 (0.2743297517299652) + | > loss_disc_real_3: 0.25226733088493347 (0.25511492788791656) + | > loss_disc_real_4: 0.27625423669815063 (0.26890647411346436) + | > loss_disc_real_5: 0.2967510223388672 (0.28138457238674164) + | > loss_0: 2.595280647277832 (2.5639156103134155) + | > loss_gen: 2.3119301795959473 (2.404972195625305) + | > loss_kl: 2.203411817550659 (2.3396459817886353) + | > loss_feat: 7.268550872802734 (6.791463136672974) + | > loss_mel: 18.718189239501953 (19.09149932861328) + | > loss_duration: 1.6233069896697998 (1.6408673524856567) + | > loss_1: 32.125389099121094 (32.26844787597656) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004598021507263184 (+0.0012334585189819336) + | > avg_loss_disc: 2.5639156103134155 (+0.06612169742584229) + | > avg_loss_disc_real_0: 0.12793423235416412 (+0.007175508886575699) + | > avg_loss_disc_real_1: 0.2548590749502182 (+0.027222678065299988) + | > avg_loss_disc_real_2: 0.2743297517299652 (+0.03251786530017853) + | > avg_loss_disc_real_3: 0.25511492788791656 (+0.020077265799045563) + | > avg_loss_disc_real_4: 0.26890647411346436 (+0.04349830746650696) + | > avg_loss_disc_real_5: 0.28138457238674164 (+0.05474202334880829) + | > avg_loss_0: 2.5639156103134155 (+0.06612169742584229) + | > avg_loss_gen: 2.404972195625305 (+0.15798413753509521) + | > avg_loss_kl: 2.3396459817886353 (+0.016164183616638184) + | > avg_loss_feat: 6.791463136672974 (-0.07652044296264648) + | > avg_loss_mel: 19.09149932861328 (+1.1775283813476562) + | > avg_loss_duration: 1.6408673524856567 (+0.011423945426940918) + | > avg_loss_1: 32.26844787597656 (+1.2865800857543945) + + + > EPOCH: 91/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:30:14)  + + --> TIME: 2025-04-19 00:30:21 -- STEP: 21/142 -- GLOBAL_STEP: 392225 + | > loss_disc: 2.424705982208252 (2.4200824215298606) + | > loss_disc_real_0: 0.10410163551568985 (0.11473265609570912) + | > loss_disc_real_1: 0.1907837688922882 (0.20249982958748228) + | > loss_disc_real_2: 0.20076468586921692 (0.2243960052728653) + | > loss_disc_real_3: 0.24546481668949127 (0.2294076383113861) + | > loss_disc_real_4: 0.2074301540851593 (0.22254354116462527) + | > loss_disc_real_5: 0.21514272689819336 (0.23016762236754099) + | > loss_0: 2.424705982208252 (2.4200824215298606) + | > grad_norm_0: tensor(21.0034, device='cuda:0') (tensor(11.0408, device='cuda:0')) + | > loss_gen: 2.373026132583618 (2.3541874431428456) + | > loss_kl: 1.8968082666397095 (1.8453796818142845) + | > loss_feat: 7.215117454528809 (7.172924291519892) + | > loss_mel: 17.765644073486328 (17.88693196432931) + | > loss_duration: 1.4810285568237305 (1.461795142718724) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.731624603271484 (30.721218381609237) + | > grad_norm_1: tensor(335.0999, device='cuda:0') (tensor(246.7650, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.3439 (0.33257189251127695) + | > loader_time: 0.0059 (0.005137080237978981) + + + --> TIME: 2025-04-19 00:30:30 -- STEP: 46/142 -- GLOBAL_STEP: 392250 + | > loss_disc: 2.3787572383880615 (2.427195035892984) + | > loss_disc_real_0: 0.09049271047115326 (0.11769462797952734) + | > loss_disc_real_1: 0.22164729237556458 (0.20105183901994125) + | > loss_disc_real_2: 0.20085963606834412 (0.2249459238803905) + | > loss_disc_real_3: 0.23826514184474945 (0.22781866335350534) + | > loss_disc_real_4: 0.1992959827184677 (0.2259197507215583) + | > loss_disc_real_5: 0.23938743770122528 (0.23292184847852457) + | > loss_0: 2.3787572383880615 (2.427195035892984) + | > grad_norm_0: tensor(6.3926, device='cuda:0') (tensor(9.6172, device='cuda:0')) + | > loss_gen: 2.5439889430999756 (2.3658654430638184) + | > loss_kl: 1.9058743715286255 (1.8784552657085916) + | > loss_feat: 7.956328392028809 (7.124039867649907) + | > loss_mel: 18.902774810791016 (17.82943053867506) + | > loss_duration: 1.4597992897033691 (1.47289806863536) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.76876449584961 (30.670689251111902) + | > grad_norm_1: tensor(238.8864, device='cuda:0') (tensor(238.9964, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.3683 (0.34549868106842047) + | > loader_time: 0.006 (0.005546512811080269) + + + --> TIME: 2025-04-19 00:30:40 -- STEP: 71/142 -- GLOBAL_STEP: 392275 + | > loss_disc: 2.4961791038513184 (2.4546088702242126) + | > loss_disc_real_0: 0.1543852984905243 (0.12107414647307195) + | > loss_disc_real_1: 0.21451745927333832 (0.20537305654774243) + | > loss_disc_real_2: 0.2384367287158966 (0.22681469753594466) + | > loss_disc_real_3: 0.23804867267608643 (0.2289546054433769) + | > loss_disc_real_4: 0.26006820797920227 (0.22944713457369467) + | > loss_disc_real_5: 0.24742670357227325 (0.23595726741871365) + | > loss_0: 2.4961791038513184 (2.4546088702242126) + | > grad_norm_0: tensor(10.4032, device='cuda:0') (tensor(9.2545, device='cuda:0')) + | > loss_gen: 2.354332447052002 (2.353566992450767) + | > loss_kl: 1.9973677396774292 (1.8994697365962283) + | > loss_feat: 6.99434232711792 (7.095765180990729) + | > loss_mel: 18.550960540771484 (17.924721838722768) + | > loss_duration: 1.5239627361297607 (1.4847660299757837) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.42096519470215 (30.758289955031703) + | > grad_norm_1: tensor(282.6654, device='cuda:0') (tensor(216.9134, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.3858 (0.3569541581919494) + | > loader_time: 0.0061 (0.005822346244059818) + + + --> TIME: 2025-04-19 00:30:50 -- STEP: 96/142 -- GLOBAL_STEP: 392300 + | > loss_disc: 2.6311936378479004 (2.457836404442786) + | > loss_disc_real_0: 0.17066915333271027 (0.12383536303726335) + | > loss_disc_real_1: 0.20948559045791626 (0.20636607504760227) + | > loss_disc_real_2: 0.2517499327659607 (0.22628145338967443) + | > loss_disc_real_3: 0.25078868865966797 (0.2288781295840939) + | > loss_disc_real_4: 0.23697026073932648 (0.22838106487567225) + | > loss_disc_real_5: 0.24816006422042847 (0.23341842585553726) + | > loss_0: 2.6311936378479004 (2.457836404442786) + | > grad_norm_0: tensor(7.5216, device='cuda:0') (tensor(9.1277, device='cuda:0')) + | > loss_gen: 2.2883453369140625 (2.3490599369009337) + | > loss_kl: 1.9342985153198242 (1.9185367561876774) + | > loss_feat: 6.849630355834961 (7.02034171919028) + | > loss_mel: 18.338985443115234 (17.91465127468109) + | > loss_duration: 1.5157389640808105 (1.495951846241951) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.926998138427734 (30.69854176044464) + | > grad_norm_1: tensor(100.2940, device='cuda:0') (tensor(202.0515, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.4031 (0.36782266696294136) + | > loader_time: 0.0071 (0.00605987012386322) + + + --> TIME: 2025-04-19 00:31:01 -- STEP: 121/142 -- GLOBAL_STEP: 392325 + | > loss_disc: 2.346057891845703 (2.463708022409233) + | > loss_disc_real_0: 0.1325623244047165 (0.12529811540172123) + | > loss_disc_real_1: 0.19169822335243225 (0.2075844722345841) + | > loss_disc_real_2: 0.20286650955677032 (0.22655642771523846) + | > loss_disc_real_3: 0.22707036137580872 (0.2288923919939798) + | > loss_disc_real_4: 0.22863629460334778 (0.22935937000207665) + | > loss_disc_real_5: 0.2314428985118866 (0.23404806741505615) + | > loss_0: 2.346057891845703 (2.463708022409233) + | > grad_norm_0: tensor(4.9611, device='cuda:0') (tensor(9.3709, device='cuda:0')) + | > loss_gen: 2.5065431594848633 (2.338284435351032) + | > loss_kl: 2.0383129119873047 (1.9262732622052028) + | > loss_feat: 7.06292200088501 (6.951095916022939) + | > loss_mel: 19.44411849975586 (17.879152967910137) + | > loss_duration: 1.5734789371490479 (1.502747290390582) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.62537384033203 (30.597554072860845) + | > grad_norm_1: tensor(101.2190, device='cuda:0') (tensor(206.9149, device='cuda:0')) + | > current_lr_0: 0.0001977377495501393 + | > current_lr_1: 0.0001977377495501393 + | > step_time: 0.4474 (0.3790621067866805) + | > loader_time: 0.008 (0.006307239374838585) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6061108112335205 (2.6061108112335205) + | > loss_disc_real_0: 0.26278653740882874 (0.26278653740882874) + | > loss_disc_real_1: 0.24063153564929962 (0.24063153564929962) + | > loss_disc_real_2: 0.2531859278678894 (0.2531859278678894) + | > loss_disc_real_3: 0.23622161149978638 (0.23622161149978638) + | > loss_disc_real_4: 0.21326325833797455 (0.21326325833797455) + | > loss_disc_real_5: 0.2660825252532959 (0.2660825252532959) + | > loss_0: 2.6061108112335205 (2.6061108112335205) + | > loss_gen: 2.4588661193847656 (2.4588661193847656) + | > loss_kl: 1.9644668102264404 (1.9644668102264404) + | > loss_feat: 5.6775102615356445 (5.6775102615356445) + | > loss_mel: 17.84385871887207 (17.84385871887207) + | > loss_duration: 1.6494642496109009 (1.6494642496109009) + | > loss_1: 29.594165802001953 (29.594165802001953) + + --> STEP: 1 + | > loss_disc: 2.415398597717285 (2.415398597717285) + | > loss_disc_real_0: 0.1820228546857834 (0.1820228546857834) + | > loss_disc_real_1: 0.2384101152420044 (0.2384101152420044) + | > loss_disc_real_2: 0.23473277688026428 (0.23473277688026428) + | > loss_disc_real_3: 0.22324924170970917 (0.22324924170970917) + | > loss_disc_real_4: 0.2102597951889038 (0.2102597951889038) + | > loss_disc_real_5: 0.24871939420700073 (0.24871939420700073) + | > loss_0: 2.415398597717285 (2.415398597717285) + | > loss_gen: 2.5323917865753174 (2.5323917865753174) + | > loss_kl: 2.6070151329040527 (2.6070151329040527) + | > loss_feat: 6.752813816070557 (6.752813816070557) + | > loss_mel: 18.36871337890625 (18.36871337890625) + | > loss_duration: 1.6752543449401855 (1.6752543449401855) + | > loss_1: 31.936189651489258 (31.936189651489258) + + --> STEP: 2 + | > loss_disc: 2.5955638885498047 (2.505481243133545) + | > loss_disc_real_0: 0.2596307098865509 (0.22082678228616714) + | > loss_disc_real_1: 0.25018781423568726 (0.24429896473884583) + | > loss_disc_real_2: 0.25817611813545227 (0.24645444750785828) + | > loss_disc_real_3: 0.2382327914237976 (0.2307410165667534) + | > loss_disc_real_4: 0.26811227202415466 (0.23918603360652924) + | > loss_disc_real_5: 0.30897971987724304 (0.2788495570421219) + | > loss_0: 2.5955638885498047 (2.505481243133545) + | > loss_gen: 2.5630578994750977 (2.5477248430252075) + | > loss_kl: 2.117526054382324 (2.3622705936431885) + | > loss_feat: 4.942718982696533 (5.847766399383545) + | > loss_mel: 15.918129920959473 (17.14342164993286) + | > loss_duration: 1.6134743690490723 (1.644364356994629) + | > loss_1: 27.154905319213867 (29.545547485351562) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003696918487548828 (-0.0009011030197143555) + | > avg_loss_disc: 2.505481243133545 (-0.058434367179870605) + | > avg_loss_disc_real_0: 0.22082678228616714 (+0.09289254993200302) + | > avg_loss_disc_real_1: 0.24429896473884583 (-0.010560110211372375) + | > avg_loss_disc_real_2: 0.24645444750785828 (-0.027875304222106934) + | > avg_loss_disc_real_3: 0.2307410165667534 (-0.024373911321163177) + | > avg_loss_disc_real_4: 0.23918603360652924 (-0.02972044050693512) + | > avg_loss_disc_real_5: 0.2788495570421219 (-0.002535015344619751) + | > avg_loss_0: 2.505481243133545 (-0.058434367179870605) + | > avg_loss_gen: 2.5477248430252075 (+0.14275264739990234) + | > avg_loss_kl: 2.3622705936431885 (+0.022624611854553223) + | > avg_loss_feat: 5.847766399383545 (-0.9436967372894287) + | > avg_loss_mel: 17.14342164993286 (-1.94807767868042) + | > avg_loss_duration: 1.644364356994629 (+0.003497004508972168) + | > avg_loss_1: 29.545547485351562 (-2.722900390625) + + + > EPOCH: 92/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:31:12)  + + --> TIME: 2025-04-19 00:31:15 -- STEP: 4/142 -- GLOBAL_STEP: 392350 + | > loss_disc: 2.4324464797973633 (2.385261356830597) + | > loss_disc_real_0: 0.0733904168009758 (0.1288769394159317) + | > loss_disc_real_1: 0.17306512594223022 (0.19528160989284515) + | > loss_disc_real_2: 0.21660292148590088 (0.2295222468674183) + | > loss_disc_real_3: 0.20871680974960327 (0.22190828621387482) + | > loss_disc_real_4: 0.21247689425945282 (0.22756264358758926) + | > loss_disc_real_5: 0.20717190206050873 (0.23582344129681587) + | > loss_0: 2.4324464797973633 (2.385261356830597) + | > grad_norm_0: tensor(11.2977, device='cuda:0') (tensor(12.2108, device='cuda:0')) + | > loss_gen: 2.3005611896514893 (2.33710378408432) + | > loss_kl: 2.030001163482666 (1.9271085262298584) + | > loss_feat: 7.112791538238525 (7.147339582443237) + | > loss_mel: 18.272232055664062 (17.63683319091797) + | > loss_duration: 1.4796457290649414 (1.470864087343216) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.195232391357422 (30.51924991607666) + | > grad_norm_1: tensor(70.7695, device='cuda:0') (tensor(212.9341, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.3204 (0.3243611454963684) + | > loader_time: 0.005 (0.004711508750915527) + + + --> TIME: 2025-04-19 00:31:23 -- STEP: 29/142 -- GLOBAL_STEP: 392375 + | > loss_disc: 2.5736491680145264 (2.469240764091755) + | > loss_disc_real_0: 0.09166911989450455 (0.12538081099247111) + | > loss_disc_real_1: 0.24548502266407013 (0.20661724692788616) + | > loss_disc_real_2: 0.2672887146472931 (0.2256237037222961) + | > loss_disc_real_3: 0.2972261905670166 (0.2307884199865933) + | > loss_disc_real_4: 0.24434491991996765 (0.23221936102571158) + | > loss_disc_real_5: 0.2603720724582672 (0.23793196472628364) + | > loss_0: 2.5736491680145264 (2.469240764091755) + | > grad_norm_0: tensor(10.0686, device='cuda:0') (tensor(8.5637, device='cuda:0')) + | > loss_gen: 2.4478330612182617 (2.37173185677364) + | > loss_kl: 1.8831042051315308 (1.8947713662838113) + | > loss_feat: 7.4240312576293945 (7.159079485926135) + | > loss_mel: 18.252946853637695 (17.883282102387525) + | > loss_duration: 1.4529447555541992 (1.4679077740373283) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.460861206054688 (30.776772531969794) + | > grad_norm_1: tensor(160.1741, device='cuda:0') (tensor(191.6628, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.3458 (0.3362425360186347) + | > loader_time: 0.0054 (0.0051812385690623314) + + + --> TIME: 2025-04-19 00:31:32 -- STEP: 54/142 -- GLOBAL_STEP: 392400 + | > loss_disc: 2.6006805896759033 (2.462130630457844) + | > loss_disc_real_0: 0.15965676307678223 (0.12607020967536503) + | > loss_disc_real_1: 0.2659716010093689 (0.2066388866967625) + | > loss_disc_real_2: 0.2513137757778168 (0.22503607637352413) + | > loss_disc_real_3: 0.23292511701583862 (0.22782212495803833) + | > loss_disc_real_4: 0.2691417634487152 (0.22941083985346336) + | > loss_disc_real_5: 0.26146790385246277 (0.23427224104051236) + | > loss_0: 2.6006805896759033 (2.462130630457844) + | > grad_norm_0: tensor(32.3378, device='cuda:0') (tensor(8.8456, device='cuda:0')) + | > loss_gen: 2.2445363998413086 (2.3618568535204294) + | > loss_kl: 2.0528347492218018 (1.9036017568023116) + | > loss_feat: 5.261398792266846 (7.134595076243083) + | > loss_mel: 15.50141429901123 (17.963748596332692) + | > loss_duration: 1.4902249574661255 (1.4747740670486733) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.5504093170166 (30.838576352154767) + | > grad_norm_1: tensor(216.1005, device='cuda:0') (tensor(196.4908, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.3687 (0.3479790422651503) + | > loader_time: 0.0059 (0.005518127370763708) + + + --> TIME: 2025-04-19 00:31:42 -- STEP: 79/142 -- GLOBAL_STEP: 392425 + | > loss_disc: 2.4701523780822754 (2.4763751422302636) + | > loss_disc_real_0: 0.12527641654014587 (0.1296773068135298) + | > loss_disc_real_1: 0.20550696551799774 (0.2073071649934672) + | > loss_disc_real_2: 0.2282915711402893 (0.2269274275514144) + | > loss_disc_real_3: 0.23693589866161346 (0.22807963079289545) + | > loss_disc_real_4: 0.2541743814945221 (0.22933846844148031) + | > loss_disc_real_5: 0.23770247399806976 (0.23497125506401062) + | > loss_0: 2.4701523780822754 (2.4763751422302636) + | > grad_norm_0: tensor(12.8889, device='cuda:0') (tensor(9.0363, device='cuda:0')) + | > loss_gen: 2.40364408493042 (2.3483089495308795) + | > loss_kl: 1.821540355682373 (1.9114992045149017) + | > loss_feat: 6.50104284286499 (7.072213378133653) + | > loss_mel: 17.756576538085938 (17.89913207669801) + | > loss_duration: 1.547684669494629 (1.4863750512086895) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.030487060546875 (30.717528644996353) + | > grad_norm_1: tensor(333.5612, device='cuda:0') (tensor(196.9077, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.3884 (0.3607434019257751) + | > loader_time: 0.0071 (0.005823262130157857) + + + --> TIME: 2025-04-19 00:31:53 -- STEP: 104/142 -- GLOBAL_STEP: 392450 + | > loss_disc: 2.461425542831421 (2.48318658883755) + | > loss_disc_real_0: 0.12245996296405792 (0.13145906814875513) + | > loss_disc_real_1: 0.20247681438922882 (0.20789258898450777) + | > loss_disc_real_2: 0.23831836879253387 (0.22723672854212615) + | > loss_disc_real_3: 0.24345077574253082 (0.22890337098103303) + | > loss_disc_real_4: 0.2347639799118042 (0.231928574351164) + | > loss_disc_real_5: 0.23246198892593384 (0.2344166124955966) + | > loss_0: 2.461425542831421 (2.48318658883755) + | > grad_norm_0: tensor(17.3297, device='cuda:0') (tensor(9.5694, device='cuda:0')) + | > loss_gen: 2.3870131969451904 (2.3382155047013216) + | > loss_kl: 1.943115234375 (1.9208311450022917) + | > loss_feat: 6.7308759689331055 (6.979966250749735) + | > loss_mel: 17.22191047668457 (17.814596295356758) + | > loss_duration: 1.5393445491790771 (1.4971659275201654) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.82225799560547 (30.550775124476505) + | > grad_norm_1: tensor(300.9600, device='cuda:0') (tensor(202.5936, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.4172 (0.37282700951282793) + | > loader_time: 0.0071 (0.006151070961585412) + + + --> TIME: 2025-04-19 00:32:04 -- STEP: 129/142 -- GLOBAL_STEP: 392475 + | > loss_disc: 2.410428524017334 (2.48434331805207) + | > loss_disc_real_0: 0.16350983083248138 (0.13144730678362446) + | > loss_disc_real_1: 0.17643816769123077 (0.20845929030762161) + | > loss_disc_real_2: 0.19014519453048706 (0.22678931623466256) + | > loss_disc_real_3: 0.21443331241607666 (0.22880992154742397) + | > loss_disc_real_4: 0.2390144169330597 (0.23144245066846064) + | > loss_disc_real_5: 0.20750072598457336 (0.23468517321486806) + | > loss_0: 2.410428524017334 (2.48434331805207) + | > grad_norm_0: tensor(11.3575, device='cuda:0') (tensor(10.0226, device='cuda:0')) + | > loss_gen: 2.2779507637023926 (2.3335749104965573) + | > loss_kl: 2.0691092014312744 (1.9270868532417356) + | > loss_feat: 7.1694488525390625 (6.941244868345039) + | > loss_mel: 17.056171417236328 (17.761809452559604) + | > loss_duration: 1.5149338245391846 (1.5030333579972734) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.087614059448242 (30.466749472211497) + | > grad_norm_1: tensor(316.7755, device='cuda:0') (tensor(218.7621, device='cuda:0')) + | > current_lr_0: 0.0001977130323314455 + | > current_lr_1: 0.0001977130323314455 + | > step_time: 0.4527 (0.3854602436686671) + | > loader_time: 0.0076 (0.006434898967890777) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3689677715301514 (2.3689677715301514) + | > loss_disc_real_0: 0.12296999990940094 (0.12296999990940094) + | > loss_disc_real_1: 0.2159562110900879 (0.2159562110900879) + | > loss_disc_real_2: 0.25577878952026367 (0.25577878952026367) + | > loss_disc_real_3: 0.2365092784166336 (0.2365092784166336) + | > loss_disc_real_4: 0.21671348810195923 (0.21671348810195923) + | > loss_disc_real_5: 0.26120227575302124 (0.26120227575302124) + | > loss_0: 2.3689677715301514 (2.3689677715301514) + | > loss_gen: 2.4234228134155273 (2.4234228134155273) + | > loss_kl: 2.2148945331573486 (2.2148945331573486) + | > loss_feat: 6.310244083404541 (6.310244083404541) + | > loss_mel: 18.051555633544922 (18.051555633544922) + | > loss_duration: 1.636501431465149 (1.636501431465149) + | > loss_1: 30.636619567871094 (30.636619567871094) + + --> STEP: 1 + | > loss_disc: 2.500417947769165 (2.500417947769165) + | > loss_disc_real_0: 0.13700103759765625 (0.13700103759765625) + | > loss_disc_real_1: 0.2375500649213791 (0.2375500649213791) + | > loss_disc_real_2: 0.27216896414756775 (0.27216896414756775) + | > loss_disc_real_3: 0.2158748358488083 (0.2158748358488083) + | > loss_disc_real_4: 0.20598919689655304 (0.20598919689655304) + | > loss_disc_real_5: 0.27550846338272095 (0.27550846338272095) + | > loss_0: 2.500417947769165 (2.500417947769165) + | > loss_gen: 2.3586201667785645 (2.3586201667785645) + | > loss_kl: 2.5056986808776855 (2.5056986808776855) + | > loss_feat: 6.68141508102417 (6.68141508102417) + | > loss_mel: 17.9774227142334 (17.9774227142334) + | > loss_duration: 1.6641604900360107 (1.6641604900360107) + | > loss_1: 31.187314987182617 (31.187314987182617) + + --> STEP: 2 + | > loss_disc: 2.628413200378418 (2.5644155740737915) + | > loss_disc_real_0: 0.19677118957042694 (0.1668861135840416) + | > loss_disc_real_1: 0.25312522053718567 (0.24533764272928238) + | > loss_disc_real_2: 0.23468565940856934 (0.25342731177806854) + | > loss_disc_real_3: 0.23111233115196228 (0.22349358350038528) + | > loss_disc_real_4: 0.2355266660451889 (0.22075793147087097) + | > loss_disc_real_5: 0.2879173159599304 (0.2817128896713257) + | > loss_0: 2.628413200378418 (2.5644155740737915) + | > loss_gen: 2.25039005279541 (2.3045051097869873) + | > loss_kl: 2.186680793762207 (2.3461897373199463) + | > loss_feat: 5.607559680938721 (6.144487380981445) + | > loss_mel: 14.905193328857422 (16.44130802154541) + | > loss_duration: 1.6236621141433716 (1.6439113020896912) + | > loss_1: 26.573488235473633 (28.880401611328125) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003423929214477539 (-0.00027298927307128906) + | > avg_loss_disc: 2.5644155740737915 (+0.05893433094024658) + | > avg_loss_disc_real_0: 0.1668861135840416 (-0.05394066870212555) + | > avg_loss_disc_real_1: 0.24533764272928238 (+0.001038677990436554) + | > avg_loss_disc_real_2: 0.25342731177806854 (+0.006972864270210266) + | > avg_loss_disc_real_3: 0.22349358350038528 (-0.007247433066368103) + | > avg_loss_disc_real_4: 0.22075793147087097 (-0.018428102135658264) + | > avg_loss_disc_real_5: 0.2817128896713257 (+0.0028633326292037964) + | > avg_loss_0: 2.5644155740737915 (+0.05893433094024658) + | > avg_loss_gen: 2.3045051097869873 (-0.24321973323822021) + | > avg_loss_kl: 2.3461897373199463 (-0.016080856323242188) + | > avg_loss_feat: 6.144487380981445 (+0.2967209815979004) + | > avg_loss_mel: 16.44130802154541 (-0.7021136283874512) + | > avg_loss_duration: 1.6439113020896912 (-0.00045305490493774414) + | > avg_loss_1: 28.880401611328125 (-0.6651458740234375) + + + > EPOCH: 93/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:32:11)  + + --> TIME: 2025-04-19 00:32:16 -- STEP: 12/142 -- GLOBAL_STEP: 392500 + | > loss_disc: 2.53381085395813 (2.4990482727686563) + | > loss_disc_real_0: 0.14598731696605682 (0.13384396396577358) + | > loss_disc_real_1: 0.1972314417362213 (0.2100690578420957) + | > loss_disc_real_2: 0.19913460314273834 (0.22321423391501108) + | > loss_disc_real_3: 0.23655590415000916 (0.22982602939009666) + | > loss_disc_real_4: 0.23157447576522827 (0.22661713262399039) + | > loss_disc_real_5: 0.26889699697494507 (0.24224812785784403) + | > loss_0: 2.53381085395813 (2.4990482727686563) + | > grad_norm_0: tensor(4.6034, device='cuda:0') (tensor(5.6292, device='cuda:0')) + | > loss_gen: 2.1369967460632324 (2.2884084979693093) + | > loss_kl: 1.8759300708770752 (1.8986720542112987) + | > loss_feat: 6.464727878570557 (7.033624887466431) + | > loss_mel: 17.273242950439453 (17.992646376291912) + | > loss_duration: 1.4669357538223267 (1.456307629744212) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.21783447265625 (30.669659932454426) + | > grad_norm_1: tensor(50.8103, device='cuda:0') (tensor(87.8702, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.3278 (0.3275720675786336) + | > loader_time: 0.0056 (0.004782497882843018) + + + --> TIME: 2025-04-19 00:32:25 -- STEP: 37/142 -- GLOBAL_STEP: 392525 + | > loss_disc: 2.4102299213409424 (2.46887045293241) + | > loss_disc_real_0: 0.11912739276885986 (0.12681824270937894) + | > loss_disc_real_1: 0.18129533529281616 (0.20894225263917768) + | > loss_disc_real_2: 0.23958350718021393 (0.22384406505404292) + | > loss_disc_real_3: 0.21579711139202118 (0.22816658744940888) + | > loss_disc_real_4: 0.21234194934368134 (0.22621835204395088) + | > loss_disc_real_5: 0.20060721039772034 (0.23532352898571943) + | > loss_0: 2.4102299213409424 (2.46887045293241) + | > grad_norm_0: tensor(8.8458, device='cuda:0') (tensor(8.0921, device='cuda:0')) + | > loss_gen: 2.2861735820770264 (2.3344014013135754) + | > loss_kl: 1.7328615188598633 (1.9017021301630381) + | > loss_feat: 7.102685928344727 (7.119008566882159) + | > loss_mel: 18.49077033996582 (17.984096372449724) + | > loss_duration: 1.510873794555664 (1.4684452462840725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.123363494873047 (30.807653736423802) + | > grad_norm_1: tensor(79.4328, device='cuda:0') (tensor(168.9767, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.3552 (0.34005052334553487) + | > loader_time: 0.0062 (0.005321464023074588) + + + --> TIME: 2025-04-19 00:32:34 -- STEP: 62/142 -- GLOBAL_STEP: 392550 + | > loss_disc: 2.566383123397827 (2.4723235714820135) + | > loss_disc_real_0: 0.07974138855934143 (0.12570540799248603) + | > loss_disc_real_1: 0.21468527615070343 (0.21019336340888853) + | > loss_disc_real_2: 0.2530365586280823 (0.22425232731526898) + | > loss_disc_real_3: 0.2649260461330414 (0.22753250863282912) + | > loss_disc_real_4: 0.21302635967731476 (0.22668103777593182) + | > loss_disc_real_5: 0.23912730813026428 (0.23597698754841281) + | > loss_0: 2.566383123397827 (2.4723235714820135) + | > grad_norm_0: tensor(10.0036, device='cuda:0') (tensor(10.3450, device='cuda:0')) + | > loss_gen: 2.45752215385437 (2.3428859864511797) + | > loss_kl: 1.9703338146209717 (1.9100274981990937) + | > loss_feat: 6.6426472663879395 (7.0758408423393) + | > loss_mel: 17.10144805908203 (17.906588492854958) + | > loss_duration: 1.5339429378509521 (1.4793531721638096) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70589256286621 (30.714696084299394) + | > grad_norm_1: tensor(365.8047, device='cuda:0') (tensor(217.0818, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.3815 (0.3514447673674553) + | > loader_time: 0.0063 (0.005631323783628402) + + + --> TIME: 2025-04-19 00:32:45 -- STEP: 87/142 -- GLOBAL_STEP: 392575 + | > loss_disc: 2.426118850708008 (2.4895107143226722) + | > loss_disc_real_0: 0.1080397292971611 (0.12973988313099433) + | > loss_disc_real_1: 0.20773884654045105 (0.2096299424596216) + | > loss_disc_real_2: 0.24468687176704407 (0.2262391035241642) + | > loss_disc_real_3: 0.23273980617523193 (0.23015352501266303) + | > loss_disc_real_4: 0.23451267182826996 (0.23027139610942754) + | > loss_disc_real_5: 0.24642416834831238 (0.2369339296530033) + | > loss_0: 2.426118850708008 (2.4895107143226722) + | > grad_norm_0: tensor(17.8788, device='cuda:0') (tensor(13.4121, device='cuda:0')) + | > loss_gen: 2.176295280456543 (2.338433627424569) + | > loss_kl: 2.0562703609466553 (1.9317132371595536) + | > loss_feat: 6.34316873550415 (6.958125087036484) + | > loss_mel: 17.345928192138672 (17.789309687998106) + | > loss_duration: 1.5558280944824219 (1.492031186476521) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.477489471435547 (30.509612795950353) + | > grad_norm_1: tensor(406.9636, device='cuda:0') (tensor(249.0796, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.4132 (0.3649541536966958) + | > loader_time: 0.008 (0.005904748522002122) + + + --> TIME: 2025-04-19 00:32:55 -- STEP: 112/142 -- GLOBAL_STEP: 392600 + | > loss_disc: 2.4890642166137695 (2.4844868821757196) + | > loss_disc_real_0: 0.10921060293912888 (0.12853941487680592) + | > loss_disc_real_1: 0.19489754736423492 (0.2095466065885765) + | > loss_disc_real_2: 0.23162929713726044 (0.22543566274855817) + | > loss_disc_real_3: 0.24497392773628235 (0.2293881925621203) + | > loss_disc_real_4: 0.23800541460514069 (0.23025378145809686) + | > loss_disc_real_5: 0.232486292719841 (0.23516082763671875) + | > loss_0: 2.4890642166137695 (2.4844868821757196) + | > grad_norm_0: tensor(11.5891, device='cuda:0') (tensor(12.7271, device='cuda:0')) + | > loss_gen: 2.2877724170684814 (2.3269535409552704) + | > loss_kl: 1.8629779815673828 (1.9370394902569907) + | > loss_feat: 7.190613269805908 (6.92153862970216) + | > loss_mel: 17.68578338623047 (17.80409135137287) + | > loss_duration: 1.4955540895462036 (1.499186938362462) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.522701263427734 (30.488809960229055) + | > grad_norm_1: tensor(413.7738, device='cuda:0') (tensor(252.1815, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.433 (0.3773454917328697) + | > loader_time: 0.0076 (0.006212302616664341) + + + --> TIME: 2025-04-19 00:33:07 -- STEP: 137/142 -- GLOBAL_STEP: 392625 + | > loss_disc: 2.469980478286743 (2.4828672861530845) + | > loss_disc_real_0: 0.12590362131595612 (0.12894315859914696) + | > loss_disc_real_1: 0.1950901448726654 (0.20918204619066555) + | > loss_disc_real_2: 0.22890691459178925 (0.22502964832921968) + | > loss_disc_real_3: 0.24073517322540283 (0.22899625151261796) + | > loss_disc_real_4: 0.23643699288368225 (0.2307451493548651) + | > loss_disc_real_5: 0.22671082615852356 (0.2347244742142893) + | > loss_0: 2.469980478286743 (2.4828672861530845) + | > grad_norm_0: tensor(10.1748, device='cuda:0') (tensor(12.0507, device='cuda:0')) + | > loss_gen: 2.2415969371795654 (2.3218373834651747) + | > loss_kl: 2.0009992122650146 (1.9393108413167244) + | > loss_feat: 6.324985504150391 (6.894849140278614) + | > loss_mel: 17.869787216186523 (17.796005555313005) + | > loss_duration: 1.521458625793457 (1.5052157414220546) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.95882797241211 (30.457218727056127) + | > grad_norm_1: tensor(268.5992, device='cuda:0') (tensor(243.9083, device='cuda:0')) + | > current_lr_0: 0.00019768831820240408 + | > current_lr_1: 0.00019768831820240408 + | > step_time: 0.4571 (0.3895912814314347) + | > loader_time: 0.0079 (0.0064722412694109615) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5066022872924805 (2.5066022872924805) + | > loss_disc_real_0: 0.1704891324043274 (0.1704891324043274) + | > loss_disc_real_1: 0.2856130003929138 (0.2856130003929138) + | > loss_disc_real_2: 0.2236386239528656 (0.2236386239528656) + | > loss_disc_real_3: 0.24785731732845306 (0.24785731732845306) + | > loss_disc_real_4: 0.2536177635192871 (0.2536177635192871) + | > loss_disc_real_5: 0.2758651673793793 (0.2758651673793793) + | > loss_0: 2.5066022872924805 (2.5066022872924805) + | > loss_gen: 2.59328293800354 (2.59328293800354) + | > loss_kl: 2.341867446899414 (2.341867446899414) + | > loss_feat: 7.184374809265137 (7.184374809265137) + | > loss_mel: 18.312227249145508 (18.312227249145508) + | > loss_duration: 1.640805959701538 (1.640805959701538) + | > loss_1: 32.07255935668945 (32.07255935668945) + + --> STEP: 1 + | > loss_disc: 2.567141532897949 (2.567141532897949) + | > loss_disc_real_0: 0.13975675404071808 (0.13975675404071808) + | > loss_disc_real_1: 0.2931700646877289 (0.2931700646877289) + | > loss_disc_real_2: 0.23451024293899536 (0.23451024293899536) + | > loss_disc_real_3: 0.26532840728759766 (0.26532840728759766) + | > loss_disc_real_4: 0.2643032670021057 (0.2643032670021057) + | > loss_disc_real_5: 0.2713446617126465 (0.2713446617126465) + | > loss_0: 2.567141532897949 (2.567141532897949) + | > loss_gen: 2.4995224475860596 (2.4995224475860596) + | > loss_kl: 2.611384630203247 (2.611384630203247) + | > loss_feat: 6.4095139503479 (6.4095139503479) + | > loss_mel: 16.86434555053711 (16.86434555053711) + | > loss_duration: 1.6696022748947144 (1.6696022748947144) + | > loss_1: 30.05436897277832 (30.05436897277832) + + --> STEP: 2 + | > loss_disc: 2.690309762954712 (2.6287256479263306) + | > loss_disc_real_0: 0.2267824113368988 (0.18326958268880844) + | > loss_disc_real_1: 0.29369640350341797 (0.2934332340955734) + | > loss_disc_real_2: 0.25053104758262634 (0.24252064526081085) + | > loss_disc_real_3: 0.31458425521850586 (0.28995633125305176) + | > loss_disc_real_4: 0.27541548013687134 (0.2698593735694885) + | > loss_disc_real_5: 0.2564384341239929 (0.2638915479183197) + | > loss_0: 2.690309762954712 (2.6287256479263306) + | > loss_gen: 2.5959911346435547 (2.547756791114807) + | > loss_kl: 1.9586434364318848 (2.285014033317566) + | > loss_feat: 6.912673473358154 (6.661093711853027) + | > loss_mel: 17.029617309570312 (16.94698143005371) + | > loss_duration: 1.5921090841293335 (1.630855679512024) + | > loss_1: 30.089035034179688 (30.071702003479004) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037949085235595703 (+0.00037097930908203125) + | > avg_loss_disc: 2.6287256479263306 (+0.06431007385253906) + | > avg_loss_disc_real_0: 0.18326958268880844 (+0.016383469104766846) + | > avg_loss_disc_real_1: 0.2934332340955734 (+0.048095591366291046) + | > avg_loss_disc_real_2: 0.24252064526081085 (-0.01090666651725769) + | > avg_loss_disc_real_3: 0.28995633125305176 (+0.06646274775266647) + | > avg_loss_disc_real_4: 0.2698593735694885 (+0.049101442098617554) + | > avg_loss_disc_real_5: 0.2638915479183197 (-0.01782134175300598) + | > avg_loss_0: 2.6287256479263306 (+0.06431007385253906) + | > avg_loss_gen: 2.547756791114807 (+0.24325168132781982) + | > avg_loss_kl: 2.285014033317566 (-0.06117570400238037) + | > avg_loss_feat: 6.661093711853027 (+0.516606330871582) + | > avg_loss_mel: 16.94698143005371 (+0.5056734085083008) + | > avg_loss_duration: 1.630855679512024 (-0.013055622577667236) + | > avg_loss_1: 30.071702003479004 (+1.191300392150879) + + + > EPOCH: 94/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:33:11)  + + --> TIME: 2025-04-19 00:33:18 -- STEP: 20/142 -- GLOBAL_STEP: 392650 + | > loss_disc: 2.507692575454712 (2.4817326784133913) + | > loss_disc_real_0: 0.15269680321216583 (0.12218278385698796) + | > loss_disc_real_1: 0.20333905518054962 (0.20912769436836243) + | > loss_disc_real_2: 0.20641863346099854 (0.23201775476336478) + | > loss_disc_real_3: 0.2397782951593399 (0.2302115738391876) + | > loss_disc_real_4: 0.24628406763076782 (0.22970607802271842) + | > loss_disc_real_5: 0.22597724199295044 (0.23826621398329734) + | > loss_0: 2.507692575454712 (2.4817326784133913) + | > grad_norm_0: tensor(9.1752, device='cuda:0') (tensor(8.3155, device='cuda:0')) + | > loss_gen: 2.3384501934051514 (2.3222151517868044) + | > loss_kl: 1.9262326955795288 (1.8903247237205505) + | > loss_feat: 6.625532150268555 (6.940991640090942) + | > loss_mel: 16.212095260620117 (17.66157398223877) + | > loss_duration: 1.5355229377746582 (1.4611384093761444) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.63783073425293 (30.276244163513184) + | > grad_norm_1: tensor(140.3360, device='cuda:0') (tensor(167.3794, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.3479 (0.33906701803207395) + | > loader_time: 0.0056 (0.005068111419677734) + + + --> TIME: 2025-04-19 00:33:27 -- STEP: 45/142 -- GLOBAL_STEP: 392675 + | > loss_disc: 2.539083242416382 (2.467303456200493) + | > loss_disc_real_0: 0.1574334353208542 (0.12149019953277376) + | > loss_disc_real_1: 0.22141171991825104 (0.20519501467545828) + | > loss_disc_real_2: 0.25846049189567566 (0.2286512593428294) + | > loss_disc_real_3: 0.21095304191112518 (0.23025188214249082) + | > loss_disc_real_4: 0.1986900120973587 (0.22835609118143718) + | > loss_disc_real_5: 0.24327245354652405 (0.2408155424727334) + | > loss_0: 2.539083242416382 (2.467303456200493) + | > grad_norm_0: tensor(13.8255, device='cuda:0') (tensor(9.2298, device='cuda:0')) + | > loss_gen: 2.1963260173797607 (2.3393204106224905) + | > loss_kl: 2.062908887863159 (1.9007418632507325) + | > loss_feat: 7.043509483337402 (6.99146998723348) + | > loss_mel: 17.942922592163086 (17.79301579793295) + | > loss_duration: 1.5021487474441528 (1.4741143200132583) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.74781608581543 (30.498662397596572) + | > grad_norm_1: tensor(340.8135, device='cuda:0') (tensor(231.9070, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.3689 (0.3514136791229248) + | > loader_time: 0.0061 (0.005430597729153104) + + + --> TIME: 2025-04-19 00:33:37 -- STEP: 70/142 -- GLOBAL_STEP: 392700 + | > loss_disc: 2.4867215156555176 (2.478664984021868) + | > loss_disc_real_0: 0.13822022080421448 (0.12329336851835251) + | > loss_disc_real_1: 0.1893872320652008 (0.20743477514811925) + | > loss_disc_real_2: 0.2043399065732956 (0.2296967572399548) + | > loss_disc_real_3: 0.20168709754943848 (0.22804976275988986) + | > loss_disc_real_4: 0.23260751366615295 (0.23124492572886604) + | > loss_disc_real_5: 0.2511442303657532 (0.2404804276568549) + | > loss_0: 2.4867215156555176 (2.478664984021868) + | > grad_norm_0: tensor(8.5323, device='cuda:0') (tensor(10.1728, device='cuda:0')) + | > loss_gen: 2.083977699279785 (2.346889250619071) + | > loss_kl: 2.005378484725952 (1.9217640314783369) + | > loss_feat: 7.177387237548828 (7.015446792330061) + | > loss_mel: 19.257625579833984 (17.838626234872002) + | > loss_duration: 1.4985730648040771 (1.484526479244232) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.02294158935547 (30.60725291115897) + | > grad_norm_1: tensor(237.8202, device='cuda:0') (tensor(235.8444, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.3888 (0.36075599534170966) + | > loader_time: 0.0066 (0.005759412901742118) + + + --> TIME: 2025-04-19 00:33:47 -- STEP: 95/142 -- GLOBAL_STEP: 392725 + | > loss_disc: 2.630589246749878 (2.482158131348459) + | > loss_disc_real_0: 0.16784845292568207 (0.1263448849320412) + | > loss_disc_real_1: 0.2345387041568756 (0.20778492061715376) + | > loss_disc_real_2: 0.21744000911712646 (0.22834668159484864) + | > loss_disc_real_3: 0.2215423882007599 (0.22872296054112284) + | > loss_disc_real_4: 0.22339138388633728 (0.23055602986561624) + | > loss_disc_real_5: 0.2029564529657364 (0.2388129948001159) + | > loss_0: 2.630589246749878 (2.482158131348459) + | > grad_norm_0: tensor(15.1965, device='cuda:0') (tensor(10.8592, device='cuda:0')) + | > loss_gen: 2.1051883697509766 (2.3401745193883006) + | > loss_kl: 1.974316120147705 (1.9380212445008127) + | > loss_feat: 5.29653263092041 (6.975679056267989) + | > loss_mel: 15.019723892211914 (17.85336089887117) + | > loss_duration: 1.520157814025879 (1.4951443509051674) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.91592025756836 (30.602380190397565) + | > grad_norm_1: tensor(191.9266, device='cuda:0') (tensor(236.7865, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.4037 (0.3704342716618588) + | > loader_time: 0.0071 (0.006058524784288908) + + + --> TIME: 2025-04-19 00:33:58 -- STEP: 120/142 -- GLOBAL_STEP: 392750 + | > loss_disc: 2.47540020942688 (2.483655939499536) + | > loss_disc_real_0: 0.1112590804696083 (0.12643017036219445) + | > loss_disc_real_1: 0.17510394752025604 (0.20765490718185903) + | > loss_disc_real_2: 0.22602573037147522 (0.2288844432681799) + | > loss_disc_real_3: 0.20650428533554077 (0.2288765486329794) + | > loss_disc_real_4: 0.24585740268230438 (0.23086332865059375) + | > loss_disc_real_5: 0.24968545138835907 (0.23847713035841783) + | > loss_0: 2.47540020942688 (2.483655939499536) + | > grad_norm_0: tensor(15.9806, device='cuda:0') (tensor(11.1480, device='cuda:0')) + | > loss_gen: 2.3926405906677246 (2.3420827448368073) + | > loss_kl: 2.07269549369812 (1.952307685216268) + | > loss_feat: 7.50266170501709 (6.977337078253428) + | > loss_mel: 18.589397430419922 (17.88595468997955) + | > loss_duration: 1.543581485748291 (1.5012713978687922) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.10097885131836 (30.658953698476157) + | > grad_norm_1: tensor(424.3793, device='cuda:0') (tensor(248.2952, device='cuda:0')) + | > current_lr_0: 0.00019766360716262876 + | > current_lr_1: 0.00019766360716262876 + | > step_time: 0.4365 (0.3810360292593638) + | > loader_time: 0.0075 (0.00629655122756958) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.543596029281616 (2.543596029281616) + | > loss_disc_real_0: 0.22641852498054504 (0.22641852498054504) + | > loss_disc_real_1: 0.22560074925422668 (0.22560074925422668) + | > loss_disc_real_2: 0.23114840686321259 (0.23114840686321259) + | > loss_disc_real_3: 0.26496103405952454 (0.26496103405952454) + | > loss_disc_real_4: 0.28292450308799744 (0.28292450308799744) + | > loss_disc_real_5: 0.2717025578022003 (0.2717025578022003) + | > loss_0: 2.543596029281616 (2.543596029281616) + | > loss_gen: 2.4833264350891113 (2.4833264350891113) + | > loss_kl: 2.221569776535034 (2.221569776535034) + | > loss_feat: 5.774478435516357 (5.774478435516357) + | > loss_mel: 17.43661117553711 (17.43661117553711) + | > loss_duration: 1.6501309871673584 (1.6501309871673584) + | > loss_1: 29.566116333007812 (29.566116333007812) + + --> STEP: 1 + | > loss_disc: 2.451643228530884 (2.451643228530884) + | > loss_disc_real_0: 0.24529802799224854 (0.24529802799224854) + | > loss_disc_real_1: 0.2152511477470398 (0.2152511477470398) + | > loss_disc_real_2: 0.2090136855840683 (0.2090136855840683) + | > loss_disc_real_3: 0.23515290021896362 (0.23515290021896362) + | > loss_disc_real_4: 0.2741936445236206 (0.2741936445236206) + | > loss_disc_real_5: 0.2487681806087494 (0.2487681806087494) + | > loss_0: 2.451643228530884 (2.451643228530884) + | > loss_gen: 2.6076607704162598 (2.6076607704162598) + | > loss_kl: 2.466907501220703 (2.466907501220703) + | > loss_feat: 6.651268482208252 (6.651268482208252) + | > loss_mel: 19.04301643371582 (19.04301643371582) + | > loss_duration: 1.6709716320037842 (1.6709716320037842) + | > loss_1: 32.439823150634766 (32.439823150634766) + + --> STEP: 2 + | > loss_disc: 2.4809234142303467 (2.4662833213806152) + | > loss_disc_real_0: 0.20915339887142181 (0.22722571343183517) + | > loss_disc_real_1: 0.215395987033844 (0.2153235673904419) + | > loss_disc_real_2: 0.20982129871845245 (0.20941749215126038) + | > loss_disc_real_3: 0.20362475514411926 (0.21938882768154144) + | > loss_disc_real_4: 0.2627817988395691 (0.26848772168159485) + | > loss_disc_real_5: 0.228933185338974 (0.2388506829738617) + | > loss_0: 2.4809234142303467 (2.4662833213806152) + | > loss_gen: 2.3486998081207275 (2.4781802892684937) + | > loss_kl: 2.1858506202697754 (2.3263790607452393) + | > loss_feat: 7.285093784332275 (6.968181133270264) + | > loss_mel: 17.501407623291016 (18.272212028503418) + | > loss_duration: 1.6237941980361938 (1.647382915019989) + | > loss_1: 30.94484519958496 (31.692334175109863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003627181053161621 (-0.00016772747039794922) + | > avg_loss_disc: 2.4662833213806152 (-0.16244232654571533) + | > avg_loss_disc_real_0: 0.22722571343183517 (+0.04395613074302673) + | > avg_loss_disc_real_1: 0.2153235673904419 (-0.07810966670513153) + | > avg_loss_disc_real_2: 0.20941749215126038 (-0.033103153109550476) + | > avg_loss_disc_real_3: 0.21938882768154144 (-0.07056750357151031) + | > avg_loss_disc_real_4: 0.26848772168159485 (-0.0013716518878936768) + | > avg_loss_disc_real_5: 0.2388506829738617 (-0.025040864944458008) + | > avg_loss_0: 2.4662833213806152 (-0.16244232654571533) + | > avg_loss_gen: 2.4781802892684937 (-0.06957650184631348) + | > avg_loss_kl: 2.3263790607452393 (+0.04136502742767334) + | > avg_loss_feat: 6.968181133270264 (+0.30708742141723633) + | > avg_loss_mel: 18.272212028503418 (+1.325230598449707) + | > avg_loss_duration: 1.647382915019989 (+0.016527235507965088) + | > avg_loss_1: 31.692334175109863 (+1.6206321716308594) + + + > EPOCH: 95/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:34:09)  + + --> TIME: 2025-04-19 00:34:11 -- STEP: 3/142 -- GLOBAL_STEP: 392775 + | > loss_disc: 2.4345805644989014 (2.464080254236857) + | > loss_disc_real_0: 0.07292456924915314 (0.12894803285598755) + | > loss_disc_real_1: 0.2515508532524109 (0.21951530377070108) + | > loss_disc_real_2: 0.2779381275177002 (0.2602897584438324) + | > loss_disc_real_3: 0.2436550110578537 (0.24113954106966654) + | > loss_disc_real_4: 0.26177194714546204 (0.26624449094136554) + | > loss_disc_real_5: 0.2575189769268036 (0.24970401326815286) + | > loss_0: 2.4345805644989014 (2.464080254236857) + | > grad_norm_0: tensor(17.0272, device='cuda:0') (tensor(21.7231, device='cuda:0')) + | > loss_gen: 2.458416223526001 (2.464815139770508) + | > loss_kl: 1.9187182188034058 (1.9227263927459717) + | > loss_feat: 7.458603382110596 (7.443866888682048) + | > loss_mel: 18.98368263244629 (18.27002016703288) + | > loss_duration: 1.486579418182373 (1.4658143122990925) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.305999755859375 (31.567243576049805) + | > grad_norm_1: tensor(454.6150, device='cuda:0') (tensor(534.1447, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.3214 (0.32355809211730957) + | > loader_time: 0.004 (0.004506349563598633) + + + --> TIME: 2025-04-19 00:34:20 -- STEP: 28/142 -- GLOBAL_STEP: 392800 + | > loss_disc: 2.5473990440368652 (2.467982462474279) + | > loss_disc_real_0: 0.0991324633359909 (0.12769518300358743) + | > loss_disc_real_1: 0.21190473437309265 (0.2090169276509966) + | > loss_disc_real_2: 0.21425116062164307 (0.2313168485249792) + | > loss_disc_real_3: 0.21243540942668915 (0.22920405492186546) + | > loss_disc_real_4: 0.21766650676727295 (0.23147121497562953) + | > loss_disc_real_5: 0.2366448938846588 (0.23481895242418563) + | > loss_0: 2.5473990440368652 (2.467982462474279) + | > grad_norm_0: tensor(13.5823, device='cuda:0') (tensor(15.7400, device='cuda:0')) + | > loss_gen: 2.1085779666900635 (2.3382783787591115) + | > loss_kl: 1.9118177890777588 (1.9117123356887273) + | > loss_feat: 6.661823272705078 (7.015282716069903) + | > loss_mel: 18.70539665222168 (18.10854312351772) + | > loss_duration: 1.5044490098953247 (1.4658920339175634) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.892065048217773 (30.83970880508423) + | > grad_norm_1: tensor(217.6782, device='cuda:0') (tensor(315.7045, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.3445 (0.3350744843482971) + | > loader_time: 0.0054 (0.005082811628069196) + + + --> TIME: 2025-04-19 00:34:29 -- STEP: 53/142 -- GLOBAL_STEP: 392825 + | > loss_disc: 2.3226280212402344 (2.4456809061878144) + | > loss_disc_real_0: 0.118648961186409 (0.1254567659266715) + | > loss_disc_real_1: 0.1546408087015152 (0.20565879148132396) + | > loss_disc_real_2: 0.19568608701229095 (0.2279731217982634) + | > loss_disc_real_3: 0.18778419494628906 (0.22568914687858438) + | > loss_disc_real_4: 0.19216422736644745 (0.22784903251899863) + | > loss_disc_real_5: 0.1508542150259018 (0.23330711840458637) + | > loss_0: 2.3226280212402344 (2.4456809061878144) + | > grad_norm_0: tensor(10.9234, device='cuda:0') (tensor(13.2533, device='cuda:0')) + | > loss_gen: 2.4313814640045166 (2.351880820292347) + | > loss_kl: 1.8960762023925781 (1.9224135065978427) + | > loss_feat: 8.665898323059082 (7.0638751533796205) + | > loss_mel: 18.976314544677734 (18.070156385313783) + | > loss_duration: 1.461216926574707 (1.4738161473904017) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.430885314941406 (30.882142120937132) + | > grad_norm_1: tensor(237.0804, device='cuda:0') (tensor(280.2764, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.3691 (0.34736736315601274) + | > loader_time: 0.006 (0.0054528398333855395) + + + --> TIME: 2025-04-19 00:34:39 -- STEP: 78/142 -- GLOBAL_STEP: 392850 + | > loss_disc: 2.576678514480591 (2.4683575660754475) + | > loss_disc_real_0: 0.13843762874603271 (0.12827431615919638) + | > loss_disc_real_1: 0.19070042669773102 (0.2104168855226957) + | > loss_disc_real_2: 0.22049130499362946 (0.2270270096950042) + | > loss_disc_real_3: 0.23352304100990295 (0.22684292457042596) + | > loss_disc_real_4: 0.22941721975803375 (0.22802250622174677) + | > loss_disc_real_5: 0.21195819973945618 (0.23416179972581375) + | > loss_0: 2.576678514480591 (2.4683575660754475) + | > grad_norm_0: tensor(8.4937, device='cuda:0') (tensor(12.4509, device='cuda:0')) + | > loss_gen: 2.3135833740234375 (2.3427890111238527) + | > loss_kl: 1.8783961534500122 (1.9311108390490215) + | > loss_feat: 6.811992645263672 (6.972798194640722) + | > loss_mel: 18.9091796875 (17.96843462723952) + | > loss_duration: 1.5359752178192139 (1.4850910107294717) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.449125289916992 (30.70022375155718) + | > grad_norm_1: tensor(69.8035, device='cuda:0') (tensor(250.0550, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.4004 (0.35942602463257606) + | > loader_time: 0.0081 (0.005806540831541405) + + + --> TIME: 2025-04-19 00:34:49 -- STEP: 103/142 -- GLOBAL_STEP: 392875 + | > loss_disc: 2.390516996383667 (2.4756891218203934) + | > loss_disc_real_0: 0.106624536216259 (0.13054785895550136) + | > loss_disc_real_1: 0.16676177084445953 (0.21040082267187174) + | > loss_disc_real_2: 0.20527303218841553 (0.22777030407225043) + | > loss_disc_real_3: 0.23038014769554138 (0.22743136139170636) + | > loss_disc_real_4: 0.2533113658428192 (0.22902312961596888) + | > loss_disc_real_5: 0.24913638830184937 (0.23436709081084983) + | > loss_0: 2.390516996383667 (2.4756891218203934) + | > grad_norm_0: tensor(5.0841, device='cuda:0') (tensor(12.0830, device='cuda:0')) + | > loss_gen: 2.281318426132202 (2.3307973320044373) + | > loss_kl: 1.9520777463912964 (1.9381296507363182) + | > loss_feat: 6.655816078186035 (6.892970913822211) + | > loss_mel: 17.97898292541504 (17.905941953936832) + | > loss_duration: 1.5709633827209473 (1.4959386621864097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.439157485961914 (30.5637786643019) + | > grad_norm_1: tensor(62.1644, device='cuda:0') (tensor(234.6955, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.4121 (0.3699680045970434) + | > loader_time: 0.007 (0.006055505530348104) + + + --> TIME: 2025-04-19 00:35:00 -- STEP: 128/142 -- GLOBAL_STEP: 392900 + | > loss_disc: 2.483891010284424 (2.4873746298253527) + | > loss_disc_real_0: 0.1065547987818718 (0.1324390989902895) + | > loss_disc_real_1: 0.231856569647789 (0.21115353528875858) + | > loss_disc_real_2: 0.24704095721244812 (0.22813264711294323) + | > loss_disc_real_3: 0.24312333762645721 (0.2287507086293772) + | > loss_disc_real_4: 0.22752848267555237 (0.2303793034516275) + | > loss_disc_real_5: 0.24823331832885742 (0.2349071140633896) + | > loss_0: 2.483891010284424 (2.4873746298253527) + | > grad_norm_0: tensor(3.7819, device='cuda:0') (tensor(11.0818, device='cuda:0')) + | > loss_gen: 2.2321627140045166 (2.317130433395506) + | > loss_kl: 1.993903636932373 (1.9444518303498626) + | > loss_feat: 6.28418493270874 (6.838801462203264) + | > loss_mel: 18.44222640991211 (17.873736962676055) + | > loss_duration: 1.541175365447998 (1.5027025388553739) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.49365234375 (30.476823315024376) + | > grad_norm_1: tensor(69.5837, device='cuda:0') (tensor(211.2379, device='cuda:0')) + | > current_lr_0: 0.00019763889921173343 + | > current_lr_1: 0.00019763889921173343 + | > step_time: 0.4391 (0.381839683279395) + | > loader_time: 0.0087 (0.006329463794827463) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.480327844619751 (2.480327844619751) + | > loss_disc_real_0: 0.14625976979732513 (0.14625976979732513) + | > loss_disc_real_1: 0.2142571210861206 (0.2142571210861206) + | > loss_disc_real_2: 0.21220946311950684 (0.21220946311950684) + | > loss_disc_real_3: 0.20709924399852753 (0.20709924399852753) + | > loss_disc_real_4: 0.19920067489147186 (0.19920067489147186) + | > loss_disc_real_5: 0.2035428136587143 (0.2035428136587143) + | > loss_0: 2.480327844619751 (2.480327844619751) + | > loss_gen: 2.2965054512023926 (2.2965054512023926) + | > loss_kl: 1.9376046657562256 (1.9376046657562256) + | > loss_feat: 6.401113033294678 (6.401113033294678) + | > loss_mel: 17.357145309448242 (17.357145309448242) + | > loss_duration: 1.630294680595398 (1.630294680595398) + | > loss_1: 29.622661590576172 (29.622661590576172) + + --> STEP: 1 + | > loss_disc: 2.5519402027130127 (2.5519402027130127) + | > loss_disc_real_0: 0.13801933825016022 (0.13801933825016022) + | > loss_disc_real_1: 0.21889762580394745 (0.21889762580394745) + | > loss_disc_real_2: 0.21941757202148438 (0.21941757202148438) + | > loss_disc_real_3: 0.21362294256687164 (0.21362294256687164) + | > loss_disc_real_4: 0.2142236977815628 (0.2142236977815628) + | > loss_disc_real_5: 0.20980770885944366 (0.20980770885944366) + | > loss_0: 2.5519402027130127 (2.5519402027130127) + | > loss_gen: 2.065080165863037 (2.065080165863037) + | > loss_kl: 2.5023908615112305 (2.5023908615112305) + | > loss_feat: 6.780916213989258 (6.780916213989258) + | > loss_mel: 18.2475643157959 (18.2475643157959) + | > loss_duration: 1.685929536819458 (1.685929536819458) + | > loss_1: 31.281883239746094 (31.281883239746094) + + --> STEP: 2 + | > loss_disc: 2.3396241664886475 (2.44578218460083) + | > loss_disc_real_0: 0.13879472017288208 (0.13840702921152115) + | > loss_disc_real_1: 0.23276804387569427 (0.22583283483982086) + | > loss_disc_real_2: 0.21178662776947021 (0.2156020998954773) + | > loss_disc_real_3: 0.21473562717437744 (0.21417928487062454) + | > loss_disc_real_4: 0.20359759032726288 (0.20891064405441284) + | > loss_disc_real_5: 0.22447292506694794 (0.2171403169631958) + | > loss_0: 2.3396241664886475 (2.44578218460083) + | > loss_gen: 2.412264108657837 (2.238672137260437) + | > loss_kl: 2.2384185791015625 (2.3704047203063965) + | > loss_feat: 7.6290178298950195 (7.204967021942139) + | > loss_mel: 17.448759078979492 (17.848161697387695) + | > loss_duration: 1.6018085479736328 (1.6438690423965454) + | > loss_1: 31.33026885986328 (31.306076049804688) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035102367401123047 (-0.0001169443130493164) + | > avg_loss_disc: 2.44578218460083 (-0.020501136779785156) + | > avg_loss_disc_real_0: 0.13840702921152115 (-0.08881868422031403) + | > avg_loss_disc_real_1: 0.22583283483982086 (+0.010509267449378967) + | > avg_loss_disc_real_2: 0.2156020998954773 (+0.006184607744216919) + | > avg_loss_disc_real_3: 0.21417928487062454 (-0.005209542810916901) + | > avg_loss_disc_real_4: 0.20891064405441284 (-0.05957707762718201) + | > avg_loss_disc_real_5: 0.2171403169631958 (-0.021710366010665894) + | > avg_loss_0: 2.44578218460083 (-0.020501136779785156) + | > avg_loss_gen: 2.238672137260437 (-0.23950815200805664) + | > avg_loss_kl: 2.3704047203063965 (+0.04402565956115723) + | > avg_loss_feat: 7.204967021942139 (+0.236785888671875) + | > avg_loss_mel: 17.848161697387695 (-0.42405033111572266) + | > avg_loss_duration: 1.6438690423965454 (-0.0035138726234436035) + | > avg_loss_1: 31.306076049804688 (-0.3862581253051758) + + + > EPOCH: 96/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:35:08)  + + --> TIME: 2025-04-19 00:35:12 -- STEP: 11/142 -- GLOBAL_STEP: 392925 + | > loss_disc: 2.3694419860839844 (2.4278797669844194) + | > loss_disc_real_0: 0.1038036122918129 (0.12205298312685707) + | > loss_disc_real_1: 0.20124588906764984 (0.20092644610188223) + | > loss_disc_real_2: 0.2115347534418106 (0.21523404527794232) + | > loss_disc_real_3: 0.22141307592391968 (0.22973179546269504) + | > loss_disc_real_4: 0.23010985553264618 (0.23264466497031125) + | > loss_disc_real_5: 0.2437758892774582 (0.23329409686001865) + | > loss_0: 2.3694419860839844 (2.4278797669844194) + | > grad_norm_0: tensor(4.2942, device='cuda:0') (tensor(5.7500, device='cuda:0')) + | > loss_gen: 2.400458812713623 (2.358194264498624) + | > loss_kl: 1.8028614521026611 (1.915039225058122) + | > loss_feat: 7.367559909820557 (7.007113023237749) + | > loss_mel: 18.04943084716797 (17.680485118519176) + | > loss_duration: 1.4365707635879517 (1.4528445980765603) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.056882858276367 (30.41367617520419) + | > grad_norm_1: tensor(57.7827, device='cuda:0') (tensor(60.7072, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.3311 (0.32799259099093353) + | > loader_time: 0.0055 (0.004746111956509677) + + + --> TIME: 2025-04-19 00:35:21 -- STEP: 36/142 -- GLOBAL_STEP: 392950 + | > loss_disc: 2.4903290271759033 (2.4411987463633213) + | > loss_disc_real_0: 0.19070829451084137 (0.12828872601191207) + | > loss_disc_real_1: 0.23676636815071106 (0.20577193796634674) + | > loss_disc_real_2: 0.25309866666793823 (0.2227728015018834) + | > loss_disc_real_3: 0.23503580689430237 (0.22522183673249352) + | > loss_disc_real_4: 0.25845974683761597 (0.22890951567225987) + | > loss_disc_real_5: 0.24497756361961365 (0.2275689293940862) + | > loss_0: 2.4903290271759033 (2.4411987463633213) + | > grad_norm_0: tensor(7.3428, device='cuda:0') (tensor(6.4032, device='cuda:0')) + | > loss_gen: 2.3914761543273926 (2.373082882828183) + | > loss_kl: 1.9930293560028076 (1.8936464256710477) + | > loss_feat: 7.262251377105713 (7.0451524125205145) + | > loss_mel: 17.8184814453125 (17.553952402538727) + | > loss_duration: 1.4669173955917358 (1.4652151101165347) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.932153701782227 (30.331049124399822) + | > grad_norm_1: tensor(91.4023, device='cuda:0') (tensor(89.0119, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.3779 (0.3446939256456163) + | > loader_time: 0.006 (0.005215340190463596) + + + --> TIME: 2025-04-19 00:35:31 -- STEP: 61/142 -- GLOBAL_STEP: 392975 + | > loss_disc: 2.430720329284668 (2.44229103698105) + | > loss_disc_real_0: 0.19753193855285645 (0.12724002157566972) + | > loss_disc_real_1: 0.23909719288349152 (0.2063596192442003) + | > loss_disc_real_2: 0.21404461562633514 (0.22468432490942908) + | > loss_disc_real_3: 0.241593137383461 (0.22649988238928748) + | > loss_disc_real_4: 0.20921459794044495 (0.228281667486566) + | > loss_disc_real_5: 0.24294477701187134 (0.22760371766129478) + | > loss_0: 2.430720329284668 (2.44229103698105) + | > grad_norm_0: tensor(13.0205, device='cuda:0') (tensor(8.4717, device='cuda:0')) + | > loss_gen: 2.355780839920044 (2.3757472038269043) + | > loss_kl: 1.946549654006958 (1.9121147843657946) + | > loss_feat: 6.738361835479736 (7.084243446099953) + | > loss_mel: 16.983428955078125 (17.69776600697002) + | > loss_duration: 1.482309103012085 (1.4774612520561843) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.506431579589844 (30.547332669867846) + | > grad_norm_1: tensor(274.4840, device='cuda:0') (tensor(153.4791, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.3852 (0.3541908420500208) + | > loader_time: 0.006 (0.005496212693511463) + + + --> TIME: 2025-04-19 00:35:41 -- STEP: 86/142 -- GLOBAL_STEP: 393000 + | > loss_disc: 2.5495357513427734 (2.4428614738375627) + | > loss_disc_real_0: 0.13646425306797028 (0.12693216513062627) + | > loss_disc_real_1: 0.1971910446882248 (0.20515522447436355) + | > loss_disc_real_2: 0.25670483708381653 (0.22477217919604722) + | > loss_disc_real_3: 0.2420612871646881 (0.2270302178208218) + | > loss_disc_real_4: 0.22110402584075928 (0.22717993429233863) + | > loss_disc_real_5: 0.24770532548427582 (0.22781990139290345) + | > loss_0: 2.5495357513427734 (2.4428614738375627) + | > grad_norm_0: tensor(11.0414, device='cuda:0') (tensor(8.8305, device='cuda:0')) + | > loss_gen: 2.3152873516082764 (2.3747240887131813) + | > loss_kl: 1.9636496305465698 (1.930985898472542) + | > loss_feat: 6.71143102645874 (7.033471861550974) + | > loss_mel: 18.291370391845703 (17.716080931730055) + | > loss_duration: 1.531111717224121 (1.4896848063136257) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.812847137451172 (30.544947535492653) + | > grad_norm_1: tensor(34.7237, device='cuda:0') (tensor(172.4686, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.405 (0.3658708611200021) + | > loader_time: 0.0067 (0.005802431771921557) + + + --> TIME: 2025-04-19 00:35:52 -- STEP: 111/142 -- GLOBAL_STEP: 393025 + | > loss_disc: 2.496492862701416 (2.460062168739937) + | > loss_disc_real_0: 0.1611137092113495 (0.12736841089822146) + | > loss_disc_real_1: 0.16012069582939148 (0.20760479676830876) + | > loss_disc_real_2: 0.2391286939382553 (0.2268370491964323) + | > loss_disc_real_3: 0.24306780099868774 (0.22858213720557927) + | > loss_disc_real_4: 0.28600049018859863 (0.22981698690234004) + | > loss_disc_real_5: 0.2728325128555298 (0.22874856048876102) + | > loss_0: 2.496492862701416 (2.460062168739937) + | > grad_norm_0: tensor(12.3332, device='cuda:0') (tensor(10.2905, device='cuda:0')) + | > loss_gen: 2.4791691303253174 (2.371398717433484) + | > loss_kl: 1.9282335042953491 (1.9361791642936501) + | > loss_feat: 6.48985481262207 (6.976388304083197) + | > loss_mel: 17.55816078186035 (17.698575217444624) + | > loss_duration: 1.539237141609192 (1.4978130271842889) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.994657516479492 (30.4803544465486) + | > grad_norm_1: tensor(340.2985, device='cuda:0') (tensor(209.4010, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.4291 (0.37696983577968823) + | > loader_time: 0.0071 (0.00607248254724451) + + + --> TIME: 2025-04-19 00:36:03 -- STEP: 136/142 -- GLOBAL_STEP: 393050 + | > loss_disc: 2.371669054031372 (2.4583019473973438) + | > loss_disc_real_0: 0.170932799577713 (0.12684212364804215) + | > loss_disc_real_1: 0.1960492581129074 (0.20676671121926868) + | > loss_disc_real_2: 0.21299006044864655 (0.2264590032179566) + | > loss_disc_real_3: 0.20809508860111237 (0.228630212299964) + | > loss_disc_real_4: 0.19255684316158295 (0.23003461726886384) + | > loss_disc_real_5: 0.18204881250858307 (0.22934584659250343) + | > loss_0: 2.371669054031372 (2.4583019473973438) + | > grad_norm_0: tensor(13.1014, device='cuda:0') (tensor(11.8085, device='cuda:0')) + | > loss_gen: 2.5209336280822754 (2.3840811831109674) + | > loss_kl: 1.8770666122436523 (1.936718520872733) + | > loss_feat: 7.024642467498779 (6.9946786761283875) + | > loss_mel: 17.878925323486328 (17.694115246043484) + | > loss_duration: 1.5315876007080078 (1.5030099171049454) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.833154678344727 (30.51260354939629) + | > grad_norm_1: tensor(396.8833, device='cuda:0') (tensor(240.6604, device='cuda:0')) + | > current_lr_0: 0.00019761419434933197 + | > current_lr_1: 0.00019761419434933197 + | > step_time: 0.4615 (0.3902989678523119) + | > loader_time: 0.0076 (0.006344483179204604) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.449035406112671 (2.449035406112671) + | > loss_disc_real_0: 0.18704591691493988 (0.18704591691493988) + | > loss_disc_real_1: 0.27411553263664246 (0.27411553263664246) + | > loss_disc_real_2: 0.23673531413078308 (0.23673531413078308) + | > loss_disc_real_3: 0.20579728484153748 (0.20579728484153748) + | > loss_disc_real_4: 0.20392383635044098 (0.20392383635044098) + | > loss_disc_real_5: 0.25409063696861267 (0.25409063696861267) + | > loss_0: 2.449035406112671 (2.449035406112671) + | > loss_gen: 2.7881126403808594 (2.7881126403808594) + | > loss_kl: 2.2397878170013428 (2.2397878170013428) + | > loss_feat: 7.452540397644043 (7.452540397644043) + | > loss_mel: 19.060087203979492 (19.060087203979492) + | > loss_duration: 1.6432994604110718 (1.6432994604110718) + | > loss_1: 33.1838264465332 (33.1838264465332) + + --> STEP: 1 + | > loss_disc: 2.458071708679199 (2.458071708679199) + | > loss_disc_real_0: 0.2647406756877899 (0.2647406756877899) + | > loss_disc_real_1: 0.2669978439807892 (0.2669978439807892) + | > loss_disc_real_2: 0.242055281996727 (0.242055281996727) + | > loss_disc_real_3: 0.21941328048706055 (0.21941328048706055) + | > loss_disc_real_4: 0.2062169760465622 (0.2062169760465622) + | > loss_disc_real_5: 0.20417657494544983 (0.20417657494544983) + | > loss_0: 2.458071708679199 (2.458071708679199) + | > loss_gen: 2.807393789291382 (2.807393789291382) + | > loss_kl: 2.4866302013397217 (2.4866302013397217) + | > loss_feat: 6.892180442810059 (6.892180442810059) + | > loss_mel: 17.950815200805664 (17.950815200805664) + | > loss_duration: 1.6572407484054565 (1.6572407484054565) + | > loss_1: 31.794260025024414 (31.794260025024414) + + --> STEP: 2 + | > loss_disc: 2.4041786193847656 (2.4311251640319824) + | > loss_disc_real_0: 0.21106135845184326 (0.2379010170698166) + | > loss_disc_real_1: 0.27689626812934875 (0.27194705605506897) + | > loss_disc_real_2: 0.20284949243068695 (0.22245238721370697) + | > loss_disc_real_3: 0.21358835697174072 (0.21650081872940063) + | > loss_disc_real_4: 0.21528048813343048 (0.21074873208999634) + | > loss_disc_real_5: 0.1957998275756836 (0.1999882012605667) + | > loss_0: 2.4041786193847656 (2.4311251640319824) + | > loss_gen: 3.0313720703125 (2.919382929801941) + | > loss_kl: 2.163378953933716 (2.3250045776367188) + | > loss_feat: 6.952354907989502 (6.92226767539978) + | > loss_mel: 18.07218360900879 (18.011499404907227) + | > loss_duration: 1.6137945652008057 (1.635517656803131) + | > loss_1: 31.833084106445312 (31.813672065734863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003418445587158203 (-9.179115295410156e-05) + | > avg_loss_disc: 2.4311251640319824 (-0.014657020568847656) + | > avg_loss_disc_real_0: 0.2379010170698166 (+0.09949398785829544) + | > avg_loss_disc_real_1: 0.27194705605506897 (+0.04611422121524811) + | > avg_loss_disc_real_2: 0.22245238721370697 (+0.006850287318229675) + | > avg_loss_disc_real_3: 0.21650081872940063 (+0.0023215338587760925) + | > avg_loss_disc_real_4: 0.21074873208999634 (+0.001838088035583496) + | > avg_loss_disc_real_5: 0.1999882012605667 (-0.01715211570262909) + | > avg_loss_0: 2.4311251640319824 (-0.014657020568847656) + | > avg_loss_gen: 2.919382929801941 (+0.6807107925415039) + | > avg_loss_kl: 2.3250045776367188 (-0.045400142669677734) + | > avg_loss_feat: 6.92226767539978 (-0.2826993465423584) + | > avg_loss_mel: 18.011499404907227 (+0.16333770751953125) + | > avg_loss_duration: 1.635517656803131 (-0.008351385593414307) + | > avg_loss_1: 31.813672065734863 (+0.5075960159301758) + + + > EPOCH: 97/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:36:07)  + + --> TIME: 2025-04-19 00:36:14 -- STEP: 19/142 -- GLOBAL_STEP: 393075 + | > loss_disc: 2.4522464275360107 (2.4450657618673226) + | > loss_disc_real_0: 0.07239437848329544 (0.11912189835780546) + | > loss_disc_real_1: 0.1759720891714096 (0.2068884827588734) + | > loss_disc_real_2: 0.2122248411178589 (0.22568236056127047) + | > loss_disc_real_3: 0.23452715575695038 (0.22618949334872396) + | > loss_disc_real_4: 0.1853688657283783 (0.23012149333953857) + | > loss_disc_real_5: 0.21572045981884003 (0.22874152817224203) + | > loss_0: 2.4522464275360107 (2.4450657618673226) + | > grad_norm_0: tensor(9.1083, device='cuda:0') (tensor(18.8315, device='cuda:0')) + | > loss_gen: 2.299985885620117 (2.437702304438541) + | > loss_kl: 1.9517635107040405 (1.8617041926634939) + | > loss_feat: 7.884295463562012 (7.343807019685444) + | > loss_mel: 18.37904930114746 (18.148896568699882) + | > loss_duration: 1.4429782629013062 (1.4559245611491956) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.958070755004883 (31.248034627814043) + | > grad_norm_1: tensor(341.1052, device='cuda:0') (tensor(389.4068, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.3394 (0.3299829834385922) + | > loader_time: 0.0049 (0.00491450962267424) + + + --> TIME: 2025-04-19 00:36:23 -- STEP: 44/142 -- GLOBAL_STEP: 393100 + | > loss_disc: 2.458664894104004 (2.4344172152605927) + | > loss_disc_real_0: 0.11246557533740997 (0.11934810800647194) + | > loss_disc_real_1: 0.23741458356380463 (0.20422620258548044) + | > loss_disc_real_2: 0.23143938183784485 (0.22479509765451605) + | > loss_disc_real_3: 0.2346070408821106 (0.2264810593968088) + | > loss_disc_real_4: 0.23654156923294067 (0.23212896727702834) + | > loss_disc_real_5: 0.21532034873962402 (0.2306972129101103) + | > loss_0: 2.458664894104004 (2.4344172152605927) + | > grad_norm_0: tensor(8.4307, device='cuda:0') (tensor(16.4136, device='cuda:0')) + | > loss_gen: 2.3344128131866455 (2.404459655284882) + | > loss_kl: 1.9364616870880127 (1.877902169119228) + | > loss_feat: 6.809741020202637 (7.222811113704335) + | > loss_mel: 16.90138053894043 (18.015533317219123) + | > loss_duration: 1.5153882503509521 (1.4698078090494329) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.497385025024414 (30.99051401831887) + | > grad_norm_1: tensor(337.6116, device='cuda:0') (tensor(353.0486, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.359 (0.3425021930174394) + | > loader_time: 0.0059 (0.005269966342232446) + + + --> TIME: 2025-04-19 00:36:33 -- STEP: 69/142 -- GLOBAL_STEP: 393125 + | > loss_disc: 2.43410587310791 (2.43535570130832) + | > loss_disc_real_0: 0.17656712234020233 (0.12091683407408604) + | > loss_disc_real_1: 0.2213912457227707 (0.20537116302960162) + | > loss_disc_real_2: 0.2249310463666916 (0.2234841917735943) + | > loss_disc_real_3: 0.23843316733837128 (0.22668219930019928) + | > loss_disc_real_4: 0.2462197095155716 (0.23033719434254413) + | > loss_disc_real_5: 0.27474862337112427 (0.23156707269558008) + | > loss_0: 2.43410587310791 (2.43535570130832) + | > grad_norm_0: tensor(7.2706, device='cuda:0') (tensor(13.6194, device='cuda:0')) + | > loss_gen: 2.365751266479492 (2.3921741851861933) + | > loss_kl: 1.9774580001831055 (1.8958949023398801) + | > loss_feat: 6.621086120605469 (7.182267997575843) + | > loss_mel: 18.655519485473633 (17.996122152908985) + | > loss_duration: 1.5679982900619507 (1.4807623849398848) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.18781280517578 (30.947221617767777) + | > grad_norm_1: tensor(258.9145, device='cuda:0') (tensor(306.1504, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.3946 (0.3545832703079002) + | > loader_time: 0.0063 (0.005594682002413101) + + + --> TIME: 2025-04-19 00:36:43 -- STEP: 94/142 -- GLOBAL_STEP: 393150 + | > loss_disc: 2.529275417327881 (2.4442232289212824) + | > loss_disc_real_0: 0.12960918247699738 (0.12171721850779463) + | > loss_disc_real_1: 0.2030133306980133 (0.20631057594684846) + | > loss_disc_real_2: 0.21185332536697388 (0.22497444774242156) + | > loss_disc_real_3: 0.22222447395324707 (0.22660291448552558) + | > loss_disc_real_4: 0.26779744029045105 (0.230495275018063) + | > loss_disc_real_5: 0.24767103791236877 (0.23135049641132355) + | > loss_0: 2.529275417327881 (2.4442232289212824) + | > grad_norm_0: tensor(13.5208, device='cuda:0') (tensor(13.4048, device='cuda:0')) + | > loss_gen: 2.3428804874420166 (2.3806161068855447) + | > loss_kl: 1.8730095624923706 (1.909806638321978) + | > loss_feat: 6.880424976348877 (7.116832829536276) + | > loss_mel: 17.960010528564453 (17.973552227020257) + | > loss_duration: 1.5316425561904907 (1.491802711436089) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.587966918945312 (30.872610416818173) + | > grad_norm_1: tensor(292.4920, device='cuda:0') (tensor(299.0475, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.4016 (0.36624355265434755) + | > loader_time: 0.0071 (0.005935019635139629) + + + --> TIME: 2025-04-19 00:36:54 -- STEP: 119/142 -- GLOBAL_STEP: 393175 + | > loss_disc: 2.3686299324035645 (2.448080980477213) + | > loss_disc_real_0: 0.08281686156988144 (0.12275353214946114) + | > loss_disc_real_1: 0.20918655395507812 (0.2067422454848009) + | > loss_disc_real_2: 0.21226729452610016 (0.2252210358611676) + | > loss_disc_real_3: 0.21539849042892456 (0.22703550555625884) + | > loss_disc_real_4: 0.23030498623847961 (0.22968296017967352) + | > loss_disc_real_5: 0.21023611724376678 (0.23011215394284545) + | > loss_0: 2.3686299324035645 (2.448080980477213) + | > grad_norm_0: tensor(11.2905, device='cuda:0') (tensor(13.3208, device='cuda:0')) + | > loss_gen: 2.398908853530884 (2.36571957484013) + | > loss_kl: 2.0066633224487305 (1.9176310881847094) + | > loss_feat: 7.546422958374023 (7.083981734363973) + | > loss_mel: 18.203622817993164 (17.96516969825038) + | > loss_duration: 1.585425853729248 (1.4989792308887513) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.741044998168945 (30.83148127644002) + | > grad_norm_1: tensor(343.2272, device='cuda:0') (tensor(300.1420, device='cuda:0')) + | > current_lr_0: 0.0001975894925750383 + | > current_lr_1: 0.0001975894925750383 + | > step_time: 0.4263 (0.3785623101627125) + | > loader_time: 0.0075 (0.006216530038529084) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.455758571624756 (2.455758571624756) + | > loss_disc_real_0: 0.14319470524787903 (0.14319470524787903) + | > loss_disc_real_1: 0.2299489974975586 (0.2299489974975586) + | > loss_disc_real_2: 0.28470689058303833 (0.28470689058303833) + | > loss_disc_real_3: 0.24502019584178925 (0.24502019584178925) + | > loss_disc_real_4: 0.20978161692619324 (0.20978161692619324) + | > loss_disc_real_5: 0.24598358571529388 (0.24598358571529388) + | > loss_0: 2.455758571624756 (2.455758571624756) + | > loss_gen: 2.3399713039398193 (2.3399713039398193) + | > loss_kl: 2.2851357460021973 (2.2851357460021973) + | > loss_feat: 5.530028820037842 (5.530028820037842) + | > loss_mel: 16.588878631591797 (16.588878631591797) + | > loss_duration: 1.62506902217865 (1.62506902217865) + | > loss_1: 28.369083404541016 (28.369083404541016) + + --> STEP: 1 + | > loss_disc: 2.594158172607422 (2.594158172607422) + | > loss_disc_real_0: 0.13462267816066742 (0.13462267816066742) + | > loss_disc_real_1: 0.2700487971305847 (0.2700487971305847) + | > loss_disc_real_2: 0.29805564880371094 (0.29805564880371094) + | > loss_disc_real_3: 0.26781171560287476 (0.26781171560287476) + | > loss_disc_real_4: 0.2260880023241043 (0.2260880023241043) + | > loss_disc_real_5: 0.2978261113166809 (0.2978261113166809) + | > loss_0: 2.594158172607422 (2.594158172607422) + | > loss_gen: 2.424036979675293 (2.424036979675293) + | > loss_kl: 2.5245771408081055 (2.5245771408081055) + | > loss_feat: 7.029361248016357 (7.029361248016357) + | > loss_mel: 19.690502166748047 (19.690502166748047) + | > loss_duration: 1.6773988008499146 (1.6773988008499146) + | > loss_1: 33.34587478637695 (33.34587478637695) + + --> STEP: 2 + | > loss_disc: 2.507324457168579 (2.5507413148880005) + | > loss_disc_real_0: 0.18602529168128967 (0.16032398492097855) + | > loss_disc_real_1: 0.23712794482707977 (0.25358837097883224) + | > loss_disc_real_2: 0.22906021773815155 (0.26355793327093124) + | > loss_disc_real_3: 0.2505856156349182 (0.2591986656188965) + | > loss_disc_real_4: 0.20846635103225708 (0.2172771766781807) + | > loss_disc_real_5: 0.28362107276916504 (0.290723592042923) + | > loss_0: 2.507324457168579 (2.5507413148880005) + | > loss_gen: 2.479642629623413 (2.451839804649353) + | > loss_kl: 1.9933884143829346 (2.25898277759552) + | > loss_feat: 6.473134994506836 (6.751248121261597) + | > loss_mel: 16.846582412719727 (18.268542289733887) + | > loss_duration: 1.6158804893493652 (1.64663964509964) + | > loss_1: 29.40863037109375 (31.37725257873535) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004110097885131836 (+0.0006916522979736328) + | > avg_loss_disc: 2.5507413148880005 (+0.11961615085601807) + | > avg_loss_disc_real_0: 0.16032398492097855 (-0.07757703214883804) + | > avg_loss_disc_real_1: 0.25358837097883224 (-0.018358685076236725) + | > avg_loss_disc_real_2: 0.26355793327093124 (+0.041105546057224274) + | > avg_loss_disc_real_3: 0.2591986656188965 (+0.04269784688949585) + | > avg_loss_disc_real_4: 0.2172771766781807 (+0.006528444588184357) + | > avg_loss_disc_real_5: 0.290723592042923 (+0.09073539078235626) + | > avg_loss_0: 2.5507413148880005 (+0.11961615085601807) + | > avg_loss_gen: 2.451839804649353 (-0.4675431251525879) + | > avg_loss_kl: 2.25898277759552 (-0.06602180004119873) + | > avg_loss_feat: 6.751248121261597 (-0.1710195541381836) + | > avg_loss_mel: 18.268542289733887 (+0.25704288482666016) + | > avg_loss_duration: 1.64663964509964 (+0.011121988296508789) + | > avg_loss_1: 31.37725257873535 (-0.4364194869995117) + + + > EPOCH: 98/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:37:06)  + + --> TIME: 2025-04-19 00:37:08 -- STEP: 2/142 -- GLOBAL_STEP: 393200 + | > loss_disc: 2.538748025894165 (2.4347745180130005) + | > loss_disc_real_0: 0.16296997666358948 (0.13598008081316948) + | > loss_disc_real_1: 0.2114347666501999 (0.20538732409477234) + | > loss_disc_real_2: 0.2256554514169693 (0.23727234452962875) + | > loss_disc_real_3: 0.287669837474823 (0.26309311389923096) + | > loss_disc_real_4: 0.24106834828853607 (0.23890162259340286) + | > loss_disc_real_5: 0.28256213665008545 (0.27326834201812744) + | > loss_0: 2.538748025894165 (2.4347745180130005) + | > grad_norm_0: tensor(11.5664, device='cuda:0') (tensor(14.4448, device='cuda:0')) + | > loss_gen: 2.315441131591797 (2.4929873943328857) + | > loss_kl: 2.014683961868286 (1.955876648426056) + | > loss_feat: 6.04261589050293 (6.936089515686035) + | > loss_mel: 15.24865436553955 (16.75556707382202) + | > loss_duration: 1.4868838787078857 (1.455937922000885) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.108278274536133 (29.59645938873291) + | > grad_norm_1: tensor(136.3610, device='cuda:0') (tensor(329.7032, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.324 (0.326119065284729) + | > loader_time: 0.0049 (0.00445401668548584) + + + --> TIME: 2025-04-19 00:37:16 -- STEP: 27/142 -- GLOBAL_STEP: 393225 + | > loss_disc: 2.562586784362793 (2.4522701369391546) + | > loss_disc_real_0: 0.1562822312116623 (0.12499906150279222) + | > loss_disc_real_1: 0.27134472131729126 (0.20562366699730908) + | > loss_disc_real_2: 0.24911528825759888 (0.22920793294906616) + | > loss_disc_real_3: 0.2037266492843628 (0.22809805307123396) + | > loss_disc_real_4: 0.19426044821739197 (0.22710647847917345) + | > loss_disc_real_5: 0.21098919212818146 (0.23292390726230763) + | > loss_0: 2.562586784362793 (2.4522701369391546) + | > grad_norm_0: tensor(16.4700, device='cuda:0') (tensor(8.9791, device='cuda:0')) + | > loss_gen: 2.3189148902893066 (2.3847076716246427) + | > loss_kl: 1.8948779106140137 (1.8702969948450725) + | > loss_feat: 7.550020694732666 (7.221637478581181) + | > loss_mel: 18.126325607299805 (17.60632391329165) + | > loss_duration: 1.478613018989563 (1.4681940917615537) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.368751525878906 (30.551160247237593) + | > grad_norm_1: tensor(378.3026, device='cuda:0') (tensor(171.0157, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.3507 (0.3356318915331805) + | > loader_time: 0.0056 (0.005034588001392507) + + + --> TIME: 2025-04-19 00:37:26 -- STEP: 52/142 -- GLOBAL_STEP: 393250 + | > loss_disc: 2.5003864765167236 (2.467317512402168) + | > loss_disc_real_0: 0.18751022219657898 (0.12638332694768908) + | > loss_disc_real_1: 0.20068976283073425 (0.206297741486476) + | > loss_disc_real_2: 0.24192209541797638 (0.22855819675784844) + | > loss_disc_real_3: 0.2460484653711319 (0.2285219794855668) + | > loss_disc_real_4: 0.22450785338878632 (0.22977162897586823) + | > loss_disc_real_5: 0.2642311155796051 (0.2346431200320904) + | > loss_0: 2.5003864765167236 (2.467317512402168) + | > grad_norm_0: tensor(26.6731, device='cuda:0') (tensor(11.0450, device='cuda:0')) + | > loss_gen: 2.38950252532959 (2.3567755359869738) + | > loss_kl: 1.8885607719421387 (1.881289555476262) + | > loss_feat: 6.649680137634277 (7.102992791395921) + | > loss_mel: 18.8768367767334 (17.783636514957138) + | > loss_duration: 1.537331461906433 (1.4737029740443597) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.34191131591797 (30.598397511702316) + | > grad_norm_1: tensor(238.0647, device='cuda:0') (tensor(221.4208, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.3677 (0.3477336351688092) + | > loader_time: 0.0058 (0.005339631667503944) + + + --> TIME: 2025-04-19 00:37:35 -- STEP: 77/142 -- GLOBAL_STEP: 393275 + | > loss_disc: 2.440833568572998 (2.4709088152105156) + | > loss_disc_real_0: 0.15906113386154175 (0.13012753543141609) + | > loss_disc_real_1: 0.19887445867061615 (0.2061867444933235) + | > loss_disc_real_2: 0.22308148443698883 (0.22635026469633177) + | > loss_disc_real_3: 0.24152378737926483 (0.22868556635720388) + | > loss_disc_real_4: 0.24549296498298645 (0.2292223148918771) + | > loss_disc_real_5: 0.19751763343811035 (0.23484389425872207) + | > loss_0: 2.440833568572998 (2.4709088152105156) + | > grad_norm_0: tensor(9.1540, device='cuda:0') (tensor(13.4597, device='cuda:0')) + | > loss_gen: 2.4250130653381348 (2.3598690683191474) + | > loss_kl: 2.0103678703308105 (1.8875830622462484) + | > loss_feat: 7.168269634246826 (7.0860872392530565) + | > loss_mel: 17.366376876831055 (17.785769487356212) + | > loss_duration: 1.487438678741455 (1.4842017022046177) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.45746421813965 (30.6035105469939) + | > grad_norm_1: tensor(292.3840, device='cuda:0') (tensor(254.9550, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.3928 (0.360163029138144) + | > loader_time: 0.0065 (0.005669268694790926) + + + --> TIME: 2025-04-19 00:37:46 -- STEP: 102/142 -- GLOBAL_STEP: 393300 + | > loss_disc: 2.619539976119995 (2.4714680746489877) + | > loss_disc_real_0: 0.1480124443769455 (0.12997176373998331) + | > loss_disc_real_1: 0.2192520797252655 (0.20571704457203546) + | > loss_disc_real_2: 0.2267957627773285 (0.2268874109083531) + | > loss_disc_real_3: 0.27767786383628845 (0.2290250888349963) + | > loss_disc_real_4: 0.2488129734992981 (0.22947627189112643) + | > loss_disc_real_5: 0.2668949067592621 (0.2341922674985493) + | > loss_0: 2.619539976119995 (2.4714680746489877) + | > grad_norm_0: tensor(12.8108, device='cuda:0') (tensor(12.5242, device='cuda:0')) + | > loss_gen: 2.310657024383545 (2.3461699649399406) + | > loss_kl: 1.9711071252822876 (1.9066455387601666) + | > loss_feat: 6.640120983123779 (7.001189367443907) + | > loss_mel: 17.304210662841797 (17.743287151935046) + | > loss_duration: 1.5073132514953613 (1.4952453024247114) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.733409881591797 (30.492537367577647) + | > grad_norm_1: tensor(260.0753, device='cuda:0') (tensor(248.3330, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.421 (0.37140529997208543) + | > loader_time: 0.0072 (0.005990395358964509) + + + --> TIME: 2025-04-19 00:37:57 -- STEP: 127/142 -- GLOBAL_STEP: 393325 + | > loss_disc: 2.5144898891448975 (2.4770674217404345) + | > loss_disc_real_0: 0.1301635503768921 (0.13008552748621918) + | > loss_disc_real_1: 0.2026788294315338 (0.20600110222035506) + | > loss_disc_real_2: 0.2486754059791565 (0.22766012974142089) + | > loss_disc_real_3: 0.25765591859817505 (0.229788973223506) + | > loss_disc_real_4: 0.24680565297603607 (0.230543623407056) + | > loss_disc_real_5: 0.21611836552619934 (0.2350892514929058) + | > loss_0: 2.5144898891448975 (2.4770674217404345) + | > grad_norm_0: tensor(4.8198, device='cuda:0') (tensor(11.4988, device='cuda:0')) + | > loss_gen: 2.3495776653289795 (2.3346303466736793) + | > loss_kl: 1.966998815536499 (1.9105874450187983) + | > loss_feat: 6.303309917449951 (6.932631030796081) + | > loss_mel: 16.65611457824707 (17.6632571558314) + | > loss_duration: 1.5263632535934448 (1.502242365221339) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.8023624420166 (30.343348345418615) + | > grad_norm_1: tensor(123.8919, device='cuda:0') (tensor(226.5177, device='cuda:0')) + | > current_lr_0: 0.0001975647938884664 + | > current_lr_1: 0.0001975647938884664 + | > step_time: 0.4397 (0.38314126044746466) + | > loader_time: 0.0072 (0.006243559319203294) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.531327724456787 (2.531327724456787) + | > loss_disc_real_0: 0.12683168053627014 (0.12683168053627014) + | > loss_disc_real_1: 0.19509099423885345 (0.19509099423885345) + | > loss_disc_real_2: 0.2546972632408142 (0.2546972632408142) + | > loss_disc_real_3: 0.23856975138187408 (0.23856975138187408) + | > loss_disc_real_4: 0.21033711731433868 (0.21033711731433868) + | > loss_disc_real_5: 0.2400602102279663 (0.2400602102279663) + | > loss_0: 2.531327724456787 (2.531327724456787) + | > loss_gen: 2.189446449279785 (2.189446449279785) + | > loss_kl: 2.31430983543396 (2.31430983543396) + | > loss_feat: 6.752213001251221 (6.752213001251221) + | > loss_mel: 18.039823532104492 (18.039823532104492) + | > loss_duration: 1.6392669677734375 (1.6392669677734375) + | > loss_1: 30.93505859375 (30.93505859375) + + --> STEP: 1 + | > loss_disc: 2.4816932678222656 (2.4816932678222656) + | > loss_disc_real_0: 0.1679336130619049 (0.1679336130619049) + | > loss_disc_real_1: 0.1986899971961975 (0.1986899971961975) + | > loss_disc_real_2: 0.2530825436115265 (0.2530825436115265) + | > loss_disc_real_3: 0.25230172276496887 (0.25230172276496887) + | > loss_disc_real_4: 0.23129720985889435 (0.23129720985889435) + | > loss_disc_real_5: 0.2358245551586151 (0.2358245551586151) + | > loss_0: 2.4816932678222656 (2.4816932678222656) + | > loss_gen: 2.2703001499176025 (2.2703001499176025) + | > loss_kl: 2.5955233573913574 (2.5955233573913574) + | > loss_feat: 6.703635215759277 (6.703635215759277) + | > loss_mel: 17.91358757019043 (17.91358757019043) + | > loss_duration: 1.6690995693206787 (1.6690995693206787) + | > loss_1: 31.152145385742188 (31.152145385742188) + + --> STEP: 2 + | > loss_disc: 2.4303271770477295 (2.4560102224349976) + | > loss_disc_real_0: 0.1446571797132492 (0.15629539638757706) + | > loss_disc_real_1: 0.19679482281208038 (0.19774241000413895) + | > loss_disc_real_2: 0.2637311518192291 (0.2584068477153778) + | > loss_disc_real_3: 0.26423823833465576 (0.2582699805498123) + | > loss_disc_real_4: 0.23419226706027985 (0.2327447384595871) + | > loss_disc_real_5: 0.23435313999652863 (0.23508884757757187) + | > loss_0: 2.4303271770477295 (2.4560102224349976) + | > loss_gen: 2.372795343399048 (2.321547746658325) + | > loss_kl: 2.2177658081054688 (2.406644582748413) + | > loss_feat: 7.175140857696533 (6.939388036727905) + | > loss_mel: 18.713979721069336 (18.313783645629883) + | > loss_duration: 1.620858907699585 (1.6449792385101318) + | > loss_1: 32.10054016113281 (31.6263427734375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036096572875976562 (-0.0005004405975341797) + | > avg_loss_disc: 2.4560102224349976 (-0.09473109245300293) + | > avg_loss_disc_real_0: 0.15629539638757706 (-0.004028588533401489) + | > avg_loss_disc_real_1: 0.19774241000413895 (-0.0558459609746933) + | > avg_loss_disc_real_2: 0.2584068477153778 (-0.005151085555553436) + | > avg_loss_disc_real_3: 0.2582699805498123 (-0.0009286850690841675) + | > avg_loss_disc_real_4: 0.2327447384595871 (+0.015467561781406403) + | > avg_loss_disc_real_5: 0.23508884757757187 (-0.055634744465351105) + | > avg_loss_0: 2.4560102224349976 (-0.09473109245300293) + | > avg_loss_gen: 2.321547746658325 (-0.13029205799102783) + | > avg_loss_kl: 2.406644582748413 (+0.14766180515289307) + | > avg_loss_feat: 6.939388036727905 (+0.1881399154663086) + | > avg_loss_mel: 18.313783645629883 (+0.045241355895996094) + | > avg_loss_duration: 1.6449792385101318 (-0.0016604065895080566) + | > avg_loss_1: 31.6263427734375 (+0.24909019470214844) + + + > EPOCH: 99/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:38:05)  + + --> TIME: 2025-04-19 00:38:09 -- STEP: 10/142 -- GLOBAL_STEP: 393350 + | > loss_disc: 2.397667407989502 (2.4265851736068726) + | > loss_disc_real_0: 0.11425447463989258 (0.12018536925315856) + | > loss_disc_real_1: 0.19996705651283264 (0.20549863278865815) + | > loss_disc_real_2: 0.24896271526813507 (0.22964493036270142) + | > loss_disc_real_3: 0.22710971534252167 (0.22578029632568358) + | > loss_disc_real_4: 0.2307923287153244 (0.23360075503587724) + | > loss_disc_real_5: 0.22388222813606262 (0.2207889065146446) + | > loss_0: 2.397667407989502 (2.4265851736068726) + | > grad_norm_0: tensor(3.5073, device='cuda:0') (tensor(7.6673, device='cuda:0')) + | > loss_gen: 2.3940374851226807 (2.364292097091675) + | > loss_kl: 1.9542949199676514 (1.84092036485672) + | > loss_feat: 7.195989608764648 (7.0128316402435305) + | > loss_mel: 18.623004913330078 (17.78066349029541) + | > loss_duration: 1.4303929805755615 (1.4542802214622497) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.597719192504883 (30.4529878616333) + | > grad_norm_1: tensor(154.1836, device='cuda:0') (tensor(162.4010, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.3338 (0.32494559288024905) + | > loader_time: 0.0052 (0.004842519760131836) + + + --> TIME: 2025-04-19 00:38:18 -- STEP: 35/142 -- GLOBAL_STEP: 393375 + | > loss_disc: 2.3695647716522217 (2.4318715299878804) + | > loss_disc_real_0: 0.09048294275999069 (0.12145184023039682) + | > loss_disc_real_1: 0.18518851697444916 (0.2021021327802113) + | > loss_disc_real_2: 0.19482877850532532 (0.22460462834153858) + | > loss_disc_real_3: 0.19866842031478882 (0.22738992699554988) + | > loss_disc_real_4: 0.20180033147335052 (0.22834851230893816) + | > loss_disc_real_5: 0.18086117506027222 (0.2275710621050426) + | > loss_0: 2.3695647716522217 (2.4318715299878804) + | > grad_norm_0: tensor(9.6373, device='cuda:0') (tensor(7.7819, device='cuda:0')) + | > loss_gen: 2.3761703968048096 (2.3618165152413506) + | > loss_kl: 1.7119367122650146 (1.8649564674922399) + | > loss_feat: 7.454139709472656 (7.1265197890145435) + | > loss_mel: 18.2481632232666 (17.93157059805734) + | > loss_duration: 1.4873502254486084 (1.4645322288785663) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.277761459350586 (30.749395697457448) + | > grad_norm_1: tensor(162.2351, device='cuda:0') (tensor(147.2734, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.3552 (0.3384114946637835) + | > loader_time: 0.0057 (0.005346775054931641) + + + --> TIME: 2025-04-19 00:38:27 -- STEP: 60/142 -- GLOBAL_STEP: 393400 + | > loss_disc: 2.564232587814331 (2.4584088921546927) + | > loss_disc_real_0: 0.1369759440422058 (0.12818821892142296) + | > loss_disc_real_1: 0.22420305013656616 (0.20601337403059006) + | > loss_disc_real_2: 0.19967716932296753 (0.2257181187470754) + | > loss_disc_real_3: 0.2089063823223114 (0.22698185990254086) + | > loss_disc_real_4: 0.22519603371620178 (0.23083791161576908) + | > loss_disc_real_5: 0.2190704643726349 (0.22926688094933828) + | > loss_0: 2.564232587814331 (2.4584088921546927) + | > grad_norm_0: tensor(5.2130, device='cuda:0') (tensor(7.7502, device='cuda:0')) + | > loss_gen: 2.230415105819702 (2.350192777315776) + | > loss_kl: 1.961674690246582 (1.8674971302350363) + | > loss_feat: 6.835384368896484 (7.0042189677556355) + | > loss_mel: 18.30767059326172 (17.78932113647461) + | > loss_duration: 1.505950689315796 (1.4750255266825358) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.841094970703125 (30.48625555038452) + | > grad_norm_1: tensor(122.6686, device='cuda:0') (tensor(149.2820, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.3748 (0.34978053967158) + | > loader_time: 0.0065 (0.0056997855504353845) + + + --> TIME: 2025-04-19 00:38:37 -- STEP: 85/142 -- GLOBAL_STEP: 393425 + | > loss_disc: 2.4686436653137207 (2.487025549832511) + | > loss_disc_real_0: 0.12112133949995041 (0.1320450495271122) + | > loss_disc_real_1: 0.19130295515060425 (0.20950191266396467) + | > loss_disc_real_2: 0.23688574135303497 (0.22798440719352048) + | > loss_disc_real_3: 0.22666877508163452 (0.2284629804246566) + | > loss_disc_real_4: 0.2091786116361618 (0.23052150572047514) + | > loss_disc_real_5: 0.18688887357711792 (0.23245155425632702) + | > loss_0: 2.4686436653137207 (2.487025549832511) + | > grad_norm_0: tensor(5.7347, device='cuda:0') (tensor(8.0549, device='cuda:0')) + | > loss_gen: 2.3589766025543213 (2.320463087979485) + | > loss_kl: 1.9360431432724 (1.88720762869891) + | > loss_feat: 5.945146083831787 (6.867385426689597) + | > loss_mel: 17.460447311401367 (17.69726667965161) + | > loss_duration: 1.5175912380218506 (1.4865532019559078) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.218204498291016 (30.258876104915842) + | > grad_norm_1: tensor(181.9442, device='cuda:0') (tensor(156.6914, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.3985 (0.36248140335083) + | > loader_time: 0.0066 (0.00594799378338982) + + + --> TIME: 2025-04-19 00:38:48 -- STEP: 110/142 -- GLOBAL_STEP: 393450 + | > loss_disc: 2.4760355949401855 (2.483826082403008) + | > loss_disc_real_0: 0.1094823032617569 (0.13163690939545628) + | > loss_disc_real_1: 0.1902342438697815 (0.20778433382511138) + | > loss_disc_real_2: 0.21552926301956177 (0.2276910280639475) + | > loss_disc_real_3: 0.24848853051662445 (0.22923781316388736) + | > loss_disc_real_4: 0.22768403589725494 (0.23096078376878393) + | > loss_disc_real_5: 0.2433278113603592 (0.23304313123226167) + | > loss_0: 2.4760355949401855 (2.483826082403008) + | > grad_norm_0: tensor(9.9829, device='cuda:0') (tensor(8.9408, device='cuda:0')) + | > loss_gen: 2.1797256469726562 (2.319849716533313) + | > loss_kl: 1.9005229473114014 (1.8969729618592697) + | > loss_feat: 6.50826358795166 (6.846950314261696) + | > loss_mel: 17.741043090820312 (17.7170269359242) + | > loss_duration: 1.5091030597686768 (1.4955622109499846) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.838659286499023 (30.276362228393555) + | > grad_norm_1: tensor(207.4543, device='cuda:0') (tensor(188.5411, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.4305 (0.37380225875160905) + | > loader_time: 0.0077 (0.006188830462369052) + + + --> TIME: 2025-04-19 00:38:59 -- STEP: 135/142 -- GLOBAL_STEP: 393475 + | > loss_disc: 2.4410529136657715 (2.484920642994067) + | > loss_disc_real_0: 0.1434779316186905 (0.13231115214250705) + | > loss_disc_real_1: 0.198491632938385 (0.20772219167815315) + | > loss_disc_real_2: 0.24017658829689026 (0.22720950058212988) + | > loss_disc_real_3: 0.2337787002325058 (0.22829828339594382) + | > loss_disc_real_4: 0.21367940306663513 (0.23096225692166222) + | > loss_disc_real_5: 0.23973241448402405 (0.2333027474306248) + | > loss_0: 2.4410529136657715 (2.484920642994067) + | > grad_norm_0: tensor(7.5749, device='cuda:0') (tensor(9.2747, device='cuda:0')) + | > loss_gen: 2.3058314323425293 (2.31761876742045) + | > loss_kl: 1.8293410539627075 (1.901753341710126) + | > loss_feat: 6.900779724121094 (6.811500220828585) + | > loss_mel: 17.966596603393555 (17.69001989717838) + | > loss_duration: 1.4890269041061401 (1.5019855490437262) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.491575241088867 (30.2228778273971) + | > grad_norm_1: tensor(87.1298, device='cuda:0') (tensor(195.1737, device='cuda:0')) + | > current_lr_0: 0.00019754009828923033 + | > current_lr_1: 0.00019754009828923033 + | > step_time: 0.4571 (0.38577509986029723) + | > loader_time: 0.0083 (0.006429096504493996) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4708967208862305 (2.4708967208862305) + | > loss_disc_real_0: 0.13401900231838226 (0.13401900231838226) + | > loss_disc_real_1: 0.2338937371969223 (0.2338937371969223) + | > loss_disc_real_2: 0.26332685351371765 (0.26332685351371765) + | > loss_disc_real_3: 0.21679314970970154 (0.21679314970970154) + | > loss_disc_real_4: 0.213451087474823 (0.213451087474823) + | > loss_disc_real_5: 0.25400209426879883 (0.25400209426879883) + | > loss_0: 2.4708967208862305 (2.4708967208862305) + | > loss_gen: 2.2797555923461914 (2.2797555923461914) + | > loss_kl: 2.2797787189483643 (2.2797787189483643) + | > loss_feat: 6.946300029754639 (6.946300029754639) + | > loss_mel: 18.61648941040039 (18.61648941040039) + | > loss_duration: 1.6442192792892456 (1.6442192792892456) + | > loss_1: 31.76654052734375 (31.76654052734375) + + --> STEP: 1 + | > loss_disc: 2.3757848739624023 (2.3757848739624023) + | > loss_disc_real_0: 0.16649962961673737 (0.16649962961673737) + | > loss_disc_real_1: 0.2223614603281021 (0.2223614603281021) + | > loss_disc_real_2: 0.25945454835891724 (0.25945454835891724) + | > loss_disc_real_3: 0.22873540222644806 (0.22873540222644806) + | > loss_disc_real_4: 0.2251230925321579 (0.2251230925321579) + | > loss_disc_real_5: 0.23697714507579803 (0.23697714507579803) + | > loss_0: 2.3757848739624023 (2.3757848739624023) + | > loss_gen: 2.4494776725769043 (2.4494776725769043) + | > loss_kl: 2.454524517059326 (2.454524517059326) + | > loss_feat: 6.738826751708984 (6.738826751708984) + | > loss_mel: 19.35772132873535 (19.35772132873535) + | > loss_duration: 1.667099118232727 (1.667099118232727) + | > loss_1: 32.66765213012695 (32.66765213012695) + + --> STEP: 2 + | > loss_disc: 2.4905636310577393 (2.433174252510071) + | > loss_disc_real_0: 0.11037091910839081 (0.1384352743625641) + | > loss_disc_real_1: 0.2840820848941803 (0.2532217726111412) + | > loss_disc_real_2: 0.266131192445755 (0.2627928704023361) + | > loss_disc_real_3: 0.22509486973285675 (0.2269151359796524) + | > loss_disc_real_4: 0.2193930447101593 (0.2222580686211586) + | > loss_disc_real_5: 0.293714314699173 (0.2653457298874855) + | > loss_0: 2.4905636310577393 (2.433174252510071) + | > loss_gen: 2.52811336517334 (2.488795518875122) + | > loss_kl: 2.2804148197174072 (2.3674696683883667) + | > loss_feat: 7.649811267852783 (7.194319009780884) + | > loss_mel: 18.71609878540039 (19.03691005706787) + | > loss_duration: 1.6408560276031494 (1.6539775729179382) + | > loss_1: 32.8152961730957 (32.74147415161133) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038083791732788086 (+0.00019872188568115234) + | > avg_loss_disc: 2.433174252510071 (-0.022835969924926758) + | > avg_loss_disc_real_0: 0.1384352743625641 (-0.01786012202501297) + | > avg_loss_disc_real_1: 0.2532217726111412 (+0.05547936260700226) + | > avg_loss_disc_real_2: 0.2627928704023361 (+0.004386022686958313) + | > avg_loss_disc_real_3: 0.2269151359796524 (-0.03135484457015991) + | > avg_loss_disc_real_4: 0.2222580686211586 (-0.010486669838428497) + | > avg_loss_disc_real_5: 0.2653457298874855 (+0.030256882309913635) + | > avg_loss_0: 2.433174252510071 (-0.022835969924926758) + | > avg_loss_gen: 2.488795518875122 (+0.16724777221679688) + | > avg_loss_kl: 2.3674696683883667 (-0.03917491436004639) + | > avg_loss_feat: 7.194319009780884 (+0.2549309730529785) + | > avg_loss_mel: 19.03691005706787 (+0.7231264114379883) + | > avg_loss_duration: 1.6539775729179382 (+0.008998334407806396) + | > avg_loss_1: 32.74147415161133 (+1.1151313781738281) + + + > EPOCH: 100/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:39:04)  + + --> TIME: 2025-04-19 00:39:11 -- STEP: 18/142 -- GLOBAL_STEP: 393500 + | > loss_disc: 2.505364179611206 (2.429251061545478) + | > loss_disc_real_0: 0.15935766696929932 (0.12417900479502147) + | > loss_disc_real_1: 0.2554113268852234 (0.20833622167507806) + | > loss_disc_real_2: 0.2456737905740738 (0.23276970287164053) + | > loss_disc_real_3: 0.22344310581684113 (0.22330421126551098) + | > loss_disc_real_4: 0.2696315348148346 (0.218497590886222) + | > loss_disc_real_5: 0.23984137177467346 (0.23195143623484504) + | > loss_0: 2.505364179611206 (2.429251061545478) + | > grad_norm_0: tensor(21.7298, device='cuda:0') (tensor(12.7195, device='cuda:0')) + | > loss_gen: 2.308041572570801 (2.3794265455669827) + | > loss_kl: 1.812565565109253 (1.8639098869429693) + | > loss_feat: 6.321872234344482 (7.039855877558391) + | > loss_mel: 17.421016693115234 (17.783012496100532) + | > loss_duration: 1.4660558700561523 (1.451693872610728) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.329551696777344 (30.51789887746175) + | > grad_norm_1: tensor(219.8651, device='cuda:0') (tensor(303.6857, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.3554 (0.33340545495351154) + | > loader_time: 0.0054 (0.005010233985053169) + + + --> TIME: 2025-04-19 00:39:20 -- STEP: 43/142 -- GLOBAL_STEP: 393525 + | > loss_disc: 2.5116989612579346 (2.4728706603826476) + | > loss_disc_real_0: 0.09797917306423187 (0.12861184342656024) + | > loss_disc_real_1: 0.23896390199661255 (0.2073643786269565) + | > loss_disc_real_2: 0.24697835743427277 (0.22978804381780846) + | > loss_disc_real_3: 0.2097654789686203 (0.228972120687019) + | > loss_disc_real_4: 0.23175901174545288 (0.22826425142066423) + | > loss_disc_real_5: 0.2399846613407135 (0.23480266991049745) + | > loss_0: 2.5116989612579346 (2.4728706603826476) + | > grad_norm_0: tensor(5.8897, device='cuda:0') (tensor(12.5058, device='cuda:0')) + | > loss_gen: 2.2863988876342773 (2.335338476092316) + | > loss_kl: 1.7688060998916626 (1.8708957544592923) + | > loss_feat: 7.176122188568115 (6.900379014569659) + | > loss_mel: 19.78620719909668 (17.675704911697743) + | > loss_duration: 1.4760363101959229 (1.468031644821167) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.49357223510742 (30.25034997629565) + | > grad_norm_1: tensor(199.5264, device='cuda:0') (tensor(287.2935, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.3621 (0.34493073197298274) + | > loader_time: 0.0054 (0.005363541980122411) + + + --> TIME: 2025-04-19 00:39:29 -- STEP: 68/142 -- GLOBAL_STEP: 393550 + | > loss_disc: 2.450744390487671 (2.4756924930740807) + | > loss_disc_real_0: 0.08734463900327682 (0.12843872617711033) + | > loss_disc_real_1: 0.2008386105298996 (0.20719513187513633) + | > loss_disc_real_2: 0.2124234437942505 (0.22748467304250775) + | > loss_disc_real_3: 0.20260819792747498 (0.22852632162325523) + | > loss_disc_real_4: 0.18676133453845978 (0.2266309351167258) + | > loss_disc_real_5: 0.23646298050880432 (0.23567770070889416) + | > loss_0: 2.450744390487671 (2.4756924930740807) + | > grad_norm_0: tensor(9.1579, device='cuda:0') (tensor(13.5273, device='cuda:0')) + | > loss_gen: 2.3575997352600098 (2.3419633647974796) + | > loss_kl: 1.850685715675354 (1.8928302666720223) + | > loss_feat: 7.343865394592285 (6.957076297086828) + | > loss_mel: 17.3989315032959 (17.778390968547146) + | > loss_duration: 1.4894853830337524 (1.4793173179906958) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.440567016601562 (30.449578229118796) + | > grad_norm_1: tensor(293.4153, device='cuda:0') (tensor(288.5572, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.4236 (0.3571716161335216) + | > loader_time: 0.0064 (0.005783536854912253) + + + --> TIME: 2025-04-19 00:39:40 -- STEP: 93/142 -- GLOBAL_STEP: 393575 + | > loss_disc: 2.5213077068328857 (2.4825272765210875) + | > loss_disc_real_0: 0.11872673034667969 (0.13053299642858965) + | > loss_disc_real_1: 0.27646252512931824 (0.20906875819288273) + | > loss_disc_real_2: 0.2397921085357666 (0.22824606732014688) + | > loss_disc_real_3: 0.23940208554267883 (0.22848646791391475) + | > loss_disc_real_4: 0.23995836079120636 (0.2277474014028426) + | > loss_disc_real_5: 0.2588052749633789 (0.23737465069499067) + | > loss_0: 2.5213077068328857 (2.4825272765210875) + | > grad_norm_0: tensor(17.8954, device='cuda:0') (tensor(14.5564, device='cuda:0')) + | > loss_gen: 2.2697103023529053 (2.3429591040457454) + | > loss_kl: 1.8352282047271729 (1.9067535631118282) + | > loss_feat: 6.385988235473633 (6.942667556065385) + | > loss_mel: 17.85972023010254 (17.84467090073452) + | > loss_duration: 1.4989440441131592 (1.4920428299134778) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.849590301513672 (30.529094019243793) + | > grad_norm_1: tensor(450.1364, device='cuda:0') (tensor(295.1310, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.3991 (0.36862555114171836) + | > loader_time: 0.0071 (0.006024681111817719) + + + --> TIME: 2025-04-19 00:39:50 -- STEP: 118/142 -- GLOBAL_STEP: 393600 + | > loss_disc: 2.4247429370880127 (2.488815085362579) + | > loss_disc_real_0: 0.1876349002122879 (0.130969859564961) + | > loss_disc_real_1: 0.21879635751247406 (0.20877453583782002) + | > loss_disc_real_2: 0.2619174122810364 (0.22831854123180195) + | > loss_disc_real_3: 0.23865120112895966 (0.22909681865219342) + | > loss_disc_real_4: 0.2336626946926117 (0.22867121812650712) + | > loss_disc_real_5: 0.25176581740379333 (0.23811670481148411) + | > loss_0: 2.4247429370880127 (2.488815085362579) + | > grad_norm_0: tensor(18.9899, device='cuda:0') (tensor(14.0708, device='cuda:0')) + | > loss_gen: 2.4592413902282715 (2.334025308237238) + | > loss_kl: 1.9221316576004028 (1.9115389221805636) + | > loss_feat: 7.060972213745117 (6.913893016718202) + | > loss_mel: 18.4792537689209 (17.86397343166804) + | > loss_duration: 1.5003329515457153 (1.4987656100321625) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.421932220458984 (30.522196349451097) + | > grad_norm_1: tensor(329.2978, device='cuda:0') (tensor(284.8021, device='cuda:0')) + | > current_lr_0: 0.00019751540577694416 + | > current_lr_1: 0.00019751540577694416 + | > step_time: 0.4421 (0.3792184954982693) + | > loader_time: 0.0072 (0.006252500970484846) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5732643604278564 (2.5732643604278564) + | > loss_disc_real_0: 0.1310644894838333 (0.1310644894838333) + | > loss_disc_real_1: 0.3069957196712494 (0.3069957196712494) + | > loss_disc_real_2: 0.26531457901000977 (0.26531457901000977) + | > loss_disc_real_3: 0.2019762247800827 (0.2019762247800827) + | > loss_disc_real_4: 0.2371675819158554 (0.2371675819158554) + | > loss_disc_real_5: 0.23409207165241241 (0.23409207165241241) + | > loss_0: 2.5732643604278564 (2.5732643604278564) + | > loss_gen: 2.299834728240967 (2.299834728240967) + | > loss_kl: 2.1346499919891357 (2.1346499919891357) + | > loss_feat: 6.617701530456543 (6.617701530456543) + | > loss_mel: 19.194002151489258 (19.194002151489258) + | > loss_duration: 1.6325747966766357 (1.6325747966766357) + | > loss_1: 31.87876319885254 (31.87876319885254) + + --> STEP: 1 + | > loss_disc: 2.4762303829193115 (2.4762303829193115) + | > loss_disc_real_0: 0.14646027982234955 (0.14646027982234955) + | > loss_disc_real_1: 0.27524450421333313 (0.27524450421333313) + | > loss_disc_real_2: 0.2611332833766937 (0.2611332833766937) + | > loss_disc_real_3: 0.2015126794576645 (0.2015126794576645) + | > loss_disc_real_4: 0.2394130378961563 (0.2394130378961563) + | > loss_disc_real_5: 0.23415710031986237 (0.23415710031986237) + | > loss_0: 2.4762303829193115 (2.4762303829193115) + | > loss_gen: 2.336473226547241 (2.336473226547241) + | > loss_kl: 2.2598047256469727 (2.2598047256469727) + | > loss_feat: 6.461400985717773 (6.461400985717773) + | > loss_mel: 17.9660587310791 (17.9660587310791) + | > loss_duration: 1.6887359619140625 (1.6887359619140625) + | > loss_1: 30.712472915649414 (30.712472915649414) + + --> STEP: 2 + | > loss_disc: 2.4138307571411133 (2.4450305700302124) + | > loss_disc_real_0: 0.11938279867172241 (0.13292153924703598) + | > loss_disc_real_1: 0.24629178643226624 (0.2607681453227997) + | > loss_disc_real_2: 0.24662770330905914 (0.25388049334287643) + | > loss_disc_real_3: 0.19812016189098358 (0.19981642067432404) + | > loss_disc_real_4: 0.21267947554588318 (0.22604625672101974) + | > loss_disc_real_5: 0.2115471214056015 (0.22285211086273193) + | > loss_0: 2.4138307571411133 (2.4450305700302124) + | > loss_gen: 2.3702220916748047 (2.353347659111023) + | > loss_kl: 1.9491475820541382 (2.1044761538505554) + | > loss_feat: 7.190662860870361 (6.826031923294067) + | > loss_mel: 19.171327590942383 (18.568693161010742) + | > loss_duration: 1.615767478942871 (1.6522517204284668) + | > loss_1: 32.29712677001953 (31.504799842834473) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003875255584716797 (+6.687641143798828e-05) + | > avg_loss_disc: 2.4450305700302124 (+0.011856317520141602) + | > avg_loss_disc_real_0: 0.13292153924703598 (-0.005513735115528107) + | > avg_loss_disc_real_1: 0.2607681453227997 (+0.007546372711658478) + | > avg_loss_disc_real_2: 0.25388049334287643 (-0.008912377059459686) + | > avg_loss_disc_real_3: 0.19981642067432404 (-0.02709871530532837) + | > avg_loss_disc_real_4: 0.22604625672101974 (+0.003788188099861145) + | > avg_loss_disc_real_5: 0.22285211086273193 (-0.04249361902475357) + | > avg_loss_0: 2.4450305700302124 (+0.011856317520141602) + | > avg_loss_gen: 2.353347659111023 (-0.13544785976409912) + | > avg_loss_kl: 2.1044761538505554 (-0.2629935145378113) + | > avg_loss_feat: 6.826031923294067 (-0.3682870864868164) + | > avg_loss_mel: 18.568693161010742 (-0.4682168960571289) + | > avg_loss_duration: 1.6522517204284668 (-0.0017258524894714355) + | > avg_loss_1: 31.504799842834473 (-1.2366743087768555) + + + > EPOCH: 101/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:40:03)  + + --> TIME: 2025-04-19 00:40:04 -- STEP: 1/142 -- GLOBAL_STEP: 393625 + | > loss_disc: 2.444016456604004 (2.444016456604004) + | > loss_disc_real_0: 0.11360664665699005 (0.11360664665699005) + | > loss_disc_real_1: 0.22683529555797577 (0.22683529555797577) + | > loss_disc_real_2: 0.26427385210990906 (0.26427385210990906) + | > loss_disc_real_3: 0.22075651586055756 (0.22075651586055756) + | > loss_disc_real_4: 0.2184109091758728 (0.2184109091758728) + | > loss_disc_real_5: 0.2401215136051178 (0.2401215136051178) + | > loss_0: 2.444016456604004 (2.444016456604004) + | > grad_norm_0: tensor(7.8823, device='cuda:0') (tensor(7.8823, device='cuda:0')) + | > loss_gen: 2.3620505332946777 (2.3620505332946777) + | > loss_kl: 1.7208670377731323 (1.7208670377731323) + | > loss_feat: 6.969982624053955 (6.969982624053955) + | > loss_mel: 18.356172561645508 (18.356172561645508) + | > loss_duration: 1.4240235090255737 (1.4240235090255737) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.83309555053711 (30.83309555053711) + | > grad_norm_1: tensor(333.1823, device='cuda:0') (tensor(333.1823, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.3278 (0.32778286933898926) + | > loader_time: 0.0041 (0.004052400588989258) + + + --> TIME: 2025-04-19 00:40:12 -- STEP: 26/142 -- GLOBAL_STEP: 393650 + | > loss_disc: 2.4541420936584473 (2.4505076500085683) + | > loss_disc_real_0: 0.09633299708366394 (0.1180472755088256) + | > loss_disc_real_1: 0.17851796746253967 (0.20843835232349542) + | > loss_disc_real_2: 0.2218952775001526 (0.22825724516923612) + | > loss_disc_real_3: 0.24793383479118347 (0.23108368137708077) + | > loss_disc_real_4: 0.23858880996704102 (0.23027735948562622) + | > loss_disc_real_5: 0.25130027532577515 (0.2368547601195482) + | > loss_0: 2.4541420936584473 (2.4505076500085683) + | > grad_norm_0: tensor(9.7519, device='cuda:0') (tensor(11.4150, device='cuda:0')) + | > loss_gen: 2.335956335067749 (2.3732456794151897) + | > loss_kl: 1.8559398651123047 (1.849227854838738) + | > loss_feat: 6.392843723297119 (7.070410196597759) + | > loss_mel: 16.393091201782227 (18.030175942641037) + | > loss_duration: 1.4796626567840576 (1.4637644428473253) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.45749282836914 (30.786824153019833) + | > grad_norm_1: tensor(356.8078, device='cuda:0') (tensor(337.2445, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.3506 (0.3334863460980929) + | > loader_time: 0.0056 (0.005071988472571741) + + + --> TIME: 2025-04-19 00:40:22 -- STEP: 51/142 -- GLOBAL_STEP: 393675 + | > loss_disc: 2.702972888946533 (2.468939290327184) + | > loss_disc_real_0: 0.3281511068344116 (0.1289349741795484) + | > loss_disc_real_1: 0.24294224381446838 (0.20939687157378478) + | > loss_disc_real_2: 0.2385014444589615 (0.2253992998716878) + | > loss_disc_real_3: 0.24041755497455597 (0.22762688322394503) + | > loss_disc_real_4: 0.22147393226623535 (0.22850814257182328) + | > loss_disc_real_5: 0.23467563092708588 (0.23362998226109674) + | > loss_0: 2.702972888946533 (2.468939290327184) + | > grad_norm_0: tensor(51.8843, device='cuda:0') (tensor(17.0631, device='cuda:0')) + | > loss_gen: 2.290286064147949 (2.389326782787547) + | > loss_kl: 1.820739984512329 (1.8584122003293504) + | > loss_feat: 6.269378662109375 (7.032057126363118) + | > loss_mel: 16.589326858520508 (17.92872945000143) + | > loss_duration: 1.4714937210083008 (1.4709646070704743) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.44122314453125 (30.679490108116) + | > grad_norm_1: tensor(196.0077, device='cuda:0') (tensor(346.1338, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.3791 (0.3485436860252829) + | > loader_time: 0.0062 (0.005463483286838907) + + + --> TIME: 2025-04-19 00:40:32 -- STEP: 76/142 -- GLOBAL_STEP: 393700 + | > loss_disc: 2.560612201690674 (2.46800045276943) + | > loss_disc_real_0: 0.10656706243753433 (0.12888315496476074) + | > loss_disc_real_1: 0.2501468360424042 (0.20840121413532053) + | > loss_disc_real_2: 0.2649724781513214 (0.2261406824384865) + | > loss_disc_real_3: 0.2509988248348236 (0.22836335121016754) + | > loss_disc_real_4: 0.27070221304893494 (0.22838685053743815) + | > loss_disc_real_5: 0.2867298126220703 (0.23522844753767314) + | > loss_0: 2.560612201690674 (2.46800045276943) + | > grad_norm_0: tensor(13.3559, device='cuda:0') (tensor(18.1470, device='cuda:0')) + | > loss_gen: 2.460986375808716 (2.3821653033557695) + | > loss_kl: 1.9322398900985718 (1.8768473440094997) + | > loss_feat: 6.689220428466797 (6.984051284037139) + | > loss_mel: 18.737449645996094 (17.897645021739752) + | > loss_duration: 1.5114829540252686 (1.4837116254003422) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.331378936767578 (30.624420592659398) + | > grad_norm_1: tensor(259.3865, device='cuda:0') (tensor(333.6669, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.4042 (0.3633214235305786) + | > loader_time: 0.0068 (0.005848734002364309) + + + --> TIME: 2025-04-19 00:40:42 -- STEP: 101/142 -- GLOBAL_STEP: 393725 + | > loss_disc: 2.4533286094665527 (2.479154782720132) + | > loss_disc_real_0: 0.1536792367696762 (0.13027783335730575) + | > loss_disc_real_1: 0.20890429615974426 (0.20907442670057316) + | > loss_disc_real_2: 0.22552065551280975 (0.22681035839094973) + | > loss_disc_real_3: 0.24537234008312225 (0.22912075687753092) + | > loss_disc_real_4: 0.2093445360660553 (0.2302574108142664) + | > loss_disc_real_5: 0.26661914587020874 (0.2366152016243132) + | > loss_0: 2.4533286094665527 (2.479154782720132) + | > grad_norm_0: tensor(5.3269, device='cuda:0') (tensor(15.7446, device='cuda:0')) + | > loss_gen: 2.392162561416626 (2.35526684723278) + | > loss_kl: 1.9752472639083862 (1.8795292719756023) + | > loss_feat: 6.7285542488098145 (6.892014975595002) + | > loss_mel: 17.81665802001953 (17.831442540234864) + | > loss_duration: 1.5385725498199463 (1.4949432103940756) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.45119285583496 (30.453196780516368) + | > grad_norm_1: tensor(221.5583, device='cuda:0') (tensor(298.3183, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.4299 (0.37568237993976855) + | > loader_time: 0.0073 (0.006152311173996122) + + + --> TIME: 2025-04-19 00:40:54 -- STEP: 126/142 -- GLOBAL_STEP: 393750 + | > loss_disc: 2.4657630920410156 (2.4787225231291763) + | > loss_disc_real_0: 0.12691856920719147 (0.13033020076534102) + | > loss_disc_real_1: 0.19030791521072388 (0.20921953862148618) + | > loss_disc_real_2: 0.23540087044239044 (0.2257881492139801) + | > loss_disc_real_3: 0.2278400957584381 (0.2285655437717362) + | > loss_disc_real_4: 0.2660519778728485 (0.23003103418482673) + | > loss_disc_real_5: 0.23194418847560883 (0.23565884987040173) + | > loss_0: 2.4657630920410156 (2.4787225231291763) + | > grad_norm_0: tensor(9.2361, device='cuda:0') (tensor(14.1060, device='cuda:0')) + | > loss_gen: 2.3587844371795654 (2.3432635977154677) + | > loss_kl: 1.8811618089675903 (1.8870358779316856) + | > loss_feat: 6.71378231048584 (6.85453664688837) + | > loss_mel: 17.834453582763672 (17.791102591015047) + | > loss_duration: 1.5017850399017334 (1.5014770977080814) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.289966583251953 (30.37741582355802) + | > grad_norm_1: tensor(173.9914, device='cuda:0') (tensor(264.3599, device='cuda:0')) + | > current_lr_0: 0.00019749071635122203 + | > current_lr_1: 0.00019749071635122203 + | > step_time: 0.4365 (0.38965103929004974) + | > loader_time: 0.0076 (0.006421399494958301) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5041985511779785 (2.5041985511779785) + | > loss_disc_real_0: 0.14645890891551971 (0.14645890891551971) + | > loss_disc_real_1: 0.2255137413740158 (0.2255137413740158) + | > loss_disc_real_2: 0.23903444409370422 (0.23903444409370422) + | > loss_disc_real_3: 0.21806974709033966 (0.21806974709033966) + | > loss_disc_real_4: 0.2394072264432907 (0.2394072264432907) + | > loss_disc_real_5: 0.2716430723667145 (0.2716430723667145) + | > loss_0: 2.5041985511779785 (2.5041985511779785) + | > loss_gen: 2.455169200897217 (2.455169200897217) + | > loss_kl: 2.2593460083007812 (2.2593460083007812) + | > loss_feat: 6.96393346786499 (6.96393346786499) + | > loss_mel: 18.501367568969727 (18.501367568969727) + | > loss_duration: 1.631619930267334 (1.631619930267334) + | > loss_1: 31.811437606811523 (31.811437606811523) + + --> STEP: 1 + | > loss_disc: 2.5789904594421387 (2.5789904594421387) + | > loss_disc_real_0: 0.2549069821834564 (0.2549069821834564) + | > loss_disc_real_1: 0.22535495460033417 (0.22535495460033417) + | > loss_disc_real_2: 0.2735341489315033 (0.2735341489315033) + | > loss_disc_real_3: 0.22640928626060486 (0.22640928626060486) + | > loss_disc_real_4: 0.2815040051937103 (0.2815040051937103) + | > loss_disc_real_5: 0.21986304223537445 (0.21986304223537445) + | > loss_0: 2.5789904594421387 (2.5789904594421387) + | > loss_gen: 2.478019952774048 (2.478019952774048) + | > loss_kl: 2.557253122329712 (2.557253122329712) + | > loss_feat: 6.581481456756592 (6.581481456756592) + | > loss_mel: 18.975086212158203 (18.975086212158203) + | > loss_duration: 1.677657127380371 (1.677657127380371) + | > loss_1: 32.26949691772461 (32.26949691772461) + + --> STEP: 2 + | > loss_disc: 2.4964120388031006 (2.5377012491226196) + | > loss_disc_real_0: 0.23455801606178284 (0.24473249912261963) + | > loss_disc_real_1: 0.21966710686683655 (0.22251103073358536) + | > loss_disc_real_2: 0.25772523880004883 (0.26562969386577606) + | > loss_disc_real_3: 0.20748315751552582 (0.21694622188806534) + | > loss_disc_real_4: 0.23501983284950256 (0.25826191902160645) + | > loss_disc_real_5: 0.246987983584404 (0.23342551290988922) + | > loss_0: 2.4964120388031006 (2.5377012491226196) + | > loss_gen: 2.391333818435669 (2.4346768856048584) + | > loss_kl: 2.378730058670044 (2.467991590499878) + | > loss_feat: 6.211746692657471 (6.396614074707031) + | > loss_mel: 16.5035343170166 (17.739310264587402) + | > loss_duration: 1.613388180732727 (1.645522654056549) + | > loss_1: 29.09873390197754 (30.684115409851074) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036635398864746094 (-0.0002117156982421875) + | > avg_loss_disc: 2.5377012491226196 (+0.09267067909240723) + | > avg_loss_disc_real_0: 0.24473249912261963 (+0.11181095987558365) + | > avg_loss_disc_real_1: 0.22251103073358536 (-0.038257114589214325) + | > avg_loss_disc_real_2: 0.26562969386577606 (+0.011749200522899628) + | > avg_loss_disc_real_3: 0.21694622188806534 (+0.017129801213741302) + | > avg_loss_disc_real_4: 0.25826191902160645 (+0.0322156623005867) + | > avg_loss_disc_real_5: 0.23342551290988922 (+0.010573402047157288) + | > avg_loss_0: 2.5377012491226196 (+0.09267067909240723) + | > avg_loss_gen: 2.4346768856048584 (+0.08132922649383545) + | > avg_loss_kl: 2.467991590499878 (+0.3635154366493225) + | > avg_loss_feat: 6.396614074707031 (-0.42941784858703613) + | > avg_loss_mel: 17.739310264587402 (-0.8293828964233398) + | > avg_loss_duration: 1.645522654056549 (-0.006729066371917725) + | > avg_loss_1: 30.684115409851074 (-0.8206844329833984) + + + > EPOCH: 102/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:41:02)  + + --> TIME: 2025-04-19 00:41:06 -- STEP: 9/142 -- GLOBAL_STEP: 393775 + | > loss_disc: 2.420078992843628 (2.4540353616078696) + | > loss_disc_real_0: 0.09679926186800003 (0.1305649760696623) + | > loss_disc_real_1: 0.17868706583976746 (0.21072428756290013) + | > loss_disc_real_2: 0.24499359726905823 (0.23468190597163308) + | > loss_disc_real_3: 0.21172939240932465 (0.22567006448904672) + | > loss_disc_real_4: 0.20328080654144287 (0.2208874606423908) + | > loss_disc_real_5: 0.23497048020362854 (0.23470461699697706) + | > loss_0: 2.420078992843628 (2.4540353616078696) + | > grad_norm_0: tensor(7.3367, device='cuda:0') (tensor(9.5697, device='cuda:0')) + | > loss_gen: 2.4366888999938965 (2.3559382226732044) + | > loss_kl: 1.7800569534301758 (1.8540215094884236) + | > loss_feat: 6.900313854217529 (7.013696776496039) + | > loss_mel: 17.954574584960938 (17.655479855007595) + | > loss_duration: 1.4487431049346924 (1.4489849938286676) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.52037811279297 (30.328121609157986) + | > grad_norm_1: tensor(55.3733, device='cuda:0') (tensor(186.5574, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.3253 (0.3246067100101047) + | > loader_time: 0.0049 (0.0044363074832492405) + + + --> TIME: 2025-04-19 00:41:15 -- STEP: 34/142 -- GLOBAL_STEP: 393800 + | > loss_disc: 2.4954872131347656 (2.4345460779526653) + | > loss_disc_real_0: 0.1187896728515625 (0.12623944769010825) + | > loss_disc_real_1: 0.19827263057231903 (0.20338871286195867) + | > loss_disc_real_2: 0.2538072466850281 (0.2244600377538625) + | > loss_disc_real_3: 0.21362052857875824 (0.22680098037509358) + | > loss_disc_real_4: 0.2357919067144394 (0.22396481519236283) + | > loss_disc_real_5: 0.2359192967414856 (0.23306787802892573) + | > loss_0: 2.4954872131347656 (2.4345460779526653) + | > grad_norm_0: tensor(5.6054, device='cuda:0') (tensor(8.3550, device='cuda:0')) + | > loss_gen: 2.4078915119171143 (2.3761837272083057) + | > loss_kl: 1.875409722328186 (1.8460111653103548) + | > loss_feat: 7.320407390594482 (7.140138163286097) + | > loss_mel: 17.836198806762695 (17.858436023487766) + | > loss_duration: 1.4806439876556396 (1.4614221769220688) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.92055320739746 (30.68219134386848) + | > grad_norm_1: tensor(147.5145, device='cuda:0') (tensor(179.1178, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.3594 (0.3391057463253246) + | > loader_time: 0.0055 (0.005123257637023926) + + + --> TIME: 2025-04-19 00:41:24 -- STEP: 59/142 -- GLOBAL_STEP: 393825 + | > loss_disc: 2.4372637271881104 (2.4526130587367687) + | > loss_disc_real_0: 0.1064099669456482 (0.1248564702474465) + | > loss_disc_real_1: 0.21621094644069672 (0.20592459303847813) + | > loss_disc_real_2: 0.23382805287837982 (0.2254323661327362) + | > loss_disc_real_3: 0.23898504674434662 (0.22908389265254392) + | > loss_disc_real_4: 0.22811026871204376 (0.2275946253942231) + | > loss_disc_real_5: 0.19592076539993286 (0.2313321035797313) + | > loss_0: 2.4372637271881104 (2.4526130587367687) + | > grad_norm_0: tensor(11.2897, device='cuda:0') (tensor(8.6525, device='cuda:0')) + | > loss_gen: 2.3309316635131836 (2.361176284693055) + | > loss_kl: 1.6911832094192505 (1.8675373974492995) + | > loss_feat: 6.863529205322266 (7.0756939225277655) + | > loss_mel: 17.164026260375977 (17.85991933790304) + | > loss_duration: 1.489877700805664 (1.4740385059582986) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.539548873901367 (30.63836547075692) + | > grad_norm_1: tensor(341.3660, device='cuda:0') (tensor(194.7101, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.3805 (0.35250871464357536) + | > loader_time: 0.0058 (0.005512322409678314) + + + --> TIME: 2025-04-19 00:41:34 -- STEP: 84/142 -- GLOBAL_STEP: 393850 + | > loss_disc: 2.557421922683716 (2.4636147561527433) + | > loss_disc_real_0: 0.12810967862606049 (0.127507419813247) + | > loss_disc_real_1: 0.22836771607398987 (0.20565326494120417) + | > loss_disc_real_2: 0.249679297208786 (0.22623344936541148) + | > loss_disc_real_3: 0.22777101397514343 (0.22970279909315564) + | > loss_disc_real_4: 0.23276612162590027 (0.22959633845658528) + | > loss_disc_real_5: 0.27260127663612366 (0.2324859585080828) + | > loss_0: 2.557421922683716 (2.4636147561527433) + | > grad_norm_0: tensor(8.2100, device='cuda:0') (tensor(9.2065, device='cuda:0')) + | > loss_gen: 2.29191517829895 (2.353188991546631) + | > loss_kl: 1.9400604963302612 (1.8928895833946409) + | > loss_feat: 5.672317028045654 (6.9917007230577015) + | > loss_mel: 17.147876739501953 (17.782918089912048) + | > loss_duration: 1.535139799118042 (1.4855735344546181) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.587308883666992 (30.506270976293656) + | > grad_norm_1: tensor(127.3854, device='cuda:0') (tensor(195.6309, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.4183 (0.3644135622751146) + | > loader_time: 0.0073 (0.005838360105242049) + + + --> TIME: 2025-04-19 00:41:45 -- STEP: 109/142 -- GLOBAL_STEP: 393875 + | > loss_disc: 2.4683802127838135 (2.466748616017332) + | > loss_disc_real_0: 0.12218978255987167 (0.1294471058960355) + | > loss_disc_real_1: 0.1932261884212494 (0.20535748650174623) + | > loss_disc_real_2: 0.21099373698234558 (0.2263287632563792) + | > loss_disc_real_3: 0.19735965132713318 (0.22916893223556903) + | > loss_disc_real_4: 0.20543231070041656 (0.22900448329404954) + | > loss_disc_real_5: 0.22702457010746002 (0.23312350068617305) + | > loss_0: 2.4683802127838135 (2.466748616017332) + | > grad_norm_0: tensor(6.0901, device='cuda:0') (tensor(8.5913, device='cuda:0')) + | > loss_gen: 2.24898099899292 (2.3411968979266806) + | > loss_kl: 1.990221381187439 (1.9052974565313496) + | > loss_feat: 6.524657726287842 (6.969207531815275) + | > loss_mel: 17.21151351928711 (17.783469051396096) + | > loss_duration: 1.5362662076950073 (1.494106988294409) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.511640548706055 (30.493277925963795) + | > grad_norm_1: tensor(141.3396, device='cuda:0') (tensor(176.6779, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.4128 (0.3747276450515888) + | > loader_time: 0.0069 (0.006134726585598168) + + + --> TIME: 2025-04-19 00:41:56 -- STEP: 134/142 -- GLOBAL_STEP: 393900 + | > loss_disc: 2.577960968017578 (2.470699296068789) + | > loss_disc_real_0: 0.08674987405538559 (0.13053279019781014) + | > loss_disc_real_1: 0.1517946720123291 (0.2060122298660563) + | > loss_disc_real_2: 0.1886727511882782 (0.22617172360865037) + | > loss_disc_real_3: 0.229211688041687 (0.228995945916247) + | > loss_disc_real_4: 0.23896966874599457 (0.2287359444507912) + | > loss_disc_real_5: 0.28856825828552246 (0.23298616551641207) + | > loss_0: 2.577960968017578 (2.470699296068789) + | > grad_norm_0: tensor(16.0169, device='cuda:0') (tensor(8.2678, device='cuda:0')) + | > loss_gen: 2.2859325408935547 (2.3357839477596) + | > loss_kl: 1.9455899000167847 (1.9080991994089156) + | > loss_feat: 7.672516345977783 (6.958299647516279) + | > loss_mel: 18.34197235107422 (17.781137594536183) + | > loss_duration: 1.5590450763702393 (1.500277075304914) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.805057525634766 (30.483597456519284) + | > grad_norm_1: tensor(234.2850, device='cuda:0') (tensor(163.7426, device='cuda:0')) + | > current_lr_0: 0.00019746603001167813 + | > current_lr_1: 0.00019746603001167813 + | > step_time: 0.4412 (0.3861616903276586) + | > loader_time: 0.0079 (0.006397894958951581) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5934598445892334 (2.5934598445892334) + | > loss_disc_real_0: 0.16056689620018005 (0.16056689620018005) + | > loss_disc_real_1: 0.19907689094543457 (0.19907689094543457) + | > loss_disc_real_2: 0.22094666957855225 (0.22094666957855225) + | > loss_disc_real_3: 0.26524534821510315 (0.26524534821510315) + | > loss_disc_real_4: 0.1953277587890625 (0.1953277587890625) + | > loss_disc_real_5: 0.25483953952789307 (0.25483953952789307) + | > loss_0: 2.5934598445892334 (2.5934598445892334) + | > loss_gen: 2.0981802940368652 (2.0981802940368652) + | > loss_kl: 2.2535626888275146 (2.2535626888275146) + | > loss_feat: 5.375808238983154 (5.375808238983154) + | > loss_mel: 15.60798454284668 (15.60798454284668) + | > loss_duration: 1.6505895853042603 (1.6505895853042603) + | > loss_1: 26.98612403869629 (26.98612403869629) + + --> STEP: 1 + | > loss_disc: 2.4747684001922607 (2.4747684001922607) + | > loss_disc_real_0: 0.161615788936615 (0.161615788936615) + | > loss_disc_real_1: 0.23760107159614563 (0.23760107159614563) + | > loss_disc_real_2: 0.21979515254497528 (0.21979515254497528) + | > loss_disc_real_3: 0.24138231575489044 (0.24138231575489044) + | > loss_disc_real_4: 0.21446698904037476 (0.21446698904037476) + | > loss_disc_real_5: 0.23045365512371063 (0.23045365512371063) + | > loss_0: 2.4747684001922607 (2.4747684001922607) + | > loss_gen: 2.326357126235962 (2.326357126235962) + | > loss_kl: 2.4197537899017334 (2.4197537899017334) + | > loss_feat: 6.288679122924805 (6.288679122924805) + | > loss_mel: 17.914201736450195 (17.914201736450195) + | > loss_duration: 1.6729421615600586 (1.6729421615600586) + | > loss_1: 30.621932983398438 (30.621932983398438) + + --> STEP: 2 + | > loss_disc: 2.75797700881958 (2.6163727045059204) + | > loss_disc_real_0: 0.23305873572826385 (0.19733726233243942) + | > loss_disc_real_1: 0.2058725208044052 (0.22173679620027542) + | > loss_disc_real_2: 0.22570721805095673 (0.222751185297966) + | > loss_disc_real_3: 0.2619101405143738 (0.2516462281346321) + | > loss_disc_real_4: 0.24605289101600647 (0.2302599400281906) + | > loss_disc_real_5: 0.2846106290817261 (0.25753214210271835) + | > loss_0: 2.75797700881958 (2.6163727045059204) + | > loss_gen: 2.1941885948181152 (2.2602728605270386) + | > loss_kl: 2.1412174701690674 (2.2804856300354004) + | > loss_feat: 6.477232456207275 (6.38295578956604) + | > loss_mel: 19.22037124633789 (18.567286491394043) + | > loss_duration: 1.6437568664550781 (1.6583495140075684) + | > loss_1: 31.676767349243164 (31.1493501663208) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036661624908447266 (+2.6226043701171875e-06) + | > avg_loss_disc: 2.6163727045059204 (+0.07867145538330078) + | > avg_loss_disc_real_0: 0.19733726233243942 (-0.047395236790180206) + | > avg_loss_disc_real_1: 0.22173679620027542 (-0.0007742345333099365) + | > avg_loss_disc_real_2: 0.222751185297966 (-0.04287850856781006) + | > avg_loss_disc_real_3: 0.2516462281346321 (+0.03470000624656677) + | > avg_loss_disc_real_4: 0.2302599400281906 (-0.028001978993415833) + | > avg_loss_disc_real_5: 0.25753214210271835 (+0.024106629192829132) + | > avg_loss_0: 2.6163727045059204 (+0.07867145538330078) + | > avg_loss_gen: 2.2602728605270386 (-0.17440402507781982) + | > avg_loss_kl: 2.2804856300354004 (-0.18750596046447754) + | > avg_loss_feat: 6.38295578956604 (-0.013658285140991211) + | > avg_loss_mel: 18.567286491394043 (+0.8279762268066406) + | > avg_loss_duration: 1.6583495140075684 (+0.012826859951019287) + | > avg_loss_1: 31.1493501663208 (+0.46523475646972656) + + + > EPOCH: 103/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:42:01)  + + --> TIME: 2025-04-19 00:42:08 -- STEP: 17/142 -- GLOBAL_STEP: 393925 + | > loss_disc: 2.474410057067871 (2.433066073585959) + | > loss_disc_real_0: 0.10030517727136612 (0.12132471405407962) + | > loss_disc_real_1: 0.19001832604408264 (0.20373030532808864) + | > loss_disc_real_2: 0.21784915030002594 (0.22427556356962988) + | > loss_disc_real_3: 0.2018052190542221 (0.2278344964279848) + | > loss_disc_real_4: 0.22839593887329102 (0.2306013764703975) + | > loss_disc_real_5: 0.2510129511356354 (0.231159169884289) + | > loss_0: 2.474410057067871 (2.433066073585959) + | > grad_norm_0: tensor(9.5088, device='cuda:0') (tensor(9.4512, device='cuda:0')) + | > loss_gen: 2.3248214721679688 (2.3681583685033463) + | > loss_kl: 1.714858889579773 (1.8304818447898417) + | > loss_feat: 7.587522506713867 (7.1714303072761085) + | > loss_mel: 18.546768188476562 (18.030737540301153) + | > loss_duration: 1.4336814880371094 (1.4548189920537613) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.60765266418457 (30.855626835542566) + | > grad_norm_1: tensor(233.0698, device='cuda:0') (tensor(244.0044, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.3365 (0.32959480846629424) + | > loader_time: 0.0054 (0.004942851908066694) + + + --> TIME: 2025-04-19 00:42:17 -- STEP: 42/142 -- GLOBAL_STEP: 393950 + | > loss_disc: 2.4482645988464355 (2.436798544157119) + | > loss_disc_real_0: 0.12113916873931885 (0.12043145866621108) + | > loss_disc_real_1: 0.23423585295677185 (0.2046103644229117) + | > loss_disc_real_2: 0.2244635373353958 (0.22586347233681453) + | > loss_disc_real_3: 0.20500393211841583 (0.22838630101510457) + | > loss_disc_real_4: 0.24396006762981415 (0.22966691248473667) + | > loss_disc_real_5: 0.19949103891849518 (0.22932148618357523) + | > loss_0: 2.4482645988464355 (2.436798544157119) + | > grad_norm_0: tensor(14.3754, device='cuda:0') (tensor(9.4315, device='cuda:0')) + | > loss_gen: 2.364647388458252 (2.364775311379205) + | > loss_kl: 1.873794436454773 (1.8552590154466175) + | > loss_feat: 6.789511203765869 (7.109602474030995) + | > loss_mel: 17.76618003845215 (17.946282818203876) + | > loss_duration: 1.5065850019454956 (1.465341567993164) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.300716400146484 (30.741261073521205) + | > grad_norm_1: tensor(280.4057, device='cuda:0') (tensor(244.9694, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.3631 (0.3433119399206979) + | > loader_time: 0.0058 (0.005370832624889556) + + + --> TIME: 2025-04-19 00:42:26 -- STEP: 67/142 -- GLOBAL_STEP: 393975 + | > loss_disc: 2.586202621459961 (2.45855252066655) + | > loss_disc_real_0: 0.2387799769639969 (0.12533916733158168) + | > loss_disc_real_1: 0.18353235721588135 (0.20581343503140692) + | > loss_disc_real_2: 0.20761360228061676 (0.2260877321015543) + | > loss_disc_real_3: 0.2075444906949997 (0.22871650644202732) + | > loss_disc_real_4: 0.20606113970279694 (0.22958935441365882) + | > loss_disc_real_5: 0.2315005213022232 (0.23022646846166298) + | > loss_0: 2.586202621459961 (2.45855252066655) + | > grad_norm_0: tensor(20.0839, device='cuda:0') (tensor(9.7562, device='cuda:0')) + | > loss_gen: 2.2811334133148193 (2.3415568800114874) + | > loss_kl: 1.9512263536453247 (1.875923670939545) + | > loss_feat: 6.824462890625 (6.998963797270362) + | > loss_mel: 18.094881057739258 (17.816546867142858) + | > loss_duration: 1.4351482391357422 (1.4764936913305253) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.58685302734375 (30.509484846200515) + | > grad_norm_1: tensor(298.9946, device='cuda:0') (tensor(246.7143, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.3831 (0.3551633464756297) + | > loader_time: 0.0066 (0.005663455422244855) + + + --> TIME: 2025-04-19 00:42:36 -- STEP: 92/142 -- GLOBAL_STEP: 394000 + | > loss_disc: 2.419015407562256 (2.461798779342486) + | > loss_disc_real_0: 0.14453251659870148 (0.126197413743838) + | > loss_disc_real_1: 0.19447270035743713 (0.20564770763335022) + | > loss_disc_real_2: 0.23482748866081238 (0.22764456434094388) + | > loss_disc_real_3: 0.21706968545913696 (0.2295121970060079) + | > loss_disc_real_4: 0.21511374413967133 (0.230591275445793) + | > loss_disc_real_5: 0.2490648329257965 (0.23171590511565623) + | > loss_0: 2.419015407562256 (2.461798779342486) + | > grad_norm_0: tensor(16.1040, device='cuda:0') (tensor(10.5031, device='cuda:0')) + | > loss_gen: 2.3908028602600098 (2.3424860239028935) + | > loss_kl: 1.855812430381775 (1.887917346280554) + | > loss_feat: 7.359541893005371 (6.964823738388393) + | > loss_mel: 18.011396408081055 (17.776085698086284) + | > loss_duration: 1.4692107439041138 (1.4888445825680443) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.086763381958008 (30.460157290748928) + | > grad_norm_1: tensor(491.1608, device='cuda:0') (tensor(268.2457, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.4141 (0.36702902420707373) + | > loader_time: 0.0066 (0.005950647851695185) + + + --> TIME: 2025-04-19 00:42:47 -- STEP: 117/142 -- GLOBAL_STEP: 394025 + | > loss_disc: 2.3947081565856934 (2.4650653060684866) + | > loss_disc_real_0: 0.11306314915418625 (0.12664465899141428) + | > loss_disc_real_1: 0.1823960691690445 (0.20495359561382195) + | > loss_disc_real_2: 0.2204342782497406 (0.22709079927358872) + | > loss_disc_real_3: 0.2675021290779114 (0.2300239286871038) + | > loss_disc_real_4: 0.21342505514621735 (0.23017785742751554) + | > loss_disc_real_5: 0.23028728365898132 (0.23178059779680693) + | > loss_0: 2.3947081565856934 (2.4650653060684866) + | > grad_norm_0: tensor(6.0612, device='cuda:0') (tensor(10.7623, device='cuda:0')) + | > loss_gen: 2.3795933723449707 (2.332487817503449) + | > loss_kl: 1.7923756837844849 (1.8998211153552063) + | > loss_feat: 6.800905704498291 (6.9163564494532395) + | > loss_mel: 17.423877716064453 (17.72763546511659) + | > loss_duration: 1.5149844884872437 (1.4961608852076733) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.91173553466797 (30.372461645012226) + | > grad_norm_1: tensor(84.7820, device='cuda:0') (tensor(272.5668, device='cuda:0')) + | > current_lr_0: 0.00019744134675792665 + | > current_lr_1: 0.00019744134675792665 + | > step_time: 0.4373 (0.37871463482196516) + | > loader_time: 0.0071 (0.0062085114992581885) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4670348167419434 (2.4670348167419434) + | > loss_disc_real_0: 0.1545618623495102 (0.1545618623495102) + | > loss_disc_real_1: 0.2443918138742447 (0.2443918138742447) + | > loss_disc_real_2: 0.27281710505485535 (0.27281710505485535) + | > loss_disc_real_3: 0.2443472445011139 (0.2443472445011139) + | > loss_disc_real_4: 0.259546160697937 (0.259546160697937) + | > loss_disc_real_5: 0.22958911955356598 (0.22958911955356598) + | > loss_0: 2.4670348167419434 (2.4670348167419434) + | > loss_gen: 2.627507209777832 (2.627507209777832) + | > loss_kl: 2.1886661052703857 (2.1886661052703857) + | > loss_feat: 6.6174540519714355 (6.6174540519714355) + | > loss_mel: 17.972789764404297 (17.972789764404297) + | > loss_duration: 1.6172595024108887 (1.6172595024108887) + | > loss_1: 31.023677825927734 (31.023677825927734) + + --> STEP: 1 + | > loss_disc: 2.5827813148498535 (2.5827813148498535) + | > loss_disc_real_0: 0.14649827778339386 (0.14649827778339386) + | > loss_disc_real_1: 0.2295435070991516 (0.2295435070991516) + | > loss_disc_real_2: 0.2869560122489929 (0.2869560122489929) + | > loss_disc_real_3: 0.22321544587612152 (0.22321544587612152) + | > loss_disc_real_4: 0.2249966561794281 (0.2249966561794281) + | > loss_disc_real_5: 0.24481728672981262 (0.24481728672981262) + | > loss_0: 2.5827813148498535 (2.5827813148498535) + | > loss_gen: 2.341615676879883 (2.341615676879883) + | > loss_kl: 2.592320203781128 (2.592320203781128) + | > loss_feat: 7.309116363525391 (7.309116363525391) + | > loss_mel: 19.547861099243164 (19.547861099243164) + | > loss_duration: 1.6690342426300049 (1.6690342426300049) + | > loss_1: 33.4599494934082 (33.4599494934082) + + --> STEP: 2 + | > loss_disc: 2.482485294342041 (2.5326333045959473) + | > loss_disc_real_0: 0.11632691323757172 (0.1314125955104828) + | > loss_disc_real_1: 0.24044936895370483 (0.23499643802642822) + | > loss_disc_real_2: 0.2622368633747101 (0.2745964378118515) + | > loss_disc_real_3: 0.2637121379375458 (0.24346379190683365) + | > loss_disc_real_4: 0.2198338359594345 (0.2224152460694313) + | > loss_disc_real_5: 0.22458328306674957 (0.2347002848982811) + | > loss_0: 2.482485294342041 (2.5326333045959473) + | > loss_gen: 2.6650595664978027 (2.5033376216888428) + | > loss_kl: 2.205385684967041 (2.3988529443740845) + | > loss_feat: 8.650259971618652 (7.9796881675720215) + | > loss_mel: 18.745071411132812 (19.14646625518799) + | > loss_duration: 1.6071877479553223 (1.6381109952926636) + | > loss_1: 33.87296676635742 (33.66645812988281) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035244226455688477 (-0.0001417398452758789) + | > avg_loss_disc: 2.5326333045959473 (-0.08373939990997314) + | > avg_loss_disc_real_0: 0.1314125955104828 (-0.06592466682195663) + | > avg_loss_disc_real_1: 0.23499643802642822 (+0.013259641826152802) + | > avg_loss_disc_real_2: 0.2745964378118515 (+0.0518452525138855) + | > avg_loss_disc_real_3: 0.24346379190683365 (-0.008182436227798462) + | > avg_loss_disc_real_4: 0.2224152460694313 (-0.007844693958759308) + | > avg_loss_disc_real_5: 0.2347002848982811 (-0.022831857204437256) + | > avg_loss_0: 2.5326333045959473 (-0.08373939990997314) + | > avg_loss_gen: 2.5033376216888428 (+0.2430647611618042) + | > avg_loss_kl: 2.3988529443740845 (+0.11836731433868408) + | > avg_loss_feat: 7.9796881675720215 (+1.5967323780059814) + | > avg_loss_mel: 19.14646625518799 (+0.5791797637939453) + | > avg_loss_duration: 1.6381109952926636 (-0.020238518714904785) + | > avg_loss_1: 33.66645812988281 (+2.5171079635620117) + + + > EPOCH: 104/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:43:00)  + + --> TIME: 2025-04-19 00:43:01 -- STEP: 0/142 -- GLOBAL_STEP: 394050 + | > loss_disc: 2.496297836303711 (2.496297836303711) + | > loss_disc_real_0: 0.1463477909564972 (0.1463477909564972) + | > loss_disc_real_1: 0.25862449407577515 (0.25862449407577515) + | > loss_disc_real_2: 0.25516223907470703 (0.25516223907470703) + | > loss_disc_real_3: 0.26346659660339355 (0.26346659660339355) + | > loss_disc_real_4: 0.2418033480644226 (0.2418033480644226) + | > loss_disc_real_5: 0.20383696258068085 (0.20383696258068085) + | > loss_0: 2.496297836303711 (2.496297836303711) + | > grad_norm_0: tensor(4.6127, device='cuda:0') (tensor(4.6127, device='cuda:0')) + | > loss_gen: 2.6736738681793213 (2.6736738681793213) + | > loss_kl: 1.811618447303772 (1.811618447303772) + | > loss_feat: 7.1838765144348145 (7.1838765144348145) + | > loss_mel: 17.19969940185547 (17.19969940185547) + | > loss_duration: 1.4725816249847412 (1.4725816249847412) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.341449737548828 (30.341449737548828) + | > grad_norm_1: tensor(353.8364, device='cuda:0') (tensor(353.8364, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.3516 (0.35162782669067383) + | > loader_time: 0.3191 (0.31913137435913086) + + + --> TIME: 2025-04-19 00:43:09 -- STEP: 25/142 -- GLOBAL_STEP: 394075 + | > loss_disc: 2.5728228092193604 (2.4193002223968505) + | > loss_disc_real_0: 0.19186091423034668 (0.11714797511696816) + | > loss_disc_real_1: 0.22628851234912872 (0.20342955529689788) + | > loss_disc_real_2: 0.2223776876926422 (0.22291890263557435) + | > loss_disc_real_3: 0.23091506958007812 (0.22425424337387084) + | > loss_disc_real_4: 0.22709262371063232 (0.2181775140762329) + | > loss_disc_real_5: 0.28298601508140564 (0.2356245619058609) + | > loss_0: 2.5728228092193604 (2.4193002223968505) + | > grad_norm_0: tensor(33.7309, device='cuda:0') (tensor(19.4074, device='cuda:0')) + | > loss_gen: 2.4831063747406006 (2.4570874214172362) + | > loss_kl: 1.8435990810394287 (1.842144145965576) + | > loss_feat: 7.101330757141113 (7.391860446929932) + | > loss_mel: 17.981138229370117 (18.15408592224121) + | > loss_duration: 1.4948878288269043 (1.461104393005371) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.904062271118164 (31.306282196044922) + | > grad_norm_1: tensor(282.9265, device='cuda:0') (tensor(373.4624, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.3493 (0.3339120388031006) + | > loader_time: 0.006 (0.005136346817016602) + + + --> TIME: 2025-04-19 00:43:18 -- STEP: 50/142 -- GLOBAL_STEP: 394100 + | > loss_disc: 2.5936315059661865 (2.4679920148849486) + | > loss_disc_real_0: 0.1916346698999405 (0.1279305685311556) + | > loss_disc_real_1: 0.229726642370224 (0.2075363329052925) + | > loss_disc_real_2: 0.24805721640586853 (0.22623293340206146) + | > loss_disc_real_3: 0.23594129085540771 (0.22766440331935883) + | > loss_disc_real_4: 0.2448018342256546 (0.22631988018751145) + | > loss_disc_real_5: 0.2562233507633209 (0.23741057634353638) + | > loss_0: 2.5936315059661865 (2.4679920148849486) + | > grad_norm_0: tensor(7.5523, device='cuda:0') (tensor(17.3582, device='cuda:0')) + | > loss_gen: 2.1257002353668213 (2.409987680912018) + | > loss_kl: 1.885024070739746 (1.8774272418022155) + | > loss_feat: 5.676602363586426 (7.256245679855347) + | > loss_mel: 16.78241539001465 (18.061896133422852) + | > loss_duration: 1.4973082542419434 (1.4692671632766723) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.967050552368164 (31.074823760986327) + | > grad_norm_1: tensor(97.8958, device='cuda:0') (tensor(334.4014, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.3692 (0.3463441514968872) + | > loader_time: 0.0065 (0.0055941009521484385) + + + --> TIME: 2025-04-19 00:43:28 -- STEP: 75/142 -- GLOBAL_STEP: 394125 + | > loss_disc: 2.3646297454833984 (2.465060796737671) + | > loss_disc_real_0: 0.11151953786611557 (0.12556341802080473) + | > loss_disc_real_1: 0.18167145550251007 (0.20621279100577036) + | > loss_disc_real_2: 0.20094093680381775 (0.22703000605106355) + | > loss_disc_real_3: 0.22578676044940948 (0.2281071259578069) + | > loss_disc_real_4: 0.20898933708667755 (0.22646346946557364) + | > loss_disc_real_5: 0.243488609790802 (0.2369638713200887) + | > loss_0: 2.3646297454833984 (2.465060796737671) + | > grad_norm_0: tensor(8.5958, device='cuda:0') (tensor(14.9833, device='cuda:0')) + | > loss_gen: 2.3695383071899414 (2.3764254840215053) + | > loss_kl: 1.9165618419647217 (1.8925355259577434) + | > loss_feat: 6.893093109130859 (7.164506047566732) + | > loss_mel: 18.319839477539062 (18.010678329467773) + | > loss_duration: 1.4721373319625854 (1.4796330785751344) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.971168518066406 (30.92377825419108) + | > grad_norm_1: tensor(166.6562, device='cuda:0') (tensor(296.2229, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.3973 (0.36015993754069003) + | > loader_time: 0.0065 (0.005869725545247398) + + + --> TIME: 2025-04-19 00:43:39 -- STEP: 100/142 -- GLOBAL_STEP: 394150 + | > loss_disc: 2.5889785289764404 (2.477331101894378) + | > loss_disc_real_0: 0.12972939014434814 (0.1284245029464364) + | > loss_disc_real_1: 0.23407696187496185 (0.2084207884967327) + | > loss_disc_real_2: 0.2079543173313141 (0.22700227051973343) + | > loss_disc_real_3: 0.22363954782485962 (0.2282511319220066) + | > loss_disc_real_4: 0.2400854229927063 (0.22817170664668082) + | > loss_disc_real_5: 0.2318131923675537 (0.2365902441740036) + | > loss_0: 2.5889785289764404 (2.477331101894378) + | > grad_norm_0: tensor(11.2785, device='cuda:0') (tensor(13.3500, device='cuda:0')) + | > loss_gen: 2.1284828186035156 (2.355047367811203) + | > loss_kl: 2.0271904468536377 (1.9008970499038695) + | > loss_feat: 6.844560146331787 (7.057429676055908) + | > loss_mel: 17.495527267456055 (17.963196182250982) + | > loss_duration: 1.512029767036438 (1.4905965805053714) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.00779151916504 (30.767166690826414) + | > grad_norm_1: tensor(90.9263, device='cuda:0') (tensor(262.0479, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.4331 (0.37148878812789915) + | > loader_time: 0.009 (0.0061504220962524435) + + + --> TIME: 2025-04-19 00:43:50 -- STEP: 125/142 -- GLOBAL_STEP: 394175 + | > loss_disc: 2.4974782466888428 (2.476715118408203) + | > loss_disc_real_0: 0.1290547251701355 (0.12950287505984312) + | > loss_disc_real_1: 0.21363188326358795 (0.20791128599643707) + | > loss_disc_real_2: 0.2569754123687744 (0.2267217711210251) + | > loss_disc_real_3: 0.23853428661823273 (0.22834502279758453) + | > loss_disc_real_4: 0.23666846752166748 (0.2277712503671646) + | > loss_disc_real_5: 0.24249328672885895 (0.2360532236099243) + | > loss_0: 2.4974782466888428 (2.476715118408203) + | > grad_norm_0: tensor(6.1592, device='cuda:0') (tensor(12.4860, device='cuda:0')) + | > loss_gen: 2.243687152862549 (2.339481188774109) + | > loss_kl: 1.908738613128662 (1.912272850036621) + | > loss_feat: 6.35416841506958 (6.950385608673096) + | > loss_mel: 16.590251922607422 (17.853967369079598) + | > loss_duration: 1.5705115795135498 (1.497606459617615) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.667356491088867 (30.553713302612305) + | > grad_norm_1: tensor(103.9245, device='cuda:0') (tensor(241.5641, device='cuda:0')) + | > current_lr_0: 0.0001974166665895819 + | > current_lr_1: 0.0001974166665895819 + | > step_time: 0.4381 (0.3837177791595459) + | > loader_time: 0.0076 (0.006430807113647462) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4409539699554443 (2.4409539699554443) + | > loss_disc_real_0: 0.1214902400970459 (0.1214902400970459) + | > loss_disc_real_1: 0.20564459264278412 (0.20564459264278412) + | > loss_disc_real_2: 0.25141769647598267 (0.25141769647598267) + | > loss_disc_real_3: 0.23398447036743164 (0.23398447036743164) + | > loss_disc_real_4: 0.24383969604969025 (0.24383969604969025) + | > loss_disc_real_5: 0.26932328939437866 (0.26932328939437866) + | > loss_0: 2.4409539699554443 (2.4409539699554443) + | > loss_gen: 2.3729164600372314 (2.3729164600372314) + | > loss_kl: 2.314852714538574 (2.314852714538574) + | > loss_feat: 6.698297500610352 (6.698297500610352) + | > loss_mel: 17.688499450683594 (17.688499450683594) + | > loss_duration: 1.6421229839324951 (1.6421229839324951) + | > loss_1: 30.71668815612793 (30.71668815612793) + + --> STEP: 1 + | > loss_disc: 2.5550479888916016 (2.5550479888916016) + | > loss_disc_real_0: 0.17829911410808563 (0.17829911410808563) + | > loss_disc_real_1: 0.20418678224086761 (0.20418678224086761) + | > loss_disc_real_2: 0.21913103759288788 (0.21913103759288788) + | > loss_disc_real_3: 0.24434834718704224 (0.24434834718704224) + | > loss_disc_real_4: 0.25064173340797424 (0.25064173340797424) + | > loss_disc_real_5: 0.21275922656059265 (0.21275922656059265) + | > loss_0: 2.5550479888916016 (2.5550479888916016) + | > loss_gen: 2.1191565990448 (2.1191565990448) + | > loss_kl: 2.3897292613983154 (2.3897292613983154) + | > loss_feat: 5.245940685272217 (5.245940685272217) + | > loss_mel: 16.378414154052734 (16.378414154052734) + | > loss_duration: 1.659521222114563 (1.659521222114563) + | > loss_1: 27.792760848999023 (27.792760848999023) + + --> STEP: 2 + | > loss_disc: 2.447678804397583 (2.5013633966445923) + | > loss_disc_real_0: 0.11460031569004059 (0.1464497148990631) + | > loss_disc_real_1: 0.23259186744689941 (0.21838932484388351) + | > loss_disc_real_2: 0.21451331675052643 (0.21682217717170715) + | > loss_disc_real_3: 0.20422743260860443 (0.22428788989782333) + | > loss_disc_real_4: 0.2381577342748642 (0.24439973384141922) + | > loss_disc_real_5: 0.23679327964782715 (0.2247762531042099) + | > loss_0: 2.447678804397583 (2.5013633966445923) + | > loss_gen: 2.231475353240967 (2.1753159761428833) + | > loss_kl: 1.9446159601211548 (2.167172610759735) + | > loss_feat: 7.908933639526367 (6.577437162399292) + | > loss_mel: 19.630460739135742 (18.00443744659424) + | > loss_duration: 1.620710015296936 (1.6401156187057495) + | > loss_1: 33.33619689941406 (30.564478874206543) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034519433975219727 (-7.2479248046875e-05) + | > avg_loss_disc: 2.5013633966445923 (-0.03126990795135498) + | > avg_loss_disc_real_0: 0.1464497148990631 (+0.015037119388580322) + | > avg_loss_disc_real_1: 0.21838932484388351 (-0.016607113182544708) + | > avg_loss_disc_real_2: 0.21682217717170715 (-0.05777426064014435) + | > avg_loss_disc_real_3: 0.22428788989782333 (-0.019175902009010315) + | > avg_loss_disc_real_4: 0.24439973384141922 (+0.021984487771987915) + | > avg_loss_disc_real_5: 0.2247762531042099 (-0.009924031794071198) + | > avg_loss_0: 2.5013633966445923 (-0.03126990795135498) + | > avg_loss_gen: 2.1753159761428833 (-0.3280216455459595) + | > avg_loss_kl: 2.167172610759735 (-0.23168033361434937) + | > avg_loss_feat: 6.577437162399292 (-1.4022510051727295) + | > avg_loss_mel: 18.00443744659424 (-1.14202880859375) + | > avg_loss_duration: 1.6401156187057495 (+0.0020046234130859375) + | > avg_loss_1: 30.564478874206543 (-3.1019792556762695) + + + > EPOCH: 105/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:43:59)  + + --> TIME: 2025-04-19 00:44:02 -- STEP: 8/142 -- GLOBAL_STEP: 394200 + | > loss_disc: 2.272965669631958 (2.415424168109894) + | > loss_disc_real_0: 0.09484304487705231 (0.11250789277255535) + | > loss_disc_real_1: 0.20599211752414703 (0.21839995123445988) + | > loss_disc_real_2: 0.19712483882904053 (0.22290993481874466) + | > loss_disc_real_3: 0.19261226058006287 (0.22110160440206528) + | > loss_disc_real_4: 0.16717147827148438 (0.21748289838433266) + | > loss_disc_real_5: 0.20034460723400116 (0.23349760472774506) + | > loss_0: 2.272965669631958 (2.415424168109894) + | > grad_norm_0: tensor(5.7785, device='cuda:0') (tensor(5.3260, device='cuda:0')) + | > loss_gen: 2.449380874633789 (2.3951241970062256) + | > loss_kl: 1.7335327863693237 (1.8161786496639252) + | > loss_feat: 7.6838226318359375 (7.263693749904633) + | > loss_mel: 18.98023223876953 (18.392536878585815) + | > loss_duration: 1.4529598951339722 (1.4566795527935028) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.2999267578125 (31.324212551116943) + | > grad_norm_1: tensor(153.7484, device='cuda:0') (tensor(125.3022, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.3266 (0.3264628052711487) + | > loader_time: 0.0053 (0.004697531461715698) + + + --> TIME: 2025-04-19 00:44:11 -- STEP: 33/142 -- GLOBAL_STEP: 394225 + | > loss_disc: 2.415584087371826 (2.4362953287182436) + | > loss_disc_real_0: 0.11117488145828247 (0.12202076862255733) + | > loss_disc_real_1: 0.19058623909950256 (0.20660220357504758) + | > loss_disc_real_2: 0.25754961371421814 (0.22724000071034287) + | > loss_disc_real_3: 0.2201167792081833 (0.2247944984472159) + | > loss_disc_real_4: 0.22835353016853333 (0.22301930053667587) + | > loss_disc_real_5: 0.2589051425457001 (0.22970596588019168) + | > loss_0: 2.415584087371826 (2.4362953287182436) + | > grad_norm_0: tensor(6.3980, device='cuda:0') (tensor(6.5386, device='cuda:0')) + | > loss_gen: 2.4078304767608643 (2.3563987775282427) + | > loss_kl: 1.8324381113052368 (1.8802635597460198) + | > loss_feat: 6.465963840484619 (7.060628789843935) + | > loss_mel: 17.37209701538086 (17.95276549368194) + | > loss_duration: 1.4735393524169922 (1.4599168481248799) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.551868438720703 (30.709973364165336) + | > grad_norm_1: tensor(83.5624, device='cuda:0') (tensor(130.6608, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.356 (0.3384656111399333) + | > loader_time: 0.0057 (0.005258473482998936) + + + --> TIME: 2025-04-19 00:44:20 -- STEP: 58/142 -- GLOBAL_STEP: 394250 + | > loss_disc: 2.492955207824707 (2.4519373219588716) + | > loss_disc_real_0: 0.15088839828968048 (0.12658938238846845) + | > loss_disc_real_1: 0.23044100403785706 (0.20480777580162574) + | > loss_disc_real_2: 0.23503534495830536 (0.2272659231876505) + | > loss_disc_real_3: 0.2300933301448822 (0.22705513393056803) + | > loss_disc_real_4: 0.23749268054962158 (0.22618535109635057) + | > loss_disc_real_5: 0.2558021545410156 (0.2347763088242761) + | > loss_0: 2.492955207824707 (2.4519373219588716) + | > grad_norm_0: tensor(4.9044, device='cuda:0') (tensor(8.2833, device='cuda:0')) + | > loss_gen: 2.196890354156494 (2.3525043684860747) + | > loss_kl: 2.0580496788024902 (1.8841701745986938) + | > loss_feat: 6.454048156738281 (7.001459294352038) + | > loss_mel: 17.928213119506836 (17.936878796281484) + | > loss_duration: 1.5090503692626953 (1.47164445498894) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.146251678466797 (30.64665699005127) + | > grad_norm_1: tensor(73.6291, device='cuda:0') (tensor(169.0231, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.3757 (0.35103855050843336) + | > loader_time: 0.006 (0.0055369878637379615) + + + --> TIME: 2025-04-19 00:44:30 -- STEP: 83/142 -- GLOBAL_STEP: 394275 + | > loss_disc: 2.4167287349700928 (2.456026669008187) + | > loss_disc_real_0: 0.07440423220396042 (0.126836066953389) + | > loss_disc_real_1: 0.17886927723884583 (0.20527645388999619) + | > loss_disc_real_2: 0.22735966742038727 (0.22596344387674905) + | > loss_disc_real_3: 0.19976705312728882 (0.2264266455747995) + | > loss_disc_real_4: 0.22141854465007782 (0.2253250054207193) + | > loss_disc_real_5: 0.2406253069639206 (0.23383980037936244) + | > loss_0: 2.4167287349700928 (2.456026669008187) + | > grad_norm_0: tensor(10.4488, device='cuda:0') (tensor(9.7240, device='cuda:0')) + | > loss_gen: 2.349881649017334 (2.3496747217982636) + | > loss_kl: 1.8673765659332275 (1.8830981570554066) + | > loss_feat: 6.945003032684326 (6.966743618608957) + | > loss_mel: 18.15459442138672 (17.845547768006835) + | > loss_duration: 1.5735687017440796 (1.4827798288988778) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.890424728393555 (30.527844026864294) + | > grad_norm_1: tensor(128.7981, device='cuda:0') (tensor(207.7984, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.3981 (0.3625166789594903) + | > loader_time: 0.0068 (0.005845305431320007) + + + --> TIME: 2025-04-19 00:44:41 -- STEP: 108/142 -- GLOBAL_STEP: 394300 + | > loss_disc: 2.61799693107605 (2.4690894815656885) + | > loss_disc_real_0: 0.23004217445850372 (0.12852230323133634) + | > loss_disc_real_1: 0.21744266152381897 (0.20757970848569163) + | > loss_disc_real_2: 0.26144227385520935 (0.22700309808607455) + | > loss_disc_real_3: 0.26338085532188416 (0.22763594391721267) + | > loss_disc_real_4: 0.24421416223049164 (0.22745921352395304) + | > loss_disc_real_5: 0.23708762228488922 (0.23447975364548188) + | > loss_0: 2.61799693107605 (2.4690894815656885) + | > grad_norm_0: tensor(22.8923, device='cuda:0') (tensor(10.3244, device='cuda:0')) + | > loss_gen: 2.343386650085449 (2.3455850217077465) + | > loss_kl: 1.983472466468811 (1.8914741664021104) + | > loss_feat: 6.231265068054199 (6.906611809024104) + | > loss_mel: 16.106149673461914 (17.772871856336227) + | > loss_duration: 1.515760898590088 (1.491475256504836) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.180036544799805 (30.408018129843253) + | > grad_norm_1: tensor(415.0153, device='cuda:0') (tensor(226.9347, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.4181 (0.3726061604641102) + | > loader_time: 0.0068 (0.006081819534301758) + + + --> TIME: 2025-04-19 00:44:52 -- STEP: 133/142 -- GLOBAL_STEP: 394325 + | > loss_disc: 2.3978185653686523 (2.4660498934580874) + | > loss_disc_real_0: 0.13380804657936096 (0.1278741448781543) + | > loss_disc_real_1: 0.2150825709104538 (0.2068936055091987) + | > loss_disc_real_2: 0.22934158146381378 (0.22616553239356307) + | > loss_disc_real_3: 0.206722229719162 (0.22761341077940805) + | > loss_disc_real_4: 0.22607746720314026 (0.22627928505714676) + | > loss_disc_real_5: 0.1982978731393814 (0.23469631777221994) + | > loss_0: 2.3978185653686523 (2.4660498934580874) + | > grad_norm_0: tensor(16.6576, device='cuda:0') (tensor(11.3620, device='cuda:0')) + | > loss_gen: 2.4230945110321045 (2.34292295283841) + | > loss_kl: 2.0705180168151855 (1.8971519936296277) + | > loss_feat: 6.946401119232178 (6.891008194227864) + | > loss_mel: 18.616540908813477 (17.807948456670996) + | > loss_duration: 1.591468334197998 (1.497691551545509) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.64802360534668 (30.436723149808724) + | > grad_norm_1: tensor(276.2549, device='cuda:0') (tensor(251.9011, device='cuda:0')) + | > current_lr_0: 0.0001973919895062582 + | > current_lr_1: 0.0001973919895062582 + | > step_time: 0.4374 (0.38431959044664427) + | > loader_time: 0.0072 (0.0063418768402329065) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5483953952789307 (2.5483953952789307) + | > loss_disc_real_0: 0.14378003776073456 (0.14378003776073456) + | > loss_disc_real_1: 0.3007585108280182 (0.3007585108280182) + | > loss_disc_real_2: 0.29578784108161926 (0.29578784108161926) + | > loss_disc_real_3: 0.26188382506370544 (0.26188382506370544) + | > loss_disc_real_4: 0.22111327946186066 (0.22111327946186066) + | > loss_disc_real_5: 0.253816694021225 (0.253816694021225) + | > loss_0: 2.5483953952789307 (2.5483953952789307) + | > loss_gen: 2.4322187900543213 (2.4322187900543213) + | > loss_kl: 2.073303699493408 (2.073303699493408) + | > loss_feat: 6.131365776062012 (6.131365776062012) + | > loss_mel: 17.4139404296875 (17.4139404296875) + | > loss_duration: 1.6448698043823242 (1.6448698043823242) + | > loss_1: 29.695697784423828 (29.695697784423828) + + --> STEP: 1 + | > loss_disc: 2.610462188720703 (2.610462188720703) + | > loss_disc_real_0: 0.14799979329109192 (0.14799979329109192) + | > loss_disc_real_1: 0.2625601589679718 (0.2625601589679718) + | > loss_disc_real_2: 0.2738829255104065 (0.2738829255104065) + | > loss_disc_real_3: 0.24580013751983643 (0.24580013751983643) + | > loss_disc_real_4: 0.22806358337402344 (0.22806358337402344) + | > loss_disc_real_5: 0.28422603011131287 (0.28422603011131287) + | > loss_0: 2.610462188720703 (2.610462188720703) + | > loss_gen: 2.303499698638916 (2.303499698638916) + | > loss_kl: 2.4361586570739746 (2.4361586570739746) + | > loss_feat: 6.1647796630859375 (6.1647796630859375) + | > loss_mel: 17.251026153564453 (17.251026153564453) + | > loss_duration: 1.6922051906585693 (1.6922051906585693) + | > loss_1: 29.84766960144043 (29.84766960144043) + + --> STEP: 2 + | > loss_disc: 2.5818803310394287 (2.596171259880066) + | > loss_disc_real_0: 0.18008054792881012 (0.16404017060995102) + | > loss_disc_real_1: 0.26120781898498535 (0.2618839889764786) + | > loss_disc_real_2: 0.27382826805114746 (0.273855596780777) + | > loss_disc_real_3: 0.22429843246936798 (0.2350492849946022) + | > loss_disc_real_4: 0.28392940759658813 (0.2559964954853058) + | > loss_disc_real_5: 0.27651387453079224 (0.28036995232105255) + | > loss_0: 2.5818803310394287 (2.596171259880066) + | > loss_gen: 2.421212911605835 (2.3623563051223755) + | > loss_kl: 2.2303690910339355 (2.333263874053955) + | > loss_feat: 6.281522750854492 (6.223151206970215) + | > loss_mel: 16.375425338745117 (16.813225746154785) + | > loss_duration: 1.60481858253479 (1.6485118865966797) + | > loss_1: 28.913349151611328 (29.38050937652588) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032682418823242188 (-0.0001837015151977539) + | > avg_loss_disc: 2.596171259880066 (+0.09480786323547363) + | > avg_loss_disc_real_0: 0.16404017060995102 (+0.01759045571088791) + | > avg_loss_disc_real_1: 0.2618839889764786 (+0.04349466413259506) + | > avg_loss_disc_real_2: 0.273855596780777 (+0.057033419609069824) + | > avg_loss_disc_real_3: 0.2350492849946022 (+0.01076139509677887) + | > avg_loss_disc_real_4: 0.2559964954853058 (+0.011596761643886566) + | > avg_loss_disc_real_5: 0.28036995232105255 (+0.05559369921684265) + | > avg_loss_0: 2.596171259880066 (+0.09480786323547363) + | > avg_loss_gen: 2.3623563051223755 (+0.1870403289794922) + | > avg_loss_kl: 2.333263874053955 (+0.16609126329421997) + | > avg_loss_feat: 6.223151206970215 (-0.35428595542907715) + | > avg_loss_mel: 16.813225746154785 (-1.1912117004394531) + | > avg_loss_duration: 1.6485118865966797 (+0.008396267890930176) + | > avg_loss_1: 29.38050937652588 (-1.183969497680664) + + + > EPOCH: 106/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:44:57)  + + --> TIME: 2025-04-19 00:45:03 -- STEP: 16/142 -- GLOBAL_STEP: 394350 + | > loss_disc: 2.434386730194092 (2.471628725528717) + | > loss_disc_real_0: 0.09970631450414658 (0.1263823849149048) + | > loss_disc_real_1: 0.2019706815481186 (0.20952293742448092) + | > loss_disc_real_2: 0.22586582601070404 (0.22942393366247416) + | > loss_disc_real_3: 0.22680416703224182 (0.22887118440121412) + | > loss_disc_real_4: 0.20322993397712708 (0.2282580304890871) + | > loss_disc_real_5: 0.22976425290107727 (0.22965680994093418) + | > loss_0: 2.434386730194092 (2.471628725528717) + | > grad_norm_0: tensor(6.4029, device='cuda:0') (tensor(11.3964, device='cuda:0')) + | > loss_gen: 2.3715059757232666 (2.3462600708007812) + | > loss_kl: 1.8165619373321533 (1.8985000476241112) + | > loss_feat: 6.839702606201172 (6.935830324888229) + | > loss_mel: 18.316171646118164 (17.4311004281044) + | > loss_duration: 1.436147928237915 (1.4482928812503815) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.780088424682617 (30.05998384952545) + | > grad_norm_1: tensor(85.9206, device='cuda:0') (tensor(254.6520, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.3388 (0.32776060700416565) + | > loader_time: 0.0049 (0.004905015230178833) + + + --> TIME: 2025-04-19 00:45:12 -- STEP: 41/142 -- GLOBAL_STEP: 394375 + | > loss_disc: 2.4848573207855225 (2.484270619183052) + | > loss_disc_real_0: 0.09655050933361053 (0.12239198495702046) + | > loss_disc_real_1: 0.19828693568706512 (0.2068960550354748) + | > loss_disc_real_2: 0.21209415793418884 (0.23281679138904665) + | > loss_disc_real_3: 0.24483616650104523 (0.23060449049240206) + | > loss_disc_real_4: 0.23365584015846252 (0.23381013295999387) + | > loss_disc_real_5: 0.21724562346935272 (0.2342465614400259) + | > loss_0: 2.4848573207855225 (2.484270619183052) + | > grad_norm_0: tensor(14.0625, device='cuda:0') (tensor(13.3171, device='cuda:0')) + | > loss_gen: 2.3961668014526367 (2.356228555121073) + | > loss_kl: 1.6944864988327026 (1.885873151988518) + | > loss_feat: 6.936606407165527 (7.0037049316778415) + | > loss_mel: 17.91838264465332 (17.750336693554388) + | > loss_duration: 1.440360188484192 (1.460472912323184) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.386005401611328 (30.45661642493271) + | > grad_norm_1: tensor(411.2225, device='cuda:0') (tensor(294.4222, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.3747 (0.3432834904368331) + | > loader_time: 0.0059 (0.005306546281023722) + + + --> TIME: 2025-04-19 00:45:22 -- STEP: 66/142 -- GLOBAL_STEP: 394400 + | > loss_disc: 2.533886432647705 (2.4753059004292344) + | > loss_disc_real_0: 0.15499646961688995 (0.12278197344505426) + | > loss_disc_real_1: 0.22661513090133667 (0.20506471847042892) + | > loss_disc_real_2: 0.23339979350566864 (0.23016023387511572) + | > loss_disc_real_3: 0.23616820573806763 (0.22999516423001434) + | > loss_disc_real_4: 0.23300406336784363 (0.23232487679430933) + | > loss_disc_real_5: 0.2377709448337555 (0.23568138231833777) + | > loss_0: 2.533886432647705 (2.4753059004292344) + | > grad_norm_0: tensor(10.0671, device='cuda:0') (tensor(14.0338, device='cuda:0')) + | > loss_gen: 2.295259952545166 (2.3514727119243517) + | > loss_kl: 1.9429247379302979 (1.888927773995833) + | > loss_feat: 5.750938892364502 (6.967047575748328) + | > loss_mel: 16.248062133789062 (17.795802448735085) + | > loss_duration: 1.466805338859558 (1.473465160890059) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.703989028930664 (30.476715810371168) + | > grad_norm_1: tensor(335.8442, device='cuda:0') (tensor(304.5124, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.3828 (0.3597164117928707) + | > loader_time: 0.0061 (0.005663438276811079) + + + --> TIME: 2025-04-19 00:45:33 -- STEP: 91/142 -- GLOBAL_STEP: 394425 + | > loss_disc: 2.460188388824463 (2.4756158917814823) + | > loss_disc_real_0: 0.11719310283660889 (0.1256556876904362) + | > loss_disc_real_1: 0.1879187673330307 (0.20603308900372014) + | > loss_disc_real_2: 0.22998161613941193 (0.22871453028458816) + | > loss_disc_real_3: 0.19639892876148224 (0.2277740770643884) + | > loss_disc_real_4: 0.1929321587085724 (0.23127373168756674) + | > loss_disc_real_5: 0.21985457837581635 (0.2349813367639269) + | > loss_0: 2.460188388824463 (2.4756158917814823) + | > grad_norm_0: tensor(6.6381, device='cuda:0') (tensor(12.5941, device='cuda:0')) + | > loss_gen: 2.352098226547241 (2.3387874233853694) + | > loss_kl: 1.969225525856018 (1.899711094059787) + | > loss_feat: 7.905236721038818 (6.948241024226933) + | > loss_mel: 18.857627868652344 (17.80643210568271) + | > loss_duration: 1.5155649185180664 (1.4862604560432853) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.599754333496094 (30.47943209553813) + | > grad_norm_1: tensor(104.5003, device='cuda:0') (tensor(271.9785, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.4006 (0.3699152705433606) + | > loader_time: 0.0067 (0.005975872605711548) + + + --> TIME: 2025-04-19 00:45:43 -- STEP: 116/142 -- GLOBAL_STEP: 394450 + | > loss_disc: 2.417480230331421 (2.478213238305058) + | > loss_disc_real_0: 0.11936555057764053 (0.12779134460564318) + | > loss_disc_real_1: 0.19561122357845306 (0.20572437731356458) + | > loss_disc_real_2: 0.23487362265586853 (0.22771065309643745) + | > loss_disc_real_3: 0.2096380591392517 (0.22870841712273401) + | > loss_disc_real_4: 0.21736854314804077 (0.23130712825162658) + | > loss_disc_real_5: 0.23005986213684082 (0.23519052295335408) + | > loss_0: 2.417480230331421 (2.478213238305058) + | > grad_norm_0: tensor(17.3077, device='cuda:0') (tensor(11.7916, device='cuda:0')) + | > loss_gen: 2.3376541137695312 (2.329200949134498) + | > loss_kl: 1.9681034088134766 (1.9101063701613197) + | > loss_feat: 7.0995025634765625 (6.911021849204754) + | > loss_mel: 17.14162826538086 (17.7511275143459) + | > loss_duration: 1.4844002723693848 (1.492772873105674) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.031288146972656 (30.394229494292162) + | > grad_norm_1: tensor(309.6154, device='cuda:0') (tensor(244.1040, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.4298 (0.3797499927981148) + | > loader_time: 0.007 (0.006203536329598262) + + + --> TIME: 2025-04-19 00:45:54 -- STEP: 141/142 -- GLOBAL_STEP: 394475 + | > loss_disc: 2.395223617553711 (2.475241874126677) + | > loss_disc_real_0: 0.07329076528549194 (0.12849580760754592) + | > loss_disc_real_1: 0.202767014503479 (0.2065000757046625) + | > loss_disc_real_2: 0.18085546791553497 (0.2263233831799622) + | > loss_disc_real_3: 0.1810310333967209 (0.2278107375117904) + | > loss_disc_real_4: 0.17769451439380646 (0.23067126692609585) + | > loss_disc_real_5: 0.19267462193965912 (0.23359313321874497) + | > loss_0: 2.395223617553711 (2.475241874126677) + | > grad_norm_0: tensor(12.7826, device='cuda:0') (tensor(11.3034, device='cuda:0')) + | > loss_gen: 2.5553805828094482 (2.3316944381023976) + | > loss_kl: 1.9555107355117798 (1.9232883749278724) + | > loss_feat: 7.27821683883667 (6.920632298111071) + | > loss_mel: 15.338977813720703 (17.744971025074634) + | > loss_duration: 1.3729255199432373 (1.49834691632724) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.501008987426758 (30.41893296207942) + | > grad_norm_1: tensor(388.8033, device='cuda:0') (tensor(238.7907, device='cuda:0')) + | > current_lr_0: 0.0001973673155075699 + | > current_lr_1: 0.0001973673155075699 + | > step_time: 0.2032 (0.3900623964079729) + | > loader_time: 0.0028 (0.006405759365000622) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3139278888702393 (2.3139278888702393) + | > loss_disc_real_0: 0.13223662972450256 (0.13223662972450256) + | > loss_disc_real_1: 0.1766243577003479 (0.1766243577003479) + | > loss_disc_real_2: 0.22822235524654388 (0.22822235524654388) + | > loss_disc_real_3: 0.27255353331565857 (0.27255353331565857) + | > loss_disc_real_4: 0.23793531954288483 (0.23793531954288483) + | > loss_disc_real_5: 0.26840293407440186 (0.26840293407440186) + | > loss_0: 2.3139278888702393 (2.3139278888702393) + | > loss_gen: 2.6610729694366455 (2.6610729694366455) + | > loss_kl: 2.196805238723755 (2.196805238723755) + | > loss_feat: 8.094634056091309 (8.094634056091309) + | > loss_mel: 20.252845764160156 (20.252845764160156) + | > loss_duration: 1.6324397325515747 (1.6324397325515747) + | > loss_1: 34.83779525756836 (34.83779525756836) + + --> STEP: 1 + | > loss_disc: 2.631202220916748 (2.631202220916748) + | > loss_disc_real_0: 0.2269309014081955 (0.2269309014081955) + | > loss_disc_real_1: 0.23924699425697327 (0.23924699425697327) + | > loss_disc_real_2: 0.22446121275424957 (0.22446121275424957) + | > loss_disc_real_3: 0.2535504996776581 (0.2535504996776581) + | > loss_disc_real_4: 0.2580990791320801 (0.2580990791320801) + | > loss_disc_real_5: 0.2717530429363251 (0.2717530429363251) + | > loss_0: 2.631202220916748 (2.631202220916748) + | > loss_gen: 2.44870662689209 (2.44870662689209) + | > loss_kl: 2.359196662902832 (2.359196662902832) + | > loss_feat: 6.277759552001953 (6.277759552001953) + | > loss_mel: 16.58701515197754 (16.58701515197754) + | > loss_duration: 1.6671041250228882 (1.6671041250228882) + | > loss_1: 29.339784622192383 (29.339784622192383) + + --> STEP: 2 + | > loss_disc: 2.4867608547210693 (2.5589815378189087) + | > loss_disc_real_0: 0.12434558570384979 (0.17563824355602264) + | > loss_disc_real_1: 0.23138092458248138 (0.23531395941972733) + | > loss_disc_real_2: 0.22825750708580017 (0.22635935992002487) + | > loss_disc_real_3: 0.2836523652076721 (0.2686014324426651) + | > loss_disc_real_4: 0.2616077959537506 (0.25985343754291534) + | > loss_disc_real_5: 0.2827964425086975 (0.2772747427225113) + | > loss_0: 2.4867608547210693 (2.5589815378189087) + | > loss_gen: 2.488516330718994 (2.468611478805542) + | > loss_kl: 2.1242358684539795 (2.2417162656784058) + | > loss_feat: 7.102128982543945 (6.689944267272949) + | > loss_mel: 17.85283660888672 (17.21992588043213) + | > loss_duration: 1.5905659198760986 (1.6288350224494934) + | > loss_1: 31.158283233642578 (30.24903392791748) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003432154655456543 (+0.00016391277313232422) + | > avg_loss_disc: 2.5589815378189087 (-0.03718972206115723) + | > avg_loss_disc_real_0: 0.17563824355602264 (+0.011598072946071625) + | > avg_loss_disc_real_1: 0.23531395941972733 (-0.02657002955675125) + | > avg_loss_disc_real_2: 0.22635935992002487 (-0.047496236860752106) + | > avg_loss_disc_real_3: 0.2686014324426651 (+0.0335521474480629) + | > avg_loss_disc_real_4: 0.25985343754291534 (+0.003856942057609558) + | > avg_loss_disc_real_5: 0.2772747427225113 (-0.0030952095985412598) + | > avg_loss_0: 2.5589815378189087 (-0.03718972206115723) + | > avg_loss_gen: 2.468611478805542 (+0.1062551736831665) + | > avg_loss_kl: 2.2417162656784058 (-0.09154760837554932) + | > avg_loss_feat: 6.689944267272949 (+0.4667930603027344) + | > avg_loss_mel: 17.21992588043213 (+0.40670013427734375) + | > avg_loss_duration: 1.6288350224494934 (-0.01967686414718628) + | > avg_loss_1: 30.24903392791748 (+0.8685245513916016) + + + > EPOCH: 107/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:45:56)  + + --> TIME: 2025-04-19 00:46:05 -- STEP: 24/142 -- GLOBAL_STEP: 394500 + | > loss_disc: 2.6528303623199463 (2.4396266937255855) + | > loss_disc_real_0: 0.1345137655735016 (0.12732619978487492) + | > loss_disc_real_1: 0.24222290515899658 (0.2045259022464355) + | > loss_disc_real_2: 0.2771371304988861 (0.22478601709008217) + | > loss_disc_real_3: 0.2572983503341675 (0.22467758692801) + | > loss_disc_real_4: 0.28926873207092285 (0.23004788470764956) + | > loss_disc_real_5: 0.2486838549375534 (0.2316936757415533) + | > loss_0: 2.6528303623199463 (2.4396266937255855) + | > grad_norm_0: tensor(11.3285, device='cuda:0') (tensor(9.8120, device='cuda:0')) + | > loss_gen: 2.346932888031006 (2.3952228923638663) + | > loss_kl: 1.9710707664489746 (1.8443467964728673) + | > loss_feat: 6.165336608886719 (7.1068787177403765) + | > loss_mel: 18.023242950439453 (17.74919080734253) + | > loss_duration: 1.510685920715332 (1.4532420138518016) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.017269134521484 (30.54888105392456) + | > grad_norm_1: tensor(203.7018, device='cuda:0') (tensor(219.1363, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.3464 (0.33387163281440735) + | > loader_time: 0.0057 (0.004779924949010213) + + + --> TIME: 2025-04-19 00:46:14 -- STEP: 49/142 -- GLOBAL_STEP: 394525 + | > loss_disc: 2.4846267700195312 (2.4602026988048937) + | > loss_disc_real_0: 0.08654099702835083 (0.12668266481890966) + | > loss_disc_real_1: 0.21159690618515015 (0.20835260834012712) + | > loss_disc_real_2: 0.2165488302707672 (0.2248195239475795) + | > loss_disc_real_3: 0.2436099350452423 (0.22798188912625217) + | > loss_disc_real_4: 0.24672403931617737 (0.22858248286101282) + | > loss_disc_real_5: 0.2322683036327362 (0.23437429934131856) + | > loss_0: 2.4846267700195312 (2.4602026988048937) + | > grad_norm_0: tensor(18.2166, device='cuda:0') (tensor(11.6181, device='cuda:0')) + | > loss_gen: 2.2488021850585938 (2.3719158221264265) + | > loss_kl: 1.7245960235595703 (1.8567142948812367) + | > loss_feat: 6.7892303466796875 (7.0821462650688325) + | > loss_mel: 16.593523025512695 (17.856041421695625) + | > loss_duration: 1.4806469678878784 (1.4625731001094895) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.8367977142334 (30.62939063870177) + | > grad_norm_1: tensor(471.7045, device='cuda:0') (tensor(269.2365, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.3718 (0.3461407447347835) + | > loader_time: 0.0057 (0.005081327594056421) + + + --> TIME: 2025-04-19 00:46:24 -- STEP: 74/142 -- GLOBAL_STEP: 394550 + | > loss_disc: 2.476231336593628 (2.4629055938205195) + | > loss_disc_real_0: 0.12785962224006653 (0.1273820585495717) + | > loss_disc_real_1: 0.18680667877197266 (0.20633490182257988) + | > loss_disc_real_2: 0.23538047075271606 (0.22555273126911474) + | > loss_disc_real_3: 0.2323499619960785 (0.22791109616692) + | > loss_disc_real_4: 0.2113318145275116 (0.22949155238834587) + | > loss_disc_real_5: 0.24998590350151062 (0.2333515278390936) + | > loss_0: 2.476231336593628 (2.4629055938205195) + | > grad_norm_0: tensor(7.2855, device='cuda:0') (tensor(14.1501, device='cuda:0')) + | > loss_gen: 2.495323896408081 (2.386465604240829) + | > loss_kl: 1.919459581375122 (1.8874854171598279) + | > loss_feat: 6.779829502105713 (7.081573189915837) + | > loss_mel: 17.581430435180664 (17.82163890632423) + | > loss_duration: 1.488525152206421 (1.476360005301398) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.264568328857422 (30.653522955404746) + | > grad_norm_1: tensor(231.7973, device='cuda:0') (tensor(287.1810, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.388 (0.3587206505440376) + | > loader_time: 0.006 (0.005377840351414035) + + + --> TIME: 2025-04-19 00:46:34 -- STEP: 99/142 -- GLOBAL_STEP: 394575 + | > loss_disc: 2.5487494468688965 (2.472053843315202) + | > loss_disc_real_0: 0.125422403216362 (0.1290079968595746) + | > loss_disc_real_1: 0.22953835129737854 (0.2072387175126509) + | > loss_disc_real_2: 0.27737662196159363 (0.22640556142185675) + | > loss_disc_real_3: 0.24862341582775116 (0.22904872608305227) + | > loss_disc_real_4: 0.2461923062801361 (0.23058680422378308) + | > loss_disc_real_5: 0.2567519247531891 (0.23398453283189524) + | > loss_0: 2.5487494468688965 (2.472053843315202) + | > grad_norm_0: tensor(20.3033, device='cuda:0') (tensor(14.9767, device='cuda:0')) + | > loss_gen: 2.336907386779785 (2.3783992998527754) + | > loss_kl: 2.059009552001953 (1.9084321489237777) + | > loss_feat: 6.252884864807129 (7.0112450676735) + | > loss_mel: 17.261985778808594 (17.81107921793003) + | > loss_duration: 1.5373692512512207 (1.4871574724563446) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.44815444946289 (30.596313091239544) + | > grad_norm_1: tensor(331.5161, device='cuda:0') (tensor(292.5107, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.4024 (0.36932983542933606) + | > loader_time: 0.0068 (0.005678610368208451) + + + --> TIME: 2025-04-19 00:46:45 -- STEP: 124/142 -- GLOBAL_STEP: 394600 + | > loss_disc: 2.3483376502990723 (2.4633189785865053) + | > loss_disc_real_0: 0.13315437734127045 (0.12758972507811367) + | > loss_disc_real_1: 0.15875408053398132 (0.20612129713258437) + | > loss_disc_real_2: 0.19039376080036163 (0.22548413084399316) + | > loss_disc_real_3: 0.190486878156662 (0.22780037611242263) + | > loss_disc_real_4: 0.21234935522079468 (0.22993306959829024) + | > loss_disc_real_5: 0.20414447784423828 (0.23408639214692578) + | > loss_0: 2.3483376502990723 (2.4633189785865053) + | > grad_norm_0: tensor(14.4506, device='cuda:0') (tensor(15.0553, device='cuda:0')) + | > loss_gen: 2.3326380252838135 (2.3736937719006694) + | > loss_kl: 2.005669593811035 (1.9218388372851956) + | > loss_feat: 7.41891622543335 (6.996830301900064) + | > loss_mel: 17.92726707458496 (17.809453710432976) + | > loss_duration: 1.545729160308838 (1.4942551245612483) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.2302188873291 (30.596071612450384) + | > grad_norm_1: tensor(493.7104, device='cuda:0') (tensor(306.2231, device='cuda:0')) + | > current_lr_0: 0.00019734264459313146 + | > current_lr_1: 0.00019734264459313146 + | > step_time: 0.4397 (0.3814080472915403) + | > loader_time: 0.0068 (0.0059365207149136455) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.405933141708374 (2.405933141708374) + | > loss_disc_real_0: 0.17887847125530243 (0.17887847125530243) + | > loss_disc_real_1: 0.19689036905765533 (0.19689036905765533) + | > loss_disc_real_2: 0.21224120259284973 (0.21224120259284973) + | > loss_disc_real_3: 0.2272062450647354 (0.2272062450647354) + | > loss_disc_real_4: 0.1889486014842987 (0.1889486014842987) + | > loss_disc_real_5: 0.2330974042415619 (0.2330974042415619) + | > loss_0: 2.405933141708374 (2.405933141708374) + | > loss_gen: 2.350743055343628 (2.350743055343628) + | > loss_kl: 2.1606743335723877 (2.1606743335723877) + | > loss_feat: 6.719719886779785 (6.719719886779785) + | > loss_mel: 19.93695831298828 (19.93695831298828) + | > loss_duration: 1.6334152221679688 (1.6334152221679688) + | > loss_1: 32.801509857177734 (32.801509857177734) + + --> STEP: 1 + | > loss_disc: 2.4270951747894287 (2.4270951747894287) + | > loss_disc_real_0: 0.15119986236095428 (0.15119986236095428) + | > loss_disc_real_1: 0.22379598021507263 (0.22379598021507263) + | > loss_disc_real_2: 0.23539358377456665 (0.23539358377456665) + | > loss_disc_real_3: 0.22568196058273315 (0.22568196058273315) + | > loss_disc_real_4: 0.24847492575645447 (0.24847492575645447) + | > loss_disc_real_5: 0.2557189464569092 (0.2557189464569092) + | > loss_0: 2.4270951747894287 (2.4270951747894287) + | > loss_gen: 2.4733457565307617 (2.4733457565307617) + | > loss_kl: 2.6880602836608887 (2.6880602836608887) + | > loss_feat: 6.76906681060791 (6.76906681060791) + | > loss_mel: 17.952144622802734 (17.952144622802734) + | > loss_duration: 1.686753749847412 (1.686753749847412) + | > loss_1: 31.569372177124023 (31.569372177124023) + + --> STEP: 2 + | > loss_disc: 2.5198593139648438 (2.4734772443771362) + | > loss_disc_real_0: 0.12781058251857758 (0.13950522243976593) + | > loss_disc_real_1: 0.19062423706054688 (0.20721010863780975) + | > loss_disc_real_2: 0.21325433254241943 (0.22432395815849304) + | > loss_disc_real_3: 0.23335665464401245 (0.2295193076133728) + | > loss_disc_real_4: 0.2559349536895752 (0.25220493972301483) + | > loss_disc_real_5: 0.23475691676139832 (0.24523793160915375) + | > loss_0: 2.5198593139648438 (2.4734772443771362) + | > loss_gen: 2.2820725440979004 (2.377709150314331) + | > loss_kl: 2.285614490509033 (2.486837387084961) + | > loss_feat: 8.619359016418457 (7.694212913513184) + | > loss_mel: 19.814847946166992 (18.883496284484863) + | > loss_duration: 1.6167633533477783 (1.6517585515975952) + | > loss_1: 34.61865997314453 (33.09401607513428) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036096572875976562 (+0.00017750263214111328) + | > avg_loss_disc: 2.4734772443771362 (-0.08550429344177246) + | > avg_loss_disc_real_0: 0.13950522243976593 (-0.036133021116256714) + | > avg_loss_disc_real_1: 0.20721010863780975 (-0.028103850781917572) + | > avg_loss_disc_real_2: 0.22432395815849304 (-0.00203540176153183) + | > avg_loss_disc_real_3: 0.2295193076133728 (-0.0390821248292923) + | > avg_loss_disc_real_4: 0.25220493972301483 (-0.007648497819900513) + | > avg_loss_disc_real_5: 0.24523793160915375 (-0.032036811113357544) + | > avg_loss_0: 2.4734772443771362 (-0.08550429344177246) + | > avg_loss_gen: 2.377709150314331 (-0.09090232849121094) + | > avg_loss_kl: 2.486837387084961 (+0.24512112140655518) + | > avg_loss_feat: 7.694212913513184 (+1.0042686462402344) + | > avg_loss_mel: 18.883496284484863 (+1.6635704040527344) + | > avg_loss_duration: 1.6517585515975952 (+0.022923529148101807) + | > avg_loss_1: 33.09401607513428 (+2.844982147216797) + + + > EPOCH: 108/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:46:55)  + + --> TIME: 2025-04-19 00:46:58 -- STEP: 7/142 -- GLOBAL_STEP: 394625 + | > loss_disc: 2.2743234634399414 (2.4139856610979353) + | > loss_disc_real_0: 0.10792624950408936 (0.11477154067584447) + | > loss_disc_real_1: 0.18059197068214417 (0.20739159839493887) + | > loss_disc_real_2: 0.22667621076107025 (0.2327701130083629) + | > loss_disc_real_3: 0.1973779946565628 (0.22437713614531926) + | > loss_disc_real_4: 0.23437459766864777 (0.22952935312475478) + | > loss_disc_real_5: 0.23420283198356628 (0.24344123687062943) + | > loss_0: 2.2743234634399414 (2.4139856610979353) + | > grad_norm_0: tensor(7.5573, device='cuda:0') (tensor(7.9899, device='cuda:0')) + | > loss_gen: 2.537165641784668 (2.438044616154262) + | > loss_kl: 1.9272387027740479 (1.858488644872393) + | > loss_feat: 7.967155456542969 (7.146909713745117) + | > loss_mel: 19.239593505859375 (17.865423474993026) + | > loss_duration: 1.441253900527954 (1.4445514678955078) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.11240768432617 (30.753418241228378) + | > grad_norm_1: tensor(256.6637, device='cuda:0') (tensor(257.9476, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.3234 (0.3239420141492571) + | > loader_time: 0.0039 (0.004073926380702427) + + + --> TIME: 2025-04-19 00:47:07 -- STEP: 32/142 -- GLOBAL_STEP: 394650 + | > loss_disc: 2.4161648750305176 (2.4582412019371986) + | > loss_disc_real_0: 0.1049593910574913 (0.12023212888743728) + | > loss_disc_real_1: 0.18196484446525574 (0.20632212748751044) + | > loss_disc_real_2: 0.19511350989341736 (0.22913918271660805) + | > loss_disc_real_3: 0.1945553421974182 (0.223495798651129) + | > loss_disc_real_4: 0.20902901887893677 (0.2341082924976945) + | > loss_disc_real_5: 0.229974165558815 (0.23516367515549064) + | > loss_0: 2.4161648750305176 (2.4582412019371986) + | > grad_norm_0: tensor(6.8363, device='cuda:0') (tensor(9.4382, device='cuda:0')) + | > loss_gen: 2.3950419425964355 (2.382920525968075) + | > loss_kl: 1.84955632686615 (1.873550035059452) + | > loss_feat: 6.82220983505249 (7.135694310069084) + | > loss_mel: 17.34604835510254 (17.88732820749283) + | > loss_duration: 1.4686017036437988 (1.457535095512867) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.881458282470703 (30.737028300762177) + | > grad_norm_1: tensor(274.8940, device='cuda:0') (tensor(244.7634, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.3546 (0.342640720307827) + | > loader_time: 0.005 (0.004784084856510164) + + + --> TIME: 2025-04-19 00:47:16 -- STEP: 57/142 -- GLOBAL_STEP: 394675 + | > loss_disc: 2.478549003601074 (2.470364106328864) + | > loss_disc_real_0: 0.10994911193847656 (0.1261131149765692) + | > loss_disc_real_1: 0.22952109575271606 (0.21005380990212424) + | > loss_disc_real_2: 0.21385978162288666 (0.22737756319213331) + | > loss_disc_real_3: 0.23139400780200958 (0.22535517681063266) + | > loss_disc_real_4: 0.22348518669605255 (0.23243669407409534) + | > loss_disc_real_5: 0.22516731917858124 (0.23452587399566382) + | > loss_0: 2.478549003601074 (2.470364106328864) + | > grad_norm_0: tensor(5.8394, device='cuda:0') (tensor(8.5765, device='cuda:0')) + | > loss_gen: 2.2204079627990723 (2.3462347315068826) + | > loss_kl: 1.969775915145874 (1.8889099966015732) + | > loss_feat: 6.7837371826171875 (6.939085508647718) + | > loss_mel: 17.70334243774414 (17.731039181090235) + | > loss_duration: 1.5140691995620728 (1.4680105866047375) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.19133186340332 (30.373280040004797) + | > grad_norm_1: tensor(90.6528, device='cuda:0') (tensor(200.3491, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.3787 (0.3521875833210192) + | > loader_time: 0.0057 (0.005161657667996592) + + + --> TIME: 2025-04-19 00:47:26 -- STEP: 82/142 -- GLOBAL_STEP: 394700 + | > loss_disc: 2.583789587020874 (2.4828042867707043) + | > loss_disc_real_0: 0.09530941396951675 (0.12870081618609952) + | > loss_disc_real_1: 0.19815649092197418 (0.20879325888505795) + | > loss_disc_real_2: 0.23972195386886597 (0.22871176135249255) + | > loss_disc_real_3: 0.21582001447677612 (0.22668561012279698) + | > loss_disc_real_4: 0.24704542756080627 (0.2316334116749647) + | > loss_disc_real_5: 0.23902954161167145 (0.23361200475838126) + | > loss_0: 2.583789587020874 (2.4828042867707043) + | > grad_norm_0: tensor(6.0554, device='cuda:0') (tensor(8.7346, device='cuda:0')) + | > loss_gen: 2.260516405105591 (2.323678522575191) + | > loss_kl: 2.268343687057495 (1.894880044751051) + | > loss_feat: 6.2616376876831055 (6.855706435878102) + | > loss_mel: 17.886417388916016 (17.670268384421746) + | > loss_duration: 1.5714404582977295 (1.4808428912627987) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.248355865478516 (30.22537636175388) + | > grad_norm_1: tensor(215.6478, device='cuda:0') (tensor(200.3883, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.3948 (0.36373960971832264) + | > loader_time: 0.0062 (0.0054240372122787844) + + + --> TIME: 2025-04-19 00:47:37 -- STEP: 107/142 -- GLOBAL_STEP: 394725 + | > loss_disc: 2.4045467376708984 (2.4915876811909907) + | > loss_disc_real_0: 0.09436743706464767 (0.1307085564830036) + | > loss_disc_real_1: 0.18612182140350342 (0.2102894581088396) + | > loss_disc_real_2: 0.21341079473495483 (0.22925852615142536) + | > loss_disc_real_3: 0.21827863156795502 (0.22834403562211544) + | > loss_disc_real_4: 0.24274100363254547 (0.23141852536491145) + | > loss_disc_real_5: 0.19907288253307343 (0.23362122684995704) + | > loss_0: 2.4045467376708984 (2.4915876811909907) + | > grad_norm_0: tensor(19.0714, device='cuda:0') (tensor(9.6846, device='cuda:0')) + | > loss_gen: 2.2859745025634766 (2.318748928676141) + | > loss_kl: 1.9042102098464966 (1.9096577902820622) + | > loss_feat: 7.650768280029297 (6.83064472786734) + | > loss_mel: 17.552148818969727 (17.695987754893075) + | > loss_duration: 1.4957275390625 (1.4911510498724252) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.888830184936523 (30.246190347404124) + | > grad_norm_1: tensor(446.5139, device='cuda:0') (tensor(223.5556, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.4191 (0.3737580085469182) + | > loader_time: 0.0062 (0.005687069670062199) + + + --> TIME: 2025-04-19 00:47:48 -- STEP: 132/142 -- GLOBAL_STEP: 394750 + | > loss_disc: 2.539987087249756 (2.4833922295859376) + | > loss_disc_real_0: 0.13336361944675446 (0.1304005554058786) + | > loss_disc_real_1: 0.24886687099933624 (0.20875506881963124) + | > loss_disc_real_2: 0.24160532653331757 (0.2279730967499993) + | > loss_disc_real_3: 0.21821202337741852 (0.22792300311001865) + | > loss_disc_real_4: 0.2700233459472656 (0.23155268089789333) + | > loss_disc_real_5: 0.23539051413536072 (0.23356417882623096) + | > loss_0: 2.539987087249756 (2.4833922295859376) + | > grad_norm_0: tensor(18.8016, device='cuda:0') (tensor(10.6763, device='cuda:0')) + | > loss_gen: 2.242208957672119 (2.327012365514581) + | > loss_kl: 1.9247678518295288 (1.9166163458968655) + | > loss_feat: 5.843289375305176 (6.834565610596628) + | > loss_mel: 17.034408569335938 (17.73137737042976) + | > loss_duration: 1.5102577209472656 (1.497596044432033) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.554931640625 (30.30716783350164) + | > grad_norm_1: tensor(305.6073, device='cuda:0') (tensor(248.7495, device='cuda:0')) + | > current_lr_0: 0.0001973179767625573 + | > current_lr_1: 0.0001973179767625573 + | > step_time: 0.4446 (0.38550417892860633) + | > loader_time: 0.0074 (0.005936071728215073) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.571503162384033 (2.571503162384033) + | > loss_disc_real_0: 0.2494470179080963 (0.2494470179080963) + | > loss_disc_real_1: 0.2595170736312866 (0.2595170736312866) + | > loss_disc_real_2: 0.258950412273407 (0.258950412273407) + | > loss_disc_real_3: 0.21569667756557465 (0.21569667756557465) + | > loss_disc_real_4: 0.24335218966007233 (0.24335218966007233) + | > loss_disc_real_5: 0.23366491496562958 (0.23366491496562958) + | > loss_0: 2.571503162384033 (2.571503162384033) + | > loss_gen: 2.561049222946167 (2.561049222946167) + | > loss_kl: 2.2543528079986572 (2.2543528079986572) + | > loss_feat: 7.436926364898682 (7.436926364898682) + | > loss_mel: 19.306867599487305 (19.306867599487305) + | > loss_duration: 1.6453553438186646 (1.6453553438186646) + | > loss_1: 33.204551696777344 (33.204551696777344) + + --> STEP: 1 + | > loss_disc: 2.5592429637908936 (2.5592429637908936) + | > loss_disc_real_0: 0.23107747733592987 (0.23107747733592987) + | > loss_disc_real_1: 0.2854635417461395 (0.2854635417461395) + | > loss_disc_real_2: 0.22793185710906982 (0.22793185710906982) + | > loss_disc_real_3: 0.24301865696907043 (0.24301865696907043) + | > loss_disc_real_4: 0.26938602328300476 (0.26938602328300476) + | > loss_disc_real_5: 0.21557411551475525 (0.21557411551475525) + | > loss_0: 2.5592429637908936 (2.5592429637908936) + | > loss_gen: 2.6368682384490967 (2.6368682384490967) + | > loss_kl: 2.4843790531158447 (2.4843790531158447) + | > loss_feat: 7.025193214416504 (7.025193214416504) + | > loss_mel: 18.941932678222656 (18.941932678222656) + | > loss_duration: 1.6744545698165894 (1.6744545698165894) + | > loss_1: 32.76282501220703 (32.76282501220703) + + --> STEP: 2 + | > loss_disc: 2.583838701248169 (2.5715408325195312) + | > loss_disc_real_0: 0.26105934381484985 (0.24606841057538986) + | > loss_disc_real_1: 0.2739041745662689 (0.2796838581562042) + | > loss_disc_real_2: 0.2789354920387268 (0.2534336745738983) + | > loss_disc_real_3: 0.2104835957288742 (0.22675112634897232) + | > loss_disc_real_4: 0.22339940071105957 (0.24639271199703217) + | > loss_disc_real_5: 0.27978965640068054 (0.2476818859577179) + | > loss_0: 2.583838701248169 (2.5715408325195312) + | > loss_gen: 2.6496310234069824 (2.6432496309280396) + | > loss_kl: 1.927416205406189 (2.205897629261017) + | > loss_feat: 7.28493595123291 (7.155064582824707) + | > loss_mel: 17.830944061279297 (18.386438369750977) + | > loss_duration: 1.6142871379852295 (1.6443708539009094) + | > loss_1: 31.307214736938477 (32.035019874572754) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003814220428466797 (+0.00020456314086914062) + | > avg_loss_disc: 2.5715408325195312 (+0.09806358814239502) + | > avg_loss_disc_real_0: 0.24606841057538986 (+0.10656318813562393) + | > avg_loss_disc_real_1: 0.2796838581562042 (+0.07247374951839447) + | > avg_loss_disc_real_2: 0.2534336745738983 (+0.029109716415405273) + | > avg_loss_disc_real_3: 0.22675112634897232 (-0.002768181264400482) + | > avg_loss_disc_real_4: 0.24639271199703217 (-0.005812227725982666) + | > avg_loss_disc_real_5: 0.2476818859577179 (+0.002443954348564148) + | > avg_loss_0: 2.5715408325195312 (+0.09806358814239502) + | > avg_loss_gen: 2.6432496309280396 (+0.2655404806137085) + | > avg_loss_kl: 2.205897629261017 (-0.2809397578239441) + | > avg_loss_feat: 7.155064582824707 (-0.5391483306884766) + | > avg_loss_mel: 18.386438369750977 (-0.4970579147338867) + | > avg_loss_duration: 1.6443708539009094 (-0.007387697696685791) + | > avg_loss_1: 32.035019874572754 (-1.0589962005615234) + + + > EPOCH: 109/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:47:54)  + + --> TIME: 2025-04-19 00:48:00 -- STEP: 15/142 -- GLOBAL_STEP: 394775 + | > loss_disc: 2.436455249786377 (2.484703699747721) + | > loss_disc_real_0: 0.11388758569955826 (0.12581451634565988) + | > loss_disc_real_1: 0.17250031232833862 (0.20909059246381123) + | > loss_disc_real_2: 0.20459328591823578 (0.22546212176481883) + | > loss_disc_real_3: 0.19174142181873322 (0.22968888978163401) + | > loss_disc_real_4: 0.23292973637580872 (0.2365737736225128) + | > loss_disc_real_5: 0.21559789776802063 (0.23383553922176362) + | > loss_0: 2.436455249786377 (2.484703699747721) + | > grad_norm_0: tensor(13.1384, device='cuda:0') (tensor(12.2732, device='cuda:0')) + | > loss_gen: 2.313260555267334 (2.338740221659343) + | > loss_kl: 1.7779858112335205 (1.8635646899541218) + | > loss_feat: 6.914430618286133 (7.002500947316488) + | > loss_mel: 18.60590362548828 (17.588580894470216) + | > loss_duration: 1.4484257698059082 (1.4492846171061198) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.060007095336914 (30.242671330769856) + | > grad_norm_1: tensor(331.1614, device='cuda:0') (tensor(287.4175, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.3398 (0.3308103084564209) + | > loader_time: 0.0052 (0.004446776707967123) + + + --> TIME: 2025-04-19 00:48:09 -- STEP: 40/142 -- GLOBAL_STEP: 394800 + | > loss_disc: 2.365927219390869 (2.460981690883636) + | > loss_disc_real_0: 0.156890869140625 (0.12232977487146854) + | > loss_disc_real_1: 0.19563698768615723 (0.20802443772554396) + | > loss_disc_real_2: 0.22286656498908997 (0.2272179637104273) + | > loss_disc_real_3: 0.253462016582489 (0.22595036402344704) + | > loss_disc_real_4: 0.20898112654685974 (0.22848555371165274) + | > loss_disc_real_5: 0.2228352576494217 (0.23213411010801793) + | > loss_0: 2.365927219390869 (2.460981690883636) + | > grad_norm_0: tensor(13.4542, device='cuda:0') (tensor(10.8762, device='cuda:0')) + | > loss_gen: 2.4450418949127197 (2.3467995703220366) + | > loss_kl: 1.8039623498916626 (1.8761766135692597) + | > loss_feat: 6.77420711517334 (6.975250506401062) + | > loss_mel: 17.586271286010742 (17.613251090049737) + | > loss_duration: 1.4366666078567505 (1.4623933970928191) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.0461483001709 (30.273871088027953) + | > grad_norm_1: tensor(274.5034, device='cuda:0') (tensor(253.5162, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.3614 (0.343299800157547) + | > loader_time: 0.006 (0.004991108179092407) + + + --> TIME: 2025-04-19 00:48:18 -- STEP: 65/142 -- GLOBAL_STEP: 394825 + | > loss_disc: 2.522268533706665 (2.4804522000826315) + | > loss_disc_real_0: 0.18190674483776093 (0.12992569013283803) + | > loss_disc_real_1: 0.2280142605304718 (0.209481670994025) + | > loss_disc_real_2: 0.21751488745212555 (0.22818197080722222) + | > loss_disc_real_3: 0.22757163643836975 (0.22848417667242196) + | > loss_disc_real_4: 0.24858726561069489 (0.23056479371511018) + | > loss_disc_real_5: 0.25252315402030945 (0.2326546238018916) + | > loss_0: 2.522268533706665 (2.4804522000826315) + | > grad_norm_0: tensor(22.7358, device='cuda:0') (tensor(11.3507, device='cuda:0')) + | > loss_gen: 2.220452308654785 (2.335106614919809) + | > loss_kl: 1.8546335697174072 (1.87818962427286) + | > loss_feat: 6.723171234130859 (6.943063405843882) + | > loss_mel: 16.744171142578125 (17.680644094027006) + | > loss_duration: 1.482928991317749 (1.474969537441547) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.02535629272461 (30.31197316096379) + | > grad_norm_1: tensor(361.3887, device='cuda:0') (tensor(255.4929, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.3894 (0.354515475493211) + | > loader_time: 0.0066 (0.005339600489689754) + + + --> TIME: 2025-04-19 00:48:28 -- STEP: 90/142 -- GLOBAL_STEP: 394850 + | > loss_disc: 2.419358730316162 (2.4777258051766298) + | > loss_disc_real_0: 0.18944787979125977 (0.12908911651207336) + | > loss_disc_real_1: 0.20725828409194946 (0.20849469585551156) + | > loss_disc_real_2: 0.2266312688589096 (0.22766384979089102) + | > loss_disc_real_3: 0.21560509502887726 (0.22806818071338866) + | > loss_disc_real_4: 0.2203025221824646 (0.23096478631099066) + | > loss_disc_real_5: 0.21900182962417603 (0.23266993429925706) + | > loss_0: 2.419358730316162 (2.4777258051766298) + | > grad_norm_0: tensor(8.8945, device='cuda:0') (tensor(11.5870, device='cuda:0')) + | > loss_gen: 2.408416748046875 (2.3400972233878243) + | > loss_kl: 1.930356502532959 (1.8859165536032783) + | > loss_feat: 6.911203384399414 (6.941638792885675) + | > loss_mel: 17.4832706451416 (17.739011838701035) + | > loss_duration: 1.5269651412963867 (1.4868132485283745) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.260211944580078 (30.393477545844185) + | > grad_norm_1: tensor(130.7776, device='cuda:0') (tensor(264.4541, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.4019 (0.3653092755211724) + | > loader_time: 0.0069 (0.005590285195244683) + + + --> TIME: 2025-04-19 00:48:39 -- STEP: 115/142 -- GLOBAL_STEP: 394875 + | > loss_disc: 2.4666357040405273 (2.4775973009026577) + | > loss_disc_real_0: 0.15427373349666595 (0.12943941280245774) + | > loss_disc_real_1: 0.2239992320537567 (0.20810687554919202) + | > loss_disc_real_2: 0.18462282419204712 (0.22682779042617135) + | > loss_disc_real_3: 0.20235376060009003 (0.22818562025609224) + | > loss_disc_real_4: 0.21567952632904053 (0.23005329409371253) + | > loss_disc_real_5: 0.17816653847694397 (0.23243365210035574) + | > loss_0: 2.4666357040405273 (2.4775973009026577) + | > grad_norm_0: tensor(10.1095, device='cuda:0') (tensor(13.0543, device='cuda:0')) + | > loss_gen: 2.413649082183838 (2.3406651268834655) + | > loss_kl: 2.003047466278076 (1.90006931035415) + | > loss_feat: 7.723954200744629 (6.922065249733303) + | > loss_mel: 19.59624671936035 (17.790968720809282) + | > loss_duration: 1.5092854499816895 (1.4939044651777849) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.24618148803711 (30.447672802469004) + | > grad_norm_1: tensor(273.7620, device='cuda:0') (tensor(282.8764, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.4234 (0.37594860118368395) + | > loader_time: 0.0072 (0.005843753400056258) + + + --> TIME: 2025-04-19 00:48:50 -- STEP: 140/142 -- GLOBAL_STEP: 394900 + | > loss_disc: 2.44264817237854 (2.4835867132459377) + | > loss_disc_real_0: 0.21355417370796204 (0.13231918500470258) + | > loss_disc_real_1: 0.18244192004203796 (0.20751679156507766) + | > loss_disc_real_2: 0.22341875731945038 (0.22762434439999718) + | > loss_disc_real_3: 0.21439068019390106 (0.229526431539229) + | > loss_disc_real_4: 0.21640151739120483 (0.23100998103618622) + | > loss_disc_real_5: 0.22139526903629303 (0.2331190743616649) + | > loss_0: 2.44264817237854 (2.4835867132459377) + | > grad_norm_0: tensor(33.2759, device='cuda:0') (tensor(14.9033, device='cuda:0')) + | > loss_gen: 2.4298839569091797 (2.343962216377258) + | > loss_kl: 1.9656254053115845 (1.9151986403124672) + | > loss_feat: 7.329928874969482 (6.9134705475398475) + | > loss_mel: 18.549156188964844 (17.79212071555002) + | > loss_duration: 1.5432407855987549 (1.5004332278456012) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.81783676147461 (30.4651853288923) + | > grad_norm_1: tensor(516.2737, device='cuda:0') (tensor(297.7209, device='cuda:0')) + | > current_lr_0: 0.00019729331201546197 + | > current_lr_1: 0.00019729331201546197 + | > step_time: 0.4583 (0.3880038619041443) + | > loader_time: 0.0076 (0.0061150687081473224) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.414252996444702 (2.414252996444702) + | > loss_disc_real_0: 0.08101842552423477 (0.08101842552423477) + | > loss_disc_real_1: 0.2182157337665558 (0.2182157337665558) + | > loss_disc_real_2: 0.24773161113262177 (0.24773161113262177) + | > loss_disc_real_3: 0.24818594753742218 (0.24818594753742218) + | > loss_disc_real_4: 0.19789372384548187 (0.19789372384548187) + | > loss_disc_real_5: 0.2276744842529297 (0.2276744842529297) + | > loss_0: 2.414252996444702 (2.414252996444702) + | > loss_gen: 2.1843090057373047 (2.1843090057373047) + | > loss_kl: 2.4252591133117676 (2.4252591133117676) + | > loss_feat: 7.033382415771484 (7.033382415771484) + | > loss_mel: 17.727073669433594 (17.727073669433594) + | > loss_duration: 1.6403483152389526 (1.6403483152389526) + | > loss_1: 31.010372161865234 (31.010372161865234) + + --> STEP: 1 + | > loss_disc: 2.4916982650756836 (2.4916982650756836) + | > loss_disc_real_0: 0.08463846147060394 (0.08463846147060394) + | > loss_disc_real_1: 0.2458273023366928 (0.2458273023366928) + | > loss_disc_real_2: 0.25323665142059326 (0.25323665142059326) + | > loss_disc_real_3: 0.2248174101114273 (0.2248174101114273) + | > loss_disc_real_4: 0.21166948974132538 (0.21166948974132538) + | > loss_disc_real_5: 0.24589715898036957 (0.24589715898036957) + | > loss_0: 2.4916982650756836 (2.4916982650756836) + | > loss_gen: 2.1968138217926025 (2.1968138217926025) + | > loss_kl: 2.370136260986328 (2.370136260986328) + | > loss_feat: 7.572748184204102 (7.572748184204102) + | > loss_mel: 18.58248519897461 (18.58248519897461) + | > loss_duration: 1.6667088270187378 (1.6667088270187378) + | > loss_1: 32.388893127441406 (32.388893127441406) + + --> STEP: 2 + | > loss_disc: 2.4296507835388184 (2.460674524307251) + | > loss_disc_real_0: 0.07817338407039642 (0.08140592277050018) + | > loss_disc_real_1: 0.21351788938045502 (0.2296725958585739) + | > loss_disc_real_2: 0.249670147895813 (0.2514533996582031) + | > loss_disc_real_3: 0.224666565656662 (0.22474198788404465) + | > loss_disc_real_4: 0.24268685281276703 (0.2271781712770462) + | > loss_disc_real_5: 0.23207031190395355 (0.23898373544216156) + | > loss_0: 2.4296507835388184 (2.460674524307251) + | > loss_gen: 2.186392068862915 (2.191602945327759) + | > loss_kl: 2.462019205093384 (2.416077733039856) + | > loss_feat: 5.791844844818115 (6.682296514511108) + | > loss_mel: 15.634942054748535 (17.108713626861572) + | > loss_duration: 1.6418068408966064 (1.6542578339576721) + | > loss_1: 27.717004776000977 (30.05294895172119) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003762364387512207 (-5.1856040954589844e-05) + | > avg_loss_disc: 2.460674524307251 (-0.11086630821228027) + | > avg_loss_disc_real_0: 0.08140592277050018 (-0.16466248780488968) + | > avg_loss_disc_real_1: 0.2296725958585739 (-0.05001126229763031) + | > avg_loss_disc_real_2: 0.2514533996582031 (-0.0019802749156951904) + | > avg_loss_disc_real_3: 0.22474198788404465 (-0.0020091384649276733) + | > avg_loss_disc_real_4: 0.2271781712770462 (-0.019214540719985962) + | > avg_loss_disc_real_5: 0.23898373544216156 (-0.008698150515556335) + | > avg_loss_0: 2.460674524307251 (-0.11086630821228027) + | > avg_loss_gen: 2.191602945327759 (-0.45164668560028076) + | > avg_loss_kl: 2.416077733039856 (+0.2101801037788391) + | > avg_loss_feat: 6.682296514511108 (-0.47276806831359863) + | > avg_loss_mel: 17.108713626861572 (-1.2777247428894043) + | > avg_loss_duration: 1.6542578339576721 (+0.009886980056762695) + | > avg_loss_1: 30.05294895172119 (-1.9820709228515625) + + + > EPOCH: 110/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:48:53)  + + --> TIME: 2025-04-19 00:49:01 -- STEP: 23/142 -- GLOBAL_STEP: 394925 + | > loss_disc: 2.539135217666626 (2.464309723480888) + | > loss_disc_real_0: 0.09091607481241226 (0.11697487597880156) + | > loss_disc_real_1: 0.21730142831802368 (0.2063096301711124) + | > loss_disc_real_2: 0.23791590332984924 (0.23016946017742157) + | > loss_disc_real_3: 0.2594924867153168 (0.22811567977718686) + | > loss_disc_real_4: 0.23761972784996033 (0.23484017926713693) + | > loss_disc_real_5: 0.30793407559394836 (0.23719034246776416) + | > loss_0: 2.539135217666626 (2.464309723480888) + | > grad_norm_0: tensor(5.3833, device='cuda:0') (tensor(8.2103, device='cuda:0')) + | > loss_gen: 2.2622828483581543 (2.3412144287772803) + | > loss_kl: 1.8858110904693604 (1.8715915472611138) + | > loss_feat: 6.656927585601807 (6.9984521036562715) + | > loss_mel: 17.5961971282959 (17.859581283901047) + | > loss_duration: 1.4994494915008545 (1.4544272059979646) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.900667190551758 (30.52526614976966) + | > grad_norm_1: tensor(224.9338, device='cuda:0') (tensor(192.0363, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.3448 (0.33248556178549066) + | > loader_time: 0.0053 (0.004610611044842264) + + + --> TIME: 2025-04-19 00:49:10 -- STEP: 48/142 -- GLOBAL_STEP: 394950 + | > loss_disc: 2.4130380153656006 (2.460183590650558) + | > loss_disc_real_0: 0.1311231106519699 (0.12045667056615154) + | > loss_disc_real_1: 0.1888953298330307 (0.20581454845766226) + | > loss_disc_real_2: 0.21753697097301483 (0.22812055082370838) + | > loss_disc_real_3: 0.2221861034631729 (0.22773594440271458) + | > loss_disc_real_4: 0.20529493689537048 (0.22947159813096127) + | > loss_disc_real_5: 0.20422957837581635 (0.23575051625569662) + | > loss_0: 2.4130380153656006 (2.460183590650558) + | > grad_norm_0: tensor(12.4888, device='cuda:0') (tensor(9.5135, device='cuda:0')) + | > loss_gen: 2.37539005279541 (2.3398094673951464) + | > loss_kl: 1.8082233667373657 (1.886129265030225) + | > loss_feat: 7.528018951416016 (7.008112003405889) + | > loss_mel: 18.299222946166992 (17.898015280564625) + | > loss_duration: 1.4636693000793457 (1.4655358244975407) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.474525451660156 (30.597601652145386) + | > grad_norm_1: tensor(237.4829, device='cuda:0') (tensor(225.3447, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.3634 (0.3450886309146881) + | > loader_time: 0.0059 (0.00496382514635722) + + + --> TIME: 2025-04-19 00:49:20 -- STEP: 73/142 -- GLOBAL_STEP: 394975 + | > loss_disc: 2.5532243251800537 (2.462480953294936) + | > loss_disc_real_0: 0.12627390027046204 (0.12447448860700816) + | > loss_disc_real_1: 0.24743899703025818 (0.2060372278298417) + | > loss_disc_real_2: 0.23830316960811615 (0.2248995549058261) + | > loss_disc_real_3: 0.22663576900959015 (0.22825809088471818) + | > loss_disc_real_4: 0.17080450057983398 (0.22824797360864404) + | > loss_disc_real_5: 0.22236360609531403 (0.23636794641409833) + | > loss_0: 2.5532243251800537 (2.462480953294936) + | > grad_norm_0: tensor(10.9915, device='cuda:0') (tensor(10.6640, device='cuda:0')) + | > loss_gen: 2.3129360675811768 (2.344623911870669) + | > loss_kl: 1.8727078437805176 (1.9064170563057676) + | > loss_feat: 6.80253267288208 (6.990766486076461) + | > loss_mel: 16.805036544799805 (17.884983820457983) + | > loss_duration: 1.5507497787475586 (1.4788967583277455) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.343963623046875 (30.605687964452457) + | > grad_norm_1: tensor(396.0023, device='cuda:0') (tensor(240.8338, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.3942 (0.358851870445356) + | > loader_time: 0.006 (0.005320010119921542) + + + --> TIME: 2025-04-19 00:49:30 -- STEP: 98/142 -- GLOBAL_STEP: 395000 + | > loss_disc: 2.4019570350646973 (2.4628815018400854) + | > loss_disc_real_0: 0.08797863125801086 (0.12467146201097232) + | > loss_disc_real_1: 0.19291843473911285 (0.20587541558304612) + | > loss_disc_real_2: 0.19824683666229248 (0.22562975755759648) + | > loss_disc_real_3: 0.2055099755525589 (0.2278638447121698) + | > loss_disc_real_4: 0.174367293715477 (0.22794773079911057) + | > loss_disc_real_5: 0.1730811595916748 (0.23445780864175486) + | > loss_0: 2.4019570350646973 (2.4628815018400854) + | > grad_norm_0: tensor(6.2938, device='cuda:0') (tensor(11.1220, device='cuda:0')) + | > loss_gen: 2.326699733734131 (2.3416869616021905) + | > loss_kl: 1.933247685432434 (1.9054590220354042) + | > loss_feat: 7.587131023406982 (6.959716743352463) + | > loss_mel: 17.392629623413086 (17.89778537166362) + | > loss_duration: 1.4662330150604248 (1.4888158586560467) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.705942153930664 (30.593463878242336) + | > grad_norm_1: tensor(302.2267, device='cuda:0') (tensor(262.3860, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.4119 (0.36947031653657253) + | > loader_time: 0.0078 (0.005571504028476015) + + + --> TIME: 2025-04-19 00:49:41 -- STEP: 123/142 -- GLOBAL_STEP: 395025 + | > loss_disc: 2.3910298347473145 (2.467123946523279) + | > loss_disc_real_0: 0.14353877305984497 (0.12679727575400976) + | > loss_disc_real_1: 0.16833966970443726 (0.20594001361509648) + | > loss_disc_real_2: 0.21909518539905548 (0.22611756804512767) + | > loss_disc_real_3: 0.2060047686100006 (0.2289786955447701) + | > loss_disc_real_4: 0.2437119483947754 (0.23019354517866925) + | > loss_disc_real_5: 0.23261594772338867 (0.23550373097745383) + | > loss_0: 2.3910298347473145 (2.467123946523279) + | > grad_norm_0: tensor(17.5516, device='cuda:0') (tensor(11.8121, device='cuda:0')) + | > loss_gen: 2.412720203399658 (2.3425741932256425) + | > loss_kl: 1.8917433023452759 (1.9142907518681471) + | > loss_feat: 7.823375225067139 (6.945126114822016) + | > loss_mel: 18.150821685791016 (17.87631660554468) + | > loss_duration: 1.507310390472412 (1.4951224850445262) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.78597068786621 (30.57343006909378) + | > grad_norm_1: tensor(480.9889, device='cuda:0') (tensor(271.5251, device='cuda:0')) + | > current_lr_0: 0.00019726865035146003 + | > current_lr_1: 0.00019726865035146003 + | > step_time: 0.4394 (0.380713203089024) + | > loader_time: 0.0069 (0.005842716713261798) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4001801013946533 (2.4001801013946533) + | > loss_disc_real_0: 0.14318831264972687 (0.14318831264972687) + | > loss_disc_real_1: 0.23798143863677979 (0.23798143863677979) + | > loss_disc_real_2: 0.2612294852733612 (0.2612294852733612) + | > loss_disc_real_3: 0.24343495070934296 (0.24343495070934296) + | > loss_disc_real_4: 0.2568240165710449 (0.2568240165710449) + | > loss_disc_real_5: 0.21194711327552795 (0.21194711327552795) + | > loss_0: 2.4001801013946533 (2.4001801013946533) + | > loss_gen: 2.6511547565460205 (2.6511547565460205) + | > loss_kl: 2.3748693466186523 (2.3748693466186523) + | > loss_feat: 6.8073039054870605 (6.8073039054870605) + | > loss_mel: 18.498476028442383 (18.498476028442383) + | > loss_duration: 1.6471223831176758 (1.6471223831176758) + | > loss_1: 31.978927612304688 (31.978927612304688) + + --> STEP: 1 + | > loss_disc: 2.638370990753174 (2.638370990753174) + | > loss_disc_real_0: 0.19572249054908752 (0.19572249054908752) + | > loss_disc_real_1: 0.29580432176589966 (0.29580432176589966) + | > loss_disc_real_2: 0.27498361468315125 (0.27498361468315125) + | > loss_disc_real_3: 0.26549413800239563 (0.26549413800239563) + | > loss_disc_real_4: 0.265242338180542 (0.265242338180542) + | > loss_disc_real_5: 0.22016099095344543 (0.22016099095344543) + | > loss_0: 2.638370990753174 (2.638370990753174) + | > loss_gen: 2.3540005683898926 (2.3540005683898926) + | > loss_kl: 2.3915698528289795 (2.3915698528289795) + | > loss_feat: 6.431270599365234 (6.431270599365234) + | > loss_mel: 17.338871002197266 (17.338871002197266) + | > loss_duration: 1.6749258041381836 (1.6749258041381836) + | > loss_1: 30.19063949584961 (30.19063949584961) + + --> STEP: 2 + | > loss_disc: 2.5467209815979004 (2.592545986175537) + | > loss_disc_real_0: 0.1849149912595749 (0.1903187409043312) + | > loss_disc_real_1: 0.2669470012187958 (0.2813756614923477) + | > loss_disc_real_2: 0.2559365928173065 (0.2654601037502289) + | > loss_disc_real_3: 0.28081849217414856 (0.2731563150882721) + | > loss_disc_real_4: 0.23202501237392426 (0.24863367527723312) + | > loss_disc_real_5: 0.2112429440021515 (0.21570196747779846) + | > loss_0: 2.5467209815979004 (2.592545986175537) + | > loss_gen: 2.439695119857788 (2.3968478441238403) + | > loss_kl: 1.9593816995620728 (2.175475776195526) + | > loss_feat: 6.454225540161133 (6.442748069763184) + | > loss_mel: 18.636083602905273 (17.98747730255127) + | > loss_duration: 1.6156916618347168 (1.6453087329864502) + | > loss_1: 31.105077743530273 (30.64785861968994) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034438371658325195 (-0.0003185272216796875) + | > avg_loss_disc: 2.592545986175537 (+0.13187146186828613) + | > avg_loss_disc_real_0: 0.1903187409043312 (+0.10891281813383102) + | > avg_loss_disc_real_1: 0.2813756614923477 (+0.051703065633773804) + | > avg_loss_disc_real_2: 0.2654601037502289 (+0.014006704092025757) + | > avg_loss_disc_real_3: 0.2731563150882721 (+0.04841432720422745) + | > avg_loss_disc_real_4: 0.24863367527723312 (+0.02145550400018692) + | > avg_loss_disc_real_5: 0.21570196747779846 (-0.023281767964363098) + | > avg_loss_0: 2.592545986175537 (+0.13187146186828613) + | > avg_loss_gen: 2.3968478441238403 (+0.20524489879608154) + | > avg_loss_kl: 2.175475776195526 (-0.24060195684432983) + | > avg_loss_feat: 6.442748069763184 (-0.2395484447479248) + | > avg_loss_mel: 17.98747730255127 (+0.8787636756896973) + | > avg_loss_duration: 1.6453087329864502 (-0.008949100971221924) + | > avg_loss_1: 30.64785861968994 (+0.59490966796875) + + + > EPOCH: 111/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:49:51)  + + --> TIME: 2025-04-19 00:49:54 -- STEP: 6/142 -- GLOBAL_STEP: 395050 + | > loss_disc: 2.474375009536743 (2.525102138519287) + | > loss_disc_real_0: 0.13552771508693695 (0.12828223158915839) + | > loss_disc_real_1: 0.23573294281959534 (0.20364731550216675) + | > loss_disc_real_2: 0.2016707807779312 (0.24144364396731058) + | > loss_disc_real_3: 0.2087451070547104 (0.25520580261945724) + | > loss_disc_real_4: 0.20422577857971191 (0.2550378491481145) + | > loss_disc_real_5: 0.22062334418296814 (0.24432276437679926) + | > loss_0: 2.474375009536743 (2.525102138519287) + | > grad_norm_0: tensor(4.0785, device='cuda:0') (tensor(11.1829, device='cuda:0')) + | > loss_gen: 2.388338565826416 (2.335096756617228) + | > loss_kl: 1.9696874618530273 (1.8370927174886067) + | > loss_feat: 7.298670768737793 (6.831361134847005) + | > loss_mel: 17.684642791748047 (18.02121067047119) + | > loss_duration: 1.4416712522506714 (1.454351782798767) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.783010482788086 (30.479112943013508) + | > grad_norm_1: tensor(108.8512, device='cuda:0') (tensor(217.1171, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.3273 (0.32446380456288654) + | > loader_time: 0.0038 (0.004208167394002278) + + + --> TIME: 2025-04-19 00:50:03 -- STEP: 31/142 -- GLOBAL_STEP: 395075 + | > loss_disc: 2.4701931476593018 (2.4847845108278337) + | > loss_disc_real_0: 0.08597251027822495 (0.1252125213223119) + | > loss_disc_real_1: 0.1770472675561905 (0.2052121969961351) + | > loss_disc_real_2: 0.21903085708618164 (0.23198784503244585) + | > loss_disc_real_3: 0.21123440563678741 (0.23370276583779243) + | > loss_disc_real_4: 0.20049896836280823 (0.23343648352930624) + | > loss_disc_real_5: 0.19988314807415009 (0.23733292231636663) + | > loss_0: 2.4701931476593018 (2.4847845108278337) + | > grad_norm_0: tensor(6.2364, device='cuda:0') (tensor(9.2223, device='cuda:0')) + | > loss_gen: 2.3174993991851807 (2.328786319301974) + | > loss_kl: 1.7677664756774902 (1.874770998954773) + | > loss_feat: 7.3692121505737305 (6.939355004218317) + | > loss_mel: 18.114017486572266 (17.72314167022705) + | > loss_duration: 1.4304580688476562 (1.4559888993540118) + | > amp_scaler: 256.0 (214.70967741935485) + | > loss_1: 30.998952865600586 (30.32204314201109) + | > grad_norm_1: tensor(272.7443, device='cuda:0') (tensor(214.5775, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.3506 (0.3391778161448817) + | > loader_time: 0.0053 (0.005057573318481445) + + + --> TIME: 2025-04-19 00:50:12 -- STEP: 56/142 -- GLOBAL_STEP: 395100 + | > loss_disc: 2.491611957550049 (2.4811322348458438) + | > loss_disc_real_0: 0.11176776885986328 (0.12847007451845066) + | > loss_disc_real_1: 0.18752287328243256 (0.20796935499778815) + | > loss_disc_real_2: 0.28179192543029785 (0.22916909786207335) + | > loss_disc_real_3: 0.2563644349575043 (0.2303174670253481) + | > loss_disc_real_4: 0.2782404124736786 (0.23334161059132644) + | > loss_disc_real_5: 0.24415838718414307 (0.23528282131467546) + | > loss_0: 2.491611957550049 (2.4811322348458438) + | > grad_norm_0: tensor(16.0759, device='cuda:0') (tensor(10.0585, device='cuda:0')) + | > loss_gen: 2.3691911697387695 (2.3286793700286315) + | > loss_kl: 1.9655542373657227 (1.8836337604693003) + | > loss_feat: 5.905014514923096 (6.933844464165824) + | > loss_mel: 16.750015258789062 (17.727730563708715) + | > loss_duration: 1.494264006614685 (1.4676214435270853) + | > amp_scaler: 256.0 (233.14285714285714) + | > loss_1: 28.484041213989258 (30.341509784970963) + | > grad_norm_1: tensor(362.4454, device='cuda:0') (tensor(226.5939, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.3748 (0.35042960728917805) + | > loader_time: 0.0063 (0.005344505820955549) + + + --> TIME: 2025-04-19 00:50:22 -- STEP: 81/142 -- GLOBAL_STEP: 395125 + | > loss_disc: 2.6422908306121826 (2.4841110529723003) + | > loss_disc_real_0: 0.1025727316737175 (0.1306172175171935) + | > loss_disc_real_1: 0.2415858656167984 (0.20856569486635704) + | > loss_disc_real_2: 0.2249581515789032 (0.22792046581521447) + | > loss_disc_real_3: 0.21344557404518127 (0.22929277077869134) + | > loss_disc_real_4: 0.26048094034194946 (0.23252608709865147) + | > loss_disc_real_5: 0.22643093764781952 (0.2350169477271445) + | > loss_0: 2.6422908306121826 (2.4841110529723003) + | > grad_norm_0: tensor(15.0027, device='cuda:0') (tensor(11.3890, device='cuda:0')) + | > loss_gen: 2.309878349304199 (2.342901353482847) + | > loss_kl: 2.060349702835083 (1.91187427809209) + | > loss_feat: 6.373208999633789 (6.961914827794205) + | > loss_mel: 18.27353858947754 (17.80326572465308) + | > loss_duration: 1.4889864921569824 (1.4798604853359274) + | > amp_scaler: 256.0 (240.19753086419752) + | > loss_1: 30.505962371826172 (30.499816800341193) + | > grad_norm_1: tensor(522.2500, device='cuda:0') (tensor(258.9316, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.415 (0.3626399481738055) + | > loader_time: 0.0066 (0.005601079375655563) + + + --> TIME: 2025-04-19 00:50:32 -- STEP: 106/142 -- GLOBAL_STEP: 395150 + | > loss_disc: 2.3659303188323975 (2.4881454953607527) + | > loss_disc_real_0: 0.07040345668792725 (0.1301913188664981) + | > loss_disc_real_1: 0.1911793202161789 (0.2104232613489313) + | > loss_disc_real_2: 0.20490621030330658 (0.22841461241807579) + | > loss_disc_real_3: 0.23119395971298218 (0.22945180535316467) + | > loss_disc_real_4: 0.17065127193927765 (0.23178159867255194) + | > loss_disc_real_5: 0.23957009613513947 (0.23533244669999717) + | > loss_0: 2.3659303188323975 (2.4881454953607527) + | > grad_norm_0: tensor(10.9774, device='cuda:0') (tensor(13.4648, device='cuda:0')) + | > loss_gen: 2.5339133739471436 (2.360126706789124) + | > loss_kl: 1.8986369371414185 (1.920828880004163) + | > loss_feat: 7.14007043838501 (7.009132889081847) + | > loss_mel: 18.00899314880371 (17.94684453280466) + | > loss_duration: 1.51932954788208 (1.4914743191791036) + | > amp_scaler: 256.0 (243.9245283018868) + | > loss_1: 31.100942611694336 (30.72840740995587) + | > grad_norm_1: tensor(407.8758, device='cuda:0') (tensor(279.7664, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.4092 (0.37281774125009215) + | > loader_time: 0.0072 (0.005866720991314583) + + + --> TIME: 2025-04-19 00:50:44 -- STEP: 131/142 -- GLOBAL_STEP: 395175 + | > loss_disc: 2.554107189178467 (2.4900628865220167) + | > loss_disc_real_0: 0.1688322126865387 (0.13214182685918485) + | > loss_disc_real_1: 0.19081708788871765 (0.21013397694999025) + | > loss_disc_real_2: 0.24989131093025208 (0.22823640374281934) + | > loss_disc_real_3: 0.24263910949230194 (0.22990065869939236) + | > loss_disc_real_4: 0.24161964654922485 (0.23218730001049187) + | > loss_disc_real_5: 0.23341520130634308 (0.2354744638650472) + | > loss_0: 2.554107189178467 (2.4900628865220167) + | > grad_norm_0: tensor(8.7812, device='cuda:0') (tensor(15.3889, device='cuda:0')) + | > loss_gen: 2.3757529258728027 (2.366153884480017) + | > loss_kl: 2.0280661582946777 (1.9212895258692384) + | > loss_feat: 6.611190319061279 (6.986432006340901) + | > loss_mel: 17.871047973632812 (17.932723380227124) + | > loss_duration: 1.5333149433135986 (1.4973568543223028) + | > amp_scaler: 256.0 (246.2290076335878) + | > loss_1: 30.41937255859375 (30.703955759528938) + | > grad_norm_1: tensor(159.9969, device='cuda:0') (tensor(302.3271, device='cuda:0')) + | > current_lr_0: 0.0001972439917701661 + | > current_lr_1: 0.0001972439917701661 + | > step_time: 0.4486 (0.3855864237283022) + | > loader_time: 0.0075 (0.006132340613212295) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6339094638824463 (2.6339094638824463) + | > loss_disc_real_0: 0.05907794460654259 (0.05907794460654259) + | > loss_disc_real_1: 0.2499486804008484 (0.2499486804008484) + | > loss_disc_real_2: 0.29364722967147827 (0.29364722967147827) + | > loss_disc_real_3: 0.2741645574569702 (0.2741645574569702) + | > loss_disc_real_4: 0.24781325459480286 (0.24781325459480286) + | > loss_disc_real_5: 0.25821635127067566 (0.25821635127067566) + | > loss_0: 2.6339094638824463 (2.6339094638824463) + | > loss_gen: 2.1378490924835205 (2.1378490924835205) + | > loss_kl: 2.5551528930664062 (2.5551528930664062) + | > loss_feat: 6.3661322593688965 (6.3661322593688965) + | > loss_mel: 18.155637741088867 (18.155637741088867) + | > loss_duration: 1.653313398361206 (1.653313398361206) + | > loss_1: 30.868085861206055 (30.868085861206055) + + --> STEP: 1 + | > loss_disc: 2.436140537261963 (2.436140537261963) + | > loss_disc_real_0: 0.04769531637430191 (0.04769531637430191) + | > loss_disc_real_1: 0.23967169225215912 (0.23967169225215912) + | > loss_disc_real_2: 0.2569786608219147 (0.2569786608219147) + | > loss_disc_real_3: 0.24298159778118134 (0.24298159778118134) + | > loss_disc_real_4: 0.20099057257175446 (0.20099057257175446) + | > loss_disc_real_5: 0.17291752994060516 (0.17291752994060516) + | > loss_0: 2.436140537261963 (2.436140537261963) + | > loss_gen: 2.09146785736084 (2.09146785736084) + | > loss_kl: 2.5846762657165527 (2.5846762657165527) + | > loss_feat: 7.1739678382873535 (7.1739678382873535) + | > loss_mel: 19.01111602783203 (19.01111602783203) + | > loss_duration: 1.6923719644546509 (1.6923719644546509) + | > loss_1: 32.5536003112793 (32.5536003112793) + + --> STEP: 2 + | > loss_disc: 2.498350143432617 (2.46724534034729) + | > loss_disc_real_0: 0.0786043033003807 (0.06314980983734131) + | > loss_disc_real_1: 0.26552465558052063 (0.2525981739163399) + | > loss_disc_real_2: 0.32167309522628784 (0.28932587802410126) + | > loss_disc_real_3: 0.2943146228790283 (0.26864811033010483) + | > loss_disc_real_4: 0.2514456808567047 (0.22621812671422958) + | > loss_disc_real_5: 0.21901728212833405 (0.1959674060344696) + | > loss_0: 2.498350143432617 (2.46724534034729) + | > loss_gen: 2.5135345458984375 (2.3025012016296387) + | > loss_kl: 2.270540237426758 (2.4276082515716553) + | > loss_feat: 7.245229244232178 (7.209598541259766) + | > loss_mel: 19.276432037353516 (19.143774032592773) + | > loss_duration: 1.6120293140411377 (1.6522006392478943) + | > loss_1: 32.91776657104492 (32.73568344116211) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036814212799072266 (+0.00023758411407470703) + | > avg_loss_disc: 2.46724534034729 (-0.12530064582824707) + | > avg_loss_disc_real_0: 0.06314980983734131 (-0.1271689310669899) + | > avg_loss_disc_real_1: 0.2525981739163399 (-0.028777487576007843) + | > avg_loss_disc_real_2: 0.28932587802410126 (+0.023865774273872375) + | > avg_loss_disc_real_3: 0.26864811033010483 (-0.004508204758167267) + | > avg_loss_disc_real_4: 0.22621812671422958 (-0.02241554856300354) + | > avg_loss_disc_real_5: 0.1959674060344696 (-0.019734561443328857) + | > avg_loss_0: 2.46724534034729 (-0.12530064582824707) + | > avg_loss_gen: 2.3025012016296387 (-0.09434664249420166) + | > avg_loss_kl: 2.4276082515716553 (+0.25213247537612915) + | > avg_loss_feat: 7.209598541259766 (+0.766850471496582) + | > avg_loss_mel: 19.143774032592773 (+1.156296730041504) + | > avg_loss_duration: 1.6522006392478943 (+0.006891906261444092) + | > avg_loss_1: 32.73568344116211 (+2.087824821472168) + + + > EPOCH: 112/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:50:50)  + + --> TIME: 2025-04-19 00:50:55 -- STEP: 14/142 -- GLOBAL_STEP: 395200 + | > loss_disc: 2.447864294052124 (2.451523150716509) + | > loss_disc_real_0: 0.11987634748220444 (0.11616725421377591) + | > loss_disc_real_1: 0.2041853815317154 (0.20496562868356705) + | > loss_disc_real_2: 0.20083248615264893 (0.23213089576789311) + | > loss_disc_real_3: 0.2008703052997589 (0.23057899624109268) + | > loss_disc_real_4: 0.2387879639863968 (0.22174326543297088) + | > loss_disc_real_5: 0.21055597066879272 (0.2368548618895667) + | > loss_0: 2.447864294052124 (2.451523150716509) + | > grad_norm_0: tensor(8.8332, device='cuda:0') (tensor(9.1882, device='cuda:0')) + | > loss_gen: 2.2869973182678223 (2.3386180230549405) + | > loss_kl: 1.839504599571228 (1.8324704340526037) + | > loss_feat: 6.638020992279053 (6.965524298804147) + | > loss_mel: 16.581819534301758 (17.656187057495114) + | > loss_duration: 1.4387385845184326 (1.4422082986150468) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.785079956054688 (30.23500837598528) + | > grad_norm_1: tensor(70.9986, device='cuda:0') (tensor(226.6083, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.334 (0.32731802122933523) + | > loader_time: 0.0049 (0.004523856299264091) + + + --> TIME: 2025-04-19 00:51:04 -- STEP: 39/142 -- GLOBAL_STEP: 395225 + | > loss_disc: 2.5374271869659424 (2.4425706496605506) + | > loss_disc_real_0: 0.08737694472074509 (0.12044396404272471) + | > loss_disc_real_1: 0.23300546407699585 (0.2023961276580126) + | > loss_disc_real_2: 0.2584656774997711 (0.22823384671639174) + | > loss_disc_real_3: 0.22808381915092468 (0.2290385453364788) + | > loss_disc_real_4: 0.2436964064836502 (0.225223571062088) + | > loss_disc_real_5: 0.2922200858592987 (0.236275525811391) + | > loss_0: 2.5374271869659424 (2.4425706496605506) + | > grad_norm_0: tensor(4.3519, device='cuda:0') (tensor(8.6866, device='cuda:0')) + | > loss_gen: 2.3046159744262695 (2.3415907407418275) + | > loss_kl: 2.0190541744232178 (1.8688696745114448) + | > loss_feat: 6.862354755401611 (6.992417176564534) + | > loss_mel: 18.800159454345703 (17.60826575450408) + | > loss_duration: 1.4906015396118164 (1.45758384007674) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.476783752441406 (30.268727327004456) + | > grad_norm_1: tensor(68.5805, device='cuda:0') (tensor(181.3700, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.3553 (0.34019034948104465) + | > loader_time: 0.0054 (0.00499723507807805) + + + --> TIME: 2025-04-19 00:51:14 -- STEP: 64/142 -- GLOBAL_STEP: 395250 + | > loss_disc: 2.5045526027679443 (2.458886805921793) + | > loss_disc_real_0: 0.14416538178920746 (0.12472412153147161) + | > loss_disc_real_1: 0.18690647184848785 (0.20479592005722225) + | > loss_disc_real_2: 0.24936246871948242 (0.22882849536836147) + | > loss_disc_real_3: 0.24015893042087555 (0.22876767395064235) + | > loss_disc_real_4: 0.2939938008785248 (0.22763378685340285) + | > loss_disc_real_5: 0.22984804213047028 (0.2332876508589834) + | > loss_0: 2.5045526027679443 (2.458886805921793) + | > grad_norm_0: tensor(4.9676, device='cuda:0') (tensor(8.8554, device='cuda:0')) + | > loss_gen: 2.287184476852417 (2.33204647898674) + | > loss_kl: 1.8947995901107788 (1.8903257548809052) + | > loss_feat: 6.8090338706970215 (6.968969330191612) + | > loss_mel: 17.680818557739258 (17.716285556554794) + | > loss_duration: 1.5313072204589844 (1.4708457812666893) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.203142166137695 (30.37847301363945) + | > grad_norm_1: tensor(162.2023, device='cuda:0') (tensor(176.0894, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.3827 (0.352071650326252) + | > loader_time: 0.006 (0.005299516022205353) + + + --> TIME: 2025-04-19 00:51:24 -- STEP: 89/142 -- GLOBAL_STEP: 395275 + | > loss_disc: 2.4809861183166504 (2.461512356661679) + | > loss_disc_real_0: 0.11477183550596237 (0.1266648867324497) + | > loss_disc_real_1: 0.21989694237709045 (0.20525371258178454) + | > loss_disc_real_2: 0.22271229326725006 (0.22770415514372708) + | > loss_disc_real_3: 0.2297375500202179 (0.22949525178148505) + | > loss_disc_real_4: 0.23881256580352783 (0.2285394062486927) + | > loss_disc_real_5: 0.24960826337337494 (0.23401115449626794) + | > loss_0: 2.4809861183166504 (2.461512356661679) + | > grad_norm_0: tensor(9.8806, device='cuda:0') (tensor(8.1997, device='cuda:0')) + | > loss_gen: 2.336245536804199 (2.328480015979724) + | > loss_kl: 1.9304730892181396 (1.9025568171833338) + | > loss_feat: 7.201034069061279 (6.952828680531362) + | > loss_mel: 18.153581619262695 (17.773812690477687) + | > loss_duration: 1.4974236488342285 (1.4831207669183113) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.118757247924805 (30.44079907020826) + | > grad_norm_1: tensor(244.2162, device='cuda:0') (tensor(157.7012, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.4011 (0.36375035328811467) + | > loader_time: 0.0063 (0.0055864741293232085) + + + --> TIME: 2025-04-19 00:51:34 -- STEP: 114/142 -- GLOBAL_STEP: 395300 + | > loss_disc: 2.365332841873169 (2.4624226867106924) + | > loss_disc_real_0: 0.1237480416893959 (0.12751771692644082) + | > loss_disc_real_1: 0.18615475296974182 (0.20546993913880565) + | > loss_disc_real_2: 0.20787693560123444 (0.22682100612866252) + | > loss_disc_real_3: 0.20415286719799042 (0.22831846995834718) + | > loss_disc_real_4: 0.18413536250591278 (0.22796935235199176) + | > loss_disc_real_5: 0.2308303415775299 (0.23305149114968485) + | > loss_0: 2.365332841873169 (2.4624226867106924) + | > grad_norm_0: tensor(5.7549, device='cuda:0') (tensor(8.2755, device='cuda:0')) + | > loss_gen: 2.4391469955444336 (2.320399474679379) + | > loss_kl: 2.000901699066162 (1.9113316996055736) + | > loss_feat: 7.432943344116211 (6.89727663157279) + | > loss_mel: 18.508419036865234 (17.738387659976357) + | > loss_duration: 1.5163040161132812 (1.490195837982914) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.897716522216797 (30.35759139479252) + | > grad_norm_1: tensor(248.5180, device='cuda:0') (tensor(157.1864, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.42 (0.3746319925575925) + | > loader_time: 0.0073 (0.005857243872525399) + + + --> TIME: 2025-04-19 00:51:46 -- STEP: 139/142 -- GLOBAL_STEP: 395325 + | > loss_disc: 2.507427453994751 (2.466524559816868) + | > loss_disc_real_0: 0.19504252076148987 (0.128977911989037) + | > loss_disc_real_1: 0.20121780037879944 (0.20659452012116958) + | > loss_disc_real_2: 0.23444144427776337 (0.2274024551935333) + | > loss_disc_real_3: 0.22663123905658722 (0.22763688808722463) + | > loss_disc_real_4: 0.23439863324165344 (0.2284681843124705) + | > loss_disc_real_5: 0.19618208706378937 (0.23220771626174022) + | > loss_0: 2.507427453994751 (2.466524559816868) + | > grad_norm_0: tensor(13.3437, device='cuda:0') (tensor(8.3646, device='cuda:0')) + | > loss_gen: 2.2344675064086914 (2.317830639777424) + | > loss_kl: 1.8838348388671875 (1.9194411408129355) + | > loss_feat: 6.9914045333862305 (6.8597632860965865) + | > loss_mel: 17.26055335998535 (17.691033109486533) + | > loss_duration: 1.5202218294143677 (1.4967462107431977) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.890480041503906 (30.284814422936748) + | > grad_norm_1: tensor(169.4403, device='cuda:0') (tensor(160.5492, device='cuda:0')) + | > current_lr_0: 0.0001972193362711948 + | > current_lr_1: 0.0001972193362711948 + | > step_time: 0.4545 (0.38680653777911506) + | > loader_time: 0.008 (0.006125559909738225) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6244750022888184 (2.6244750022888184) + | > loss_disc_real_0: 0.21142084896564484 (0.21142084896564484) + | > loss_disc_real_1: 0.21179118752479553 (0.21179118752479553) + | > loss_disc_real_2: 0.24284011125564575 (0.24284011125564575) + | > loss_disc_real_3: 0.2735907733440399 (0.2735907733440399) + | > loss_disc_real_4: 0.21199047565460205 (0.21199047565460205) + | > loss_disc_real_5: 0.28377074003219604 (0.28377074003219604) + | > loss_0: 2.6244750022888184 (2.6244750022888184) + | > loss_gen: 2.321108102798462 (2.321108102798462) + | > loss_kl: 2.3114335536956787 (2.3114335536956787) + | > loss_feat: 6.252263069152832 (6.252263069152832) + | > loss_mel: 16.825674057006836 (16.825674057006836) + | > loss_duration: 1.639613389968872 (1.639613389968872) + | > loss_1: 29.3500919342041 (29.3500919342041) + + --> STEP: 1 + | > loss_disc: 2.5779788494110107 (2.5779788494110107) + | > loss_disc_real_0: 0.14794684946537018 (0.14794684946537018) + | > loss_disc_real_1: 0.21522517502307892 (0.21522517502307892) + | > loss_disc_real_2: 0.2530076205730438 (0.2530076205730438) + | > loss_disc_real_3: 0.26324462890625 (0.26324462890625) + | > loss_disc_real_4: 0.2659369111061096 (0.2659369111061096) + | > loss_disc_real_5: 0.2834146022796631 (0.2834146022796631) + | > loss_0: 2.5779788494110107 (2.5779788494110107) + | > loss_gen: 2.348646640777588 (2.348646640777588) + | > loss_kl: 2.466866970062256 (2.466866970062256) + | > loss_feat: 5.756785869598389 (5.756785869598389) + | > loss_mel: 17.789827346801758 (17.789827346801758) + | > loss_duration: 1.6861751079559326 (1.6861751079559326) + | > loss_1: 30.048301696777344 (30.048301696777344) + + --> STEP: 2 + | > loss_disc: 2.435405731201172 (2.5066922903060913) + | > loss_disc_real_0: 0.13051901757717133 (0.13923293352127075) + | > loss_disc_real_1: 0.17353105545043945 (0.19437811523675919) + | > loss_disc_real_2: 0.23208916187286377 (0.2425483912229538) + | > loss_disc_real_3: 0.2744901180267334 (0.2688673734664917) + | > loss_disc_real_4: 0.23385217785835266 (0.24989454448223114) + | > loss_disc_real_5: 0.21901068091392517 (0.25121264159679413) + | > loss_0: 2.435405731201172 (2.5066922903060913) + | > loss_gen: 2.3272345066070557 (2.3379405736923218) + | > loss_kl: 2.1119704246520996 (2.2894186973571777) + | > loss_feat: 7.408575057983398 (6.5826804637908936) + | > loss_mel: 17.456130981445312 (17.622979164123535) + | > loss_duration: 1.6335718631744385 (1.6598734855651855) + | > loss_1: 30.937482833862305 (30.492892265319824) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003262162208557129 (-0.00041925907135009766) + | > avg_loss_disc: 2.5066922903060913 (+0.03944694995880127) + | > avg_loss_disc_real_0: 0.13923293352127075 (+0.07608312368392944) + | > avg_loss_disc_real_1: 0.19437811523675919 (-0.05822005867958069) + | > avg_loss_disc_real_2: 0.2425483912229538 (-0.04677748680114746) + | > avg_loss_disc_real_3: 0.2688673734664917 (+0.00021926313638687134) + | > avg_loss_disc_real_4: 0.24989454448223114 (+0.023676417768001556) + | > avg_loss_disc_real_5: 0.25121264159679413 (+0.055245235562324524) + | > avg_loss_0: 2.5066922903060913 (+0.03944694995880127) + | > avg_loss_gen: 2.3379405736923218 (+0.035439372062683105) + | > avg_loss_kl: 2.2894186973571777 (-0.13818955421447754) + | > avg_loss_feat: 6.5826804637908936 (-0.6269180774688721) + | > avg_loss_mel: 17.622979164123535 (-1.5207948684692383) + | > avg_loss_duration: 1.6598734855651855 (+0.00767284631729126) + | > avg_loss_1: 30.492892265319824 (-2.242791175842285) + + + > EPOCH: 113/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:51:49)  + + --> TIME: 2025-04-19 00:51:57 -- STEP: 22/142 -- GLOBAL_STEP: 395350 + | > loss_disc: 2.5962531566619873 (2.450137593529441) + | > loss_disc_real_0: 0.14009685814380646 (0.12403493171388452) + | > loss_disc_real_1: 0.21444374322891235 (0.20887882601131091) + | > loss_disc_real_2: 0.23043319582939148 (0.2289535545490005) + | > loss_disc_real_3: 0.24554529786109924 (0.22838795659216968) + | > loss_disc_real_4: 0.24448904395103455 (0.23054879429665479) + | > loss_disc_real_5: 0.22082769870758057 (0.23180906068194995) + | > loss_0: 2.5962531566619873 (2.450137593529441) + | > grad_norm_0: tensor(16.0567, device='cuda:0') (tensor(13.2229, device='cuda:0')) + | > loss_gen: 2.1432816982269287 (2.3775788003748115) + | > loss_kl: 1.7529388666152954 (1.8185419602827593) + | > loss_feat: 6.681317329406738 (7.176186973398382) + | > loss_mel: 16.116607666015625 (18.26227430863814) + | > loss_duration: 1.4106440544128418 (1.4517608826810664) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.10479164123535 (31.086343591863457) + | > grad_norm_1: tensor(464.1797, device='cuda:0') (tensor(261.0901, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.3411 (0.3332946842366999) + | > loader_time: 0.005 (0.004647580060091886) + + + --> TIME: 2025-04-19 00:52:06 -- STEP: 47/142 -- GLOBAL_STEP: 395375 + | > loss_disc: 2.422954797744751 (2.4507966041564937) + | > loss_disc_real_0: 0.1164434477686882 (0.12382980451938953) + | > loss_disc_real_1: 0.2334725260734558 (0.2047716594756918) + | > loss_disc_real_2: 0.1980353146791458 (0.22673698879302817) + | > loss_disc_real_3: 0.21849162876605988 (0.22821447633682412) + | > loss_disc_real_4: 0.24239540100097656 (0.23139578579588138) + | > loss_disc_real_5: 0.23738794028759003 (0.2338656025364044) + | > loss_0: 2.422954797744751 (2.4507966041564937) + | > grad_norm_0: tensor(13.9816, device='cuda:0') (tensor(12.2671, device='cuda:0')) + | > loss_gen: 2.356968879699707 (2.3748569691434818) + | > loss_kl: 1.9240614175796509 (1.8439353323997336) + | > loss_feat: 6.898351192474365 (7.061463315436181) + | > loss_mel: 17.241357803344727 (18.037119520471453) + | > loss_duration: 1.4416687488555908 (1.464368759317601) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.862407684326172 (30.781744246787213) + | > grad_norm_1: tensor(369.3517, device='cuda:0') (tensor(278.1576, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.3871 (0.3496927251207068) + | > loader_time: 0.0062 (0.005185294658579725) + + + --> TIME: 2025-04-19 00:52:16 -- STEP: 72/142 -- GLOBAL_STEP: 395400 + | > loss_disc: 2.457939624786377 (2.459011200401518) + | > loss_disc_real_0: 0.10372990369796753 (0.12594981770962477) + | > loss_disc_real_1: 0.18935823440551758 (0.20529416224194896) + | > loss_disc_real_2: 0.23319460451602936 (0.2275982942018244) + | > loss_disc_real_3: 0.20771268010139465 (0.22778630877534548) + | > loss_disc_real_4: 0.24450375139713287 (0.23033420886430475) + | > loss_disc_real_5: 0.26039883494377136 (0.23271550486485162) + | > loss_0: 2.457939624786377 (2.459011200401518) + | > grad_norm_0: tensor(7.3719, device='cuda:0') (tensor(13.1149, device='cuda:0')) + | > loss_gen: 2.560070037841797 (2.3661335276232824) + | > loss_kl: 1.8995290994644165 (1.8595854590336482) + | > loss_feat: 7.42077112197876 (7.045603877968259) + | > loss_mel: 17.93732261657715 (18.087890677981907) + | > loss_duration: 1.4717767238616943 (1.4764880140622456) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.28946876525879 (30.835701677534317) + | > grad_norm_1: tensor(262.4931, device='cuda:0') (tensor(285.2377, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.407 (0.36329866449038195) + | > loader_time: 0.0063 (0.005525400241216025) + + + --> TIME: 2025-04-19 00:52:27 -- STEP: 97/142 -- GLOBAL_STEP: 395425 + | > loss_disc: 2.443607807159424 (2.466238560135832) + | > loss_disc_real_0: 0.12352172285318375 (0.12728967340951108) + | > loss_disc_real_1: 0.20579367876052856 (0.20668764350955018) + | > loss_disc_real_2: 0.20140507817268372 (0.2270445020235691) + | > loss_disc_real_3: 0.2396368682384491 (0.2284364709534596) + | > loss_disc_real_4: 0.23779240250587463 (0.23081444664714262) + | > loss_disc_real_5: 0.20352792739868164 (0.23218141788059904) + | > loss_0: 2.443607807159424 (2.466238560135832) + | > grad_norm_0: tensor(8.4332, device='cuda:0') (tensor(13.0633, device='cuda:0')) + | > loss_gen: 2.2925095558166504 (2.349587506854657) + | > loss_kl: 1.9203652143478394 (1.877476439033587) + | > loss_feat: 6.4049553871154785 (6.979082618792033) + | > loss_mel: 16.080564498901367 (18.074228827486333) + | > loss_duration: 1.5028517246246338 (1.4884268512430878) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.201244354248047 (30.768802269217893) + | > grad_norm_1: tensor(121.8289, device='cuda:0') (tensor(278.1088, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.4238 (0.377072870116873) + | > loader_time: 0.0066 (0.005818661955214038) + + + --> TIME: 2025-04-19 00:52:37 -- STEP: 122/142 -- GLOBAL_STEP: 395450 + | > loss_disc: 2.594060182571411 (2.472186745190231) + | > loss_disc_real_0: 0.1561541110277176 (0.1293266805224731) + | > loss_disc_real_1: 0.22955086827278137 (0.20637888791131193) + | > loss_disc_real_2: 0.20140856504440308 (0.2269300068499612) + | > loss_disc_real_3: 0.21342070400714874 (0.22926877768801862) + | > loss_disc_real_4: 0.20212984085083008 (0.23136545326866087) + | > loss_disc_real_5: 0.20580154657363892 (0.23350305476638136) + | > loss_0: 2.594060182571411 (2.472186745190231) + | > grad_norm_0: tensor(9.3679, device='cuda:0') (tensor(13.3056, device='cuda:0')) + | > loss_gen: 2.0176806449890137 (2.3406533315533498) + | > loss_kl: 1.8302850723266602 (1.893065433033177) + | > loss_feat: 5.0983428955078125 (6.926985599955574) + | > loss_mel: 13.975801467895508 (18.001634449255285) + | > loss_duration: 1.4911344051361084 (1.4955507370292163) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 24.413246154785156 (30.65788969446401) + | > grad_norm_1: tensor(215.0742, device='cuda:0') (tensor(276.8614, device='cuda:0')) + | > current_lr_0: 0.0001971946838541609 + | > current_lr_1: 0.0001971946838541609 + | > step_time: 0.4512 (0.38665521926567215) + | > loader_time: 0.0074 (0.006076828378145812) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.510096788406372 (2.510096788406372) + | > loss_disc_real_0: 0.11368139088153839 (0.11368139088153839) + | > loss_disc_real_1: 0.273347944021225 (0.273347944021225) + | > loss_disc_real_2: 0.1953134387731552 (0.1953134387731552) + | > loss_disc_real_3: 0.19970092177391052 (0.19970092177391052) + | > loss_disc_real_4: 0.19198985397815704 (0.19198985397815704) + | > loss_disc_real_5: 0.22591978311538696 (0.22591978311538696) + | > loss_0: 2.510096788406372 (2.510096788406372) + | > loss_gen: 2.198556661605835 (2.198556661605835) + | > loss_kl: 2.2940783500671387 (2.2940783500671387) + | > loss_feat: 6.936286926269531 (6.936286926269531) + | > loss_mel: 19.004301071166992 (19.004301071166992) + | > loss_duration: 1.654159426689148 (1.654159426689148) + | > loss_1: 32.08738327026367 (32.08738327026367) + + --> STEP: 1 + | > loss_disc: 2.5928218364715576 (2.5928218364715576) + | > loss_disc_real_0: 0.15457360446453094 (0.15457360446453094) + | > loss_disc_real_1: 0.2751210629940033 (0.2751210629940033) + | > loss_disc_real_2: 0.21740317344665527 (0.21740317344665527) + | > loss_disc_real_3: 0.23863738775253296 (0.23863738775253296) + | > loss_disc_real_4: 0.1983141452074051 (0.1983141452074051) + | > loss_disc_real_5: 0.22401897609233856 (0.22401897609233856) + | > loss_0: 2.5928218364715576 (2.5928218364715576) + | > loss_gen: 2.223459005355835 (2.223459005355835) + | > loss_kl: 2.587834596633911 (2.587834596633911) + | > loss_feat: 7.099577903747559 (7.099577903747559) + | > loss_mel: 17.703617095947266 (17.703617095947266) + | > loss_duration: 1.702113151550293 (1.702113151550293) + | > loss_1: 31.316600799560547 (31.316600799560547) + + --> STEP: 2 + | > loss_disc: 2.647355556488037 (2.6200886964797974) + | > loss_disc_real_0: 0.17928074300289154 (0.16692717373371124) + | > loss_disc_real_1: 0.38270461559295654 (0.3289128392934799) + | > loss_disc_real_2: 0.19887064397335052 (0.2081369087100029) + | > loss_disc_real_3: 0.19537325203418732 (0.21700531989336014) + | > loss_disc_real_4: 0.2305910289287567 (0.2144525870680809) + | > loss_disc_real_5: 0.25631123781204224 (0.2401651069521904) + | > loss_0: 2.647355556488037 (2.6200886964797974) + | > loss_gen: 2.456308126449585 (2.33988356590271) + | > loss_kl: 2.0508949756622314 (2.3193647861480713) + | > loss_feat: 8.177504539489746 (7.638541221618652) + | > loss_mel: 18.77046775817871 (18.23704242706299) + | > loss_duration: 1.6241611242294312 (1.663137137889862) + | > loss_1: 33.07933807373047 (32.19796943664551) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003325343132019043 (+6.318092346191406e-05) + | > avg_loss_disc: 2.6200886964797974 (+0.11339640617370605) + | > avg_loss_disc_real_0: 0.16692717373371124 (+0.02769424021244049) + | > avg_loss_disc_real_1: 0.3289128392934799 (+0.13453472405672073) + | > avg_loss_disc_real_2: 0.2081369087100029 (-0.0344114825129509) + | > avg_loss_disc_real_3: 0.21700531989336014 (-0.05186205357313156) + | > avg_loss_disc_real_4: 0.2144525870680809 (-0.03544195741415024) + | > avg_loss_disc_real_5: 0.2401651069521904 (-0.01104753464460373) + | > avg_loss_0: 2.6200886964797974 (+0.11339640617370605) + | > avg_loss_gen: 2.33988356590271 (+0.0019429922103881836) + | > avg_loss_kl: 2.3193647861480713 (+0.029946088790893555) + | > avg_loss_feat: 7.638541221618652 (+1.0558607578277588) + | > avg_loss_mel: 18.23704242706299 (+0.6140632629394531) + | > avg_loss_duration: 1.663137137889862 (+0.0032636523246765137) + | > avg_loss_1: 32.19796943664551 (+1.7050771713256836) + + + > EPOCH: 114/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:52:48)  + + --> TIME: 2025-04-19 00:52:50 -- STEP: 5/142 -- GLOBAL_STEP: 395475 + | > loss_disc: 2.4489030838012695 (2.498079299926758) + | > loss_disc_real_0: 0.12843337655067444 (0.1297230139374733) + | > loss_disc_real_1: 0.23042532801628113 (0.22421941459178923) + | > loss_disc_real_2: 0.21041075885295868 (0.2373064786195755) + | > loss_disc_real_3: 0.2156076431274414 (0.22455516159534455) + | > loss_disc_real_4: 0.23706108331680298 (0.22505856454372405) + | > loss_disc_real_5: 0.1993788182735443 (0.21735727787017822) + | > loss_0: 2.4489030838012695 (2.498079299926758) + | > grad_norm_0: tensor(13.7845, device='cuda:0') (tensor(10.2361, device='cuda:0')) + | > loss_gen: 2.445174217224121 (2.34681715965271) + | > loss_kl: 1.8686370849609375 (1.8999419450759887) + | > loss_feat: 6.701590061187744 (7.094830894470215) + | > loss_mel: 17.55375862121582 (18.200833129882813) + | > loss_duration: 1.4669055938720703 (1.4622669458389281) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.03606605529785 (31.00468978881836) + | > grad_norm_1: tensor(330.5504, device='cuda:0') (tensor(210.9204, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.3241 (0.3233675956726074) + | > loader_time: 0.0044 (0.004372930526733399) + + + --> TIME: 2025-04-19 00:52:59 -- STEP: 30/142 -- GLOBAL_STEP: 395500 + | > loss_disc: 2.388917922973633 (2.4983508586883545) + | > loss_disc_real_0: 0.10894069820642471 (0.12563799222310384) + | > loss_disc_real_1: 0.21721740067005157 (0.2135165865222613) + | > loss_disc_real_2: 0.22488455474376678 (0.23542526563008626) + | > loss_disc_real_3: 0.25783684849739075 (0.23256781846284866) + | > loss_disc_real_4: 0.23736248910427094 (0.2341865638891856) + | > loss_disc_real_5: 0.2384078949689865 (0.22971131950616835) + | > loss_0: 2.388917922973633 (2.4983508586883545) + | > grad_norm_0: tensor(15.2598, device='cuda:0') (tensor(12.0783, device='cuda:0')) + | > loss_gen: 2.399472236633301 (2.35840425491333) + | > loss_kl: 1.7652357816696167 (1.8505138794581095) + | > loss_feat: 7.181389331817627 (6.994924592971802) + | > loss_mel: 18.39144515991211 (17.949520810445158) + | > loss_duration: 1.467077612876892 (1.457529624303182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.204618453979492 (30.61089318593343) + | > grad_norm_1: tensor(342.9850, device='cuda:0') (tensor(302.6981, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.3485 (0.3366484006245931) + | > loader_time: 0.0055 (0.00486297607421875) + + + --> TIME: 2025-04-19 00:53:08 -- STEP: 55/142 -- GLOBAL_STEP: 395525 + | > loss_disc: 2.604456663131714 (2.492396502061324) + | > loss_disc_real_0: 0.11925363540649414 (0.129459763656963) + | > loss_disc_real_1: 0.23064017295837402 (0.21201865429228003) + | > loss_disc_real_2: 0.2736416161060333 (0.2306951633908532) + | > loss_disc_real_3: 0.25531840324401855 (0.23039155277338894) + | > loss_disc_real_4: 0.259662002325058 (0.23195090185512196) + | > loss_disc_real_5: 0.28389453887939453 (0.23302916911515323) + | > loss_0: 2.604456663131714 (2.492396502061324) + | > grad_norm_0: tensor(12.5862, device='cuda:0') (tensor(14.2636, device='cuda:0')) + | > loss_gen: 2.294001579284668 (2.351485945961692) + | > loss_kl: 1.8977653980255127 (1.8648275917226618) + | > loss_feat: 6.076647758483887 (6.937854515422474) + | > loss_mel: 17.61411476135254 (17.883533217690218) + | > loss_duration: 1.5291552543640137 (1.4681730357083407) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.411685943603516 (30.505874356356536) + | > grad_norm_1: tensor(287.7623, device='cuda:0') (tensor(329.0692, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.3757 (0.34839553399519485) + | > loader_time: 0.0059 (0.005247181112116033) + + + --> TIME: 2025-04-19 00:53:18 -- STEP: 80/142 -- GLOBAL_STEP: 395550 + | > loss_disc: 2.573277235031128 (2.4894318252801897) + | > loss_disc_real_0: 0.15538670122623444 (0.1308283681981266) + | > loss_disc_real_1: 0.25375401973724365 (0.21126520521938802) + | > loss_disc_real_2: 0.2379039227962494 (0.2300284316763282) + | > loss_disc_real_3: 0.2499019354581833 (0.2305059464648366) + | > loss_disc_real_4: 0.2623012065887451 (0.23181007150560617) + | > loss_disc_real_5: 0.2583732306957245 (0.23088112976402045) + | > loss_0: 2.573277235031128 (2.4894318252801897) + | > grad_norm_0: tensor(8.2383, device='cuda:0') (tensor(13.9115, device='cuda:0')) + | > loss_gen: 2.2134037017822266 (2.3434177309274666) + | > loss_kl: 1.8685208559036255 (1.8818120017647744) + | > loss_feat: 5.825362205505371 (6.8859218001365665) + | > loss_mel: 16.972383499145508 (17.824115169048323) + | > loss_duration: 1.5016489028930664 (1.479926064610481) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.38132095336914 (30.415192914009094) + | > grad_norm_1: tensor(180.1656, device='cuda:0') (tensor(298.8994, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.4086 (0.3603640973567963) + | > loader_time: 0.0065 (0.005532231926918029) + + + --> TIME: 2025-04-19 00:53:28 -- STEP: 105/142 -- GLOBAL_STEP: 395575 + | > loss_disc: 2.4985029697418213 (2.4907903943743026) + | > loss_disc_real_0: 0.14569135010242462 (0.13003999945663275) + | > loss_disc_real_1: 0.18899977207183838 (0.21186779226575578) + | > loss_disc_real_2: 0.19487810134887695 (0.23050859854334876) + | > loss_disc_real_3: 0.23072774708271027 (0.22986453345843724) + | > loss_disc_real_4: 0.23635393381118774 (0.2314180218038105) + | > loss_disc_real_5: 0.23723866045475006 (0.23167919573329743) + | > loss_0: 2.4985029697418213 (2.4907903943743026) + | > grad_norm_0: tensor(11.8880, device='cuda:0') (tensor(12.6716, device='cuda:0')) + | > loss_gen: 2.228010892868042 (2.32837164742606) + | > loss_kl: 1.9598157405853271 (1.8857265336172921) + | > loss_feat: 7.715520858764648 (6.845089844294956) + | > loss_mel: 19.03907012939453 (17.796895117986786) + | > loss_duration: 1.5476617813110352 (1.490534908430917) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.49007797241211 (30.346618143717446) + | > grad_norm_1: tensor(350.0550, device='cuda:0') (tensor(269.7224, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.4185 (0.3708974883669899) + | > loader_time: 0.0069 (0.0057820615314301965) + + + --> TIME: 2025-04-19 00:53:40 -- STEP: 130/142 -- GLOBAL_STEP: 395600 + | > loss_disc: 2.4742274284362793 (2.4859695012752825) + | > loss_disc_real_0: 0.12532874941825867 (0.13106406417030556) + | > loss_disc_real_1: 0.2027805596590042 (0.2105655312538147) + | > loss_disc_real_2: 0.21997153759002686 (0.22907876120163845) + | > loss_disc_real_3: 0.2168755829334259 (0.2292924517622361) + | > loss_disc_real_4: 0.22365699708461761 (0.23124248660527744) + | > loss_disc_real_5: 0.22231043875217438 (0.2302948623895645) + | > loss_0: 2.4742274284362793 (2.4859695012752825) + | > grad_norm_0: tensor(17.3416, device='cuda:0') (tensor(12.4948, device='cuda:0')) + | > loss_gen: 2.267843723297119 (2.3264316962315483) + | > loss_kl: 2.0188944339752197 (1.8875435535724348) + | > loss_feat: 6.18693733215332 (6.817217779159546) + | > loss_mel: 17.132707595825195 (17.785055343921385) + | > loss_duration: 1.4917700290679932 (1.4960367734615618) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.09815216064453 (30.312285305903508) + | > grad_norm_1: tensor(332.0990, device='cuda:0') (tensor(261.6216, device='cuda:0')) + | > current_lr_0: 0.0001971700345186791 + | > current_lr_1: 0.0001971700345186791 + | > step_time: 0.4482 (0.38258398129389837) + | > loader_time: 0.0074 (0.006078819128183217) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6628429889678955 (2.6628429889678955) + | > loss_disc_real_0: 0.19312863051891327 (0.19312863051891327) + | > loss_disc_real_1: 0.2102414071559906 (0.2102414071559906) + | > loss_disc_real_2: 0.24679464101791382 (0.24679464101791382) + | > loss_disc_real_3: 0.26489654183387756 (0.26489654183387756) + | > loss_disc_real_4: 0.24596039950847626 (0.24596039950847626) + | > loss_disc_real_5: 0.26113298535346985 (0.26113298535346985) + | > loss_0: 2.6628429889678955 (2.6628429889678955) + | > loss_gen: 2.2001898288726807 (2.2001898288726807) + | > loss_kl: 2.1893911361694336 (2.1893911361694336) + | > loss_feat: 5.513656139373779 (5.513656139373779) + | > loss_mel: 14.779071807861328 (14.779071807861328) + | > loss_duration: 1.6583716869354248 (1.6583716869354248) + | > loss_1: 26.340679168701172 (26.340679168701172) + + --> STEP: 1 + | > loss_disc: 2.4174633026123047 (2.4174633026123047) + | > loss_disc_real_0: 0.14239203929901123 (0.14239203929901123) + | > loss_disc_real_1: 0.18333373963832855 (0.18333373963832855) + | > loss_disc_real_2: 0.2886982262134552 (0.2886982262134552) + | > loss_disc_real_3: 0.25507229566574097 (0.25507229566574097) + | > loss_disc_real_4: 0.222495898604393 (0.222495898604393) + | > loss_disc_real_5: 0.18368002772331238 (0.18368002772331238) + | > loss_0: 2.4174633026123047 (2.4174633026123047) + | > loss_gen: 2.349555492401123 (2.349555492401123) + | > loss_kl: 2.4135680198669434 (2.4135680198669434) + | > loss_feat: 7.1433515548706055 (7.1433515548706055) + | > loss_mel: 19.433643341064453 (19.433643341064453) + | > loss_duration: 1.689403772354126 (1.689403772354126) + | > loss_1: 33.02952194213867 (33.02952194213867) + + --> STEP: 2 + | > loss_disc: 2.5160937309265137 (2.466778516769409) + | > loss_disc_real_0: 0.2016989290714264 (0.1720454841852188) + | > loss_disc_real_1: 0.16601862013339996 (0.17467617988586426) + | > loss_disc_real_2: 0.23645499348640442 (0.2625766098499298) + | > loss_disc_real_3: 0.2243930548429489 (0.23973267525434494) + | > loss_disc_real_4: 0.25340211391448975 (0.23794900625944138) + | > loss_disc_real_5: 0.21231026947498322 (0.1979951485991478) + | > loss_0: 2.5160937309265137 (2.466778516769409) + | > loss_gen: 2.2749385833740234 (2.3122470378875732) + | > loss_kl: 2.066263437271118 (2.2399157285690308) + | > loss_feat: 6.907344341278076 (7.025347948074341) + | > loss_mel: 18.470176696777344 (18.9519100189209) + | > loss_duration: 1.6119475364685059 (1.650675654411316) + | > loss_1: 31.330671310424805 (32.18009662628174) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004143118858337402 (+0.0008177757263183594) + | > avg_loss_disc: 2.466778516769409 (-0.15331017971038818) + | > avg_loss_disc_real_0: 0.1720454841852188 (+0.005118310451507568) + | > avg_loss_disc_real_1: 0.17467617988586426 (-0.15423665940761566) + | > avg_loss_disc_real_2: 0.2625766098499298 (+0.05443970113992691) + | > avg_loss_disc_real_3: 0.23973267525434494 (+0.022727355360984802) + | > avg_loss_disc_real_4: 0.23794900625944138 (+0.023496419191360474) + | > avg_loss_disc_real_5: 0.1979951485991478 (-0.0421699583530426) + | > avg_loss_0: 2.466778516769409 (-0.15331017971038818) + | > avg_loss_gen: 2.3122470378875732 (-0.02763652801513672) + | > avg_loss_kl: 2.2399157285690308 (-0.07944905757904053) + | > avg_loss_feat: 7.025347948074341 (-0.6131932735443115) + | > avg_loss_mel: 18.9519100189209 (+0.7148675918579102) + | > avg_loss_duration: 1.650675654411316 (-0.012461483478546143) + | > avg_loss_1: 32.18009662628174 (-0.01787281036376953) + + + > EPOCH: 115/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:53:47)  + + --> TIME: 2025-04-19 00:53:52 -- STEP: 13/142 -- GLOBAL_STEP: 395625 + | > loss_disc: 2.3513083457946777 (2.3997607781336856) + | > loss_disc_real_0: 0.09753023833036423 (0.11475423322274135) + | > loss_disc_real_1: 0.2005559802055359 (0.20157849559417138) + | > loss_disc_real_2: 0.24044767022132874 (0.2237580166413234) + | > loss_disc_real_3: 0.22007951140403748 (0.22404798750693983) + | > loss_disc_real_4: 0.21986185014247894 (0.2204595941763658) + | > loss_disc_real_5: 0.21319128572940826 (0.23073721849001372) + | > loss_0: 2.3513083457946777 (2.3997607781336856) + | > grad_norm_0: tensor(6.7791, device='cuda:0') (tensor(13.6627, device='cuda:0')) + | > loss_gen: 2.4413676261901855 (2.4010866605318513) + | > loss_kl: 1.7632765769958496 (1.7890872771923358) + | > loss_feat: 7.747433185577393 (7.297310499044565) + | > loss_mel: 18.84233283996582 (18.206379817082333) + | > loss_duration: 1.4618452787399292 (1.44552481174469) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.256256103515625 (31.139388597928562) + | > grad_norm_1: tensor(81.7794, device='cuda:0') (tensor(242.6830, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.3305 (0.32752392842219424) + | > loader_time: 0.0049 (0.004589190849891076) + + + --> TIME: 2025-04-19 00:54:01 -- STEP: 38/142 -- GLOBAL_STEP: 395650 + | > loss_disc: 2.4929471015930176 (2.4164831889303104) + | > loss_disc_real_0: 0.11052211374044418 (0.11252620482915326) + | > loss_disc_real_1: 0.21801257133483887 (0.20297577820326151) + | > loss_disc_real_2: 0.24446794390678406 (0.22335921031864067) + | > loss_disc_real_3: 0.27935081720352173 (0.22620381610958198) + | > loss_disc_real_4: 0.222327321767807 (0.22528751116049917) + | > loss_disc_real_5: 0.23125341534614563 (0.22627226655420504) + | > loss_0: 2.4929471015930176 (2.4164831889303104) + | > grad_norm_0: tensor(19.8770, device='cuda:0') (tensor(12.2285, device='cuda:0')) + | > loss_gen: 2.427082061767578 (2.400660414444772) + | > loss_kl: 1.8349149227142334 (1.841051104821657) + | > loss_feat: 7.256010055541992 (7.266331020154451) + | > loss_mel: 18.819555282592773 (18.088355014198708) + | > loss_duration: 1.4887824058532715 (1.4587455140916925) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.826345443725586 (31.055142954776162) + | > grad_norm_1: tensor(278.1098, device='cuda:0') (tensor(284.7799, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.3542 (0.34294932139547246) + | > loader_time: 0.0061 (0.005054800133956106) + + + --> TIME: 2025-04-19 00:54:10 -- STEP: 63/142 -- GLOBAL_STEP: 395675 + | > loss_disc: 2.4243111610412598 (2.451251094303433) + | > loss_disc_real_0: 0.10718884319067001 (0.12234086720716386) + | > loss_disc_real_1: 0.17720672488212585 (0.2031341828997173) + | > loss_disc_real_2: 0.24500757455825806 (0.2265409001755336) + | > loss_disc_real_3: 0.2202812284231186 (0.227917080597272) + | > loss_disc_real_4: 0.22799991071224213 (0.22895774387177967) + | > loss_disc_real_5: 0.24763166904449463 (0.23080408052792625) + | > loss_0: 2.4243111610412598 (2.451251094303433) + | > grad_norm_0: tensor(7.7275, device='cuda:0') (tensor(11.7439, device='cuda:0')) + | > loss_gen: 2.5040199756622314 (2.365574083631) + | > loss_kl: 1.893778920173645 (1.8608130424741716) + | > loss_feat: 6.861485004425049 (7.078429547567216) + | > loss_mel: 18.450172424316406 (17.91551319001213) + | > loss_duration: 1.5449647903442383 (1.4711625027278112) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.25442123413086 (30.691492201789977) + | > grad_norm_1: tensor(216.6577, device='cuda:0') (tensor(268.7769, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.3879 (0.3547833457825676) + | > loader_time: 0.0064 (0.005505501277863033) + + + --> TIME: 2025-04-19 00:54:20 -- STEP: 88/142 -- GLOBAL_STEP: 395700 + | > loss_disc: 2.43253755569458 (2.4569405263120476) + | > loss_disc_real_0: 0.12378852814435959 (0.1272184621034698) + | > loss_disc_real_1: 0.23097842931747437 (0.20365160432728854) + | > loss_disc_real_2: 0.229591503739357 (0.22560971907593988) + | > loss_disc_real_3: 0.2085535228252411 (0.22763197208670052) + | > loss_disc_real_4: 0.22115477919578552 (0.22959583608264272) + | > loss_disc_real_5: 0.20577892661094666 (0.23232491585341367) + | > loss_0: 2.43253755569458 (2.4569405263120476) + | > grad_norm_0: tensor(4.7668, device='cuda:0') (tensor(12.7938, device='cuda:0')) + | > loss_gen: 2.45106840133667 (2.363474369049072) + | > loss_kl: 2.057986259460449 (1.8779934631152586) + | > loss_feat: 6.964561939239502 (7.042067354375666) + | > loss_mel: 18.217296600341797 (17.939662911675192) + | > loss_duration: 1.5590012073516846 (1.48390232026577) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.249916076660156 (30.707100348039106) + | > grad_norm_1: tensor(201.1594, device='cuda:0') (tensor(280.4168, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.3983 (0.3662183555689725) + | > loader_time: 0.0064 (0.005769859660755505) + + + --> TIME: 2025-04-19 00:54:31 -- STEP: 113/142 -- GLOBAL_STEP: 395725 + | > loss_disc: 2.499359130859375 (2.466089016568344) + | > loss_disc_real_0: 0.13372759521007538 (0.12859792743636445) + | > loss_disc_real_1: 0.22801566123962402 (0.20500354125963904) + | > loss_disc_real_2: 0.22612252831459045 (0.22552972227598714) + | > loss_disc_real_3: 0.19976738095283508 (0.22906875742220245) + | > loss_disc_real_4: 0.23903872072696686 (0.2292922464883433) + | > loss_disc_real_5: 0.19627411663532257 (0.23218484526186917) + | > loss_0: 2.499359130859375 (2.466089016568344) + | > grad_norm_0: tensor(8.2770, device='cuda:0') (tensor(12.4292, device='cuda:0')) + | > loss_gen: 2.227282762527466 (2.349954776004352) + | > loss_kl: 1.9456866979599 (1.8829212916635834) + | > loss_feat: 6.624807357788086 (6.984611772857936) + | > loss_mel: 16.889982223510742 (17.87679824153933) + | > loss_duration: 1.5286459922790527 (1.4906365080217345) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.216405868530273 (30.584922554218664) + | > grad_norm_1: tensor(269.6259, device='cuda:0') (tensor(275.2519, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.4253 (0.37637753191247453) + | > loader_time: 0.007 (0.006007873906498462) + + + --> TIME: 2025-04-19 00:54:42 -- STEP: 138/142 -- GLOBAL_STEP: 395750 + | > loss_disc: 2.582977294921875 (2.471513331800267) + | > loss_disc_real_0: 0.17738166451454163 (0.13092725583608603) + | > loss_disc_real_1: 0.2175503820180893 (0.20617123088542966) + | > loss_disc_real_2: 0.24367506802082062 (0.22621969198403152) + | > loss_disc_real_3: 0.24643868207931519 (0.2290081267339596) + | > loss_disc_real_4: 0.26595190167427063 (0.22944667762604312) + | > loss_disc_real_5: 0.26729515194892883 (0.23117327970871027) + | > loss_0: 2.582977294921875 (2.471513331800267) + | > grad_norm_0: tensor(10.5777, device='cuda:0') (tensor(12.3228, device='cuda:0')) + | > loss_gen: 2.2383182048797607 (2.339414850525234) + | > loss_kl: 1.9353084564208984 (1.8899420396141384) + | > loss_feat: 5.7994771003723145 (6.936809878418411) + | > loss_mel: 16.80521583557129 (17.81047179733497) + | > loss_duration: 1.5651509761810303 (1.496969058893729) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.343469619750977 (30.473607657612234) + | > grad_norm_1: tensor(251.6412, device='cuda:0') (tensor(273.8437, device='cuda:0')) + | > current_lr_0: 0.00019714538826436426 + | > current_lr_1: 0.00019714538826436426 + | > step_time: 0.4589 (0.3880337580390599) + | > loader_time: 0.0076 (0.006261694258537846) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.525665760040283 (2.525665760040283) + | > loss_disc_real_0: 0.14068438112735748 (0.14068438112735748) + | > loss_disc_real_1: 0.19884051382541656 (0.19884051382541656) + | > loss_disc_real_2: 0.2252659946680069 (0.2252659946680069) + | > loss_disc_real_3: 0.29832330346107483 (0.29832330346107483) + | > loss_disc_real_4: 0.2760063707828522 (0.2760063707828522) + | > loss_disc_real_5: 0.32513904571533203 (0.32513904571533203) + | > loss_0: 2.525665760040283 (2.525665760040283) + | > loss_gen: 2.5461232662200928 (2.5461232662200928) + | > loss_kl: 2.0517475605010986 (2.0517475605010986) + | > loss_feat: 7.168323516845703 (7.168323516845703) + | > loss_mel: 19.725217819213867 (19.725217819213867) + | > loss_duration: 1.6326515674591064 (1.6326515674591064) + | > loss_1: 33.12406539916992 (33.12406539916992) + + --> STEP: 1 + | > loss_disc: 2.644052028656006 (2.644052028656006) + | > loss_disc_real_0: 0.15176890790462494 (0.15176890790462494) + | > loss_disc_real_1: 0.2884104549884796 (0.2884104549884796) + | > loss_disc_real_2: 0.21258345246315002 (0.21258345246315002) + | > loss_disc_real_3: 0.2226092517375946 (0.2226092517375946) + | > loss_disc_real_4: 0.2340313345193863 (0.2340313345193863) + | > loss_disc_real_5: 0.20284466445446014 (0.20284466445446014) + | > loss_0: 2.644052028656006 (2.644052028656006) + | > loss_gen: 2.175628423690796 (2.175628423690796) + | > loss_kl: 2.477287769317627 (2.477287769317627) + | > loss_feat: 6.747718811035156 (6.747718811035156) + | > loss_mel: 18.080671310424805 (18.080671310424805) + | > loss_duration: 1.654119610786438 (1.654119610786438) + | > loss_1: 31.135425567626953 (31.135425567626953) + + --> STEP: 2 + | > loss_disc: 2.4893455505371094 (2.5666987895965576) + | > loss_disc_real_0: 0.12209931015968323 (0.13693410903215408) + | > loss_disc_real_1: 0.24259313941001892 (0.26550179719924927) + | > loss_disc_real_2: 0.20378145575523376 (0.2081824541091919) + | > loss_disc_real_3: 0.22769489884376526 (0.22515207529067993) + | > loss_disc_real_4: 0.225685253739357 (0.22985829412937164) + | > loss_disc_real_5: 0.18672366440296173 (0.19478416442871094) + | > loss_0: 2.4893455505371094 (2.5666987895965576) + | > loss_gen: 2.1966564655303955 (2.1861424446105957) + | > loss_kl: 2.043947219848633 (2.26061749458313) + | > loss_feat: 6.3370819091796875 (6.542400360107422) + | > loss_mel: 17.578113555908203 (17.829392433166504) + | > loss_duration: 1.6066197156906128 (1.6303696632385254) + | > loss_1: 29.762418746948242 (30.448922157287598) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033627748489379883 (-0.0007803440093994141) + | > avg_loss_disc: 2.5666987895965576 (+0.09992027282714844) + | > avg_loss_disc_real_0: 0.13693410903215408 (-0.03511137515306473) + | > avg_loss_disc_real_1: 0.26550179719924927 (+0.09082561731338501) + | > avg_loss_disc_real_2: 0.2081824541091919 (-0.054394155740737915) + | > avg_loss_disc_real_3: 0.22515207529067993 (-0.014580599963665009) + | > avg_loss_disc_real_4: 0.22985829412937164 (-0.008090712130069733) + | > avg_loss_disc_real_5: 0.19478416442871094 (-0.003210984170436859) + | > avg_loss_0: 2.5666987895965576 (+0.09992027282714844) + | > avg_loss_gen: 2.1861424446105957 (-0.12610459327697754) + | > avg_loss_kl: 2.26061749458313 (+0.02070176601409912) + | > avg_loss_feat: 6.542400360107422 (-0.48294758796691895) + | > avg_loss_mel: 17.829392433166504 (-1.1225175857543945) + | > avg_loss_duration: 1.6303696632385254 (-0.020305991172790527) + | > avg_loss_1: 30.448922157287598 (-1.7311744689941406) + + + > EPOCH: 116/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:54:45)  + + --> TIME: 2025-04-19 00:54:53 -- STEP: 21/142 -- GLOBAL_STEP: 395775 + | > loss_disc: 2.4564766883850098 (2.4370326087588357) + | > loss_disc_real_0: 0.13618463277816772 (0.12500917343866258) + | > loss_disc_real_1: 0.240630641579628 (0.2052392682858876) + | > loss_disc_real_2: 0.22272169589996338 (0.2248767273766654) + | > loss_disc_real_3: 0.2415352761745453 (0.22993932380562737) + | > loss_disc_real_4: 0.2239181250333786 (0.23037802889233544) + | > loss_disc_real_5: 0.20568209886550903 (0.228618928364345) + | > loss_0: 2.4564766883850098 (2.4370326087588357) + | > grad_norm_0: tensor(6.1997, device='cuda:0') (tensor(8.9715, device='cuda:0')) + | > loss_gen: 2.3256924152374268 (2.3658328510466076) + | > loss_kl: 1.761048674583435 (1.861513342176165) + | > loss_feat: 6.770885467529297 (7.204601923624675) + | > loss_mel: 17.52057647705078 (17.982764561971027) + | > loss_duration: 1.4630091190338135 (1.4509808108920144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.841211318969727 (30.86569359188988) + | > grad_norm_1: tensor(208.1322, device='cuda:0') (tensor(228.1989, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.3412 (0.3331387270064581) + | > loader_time: 0.0052 (0.004805916831606909) + + + --> TIME: 2025-04-19 00:55:02 -- STEP: 46/142 -- GLOBAL_STEP: 395800 + | > loss_disc: 2.377535581588745 (2.4589829548545508) + | > loss_disc_real_0: 0.0776725634932518 (0.12685264047721162) + | > loss_disc_real_1: 0.17675749957561493 (0.20668123856834744) + | > loss_disc_real_2: 0.19428524374961853 (0.22605631338513416) + | > loss_disc_real_3: 0.21736817061901093 (0.23087974443383838) + | > loss_disc_real_4: 0.21744012832641602 (0.23190998026858206) + | > loss_disc_real_5: 0.21991996467113495 (0.2310880429070929) + | > loss_0: 2.377535581588745 (2.4589829548545508) + | > grad_norm_0: tensor(10.4789, device='cuda:0') (tensor(8.7674, device='cuda:0')) + | > loss_gen: 2.460359811782837 (2.358324641766756) + | > loss_kl: 1.8141753673553467 (1.865877029688462) + | > loss_feat: 7.4757537841796875 (7.105380628419959) + | > loss_mel: 18.8807430267334 (17.889326510222062) + | > loss_duration: 1.4543343782424927 (1.4633415470952573) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.08536911010742 (30.682250561921492) + | > grad_norm_1: tensor(186.9300, device='cuda:0') (tensor(183.1571, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.3676 (0.34455047483029577) + | > loader_time: 0.0058 (0.005261727001356042) + + + --> TIME: 2025-04-19 00:55:12 -- STEP: 71/142 -- GLOBAL_STEP: 395825 + | > loss_disc: 2.6454641819000244 (2.4693844721350877) + | > loss_disc_real_0: 0.18918465077877045 (0.1286776756316843) + | > loss_disc_real_1: 0.23253020644187927 (0.20960298978107075) + | > loss_disc_real_2: 0.25915879011154175 (0.22636052441429083) + | > loss_disc_real_3: 0.24383991956710815 (0.22958348481587962) + | > loss_disc_real_4: 0.2676999270915985 (0.2319509013437889) + | > loss_disc_real_5: 0.261406809091568 (0.2302065283899576) + | > loss_0: 2.6454641819000244 (2.4693844721350877) + | > grad_norm_0: tensor(10.4566, device='cuda:0') (tensor(8.8856, device='cuda:0')) + | > loss_gen: 2.25128173828125 (2.3464505370234106) + | > loss_kl: 1.971214771270752 (1.885160632536445) + | > loss_feat: 6.228939056396484 (6.9954907659073955) + | > loss_mel: 17.54395294189453 (17.877921789464814) + | > loss_duration: 1.494359016418457 (1.4743954363003584) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.48974609375 (30.579419283799723) + | > grad_norm_1: tensor(252.0405, device='cuda:0') (tensor(193.2239, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.3852 (0.35612725204145407) + | > loader_time: 0.0063 (0.005601318789200044) + + + --> TIME: 2025-04-19 00:55:22 -- STEP: 96/142 -- GLOBAL_STEP: 395850 + | > loss_disc: 2.507378101348877 (2.4753041888276734) + | > loss_disc_real_0: 0.10855259001255035 (0.13113413184570774) + | > loss_disc_real_1: 0.16914710402488708 (0.20824761595577002) + | > loss_disc_real_2: 0.1984444111585617 (0.22596982649217048) + | > loss_disc_real_3: 0.21809856593608856 (0.23065570555627346) + | > loss_disc_real_4: 0.23128612339496613 (0.23212837132935724) + | > loss_disc_real_5: 0.2173517793416977 (0.23059719738860926) + | > loss_0: 2.507378101348877 (2.4753041888276734) + | > grad_norm_0: tensor(11.8629, device='cuda:0') (tensor(8.8522, device='cuda:0')) + | > loss_gen: 2.1406147480010986 (2.3297486280401545) + | > loss_kl: 1.9148576259613037 (1.9031683256228764) + | > loss_feat: 6.638641357421875 (6.9162832250197726) + | > loss_mel: 16.834226608276367 (17.8212233086427) + | > loss_duration: 1.5105408430099487 (1.4858775946001213) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.038881301879883 (30.4563010931015) + | > grad_norm_1: tensor(219.1310, device='cuda:0') (tensor(190.2463, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.4047 (0.3673296198248864) + | > loader_time: 0.0064 (0.0058399414022763585) + + + --> TIME: 2025-04-19 00:55:33 -- STEP: 121/142 -- GLOBAL_STEP: 395875 + | > loss_disc: 2.3716506958007812 (2.4747594604807452) + | > loss_disc_real_0: 0.10625169426202774 (0.13216548406880763) + | > loss_disc_real_1: 0.21883828938007355 (0.20747644398823256) + | > loss_disc_real_2: 0.22646138072013855 (0.22562106655648917) + | > loss_disc_real_3: 0.24215054512023926 (0.22885168904115347) + | > loss_disc_real_4: 0.22231756150722504 (0.23159344174152563) + | > loss_disc_real_5: 0.2709643244743347 (0.23002690889618613) + | > loss_0: 2.3716506958007812 (2.4747594604807452) + | > grad_norm_0: tensor(13.8917, device='cuda:0') (tensor(8.8823, device='cuda:0')) + | > loss_gen: 2.674816608428955 (2.3217959364583676) + | > loss_kl: 1.972007155418396 (1.9087620973587036) + | > loss_feat: 7.2046308517456055 (6.884364813812508) + | > loss_mel: 18.70343017578125 (17.77513725501446) + | > loss_duration: 1.5628149509429932 (1.4931237017812808) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.117698669433594 (30.38318386550777) + | > grad_norm_1: tensor(340.4357, device='cuda:0') (tensor(186.9935, device='cuda:0')) + | > current_lr_0: 0.0001971207450908312 + | > current_lr_1: 0.0001971207450908312 + | > step_time: 0.4427 (0.37842957244431685) + | > loader_time: 0.0075 (0.006099340344263503) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.57670259475708 (2.57670259475708) + | > loss_disc_real_0: 0.1572321206331253 (0.1572321206331253) + | > loss_disc_real_1: 0.26936084032058716 (0.26936084032058716) + | > loss_disc_real_2: 0.2788521349430084 (0.2788521349430084) + | > loss_disc_real_3: 0.31667256355285645 (0.31667256355285645) + | > loss_disc_real_4: 0.23743776977062225 (0.23743776977062225) + | > loss_disc_real_5: 0.24704484641551971 (0.24704484641551971) + | > loss_0: 2.57670259475708 (2.57670259475708) + | > loss_gen: 2.4565951824188232 (2.4565951824188232) + | > loss_kl: 2.547593355178833 (2.547593355178833) + | > loss_feat: 5.870062351226807 (5.870062351226807) + | > loss_mel: 18.116817474365234 (18.116817474365234) + | > loss_duration: 1.6589703559875488 (1.6589703559875488) + | > loss_1: 30.65003776550293 (30.65003776550293) + + --> STEP: 1 + | > loss_disc: 2.4797868728637695 (2.4797868728637695) + | > loss_disc_real_0: 0.1836473047733307 (0.1836473047733307) + | > loss_disc_real_1: 0.2330041229724884 (0.2330041229724884) + | > loss_disc_real_2: 0.2706618905067444 (0.2706618905067444) + | > loss_disc_real_3: 0.25925904512405396 (0.25925904512405396) + | > loss_disc_real_4: 0.23821309208869934 (0.23821309208869934) + | > loss_disc_real_5: 0.2454347163438797 (0.2454347163438797) + | > loss_0: 2.4797868728637695 (2.4797868728637695) + | > loss_gen: 2.4389376640319824 (2.4389376640319824) + | > loss_kl: 2.616870880126953 (2.616870880126953) + | > loss_feat: 6.655685901641846 (6.655685901641846) + | > loss_mel: 17.235837936401367 (17.235837936401367) + | > loss_duration: 1.6864006519317627 (1.6864006519317627) + | > loss_1: 30.63373374938965 (30.63373374938965) + + --> STEP: 2 + | > loss_disc: 2.421854019165039 (2.4508204460144043) + | > loss_disc_real_0: 0.1594092696905136 (0.17152828723192215) + | > loss_disc_real_1: 0.23033319413661957 (0.23166865855455399) + | > loss_disc_real_2: 0.2790604531764984 (0.2748611718416214) + | > loss_disc_real_3: 0.25613662600517273 (0.25769783556461334) + | > loss_disc_real_4: 0.21607303619384766 (0.2271430641412735) + | > loss_disc_real_5: 0.2323835790157318 (0.23890914767980576) + | > loss_0: 2.421854019165039 (2.4508204460144043) + | > loss_gen: 2.392543077468872 (2.4157403707504272) + | > loss_kl: 2.4224393367767334 (2.5196551084518433) + | > loss_feat: 5.762736797332764 (6.209211349487305) + | > loss_mel: 16.57718849182129 (16.906513214111328) + | > loss_duration: 1.6277916431427002 (1.6570961475372314) + | > loss_1: 28.782699584960938 (29.708216667175293) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035763978958129883 (+0.000213623046875) + | > avg_loss_disc: 2.4508204460144043 (-0.11587834358215332) + | > avg_loss_disc_real_0: 0.17152828723192215 (+0.034594178199768066) + | > avg_loss_disc_real_1: 0.23166865855455399 (-0.03383313864469528) + | > avg_loss_disc_real_2: 0.2748611718416214 (+0.0666787177324295) + | > avg_loss_disc_real_3: 0.25769783556461334 (+0.03254576027393341) + | > avg_loss_disc_real_4: 0.2271430641412735 (-0.0027152299880981445) + | > avg_loss_disc_real_5: 0.23890914767980576 (+0.04412498325109482) + | > avg_loss_0: 2.4508204460144043 (-0.11587834358215332) + | > avg_loss_gen: 2.4157403707504272 (+0.22959792613983154) + | > avg_loss_kl: 2.5196551084518433 (+0.2590376138687134) + | > avg_loss_feat: 6.209211349487305 (-0.3331890106201172) + | > avg_loss_mel: 16.906513214111328 (-0.9228792190551758) + | > avg_loss_duration: 1.6570961475372314 (+0.026726484298706055) + | > avg_loss_1: 29.708216667175293 (-0.7407054901123047) + + + > EPOCH: 117/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:55:44)  + + --> TIME: 2025-04-19 00:55:46 -- STEP: 4/142 -- GLOBAL_STEP: 395900 + | > loss_disc: 2.442401885986328 (2.4540143609046936) + | > loss_disc_real_0: 0.074582040309906 (0.11946785636246204) + | > loss_disc_real_1: 0.2057003527879715 (0.21118924766778946) + | > loss_disc_real_2: 0.1700408160686493 (0.24403990805149078) + | > loss_disc_real_3: 0.23463459312915802 (0.25645263120532036) + | > loss_disc_real_4: 0.2967868745326996 (0.2759812921285629) + | > loss_disc_real_5: 0.2541232705116272 (0.2534460425376892) + | > loss_0: 2.442401885986328 (2.4540143609046936) + | > grad_norm_0: tensor(15.8899, device='cuda:0') (tensor(12.5103, device='cuda:0')) + | > loss_gen: 2.2810261249542236 (2.4478684663772583) + | > loss_kl: 1.8960374593734741 (1.9133704006671906) + | > loss_feat: 6.9163312911987305 (7.173220276832581) + | > loss_mel: 16.699691772460938 (17.758412837982178) + | > loss_duration: 1.4594612121582031 (1.4509261548519135) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.252548217773438 (30.743797779083252) + | > grad_norm_1: tensor(301.9270, device='cuda:0') (tensor(323.6436, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.3349 (0.33524107933044434) + | > loader_time: 0.0052 (0.004438459873199463) + + + --> TIME: 2025-04-19 00:55:55 -- STEP: 29/142 -- GLOBAL_STEP: 395925 + | > loss_disc: 2.543381452560425 (2.459083524243586) + | > loss_disc_real_0: 0.16278807818889618 (0.12079847912336218) + | > loss_disc_real_1: 0.24949362874031067 (0.20600187367406383) + | > loss_disc_real_2: 0.24071894586086273 (0.23132459973466807) + | > loss_disc_real_3: 0.24008068442344666 (0.22925810053430754) + | > loss_disc_real_4: 0.2363823503255844 (0.23397555073787427) + | > loss_disc_real_5: 0.22459270060062408 (0.23634813725948334) + | > loss_0: 2.543381452560425 (2.459083524243586) + | > grad_norm_0: tensor(9.7119, device='cuda:0') (tensor(10.8387, device='cuda:0')) + | > loss_gen: 2.1268935203552246 (2.38565652945946) + | > loss_kl: 1.5773855447769165 (1.86751481171312) + | > loss_feat: 6.252838611602783 (7.067856048715526) + | > loss_mel: 15.873241424560547 (17.803341174947803) + | > loss_duration: 1.4496614933013916 (1.455635905265808) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.280019760131836 (30.58000452765103) + | > grad_norm_1: tensor(222.4572, device='cuda:0') (tensor(289.5089, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.347 (0.33664131986683815) + | > loader_time: 0.0051 (0.004911110318940261) + + + --> TIME: 2025-04-19 00:56:04 -- STEP: 54/142 -- GLOBAL_STEP: 395950 + | > loss_disc: 2.4387993812561035 (2.4662991718009666) + | > loss_disc_real_0: 0.12875321507453918 (0.1217977110710409) + | > loss_disc_real_1: 0.23852238059043884 (0.2056143449412452) + | > loss_disc_real_2: 0.2420005202293396 (0.22890250743539245) + | > loss_disc_real_3: 0.17751699686050415 (0.2287368012799157) + | > loss_disc_real_4: 0.21240082383155823 (0.23205524379456485) + | > loss_disc_real_5: 0.22001519799232483 (0.2353498739776788) + | > loss_0: 2.4387993812561035 (2.4662991718009666) + | > grad_norm_0: tensor(11.4665, device='cuda:0') (tensor(10.9748, device='cuda:0')) + | > loss_gen: 2.3587727546691895 (2.354131689778081) + | > loss_kl: 1.916041374206543 (1.8783855261626068) + | > loss_feat: 6.663633823394775 (6.975071076993589) + | > loss_mel: 16.84898567199707 (17.707303947872585) + | > loss_duration: 1.4994280338287354 (1.4634466988069037) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.286861419677734 (30.378339096351905) + | > grad_norm_1: tensor(401.7050, device='cuda:0') (tensor(280.3585, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.3694 (0.34813525058605055) + | > loader_time: 0.0055 (0.005274653434753418) + + + --> TIME: 2025-04-19 00:56:14 -- STEP: 79/142 -- GLOBAL_STEP: 395975 + | > loss_disc: 2.4902184009552 (2.4684286027015014) + | > loss_disc_real_0: 0.1159595400094986 (0.12330900528762914) + | > loss_disc_real_1: 0.23389002680778503 (0.20743188620368136) + | > loss_disc_real_2: 0.2328147292137146 (0.22776887556420097) + | > loss_disc_real_3: 0.25968948006629944 (0.22798551732226263) + | > loss_disc_real_4: 0.2473851889371872 (0.23124857228013534) + | > loss_disc_real_5: 0.2215852290391922 (0.23386982328529599) + | > loss_0: 2.4902184009552 (2.4684286027015014) + | > grad_norm_0: tensor(13.0223, device='cuda:0') (tensor(10.7138, device='cuda:0')) + | > loss_gen: 2.415381908416748 (2.3422020628482483) + | > loss_kl: 1.8815428018569946 (1.8847925693173952) + | > loss_feat: 6.859105110168457 (6.953944701182691) + | > loss_mel: 18.535106658935547 (17.797688266899012) + | > loss_duration: 1.545257806777954 (1.4766145099567458) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.23639488220215 (30.455242181126074) + | > grad_norm_1: tensor(306.7657, device='cuda:0') (tensor(260.6596, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.386 (0.36032071898255175) + | > loader_time: 0.0061 (0.0055847650841821615) + + + --> TIME: 2025-04-19 00:56:24 -- STEP: 104/142 -- GLOBAL_STEP: 396000 + | > loss_disc: 2.4461963176727295 (2.4698189221895657) + | > loss_disc_real_0: 0.12966123223304749 (0.12483014517392103) + | > loss_disc_real_1: 0.21193303167819977 (0.20738540059671953) + | > loss_disc_real_2: 0.197574645280838 (0.22785387746989727) + | > loss_disc_real_3: 0.2110154926776886 (0.2282516539383393) + | > loss_disc_real_4: 0.2277735471725464 (0.231555627945524) + | > loss_disc_real_5: 0.19561941921710968 (0.234418879621304) + | > loss_0: 2.4461963176727295 (2.4698189221895657) + | > grad_norm_0: tensor(24.5929, device='cuda:0') (tensor(11.4978, device='cuda:0')) + | > loss_gen: 2.3243188858032227 (2.344800082536845) + | > loss_kl: 1.9508596658706665 (1.8983687666746287) + | > loss_feat: 6.965175628662109 (6.949029381458576) + | > loss_mel: 18.396015167236328 (17.800758251777058) + | > loss_duration: 1.5140535831451416 (1.4867147230184996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.150421142578125 (30.479671239852905) + | > grad_norm_1: tensor(234.7346, device='cuda:0') (tensor(273.4214, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.4088 (0.37147284471071695) + | > loader_time: 0.0065 (0.0058737947390629704) + + + --> TIME: 2025-04-19 00:56:35 -- STEP: 129/142 -- GLOBAL_STEP: 396025 + | > loss_disc: 2.451772689819336 (2.4757722551508468) + | > loss_disc_real_0: 0.11352924257516861 (0.12570397509622938) + | > loss_disc_real_1: 0.13554427027702332 (0.2070345243280248) + | > loss_disc_real_2: 0.20449288189411163 (0.22807351274545803) + | > loss_disc_real_3: 0.173116073012352 (0.22805044898229052) + | > loss_disc_real_4: 0.21321913599967957 (0.23149685804233994) + | > loss_disc_real_5: 0.22790393233299255 (0.23408898445524912) + | > loss_0: 2.451772689819336 (2.4757722551508468) + | > grad_norm_0: tensor(6.2727, device='cuda:0') (tensor(11.7074, device='cuda:0')) + | > loss_gen: 2.255713939666748 (2.3372628467027545) + | > loss_kl: 1.995832920074463 (1.9041388496872067) + | > loss_feat: 7.303083419799805 (6.900206643481587) + | > loss_mel: 17.950912475585938 (17.74979103443234) + | > loss_duration: 1.5133144855499268 (1.4933005386544753) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.018857955932617 (30.38469993236453) + | > grad_norm_1: tensor(194.3674, device='cuda:0') (tensor(273.7404, device='cuda:0')) + | > current_lr_0: 0.00019709610499769482 + | > current_lr_1: 0.00019709610499769482 + | > step_time: 0.4449 (0.3830792515776879) + | > loader_time: 0.0076 (0.006138613057690998) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.650257110595703 (2.650257110595703) + | > loss_disc_real_0: 0.1813446283340454 (0.1813446283340454) + | > loss_disc_real_1: 0.28979378938674927 (0.28979378938674927) + | > loss_disc_real_2: 0.2192120999097824 (0.2192120999097824) + | > loss_disc_real_3: 0.28394508361816406 (0.28394508361816406) + | > loss_disc_real_4: 0.23528973758220673 (0.23528973758220673) + | > loss_disc_real_5: 0.2440723180770874 (0.2440723180770874) + | > loss_0: 2.650257110595703 (2.650257110595703) + | > loss_gen: 2.343137264251709 (2.343137264251709) + | > loss_kl: 2.3457305431365967 (2.3457305431365967) + | > loss_feat: 5.087991714477539 (5.087991714477539) + | > loss_mel: 13.443663597106934 (13.443663597106934) + | > loss_duration: 1.647701621055603 (1.647701621055603) + | > loss_1: 24.86822509765625 (24.86822509765625) + + --> STEP: 1 + | > loss_disc: 2.485751152038574 (2.485751152038574) + | > loss_disc_real_0: 0.14086364209651947 (0.14086364209651947) + | > loss_disc_real_1: 0.27638107538223267 (0.27638107538223267) + | > loss_disc_real_2: 0.23338288068771362 (0.23338288068771362) + | > loss_disc_real_3: 0.2512592375278473 (0.2512592375278473) + | > loss_disc_real_4: 0.22634805738925934 (0.22634805738925934) + | > loss_disc_real_5: 0.2283353954553604 (0.2283353954553604) + | > loss_0: 2.485751152038574 (2.485751152038574) + | > loss_gen: 2.5460641384124756 (2.5460641384124756) + | > loss_kl: 2.591886043548584 (2.591886043548584) + | > loss_feat: 6.97696590423584 (6.97696590423584) + | > loss_mel: 19.590383529663086 (19.590383529663086) + | > loss_duration: 1.666404128074646 (1.666404128074646) + | > loss_1: 33.3717041015625 (33.3717041015625) + + --> STEP: 2 + | > loss_disc: 2.659804582595825 (2.5727778673171997) + | > loss_disc_real_0: 0.20732560753822327 (0.17409462481737137) + | > loss_disc_real_1: 0.27962812781333923 (0.27800460159778595) + | > loss_disc_real_2: 0.20863494277000427 (0.22100891172885895) + | > loss_disc_real_3: 0.30161038041114807 (0.2764348089694977) + | > loss_disc_real_4: 0.27161696553230286 (0.2489825114607811) + | > loss_disc_real_5: 0.25992870330810547 (0.24413204938173294) + | > loss_0: 2.659804582595825 (2.5727778673171997) + | > loss_gen: 2.500706911087036 (2.523385524749756) + | > loss_kl: 2.2242839336395264 (2.408084988594055) + | > loss_feat: 6.693868160247803 (6.835417032241821) + | > loss_mel: 18.478378295898438 (19.03438091278076) + | > loss_duration: 1.641485571861267 (1.6539448499679565) + | > loss_1: 31.53872299194336 (32.45521354675293) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032074451446533203 (-0.00036895275115966797) + | > avg_loss_disc: 2.5727778673171997 (+0.12195742130279541) + | > avg_loss_disc_real_0: 0.17409462481737137 (+0.0025663375854492188) + | > avg_loss_disc_real_1: 0.27800460159778595 (+0.046335943043231964) + | > avg_loss_disc_real_2: 0.22100891172885895 (-0.05385226011276245) + | > avg_loss_disc_real_3: 0.2764348089694977 (+0.01873697340488434) + | > avg_loss_disc_real_4: 0.2489825114607811 (+0.0218394473195076) + | > avg_loss_disc_real_5: 0.24413204938173294 (+0.005222901701927185) + | > avg_loss_0: 2.5727778673171997 (+0.12195742130279541) + | > avg_loss_gen: 2.523385524749756 (+0.10764515399932861) + | > avg_loss_kl: 2.408084988594055 (-0.11157011985778809) + | > avg_loss_feat: 6.835417032241821 (+0.6262056827545166) + | > avg_loss_mel: 19.03438091278076 (+2.1278676986694336) + | > avg_loss_duration: 1.6539448499679565 (-0.0031512975692749023) + | > avg_loss_1: 32.45521354675293 (+2.7469968795776367) + + + > EPOCH: 118/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:56:43)  + + --> TIME: 2025-04-19 00:56:47 -- STEP: 12/142 -- GLOBAL_STEP: 396050 + | > loss_disc: 2.510676145553589 (2.496585249900818) + | > loss_disc_real_0: 0.1534503996372223 (0.13074310248096785) + | > loss_disc_real_1: 0.21978357434272766 (0.20744589219490686) + | > loss_disc_real_2: 0.2365817278623581 (0.23312605544924736) + | > loss_disc_real_3: 0.19399358332157135 (0.22525374963879585) + | > loss_disc_real_4: 0.2061583250761032 (0.23510259886582693) + | > loss_disc_real_5: 0.2157580405473709 (0.23514278357227644) + | > loss_0: 2.510676145553589 (2.496585249900818) + | > grad_norm_0: tensor(14.7676, device='cuda:0') (tensor(9.4170, device='cuda:0')) + | > loss_gen: 2.2096664905548096 (2.3098606069882712) + | > loss_kl: 1.7531182765960693 (1.8650504151980083) + | > loss_feat: 6.400973320007324 (6.978805104891459) + | > loss_mel: 17.819189071655273 (17.991921583811443) + | > loss_duration: 1.4718413352966309 (1.4513485332330067) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.6547908782959 (30.596986611684162) + | > grad_norm_1: tensor(325.3716, device='cuda:0') (tensor(217.1376, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.3287 (0.3264325062433879) + | > loader_time: 0.0049 (0.004563728968302409) + + + --> TIME: 2025-04-19 00:56:56 -- STEP: 37/142 -- GLOBAL_STEP: 396075 + | > loss_disc: 2.4399781227111816 (2.477674116959443) + | > loss_disc_real_0: 0.0869075357913971 (0.1240304758822596) + | > loss_disc_real_1: 0.2024456411600113 (0.20899840102002426) + | > loss_disc_real_2: 0.23575279116630554 (0.22964522725826986) + | > loss_disc_real_3: 0.23096083104610443 (0.22680618714641881) + | > loss_disc_real_4: 0.2310575693845749 (0.23158524849930326) + | > loss_disc_real_5: 0.2245139479637146 (0.23244173180412603) + | > loss_0: 2.4399781227111816 (2.477674116959443) + | > grad_norm_0: tensor(18.2199, device='cuda:0') (tensor(10.5188, device='cuda:0')) + | > loss_gen: 2.3379364013671875 (2.3276400050601445) + | > loss_kl: 1.9275267124176025 (1.8618279856604498) + | > loss_feat: 7.073973178863525 (6.9589787560540275) + | > loss_mel: 18.845928192138672 (17.877860095049886) + | > loss_duration: 1.4686946868896484 (1.4589103395874436) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.6540584564209 (30.485217738795924) + | > grad_norm_1: tensor(344.8206, device='cuda:0') (tensor(250.7823, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.3643 (0.3438020976814064) + | > loader_time: 0.0054 (0.005081144539085594) + + + --> TIME: 2025-04-19 00:57:06 -- STEP: 62/142 -- GLOBAL_STEP: 396100 + | > loss_disc: 2.448971748352051 (2.4711696409410044) + | > loss_disc_real_0: 0.14028142392635345 (0.12561986847750609) + | > loss_disc_real_1: 0.22700339555740356 (0.20828912911876554) + | > loss_disc_real_2: 0.2335863709449768 (0.22764551110806003) + | > loss_disc_real_3: 0.21151360869407654 (0.2269973218921692) + | > loss_disc_real_4: 0.22043956816196442 (0.22951037869338067) + | > loss_disc_real_5: 0.20184390246868134 (0.23025752195427496) + | > loss_0: 2.448971748352051 (2.4711696409410044) + | > grad_norm_0: tensor(5.0843, device='cuda:0') (tensor(11.2937, device='cuda:0')) + | > loss_gen: 2.4300529956817627 (2.3329927229112193) + | > loss_kl: 1.847825527191162 (1.8802488023234951) + | > loss_feat: 7.251136779785156 (6.91910590664033) + | > loss_mel: 18.215198516845703 (17.776300461061542) + | > loss_duration: 1.5224413871765137 (1.4704103681348986) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.26665687561035 (30.379058653308498) + | > grad_norm_1: tensor(186.4492, device='cuda:0') (tensor(264.2054, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.3846 (0.35950391907845775) + | > loader_time: 0.0062 (0.0054232843460575225) + + + --> TIME: 2025-04-19 00:57:16 -- STEP: 87/142 -- GLOBAL_STEP: 396125 + | > loss_disc: 2.4321415424346924 (2.4709210998710547) + | > loss_disc_real_0: 0.1293981671333313 (0.1273264293184226) + | > loss_disc_real_1: 0.2165028601884842 (0.20876644979948286) + | > loss_disc_real_2: 0.2269142121076584 (0.22734657032736416) + | > loss_disc_real_3: 0.22179527580738068 (0.22735809731757503) + | > loss_disc_real_4: 0.22122374176979065 (0.2299292217725995) + | > loss_disc_real_5: 0.21301455795764923 (0.23189898503237757) + | > loss_0: 2.4321415424346924 (2.4709210998710547) + | > grad_norm_0: tensor(11.4693, device='cuda:0') (tensor(11.1028, device='cuda:0')) + | > loss_gen: 2.2542612552642822 (2.3332740465799966) + | > loss_kl: 1.9799126386642456 (1.898582637995139) + | > loss_feat: 6.305905342102051 (6.894437504910875) + | > loss_mel: 16.334814071655273 (17.783229597683608) + | > loss_duration: 1.5468521118164062 (1.4831403658307831) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.42174530029297 (30.392664438006523) + | > grad_norm_1: tensor(200.8446, device='cuda:0') (tensor(254.8937, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.4189 (0.36927586314321936) + | > loader_time: 0.0066 (0.005679629314904927) + + + --> TIME: 2025-04-19 00:57:27 -- STEP: 112/142 -- GLOBAL_STEP: 396150 + | > loss_disc: 2.406656503677368 (2.4721311884266988) + | > loss_disc_real_0: 0.09071358293294907 (0.12846558760585533) + | > loss_disc_real_1: 0.1890074759721756 (0.2085672521165439) + | > loss_disc_real_2: 0.19327597320079803 (0.22671370740447724) + | > loss_disc_real_3: 0.18786758184432983 (0.22772388333188637) + | > loss_disc_real_4: 0.19879966974258423 (0.22932299279740878) + | > loss_disc_real_5: 0.2575671374797821 (0.23168526430215156) + | > loss_0: 2.406656503677368 (2.4721311884266988) + | > grad_norm_0: tensor(5.0113, device='cuda:0') (tensor(10.8422, device='cuda:0')) + | > loss_gen: 2.2529921531677246 (2.3231903250728334) + | > loss_kl: 1.8592814207077026 (1.911810195871762) + | > loss_feat: 7.503293514251709 (6.868853837251663) + | > loss_mel: 18.14232635498047 (17.740884448800767) + | > loss_duration: 1.4731552600860596 (1.490727263901915) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.231050491333008 (30.33546621458871) + | > grad_norm_1: tensor(246.1774, device='cuda:0') (tensor(252.7835, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.4268 (0.37924711406230927) + | > loader_time: 0.0069 (0.005916154810360501) + + + --> TIME: 2025-04-19 00:57:38 -- STEP: 137/142 -- GLOBAL_STEP: 396175 + | > loss_disc: 2.6405961513519287 (2.4768574498865727) + | > loss_disc_real_0: 0.20968206226825714 (0.13013996145803564) + | > loss_disc_real_1: 0.22815483808517456 (0.20894654856546083) + | > loss_disc_real_2: 0.26082971692085266 (0.22737331209826644) + | > loss_disc_real_3: 0.2487252950668335 (0.22855266667630553) + | > loss_disc_real_4: 0.23217082023620605 (0.22923888505375298) + | > loss_disc_real_5: 0.23228509724140167 (0.23154175966760537) + | > loss_0: 2.6405961513519287 (2.4768574498865727) + | > grad_norm_0: tensor(9.1066, device='cuda:0') (tensor(10.6027, device='cuda:0')) + | > loss_gen: 2.240657329559326 (2.321415835053381) + | > loss_kl: 1.88580322265625 (1.918457878767139) + | > loss_feat: 6.126440525054932 (6.871468199430591) + | > loss_mel: 17.057231903076172 (17.734761850677263) + | > loss_duration: 1.5121772289276123 (1.496631006254767) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.822309494018555 (30.34273491462652) + | > grad_norm_1: tensor(206.3721, device='cuda:0') (tensor(245.7676, device='cuda:0')) + | > current_lr_0: 0.0001970714679845701 + | > current_lr_1: 0.0001970714679845701 + | > step_time: 0.4543 (0.3902287396201252) + | > loader_time: 0.0075 (0.006195125788667776) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.581482172012329 (2.581482172012329) + | > loss_disc_real_0: 0.19004429876804352 (0.19004429876804352) + | > loss_disc_real_1: 0.2138565629720688 (0.2138565629720688) + | > loss_disc_real_2: 0.21563410758972168 (0.21563410758972168) + | > loss_disc_real_3: 0.2682455778121948 (0.2682455778121948) + | > loss_disc_real_4: 0.2764885425567627 (0.2764885425567627) + | > loss_disc_real_5: 0.3057456910610199 (0.3057456910610199) + | > loss_0: 2.581482172012329 (2.581482172012329) + | > loss_gen: 2.3510849475860596 (2.3510849475860596) + | > loss_kl: 2.361027240753174 (2.361027240753174) + | > loss_feat: 5.729088306427002 (5.729088306427002) + | > loss_mel: 17.955358505249023 (17.955358505249023) + | > loss_duration: 1.6516276597976685 (1.6516276597976685) + | > loss_1: 30.048189163208008 (30.048189163208008) + + --> STEP: 1 + | > loss_disc: 2.299057722091675 (2.299057722091675) + | > loss_disc_real_0: 0.12950341403484344 (0.12950341403484344) + | > loss_disc_real_1: 0.207353875041008 (0.207353875041008) + | > loss_disc_real_2: 0.20593300461769104 (0.20593300461769104) + | > loss_disc_real_3: 0.23889422416687012 (0.23889422416687012) + | > loss_disc_real_4: 0.22992797195911407 (0.22992797195911407) + | > loss_disc_real_5: 0.2341083288192749 (0.2341083288192749) + | > loss_0: 2.299057722091675 (2.299057722091675) + | > loss_gen: 2.465163230895996 (2.465163230895996) + | > loss_kl: 2.498131275177002 (2.498131275177002) + | > loss_feat: 7.566171169281006 (7.566171169281006) + | > loss_mel: 19.02895736694336 (19.02895736694336) + | > loss_duration: 1.6919519901275635 (1.6919519901275635) + | > loss_1: 33.25037384033203 (33.25037384033203) + + --> STEP: 2 + | > loss_disc: 2.528393030166626 (2.4137253761291504) + | > loss_disc_real_0: 0.14701628684997559 (0.13825985044240952) + | > loss_disc_real_1: 0.23566976189613342 (0.2215118184685707) + | > loss_disc_real_2: 0.22440211474895477 (0.2151675596833229) + | > loss_disc_real_3: 0.23287713527679443 (0.23588567972183228) + | > loss_disc_real_4: 0.20710048079490662 (0.21851422637701035) + | > loss_disc_real_5: 0.2505907118320465 (0.2423495203256607) + | > loss_0: 2.528393030166626 (2.4137253761291504) + | > loss_gen: 2.1737122535705566 (2.3194377422332764) + | > loss_kl: 2.286710262298584 (2.392420768737793) + | > loss_feat: 6.9025750160217285 (7.234373092651367) + | > loss_mel: 19.15488624572754 (19.09192180633545) + | > loss_duration: 1.6103522777557373 (1.6511521339416504) + | > loss_1: 32.12823486328125 (32.68930435180664) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003322124481201172 (+0.00011467933654785156) + | > avg_loss_disc: 2.4137253761291504 (-0.15905249118804932) + | > avg_loss_disc_real_0: 0.13825985044240952 (-0.03583477437496185) + | > avg_loss_disc_real_1: 0.2215118184685707 (-0.05649278312921524) + | > avg_loss_disc_real_2: 0.2151675596833229 (-0.005841352045536041) + | > avg_loss_disc_real_3: 0.23588567972183228 (-0.040549129247665405) + | > avg_loss_disc_real_4: 0.21851422637701035 (-0.030468285083770752) + | > avg_loss_disc_real_5: 0.2423495203256607 (-0.001782529056072235) + | > avg_loss_0: 2.4137253761291504 (-0.15905249118804932) + | > avg_loss_gen: 2.3194377422332764 (-0.2039477825164795) + | > avg_loss_kl: 2.392420768737793 (-0.015664219856262207) + | > avg_loss_feat: 7.234373092651367 (+0.3989560604095459) + | > avg_loss_mel: 19.09192180633545 (+0.0575408935546875) + | > avg_loss_duration: 1.6511521339416504 (-0.0027927160263061523) + | > avg_loss_1: 32.68930435180664 (+0.23409080505371094) + + + > EPOCH: 119/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:57:42)  + + --> TIME: 2025-04-19 00:57:49 -- STEP: 20/142 -- GLOBAL_STEP: 396200 + | > loss_disc: 2.474882125854492 (2.409377634525299) + | > loss_disc_real_0: 0.17173685133457184 (0.1201639786362648) + | > loss_disc_real_1: 0.18832477927207947 (0.19997188374400138) + | > loss_disc_real_2: 0.20952925086021423 (0.2225884109735489) + | > loss_disc_real_3: 0.231326162815094 (0.22397089675068854) + | > loss_disc_real_4: 0.2372373342514038 (0.219754558801651) + | > loss_disc_real_5: 0.2342062145471573 (0.23001771196722984) + | > loss_0: 2.474882125854492 (2.409377634525299) + | > grad_norm_0: tensor(15.5432, device='cuda:0') (tensor(10.1532, device='cuda:0')) + | > loss_gen: 2.3801300525665283 (2.3916362881660462) + | > loss_kl: 1.8964306116104126 (1.8115681111812592) + | > loss_feat: 6.651994228363037 (7.2864134311676025) + | > loss_mel: 16.661394119262695 (18.013276767730712) + | > loss_duration: 1.493675947189331 (1.4450027763843538) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.08362579345703 (30.94789743423462) + | > grad_norm_1: tensor(134.3617, device='cuda:0') (tensor(222.2429, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.3432 (0.3303400158882141) + | > loader_time: 0.0052 (0.004790341854095459) + + + --> TIME: 2025-04-19 00:57:58 -- STEP: 45/142 -- GLOBAL_STEP: 396225 + | > loss_disc: 2.4752893447875977 (2.4433700402577716) + | > loss_disc_real_0: 0.1110859140753746 (0.1239735010597441) + | > loss_disc_real_1: 0.18629565834999084 (0.20211243795024025) + | > loss_disc_real_2: 0.22873961925506592 (0.225965306825108) + | > loss_disc_real_3: 0.22060714662075043 (0.2269108815325631) + | > loss_disc_real_4: 0.207515686750412 (0.22716198232438828) + | > loss_disc_real_5: 0.2888345718383789 (0.23438291019863552) + | > loss_0: 2.4752893447875977 (2.4433700402577716) + | > grad_norm_0: tensor(28.0328, device='cuda:0') (tensor(11.4591, device='cuda:0')) + | > loss_gen: 2.379340171813965 (2.379615741305881) + | > loss_kl: 1.873105764389038 (1.8299399111005994) + | > loss_feat: 7.925124645233154 (7.240371661716037) + | > loss_mel: 19.152860641479492 (18.005658043755425) + | > loss_duration: 1.4908676147460938 (1.4591618749830457) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.8213005065918 (30.914747280544706) + | > grad_norm_1: tensor(447.5531, device='cuda:0') (tensor(258.6900, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.3654 (0.3435102409786648) + | > loader_time: 0.0059 (0.005140034357706705) + + + --> TIME: 2025-04-19 00:58:08 -- STEP: 70/142 -- GLOBAL_STEP: 396250 + | > loss_disc: 2.400172710418701 (2.455207163946969) + | > loss_disc_real_0: 0.2011912316083908 (0.12911067807248663) + | > loss_disc_real_1: 0.1726098656654358 (0.20347865415470942) + | > loss_disc_real_2: 0.2284782975912094 (0.22641841300896237) + | > loss_disc_real_3: 0.2333838790655136 (0.22740617735045296) + | > loss_disc_real_4: 0.2550073266029358 (0.22624045418841499) + | > loss_disc_real_5: 0.2340203821659088 (0.23269241941826685) + | > loss_0: 2.400172710418701 (2.455207163946969) + | > grad_norm_0: tensor(33.7220, device='cuda:0') (tensor(14.8423, device='cuda:0')) + | > loss_gen: 2.437877893447876 (2.3846136672156195) + | > loss_kl: 1.9034640789031982 (1.8541612727301462) + | > loss_feat: 7.082239627838135 (7.167691707611084) + | > loss_mel: 18.92979621887207 (17.873443889617928) + | > loss_duration: 1.5039650201797485 (1.4710873331342424) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.857343673706055 (30.750997843061175) + | > grad_norm_1: tensor(357.9292, device='cuda:0') (tensor(300.4122, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.3893 (0.3556426286697388) + | > loader_time: 0.0062 (0.005424860545567104) + + + --> TIME: 2025-04-19 00:58:18 -- STEP: 95/142 -- GLOBAL_STEP: 396275 + | > loss_disc: 2.4663498401641846 (2.456926305670486) + | > loss_disc_real_0: 0.18126732110977173 (0.12883945601551158) + | > loss_disc_real_1: 0.20072175562381744 (0.20399155899098045) + | > loss_disc_real_2: 0.2359861582517624 (0.22686473491944764) + | > loss_disc_real_3: 0.23026195168495178 (0.22741427092175734) + | > loss_disc_real_4: 0.22018882632255554 (0.22673309351268567) + | > loss_disc_real_5: 0.21106545627117157 (0.2314710010039179) + | > loss_0: 2.4663498401641846 (2.456926305670486) + | > grad_norm_0: tensor(25.7287, device='cuda:0') (tensor(16.4019, device='cuda:0')) + | > loss_gen: 2.2608985900878906 (2.3969483676709626) + | > loss_kl: 1.9461454153060913 (1.8656693671879014) + | > loss_feat: 6.069747447967529 (7.1360369330958315) + | > loss_mel: 17.401857376098633 (17.87392599206223) + | > loss_duration: 1.5393309593200684 (1.483336151273627) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.217981338500977 (30.755916796232526) + | > grad_norm_1: tensor(335.4690, device='cuda:0') (tensor(328.4228, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.4037 (0.366551675294575) + | > loader_time: 0.0067 (0.005647338064093337) + + + --> TIME: 2025-04-19 00:58:29 -- STEP: 120/142 -- GLOBAL_STEP: 396300 + | > loss_disc: 2.55810546875 (2.463143922885258) + | > loss_disc_real_0: 0.2592908442020416 (0.1320470955222845) + | > loss_disc_real_1: 0.1941472887992859 (0.20324474349617958) + | > loss_disc_real_2: 0.2168114334344864 (0.22583372344573338) + | > loss_disc_real_3: 0.2276495099067688 (0.22815336274604003) + | > loss_disc_real_4: 0.24817968904972076 (0.22804925565918285) + | > loss_disc_real_5: 0.27579954266548157 (0.23164785044888656) + | > loss_0: 2.55810546875 (2.463143922885258) + | > grad_norm_0: tensor(30.6193, device='cuda:0') (tensor(18.0492, device='cuda:0')) + | > loss_gen: 2.469144821166992 (2.3925334791342414) + | > loss_kl: 1.9550646543502808 (1.881274762749672) + | > loss_feat: 6.162536144256592 (7.0469082474708555) + | > loss_mel: 16.62104034423828 (17.718635733922337) + | > loss_duration: 1.5324299335479736 (1.4904059489568076) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.740215301513672 (30.52975819905599) + | > grad_norm_1: tensor(354.0896, device='cuda:0') (tensor(335.0402, device='cuda:0')) + | > current_lr_0: 0.000197046834051072 + | > current_lr_1: 0.000197046834051072 + | > step_time: 0.4484 (0.37820432583491015) + | > loader_time: 0.0069 (0.005894853671391804) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3570358753204346 (2.3570358753204346) + | > loss_disc_real_0: 0.13247527182102203 (0.13247527182102203) + | > loss_disc_real_1: 0.23935125768184662 (0.23935125768184662) + | > loss_disc_real_2: 0.22322116792201996 (0.22322116792201996) + | > loss_disc_real_3: 0.23804228007793427 (0.23804228007793427) + | > loss_disc_real_4: 0.20313139259815216 (0.20313139259815216) + | > loss_disc_real_5: 0.21448439359664917 (0.21448439359664917) + | > loss_0: 2.3570358753204346 (2.3570358753204346) + | > loss_gen: 2.4505622386932373 (2.4505622386932373) + | > loss_kl: 2.443709373474121 (2.443709373474121) + | > loss_feat: 6.355750560760498 (6.355750560760498) + | > loss_mel: 18.164825439453125 (18.164825439453125) + | > loss_duration: 1.6458021402359009 (1.6458021402359009) + | > loss_1: 31.06064796447754 (31.06064796447754) + + --> STEP: 1 + | > loss_disc: 2.3626930713653564 (2.3626930713653564) + | > loss_disc_real_0: 0.1216140016913414 (0.1216140016913414) + | > loss_disc_real_1: 0.20831716060638428 (0.20831716060638428) + | > loss_disc_real_2: 0.23479804396629333 (0.23479804396629333) + | > loss_disc_real_3: 0.2516130208969116 (0.2516130208969116) + | > loss_disc_real_4: 0.22463274002075195 (0.22463274002075195) + | > loss_disc_real_5: 0.21210218966007233 (0.21210218966007233) + | > loss_0: 2.3626930713653564 (2.3626930713653564) + | > loss_gen: 2.3730435371398926 (2.3730435371398926) + | > loss_kl: 2.560248374938965 (2.560248374938965) + | > loss_feat: 6.823999404907227 (6.823999404907227) + | > loss_mel: 19.141742706298828 (19.141742706298828) + | > loss_duration: 1.6621630191802979 (1.6621630191802979) + | > loss_1: 32.561195373535156 (32.561195373535156) + + --> STEP: 2 + | > loss_disc: 2.5177149772644043 (2.4402040243148804) + | > loss_disc_real_0: 0.20934054255485535 (0.16547727212309837) + | > loss_disc_real_1: 0.20857271552085876 (0.20844493806362152) + | > loss_disc_real_2: 0.21481816470623016 (0.22480810433626175) + | > loss_disc_real_3: 0.250705361366272 (0.2511591911315918) + | > loss_disc_real_4: 0.21744120121002197 (0.22103697061538696) + | > loss_disc_real_5: 0.23991504311561584 (0.22600861638784409) + | > loss_0: 2.5177149772644043 (2.4402040243148804) + | > loss_gen: 2.676513671875 (2.5247786045074463) + | > loss_kl: 2.0972707271575928 (2.328759551048279) + | > loss_feat: 7.537418842315674 (7.18070912361145) + | > loss_mel: 18.76355743408203 (18.95265007019043) + | > loss_duration: 1.6129754781723022 (1.6375692486763) + | > loss_1: 32.68773651123047 (32.62446594238281) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003434896469116211 (+0.00011277198791503906) + | > avg_loss_disc: 2.4402040243148804 (+0.02647864818572998) + | > avg_loss_disc_real_0: 0.16547727212309837 (+0.027217421680688858) + | > avg_loss_disc_real_1: 0.20844493806362152 (-0.013066880404949188) + | > avg_loss_disc_real_2: 0.22480810433626175 (+0.009640544652938843) + | > avg_loss_disc_real_3: 0.2511591911315918 (+0.015273511409759521) + | > avg_loss_disc_real_4: 0.22103697061538696 (+0.0025227442383766174) + | > avg_loss_disc_real_5: 0.22600861638784409 (-0.01634090393781662) + | > avg_loss_0: 2.4402040243148804 (+0.02647864818572998) + | > avg_loss_gen: 2.5247786045074463 (+0.20534086227416992) + | > avg_loss_kl: 2.328759551048279 (-0.06366121768951416) + | > avg_loss_feat: 7.18070912361145 (-0.05366396903991699) + | > avg_loss_mel: 18.95265007019043 (-0.13927173614501953) + | > avg_loss_duration: 1.6375692486763 (-0.013582885265350342) + | > avg_loss_1: 32.62446594238281 (-0.06483840942382812) + + + > EPOCH: 120/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:58:40)  + + --> TIME: 2025-04-19 00:58:42 -- STEP: 3/142 -- GLOBAL_STEP: 396325 + | > loss_disc: 2.433835029602051 (2.4104246298472085) + | > loss_disc_real_0: 0.08128922432661057 (0.10331144432226817) + | > loss_disc_real_1: 0.19751325249671936 (0.2030485967795054) + | > loss_disc_real_2: 0.2286389023065567 (0.23108196258544922) + | > loss_disc_real_3: 0.24853800237178802 (0.24053338170051575) + | > loss_disc_real_4: 0.2431238293647766 (0.22889800369739532) + | > loss_disc_real_5: 0.24687691032886505 (0.21992495159308115) + | > loss_0: 2.433835029602051 (2.4104246298472085) + | > grad_norm_0: tensor(9.1140, device='cuda:0') (tensor(8.0631, device='cuda:0')) + | > loss_gen: 2.377802848815918 (2.3636749585469565) + | > loss_kl: 1.7741986513137817 (1.8291914463043213) + | > loss_feat: 7.199642181396484 (7.259811878204346) + | > loss_mel: 19.430904388427734 (18.519333521525066) + | > loss_duration: 1.4510399103164673 (1.4466950098673503) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.233585357666016 (31.41870625813802) + | > grad_norm_1: tensor(234.8120, device='cuda:0') (tensor(228.3061, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.3288 (0.33040523529052734) + | > loader_time: 0.0047 (0.004372358322143555) + + + --> TIME: 2025-04-19 00:58:51 -- STEP: 28/142 -- GLOBAL_STEP: 396350 + | > loss_disc: 2.436352252960205 (2.421923765114375) + | > loss_disc_real_0: 0.09442051500082016 (0.1120892081941877) + | > loss_disc_real_1: 0.17973029613494873 (0.19856665283441544) + | > loss_disc_real_2: 0.2115928828716278 (0.22993499200258935) + | > loss_disc_real_3: 0.19851739704608917 (0.22755091371280806) + | > loss_disc_real_4: 0.20384100079536438 (0.23037526650088175) + | > loss_disc_real_5: 0.2539028823375702 (0.22659932130149432) + | > loss_0: 2.436352252960205 (2.421923765114375) + | > grad_norm_0: tensor(7.3132, device='cuda:0') (tensor(6.9639, device='cuda:0')) + | > loss_gen: 2.3551058769226074 (2.358084644590105) + | > loss_kl: 1.8377830982208252 (1.8450476314340318) + | > loss_feat: 6.948801517486572 (7.122099961553301) + | > loss_mel: 18.898784637451172 (17.99557965142387) + | > loss_duration: 1.5009815692901611 (1.4552988580295019) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.541458129882812 (30.776110853467667) + | > grad_norm_1: tensor(92.9479, device='cuda:0') (tensor(154.6417, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.3453 (0.33659703390938894) + | > loader_time: 0.0054 (0.004932590893336705) + + + --> TIME: 2025-04-19 00:59:00 -- STEP: 53/142 -- GLOBAL_STEP: 396375 + | > loss_disc: 2.494525909423828 (2.4203151531939238) + | > loss_disc_real_0: 0.16617238521575928 (0.11747368172092258) + | > loss_disc_real_1: 0.1750335395336151 (0.2008311222746687) + | > loss_disc_real_2: 0.18536381423473358 (0.22460520436178963) + | > loss_disc_real_3: 0.21132224798202515 (0.2287055383313377) + | > loss_disc_real_4: 0.21978159248828888 (0.22815442000919917) + | > loss_disc_real_5: 0.22766956686973572 (0.2300001436247016) + | > loss_0: 2.494525909423828 (2.4203151531939238) + | > grad_norm_0: tensor(5.2286, device='cuda:0') (tensor(6.7765, device='cuda:0')) + | > loss_gen: 2.320610523223877 (2.3757200106134952) + | > loss_kl: 1.8669220209121704 (1.8600416565841098) + | > loss_feat: 7.658710956573486 (7.243463129367468) + | > loss_mel: 18.237018585205078 (18.01996258069885) + | > loss_duration: 1.4663519859313965 (1.4642688238395836) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.54961395263672 (30.963456387789744) + | > grad_norm_1: tensor(146.9586, device='cuda:0') (tensor(143.4972, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.3703 (0.3483157742698237) + | > loader_time: 0.0061 (0.00530116063243938) + + + --> TIME: 2025-04-19 00:59:10 -- STEP: 78/142 -- GLOBAL_STEP: 396400 + | > loss_disc: 2.343292713165283 (2.4434741062995715) + | > loss_disc_real_0: 0.1519039273262024 (0.12332822754979134) + | > loss_disc_real_1: 0.20482276380062103 (0.2048135004364527) + | > loss_disc_real_2: 0.22576916217803955 (0.2248401124125872) + | > loss_disc_real_3: 0.19773851335048676 (0.22858876219162574) + | > loss_disc_real_4: 0.2223062664270401 (0.2278491212771489) + | > loss_disc_real_5: 0.2045217901468277 (0.23199551896407053) + | > loss_0: 2.343292713165283 (2.4434741062995715) + | > grad_norm_0: tensor(6.3290, device='cuda:0') (tensor(7.3202, device='cuda:0')) + | > loss_gen: 2.3514840602874756 (2.3566007155638466) + | > loss_kl: 1.9634336233139038 (1.8749477236698835) + | > loss_feat: 7.632275581359863 (7.164111088483762) + | > loss_mel: 19.036832809448242 (17.92038774490357) + | > loss_duration: 1.5256149768829346 (1.4768759455436316) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.509639739990234 (30.792923413790188) + | > grad_norm_1: tensor(94.6815, device='cuda:0') (tensor(153.6898, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.3933 (0.3602934403297227) + | > loader_time: 0.0058 (0.005571548755352312) + + + --> TIME: 2025-04-19 00:59:20 -- STEP: 103/142 -- GLOBAL_STEP: 396425 + | > loss_disc: 2.460905075073242 (2.4580983652651893) + | > loss_disc_real_0: 0.18432137370109558 (0.1259490919199962) + | > loss_disc_real_1: 0.2180081009864807 (0.2058530282048346) + | > loss_disc_real_2: 0.21912190318107605 (0.22589399846433436) + | > loss_disc_real_3: 0.24077962338924408 (0.2297174298358195) + | > loss_disc_real_4: 0.23377057909965515 (0.23038818159149688) + | > loss_disc_real_5: 0.28369826078414917 (0.23328531772187613) + | > loss_0: 2.460905075073242 (2.4580983652651893) + | > grad_norm_0: tensor(7.6567, device='cuda:0') (tensor(8.0369, device='cuda:0')) + | > loss_gen: 2.4239633083343506 (2.3473692750467827) + | > loss_kl: 1.8935400247573853 (1.893296417680759) + | > loss_feat: 6.614174842834473 (7.0604868305539625) + | > loss_mel: 17.93437385559082 (17.85668924942757) + | > loss_duration: 1.558924674987793 (1.4881933892814858) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.424976348876953 (30.64603535179953) + | > grad_norm_1: tensor(313.4451, device='cuda:0') (tensor(173.8447, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.4149 (0.3708561670432969) + | > loader_time: 0.0066 (0.005826213984813505) + + + --> TIME: 2025-04-19 00:59:31 -- STEP: 128/142 -- GLOBAL_STEP: 396450 + | > loss_disc: 2.4890007972717285 (2.4596463534981012) + | > loss_disc_real_0: 0.20339131355285645 (0.1277094434481114) + | > loss_disc_real_1: 0.19325532019138336 (0.20521913969423622) + | > loss_disc_real_2: 0.20785856246948242 (0.22568488493561745) + | > loss_disc_real_3: 0.2456563264131546 (0.2293295996496454) + | > loss_disc_real_4: 0.23922987282276154 (0.22917367797344923) + | > loss_disc_real_5: 0.284463495016098 (0.23353864334058017) + | > loss_0: 2.4890007972717285 (2.4596463534981012) + | > grad_norm_0: tensor(9.5618, device='cuda:0') (tensor(8.3495, device='cuda:0')) + | > loss_gen: 2.409825086593628 (2.3377205394208422) + | > loss_kl: 1.9052881002426147 (1.8950439672917128) + | > loss_feat: 7.079460144042969 (7.001101154834032) + | > loss_mel: 18.60140609741211 (17.827241927385334) + | > loss_duration: 1.5241835117340088 (1.4952162588015199) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.52016258239746 (30.556324049830437) + | > grad_norm_1: tensor(314.8587, device='cuda:0') (tensor(175.3167, device='cuda:0')) + | > current_lr_0: 0.00019702220319681561 + | > current_lr_1: 0.00019702220319681561 + | > step_time: 0.4358 (0.38221052289009083) + | > loader_time: 0.0071 (0.00606950931251049) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4320967197418213 (2.4320967197418213) + | > loss_disc_real_0: 0.09630618244409561 (0.09630618244409561) + | > loss_disc_real_1: 0.2080777883529663 (0.2080777883529663) + | > loss_disc_real_2: 0.2367585152387619 (0.2367585152387619) + | > loss_disc_real_3: 0.24961639940738678 (0.24961639940738678) + | > loss_disc_real_4: 0.2355327159166336 (0.2355327159166336) + | > loss_disc_real_5: 0.24081732332706451 (0.24081732332706451) + | > loss_0: 2.4320967197418213 (2.4320967197418213) + | > loss_gen: 2.3009705543518066 (2.3009705543518066) + | > loss_kl: 2.345104217529297 (2.345104217529297) + | > loss_feat: 6.69168758392334 (6.69168758392334) + | > loss_mel: 18.470855712890625 (18.470855712890625) + | > loss_duration: 1.6219367980957031 (1.6219367980957031) + | > loss_1: 31.43055534362793 (31.43055534362793) + + --> STEP: 1 + | > loss_disc: 2.6076560020446777 (2.6076560020446777) + | > loss_disc_real_0: 0.13670314848423004 (0.13670314848423004) + | > loss_disc_real_1: 0.2261679321527481 (0.2261679321527481) + | > loss_disc_real_2: 0.25584328174591064 (0.25584328174591064) + | > loss_disc_real_3: 0.23269949853420258 (0.23269949853420258) + | > loss_disc_real_4: 0.2450132519006729 (0.2450132519006729) + | > loss_disc_real_5: 0.2210780829191208 (0.2210780829191208) + | > loss_0: 2.6076560020446777 (2.6076560020446777) + | > loss_gen: 1.9563220739364624 (1.9563220739364624) + | > loss_kl: 2.433586597442627 (2.433586597442627) + | > loss_feat: 5.687936782836914 (5.687936782836914) + | > loss_mel: 16.058996200561523 (16.058996200561523) + | > loss_duration: 1.6590023040771484 (1.6590023040771484) + | > loss_1: 27.79584312438965 (27.79584312438965) + + --> STEP: 2 + | > loss_disc: 2.5072083473205566 (2.557432174682617) + | > loss_disc_real_0: 0.14786505699157715 (0.1422841027379036) + | > loss_disc_real_1: 0.2563241422176361 (0.2412460371851921) + | > loss_disc_real_2: 0.24076393246650696 (0.2483036071062088) + | > loss_disc_real_3: 0.2330205887556076 (0.2328600436449051) + | > loss_disc_real_4: 0.21700023114681244 (0.23100674152374268) + | > loss_disc_real_5: 0.2651630640029907 (0.24312057346105576) + | > loss_0: 2.5072083473205566 (2.557432174682617) + | > loss_gen: 2.347628593444824 (2.1519753336906433) + | > loss_kl: 2.055410385131836 (2.2444984912872314) + | > loss_feat: 6.58409309387207 (6.136014938354492) + | > loss_mel: 18.57965087890625 (17.319323539733887) + | > loss_duration: 1.6179423332214355 (1.638472318649292) + | > loss_1: 31.18472671508789 (29.49028491973877) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003161311149597168 (-0.00027358531951904297) + | > avg_loss_disc: 2.557432174682617 (+0.11722815036773682) + | > avg_loss_disc_real_0: 0.1422841027379036 (-0.02319316938519478) + | > avg_loss_disc_real_1: 0.2412460371851921 (+0.03280109912157059) + | > avg_loss_disc_real_2: 0.2483036071062088 (+0.023495502769947052) + | > avg_loss_disc_real_3: 0.2328600436449051 (-0.018299147486686707) + | > avg_loss_disc_real_4: 0.23100674152374268 (+0.009969770908355713) + | > avg_loss_disc_real_5: 0.24312057346105576 (+0.01711195707321167) + | > avg_loss_0: 2.557432174682617 (+0.11722815036773682) + | > avg_loss_gen: 2.1519753336906433 (-0.372803270816803) + | > avg_loss_kl: 2.2444984912872314 (-0.08426105976104736) + | > avg_loss_feat: 6.136014938354492 (-1.044694185256958) + | > avg_loss_mel: 17.319323539733887 (-1.633326530456543) + | > avg_loss_duration: 1.638472318649292 (+0.0009030699729919434) + | > avg_loss_1: 29.49028491973877 (-3.134181022644043) + + + > EPOCH: 121/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 00:59:39)  + + --> TIME: 2025-04-19 00:59:43 -- STEP: 11/142 -- GLOBAL_STEP: 396475 + | > loss_disc: 2.491032838821411 (2.458917639472268) + | > loss_disc_real_0: 0.15354302525520325 (0.11746383932503787) + | > loss_disc_real_1: 0.20353181660175323 (0.19968286698514764) + | > loss_disc_real_2: 0.24776344001293182 (0.23040785030885177) + | > loss_disc_real_3: 0.25491970777511597 (0.22890421070835806) + | > loss_disc_real_4: 0.2615983188152313 (0.23681059886108746) + | > loss_disc_real_5: 0.2417268306016922 (0.23846835575320505) + | > loss_0: 2.491032838821411 (2.458917639472268) + | > grad_norm_0: tensor(11.0848, device='cuda:0') (tensor(6.9618, device='cuda:0')) + | > loss_gen: 2.45835018157959 (2.410226431759921) + | > loss_kl: 1.7963966131210327 (1.8505102721127598) + | > loss_feat: 6.836332321166992 (7.193126114931974) + | > loss_mel: 17.883689880371094 (17.767840818925336) + | > loss_duration: 1.4285626411437988 (1.4415944814682007) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.403331756591797 (30.663298173384234) + | > grad_norm_1: tensor(219.5160, device='cuda:0') (tensor(208.1678, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.3304 (0.3279367576945912) + | > loader_time: 0.0048 (0.004404739900068803) + + + --> TIME: 2025-04-19 00:59:52 -- STEP: 36/142 -- GLOBAL_STEP: 396500 + | > loss_disc: 2.5558114051818848 (2.4652229613727994) + | > loss_disc_real_0: 0.14968779683113098 (0.11757246177229616) + | > loss_disc_real_1: 0.26888716220855713 (0.20630822951594988) + | > loss_disc_real_2: 0.23728792369365692 (0.22811687654919094) + | > loss_disc_real_3: 0.2226955145597458 (0.22802788019180298) + | > loss_disc_real_4: 0.22222450375556946 (0.2308534992237886) + | > loss_disc_real_5: 0.28538763523101807 (0.23775058570835325) + | > loss_0: 2.5558114051818848 (2.4652229613727994) + | > grad_norm_0: tensor(17.6019, device='cuda:0') (tensor(9.0536, device='cuda:0')) + | > loss_gen: 2.456882953643799 (2.3802890578905744) + | > loss_kl: 1.964703917503357 (1.8727618753910065) + | > loss_feat: 7.055868625640869 (7.112011141247219) + | > loss_mel: 18.32683563232422 (17.720793935987686) + | > loss_duration: 1.4436678886413574 (1.455626447995504) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.24795913696289 (30.541482872433132) + | > grad_norm_1: tensor(522.1733, device='cuda:0') (tensor(261.9602, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.3549 (0.34039968252182007) + | > loader_time: 0.0054 (0.0048450032869974775) + + + --> TIME: 2025-04-19 01:00:01 -- STEP: 61/142 -- GLOBAL_STEP: 396525 + | > loss_disc: 2.5394253730773926 (2.485710745952169) + | > loss_disc_real_0: 0.13482800126075745 (0.12022334459375163) + | > loss_disc_real_1: 0.1772007793188095 (0.21014474356760743) + | > loss_disc_real_2: 0.2508358359336853 (0.22894946351403095) + | > loss_disc_real_3: 0.2561417520046234 (0.2304038595957834) + | > loss_disc_real_4: 0.2511019706726074 (0.23321972909520883) + | > loss_disc_real_5: 0.24638988077640533 (0.2356172431199277) + | > loss_0: 2.5394253730773926 (2.485710745952169) + | > grad_norm_0: tensor(9.1291, device='cuda:0') (tensor(10.4914, device='cuda:0')) + | > loss_gen: 2.1647531986236572 (2.3626756824430872) + | > loss_kl: 1.8619385957717896 (1.8798125708689455) + | > loss_feat: 6.974387168884277 (7.099746289800425) + | > loss_mel: 18.236534118652344 (17.866093244708953) + | > loss_duration: 1.4775718450546265 (1.4675770099045802) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.715185165405273 (30.67590497751705) + | > grad_norm_1: tensor(86.7180, device='cuda:0') (tensor(280.2916, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.3819 (0.3513463598782899) + | > loader_time: 0.0064 (0.005201902545866419) + + + --> TIME: 2025-04-19 01:00:12 -- STEP: 86/142 -- GLOBAL_STEP: 396550 + | > loss_disc: 2.447509527206421 (2.479877699253171) + | > loss_disc_real_0: 0.14193344116210938 (0.1230867421037929) + | > loss_disc_real_1: 0.19234046339988708 (0.2101281479347584) + | > loss_disc_real_2: 0.21370349824428558 (0.22934326269598895) + | > loss_disc_real_3: 0.2405879944562912 (0.22757455308076946) + | > loss_disc_real_4: 0.19726352393627167 (0.2307617286610049) + | > loss_disc_real_5: 0.2328752875328064 (0.23418177716260732) + | > loss_0: 2.447509527206421 (2.479877699253171) + | > grad_norm_0: tensor(13.7490, device='cuda:0') (tensor(10.5226, device='cuda:0')) + | > loss_gen: 2.217122793197632 (2.354468309602072) + | > loss_kl: 1.959864616394043 (1.882493490396544) + | > loss_feat: 7.546220302581787 (7.044942201570023) + | > loss_mel: 19.441654205322266 (17.93092730987903) + | > loss_duration: 1.5576128959655762 (1.4811691697253737) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.72247314453125 (30.694000532460766) + | > grad_norm_1: tensor(344.0993, device='cuda:0') (tensor(264.2794, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.3981 (0.36328826394192) + | > loader_time: 0.0064 (0.005479795988215956) + + + --> TIME: 2025-04-19 01:00:22 -- STEP: 111/142 -- GLOBAL_STEP: 396575 + | > loss_disc: 2.49649715423584 (2.4929862881565943) + | > loss_disc_real_0: 0.18327154219150543 (0.12483338427704732) + | > loss_disc_real_1: 0.22953547537326813 (0.211440751547212) + | > loss_disc_real_2: 0.23076249659061432 (0.2308942143981521) + | > loss_disc_real_3: 0.24084514379501343 (0.23000002336931658) + | > loss_disc_real_4: 0.20933844149112701 (0.23198271415255092) + | > loss_disc_real_5: 0.2706056535243988 (0.23514209150731027) + | > loss_0: 2.49649715423584 (2.4929862881565943) + | > grad_norm_0: tensor(10.4333, device='cuda:0') (tensor(10.7272, device='cuda:0')) + | > loss_gen: 2.2445590496063232 (2.336574172114467) + | > loss_kl: 1.9760313034057617 (1.888071732478099) + | > loss_feat: 5.581741809844971 (6.953747031924961) + | > loss_mel: 16.53345489501953 (17.866465061634493) + | > loss_duration: 1.532365322113037 (1.4897786282204293) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.868152618408203 (30.53463667792243) + | > grad_norm_1: tensor(290.0425, device='cuda:0') (tensor(267.1951, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.4263 (0.3740330043139758) + | > loader_time: 0.007 (0.005728212562767235) + + + --> TIME: 2025-04-19 01:00:33 -- STEP: 136/142 -- GLOBAL_STEP: 396600 + | > loss_disc: 2.4179787635803223 (2.4909655311528365) + | > loss_disc_real_0: 0.14246675372123718 (0.12591729565140078) + | > loss_disc_real_1: 0.15973293781280518 (0.21090288449297934) + | > loss_disc_real_2: 0.2218366116285324 (0.2301222096909495) + | > loss_disc_real_3: 0.21348091959953308 (0.22930999779525926) + | > loss_disc_real_4: 0.2301175445318222 (0.23088888539110913) + | > loss_disc_real_5: 0.20533768832683563 (0.23453150930650093) + | > loss_0: 2.4179787635803223 (2.4909655311528365) + | > grad_norm_0: tensor(11.5769, device='cuda:0') (tensor(10.8631, device='cuda:0')) + | > loss_gen: 2.1863131523132324 (2.3304092550978943) + | > loss_kl: 1.9284722805023193 (1.8922097639125937) + | > loss_feat: 7.133143424987793 (6.926108893226174) + | > loss_mel: 17.996841430664062 (17.799030956099998) + | > loss_duration: 1.5190731287002563 (1.4954041014699375) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.763843536376953 (30.44316296016469) + | > grad_norm_1: tensor(441.2982, device='cuda:0') (tensor(272.3499, device='cuda:0')) + | > current_lr_0: 0.000196997575421416 + | > current_lr_1: 0.000196997575421416 + | > step_time: 0.4671 (0.3863219478551079) + | > loader_time: 0.0078 (0.0059974947396446675) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.385262966156006 (2.385262966156006) + | > loss_disc_real_0: 0.08851595968008041 (0.08851595968008041) + | > loss_disc_real_1: 0.19778461754322052 (0.19778461754322052) + | > loss_disc_real_2: 0.22158803045749664 (0.22158803045749664) + | > loss_disc_real_3: 0.25384196639060974 (0.25384196639060974) + | > loss_disc_real_4: 0.2210831344127655 (0.2210831344127655) + | > loss_disc_real_5: 0.2508881986141205 (0.2508881986141205) + | > loss_0: 2.385262966156006 (2.385262966156006) + | > loss_gen: 2.2902724742889404 (2.2902724742889404) + | > loss_kl: 2.248882293701172 (2.248882293701172) + | > loss_feat: 7.153716564178467 (7.153716564178467) + | > loss_mel: 19.631668090820312 (19.631668090820312) + | > loss_duration: 1.6363849639892578 (1.6363849639892578) + | > loss_1: 32.96092224121094 (32.96092224121094) + + --> STEP: 1 + | > loss_disc: 2.5913681983947754 (2.5913681983947754) + | > loss_disc_real_0: 0.15710881352424622 (0.15710881352424622) + | > loss_disc_real_1: 0.20102082192897797 (0.20102082192897797) + | > loss_disc_real_2: 0.2351740300655365 (0.2351740300655365) + | > loss_disc_real_3: 0.25239434838294983 (0.25239434838294983) + | > loss_disc_real_4: 0.22484003007411957 (0.22484003007411957) + | > loss_disc_real_5: 0.22986093163490295 (0.22986093163490295) + | > loss_0: 2.5913681983947754 (2.5913681983947754) + | > loss_gen: 2.0939548015594482 (2.0939548015594482) + | > loss_kl: 2.5988616943359375 (2.5988616943359375) + | > loss_feat: 6.088581085205078 (6.088581085205078) + | > loss_mel: 17.168893814086914 (17.168893814086914) + | > loss_duration: 1.6655473709106445 (1.6655473709106445) + | > loss_1: 29.61583709716797 (29.61583709716797) + + --> STEP: 2 + | > loss_disc: 2.516911029815674 (2.5541396141052246) + | > loss_disc_real_0: 0.0949079766869545 (0.12600839510560036) + | > loss_disc_real_1: 0.20806311070919037 (0.20454196631908417) + | > loss_disc_real_2: 0.21089687943458557 (0.22303545475006104) + | > loss_disc_real_3: 0.2377549558877945 (0.24507465213537216) + | > loss_disc_real_4: 0.23671658337116241 (0.230778306722641) + | > loss_disc_real_5: 0.19985662400722504 (0.214858777821064) + | > loss_0: 2.516911029815674 (2.5541396141052246) + | > loss_gen: 2.014897346496582 (2.054426074028015) + | > loss_kl: 2.2049708366394043 (2.401916265487671) + | > loss_feat: 7.83685302734375 (6.962717056274414) + | > loss_mel: 18.16583824157715 (17.66736602783203) + | > loss_duration: 1.6222647428512573 (1.643906056880951) + | > loss_1: 31.844825744628906 (30.730331420898438) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004570364952087402 (+0.0014090538024902344) + | > avg_loss_disc: 2.5541396141052246 (-0.003292560577392578) + | > avg_loss_disc_real_0: 0.12600839510560036 (-0.016275707632303238) + | > avg_loss_disc_real_1: 0.20454196631908417 (-0.03670407086610794) + | > avg_loss_disc_real_2: 0.22303545475006104 (-0.025268152356147766) + | > avg_loss_disc_real_3: 0.24507465213537216 (+0.012214608490467072) + | > avg_loss_disc_real_4: 0.230778306722641 (-0.00022843480110168457) + | > avg_loss_disc_real_5: 0.214858777821064 (-0.02826179563999176) + | > avg_loss_0: 2.5541396141052246 (-0.003292560577392578) + | > avg_loss_gen: 2.054426074028015 (-0.09754925966262817) + | > avg_loss_kl: 2.401916265487671 (+0.15741777420043945) + | > avg_loss_feat: 6.962717056274414 (+0.8267021179199219) + | > avg_loss_mel: 17.66736602783203 (+0.34804248809814453) + | > avg_loss_duration: 1.643906056880951 (+0.0054337382316589355) + | > avg_loss_1: 30.730331420898438 (+1.240046501159668) + + + > EPOCH: 122/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:00:37)  + + --> TIME: 2025-04-19 01:00:45 -- STEP: 19/142 -- GLOBAL_STEP: 396625 + | > loss_disc: 2.4684927463531494 (2.4861153803373637) + | > loss_disc_real_0: 0.14838793873786926 (0.12890231452490153) + | > loss_disc_real_1: 0.23044298589229584 (0.21472612258635068) + | > loss_disc_real_2: 0.22269339859485626 (0.2269906001655679) + | > loss_disc_real_3: 0.2313249558210373 (0.23016946331450813) + | > loss_disc_real_4: 0.2362515777349472 (0.2291743504373651) + | > loss_disc_real_5: 0.2320643663406372 (0.22930532929144407) + | > loss_0: 2.4684927463531494 (2.4861153803373637) + | > grad_norm_0: tensor(19.2687, device='cuda:0') (tensor(14.0020, device='cuda:0')) + | > loss_gen: 2.250403642654419 (2.357679580387316) + | > loss_kl: 1.8542195558547974 (1.8468204924934788) + | > loss_feat: 7.26106071472168 (7.149031513615658) + | > loss_mel: 17.789098739624023 (18.115801861411647) + | > loss_duration: 1.4660357236862183 (1.4442419001930638) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.620819091796875 (30.913575323004473) + | > grad_norm_1: tensor(489.1462, device='cuda:0') (tensor(273.3903, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.339 (0.3363221193614759) + | > loader_time: 0.0053 (0.004698088294581363) + + + --> TIME: 2025-04-19 01:00:54 -- STEP: 44/142 -- GLOBAL_STEP: 396650 + | > loss_disc: 2.4261553287506104 (2.4787413803013894) + | > loss_disc_real_0: 0.10367736965417862 (0.124744000421329) + | > loss_disc_real_1: 0.20574301481246948 (0.21155275438319554) + | > loss_disc_real_2: 0.20910729467868805 (0.22581946104764938) + | > loss_disc_real_3: 0.21951258182525635 (0.23059664700518956) + | > loss_disc_real_4: 0.19619861245155334 (0.2312763279134577) + | > loss_disc_real_5: 0.20382748544216156 (0.23265393552455035) + | > loss_0: 2.4261553287506104 (2.4787413803013894) + | > grad_norm_0: tensor(44.8697, device='cuda:0') (tensor(15.2101, device='cuda:0')) + | > loss_gen: 2.2759017944335938 (2.3465642007914456) + | > loss_kl: 1.7409589290618896 (1.8583357252857902) + | > loss_feat: 6.845258712768555 (6.923754399473017) + | > loss_mel: 16.8997802734375 (17.692422888495713) + | > loss_duration: 1.5128536224365234 (1.4594787142493508) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.27475357055664 (30.280555855144154) + | > grad_norm_1: tensor(433.9826, device='cuda:0') (tensor(305.0161, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.3584 (0.3466584953394803) + | > loader_time: 0.0058 (0.005134105682373046) + + + --> TIME: 2025-04-19 01:01:03 -- STEP: 69/142 -- GLOBAL_STEP: 396675 + | > loss_disc: 2.4540905952453613 (2.4789042334625693) + | > loss_disc_real_0: 0.11250489205121994 (0.12672145373147467) + | > loss_disc_real_1: 0.20589205622673035 (0.21089155907216278) + | > loss_disc_real_2: 0.2538411319255829 (0.22551703517851623) + | > loss_disc_real_3: 0.20448344945907593 (0.23101538462915283) + | > loss_disc_real_4: 0.28089475631713867 (0.23253676403259885) + | > loss_disc_real_5: 0.23113584518432617 (0.23127710581689642) + | > loss_0: 2.4540905952453613 (2.4789042334625693) + | > grad_norm_0: tensor(19.4024, device='cuda:0') (tensor(14.2157, device='cuda:0')) + | > loss_gen: 2.4325907230377197 (2.3449137279952774) + | > loss_kl: 1.9453303813934326 (1.8751744066459546) + | > loss_feat: 7.206258773803711 (6.962181975876075) + | > loss_mel: 18.9617862701416 (17.823150565658793) + | > loss_duration: 1.563453197479248 (1.4711984413257544) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.10942077636719 (30.47661911231884) + | > grad_norm_1: tensor(422.4275, device='cuda:0') (tensor(295.9849, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.3932 (0.3571982383728027) + | > loader_time: 0.0062 (0.005465224169302676) + + + --> TIME: 2025-04-19 01:01:13 -- STEP: 94/142 -- GLOBAL_STEP: 396700 + | > loss_disc: 2.434508800506592 (2.4764115911849003) + | > loss_disc_real_0: 0.10840979963541031 (0.1264308393001556) + | > loss_disc_real_1: 0.22844773530960083 (0.20944921244332132) + | > loss_disc_real_2: 0.21286197006702423 (0.22601185295176) + | > loss_disc_real_3: 0.2232726663351059 (0.23089777028307001) + | > loss_disc_real_4: 0.2750878632068634 (0.2311405767785742) + | > loss_disc_real_5: 0.23833762109279633 (0.23103748753349831) + | > loss_0: 2.434508800506592 (2.4764115911849003) + | > grad_norm_0: tensor(11.6736, device='cuda:0') (tensor(13.6445, device='cuda:0')) + | > loss_gen: 2.5475692749023438 (2.338936174169498) + | > loss_kl: 1.887794017791748 (1.8968779292512448) + | > loss_feat: 7.507134437561035 (6.949967105337914) + | > loss_mel: 19.57593536376953 (17.83085986401173) + | > loss_duration: 1.537022352218628 (1.4837818095024595) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.05545425415039 (30.500422883541027) + | > grad_norm_1: tensor(320.8458, device='cuda:0') (tensor(299.7484, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.3972 (0.36731970563847965) + | > loader_time: 0.0065 (0.005715309305393948) + + + --> TIME: 2025-04-19 01:01:24 -- STEP: 119/142 -- GLOBAL_STEP: 396725 + | > loss_disc: 2.5519254207611084 (2.4859814062839796) + | > loss_disc_real_0: 0.1871495246887207 (0.1285296407129084) + | > loss_disc_real_1: 0.2353140115737915 (0.21033712019439504) + | > loss_disc_real_2: 0.22280263900756836 (0.2265204550588832) + | > loss_disc_real_3: 0.22442373633384705 (0.2311224982518108) + | > loss_disc_real_4: 0.22751231491565704 (0.2310552509392009) + | > loss_disc_real_5: 0.24436801671981812 (0.23331058050404077) + | > loss_0: 2.5519254207611084 (2.4859814062839796) + | > grad_norm_0: tensor(31.3521, device='cuda:0') (tensor(14.2203, device='cuda:0')) + | > loss_gen: 2.238804817199707 (2.3380457693789167) + | > loss_kl: 1.9813504219055176 (1.9098849126270838) + | > loss_feat: 6.504971027374268 (6.910818352418787) + | > loss_mel: 16.868465423583984 (17.759816730723664) + | > loss_duration: 1.567753791809082 (1.490459013385933) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.16134262084961 (30.409024839641667) + | > grad_norm_1: tensor(214.5069, device='cuda:0') (tensor(307.2076, device='cuda:0')) + | > current_lr_0: 0.00019697295072448832 + | > current_lr_1: 0.00019697295072448832 + | > step_time: 0.4246 (0.378152222192588) + | > loader_time: 0.0067 (0.005963427679879324) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.558415412902832 (2.558415412902832) + | > loss_disc_real_0: 0.20091314613819122 (0.20091314613819122) + | > loss_disc_real_1: 0.26792991161346436 (0.26792991161346436) + | > loss_disc_real_2: 0.2770216464996338 (0.2770216464996338) + | > loss_disc_real_3: 0.27666911482810974 (0.27666911482810974) + | > loss_disc_real_4: 0.21580420434474945 (0.21580420434474945) + | > loss_disc_real_5: 0.2842278480529785 (0.2842278480529785) + | > loss_0: 2.558415412902832 (2.558415412902832) + | > loss_gen: 2.4982070922851562 (2.4982070922851562) + | > loss_kl: 2.2512669563293457 (2.2512669563293457) + | > loss_feat: 6.1968488693237305 (6.1968488693237305) + | > loss_mel: 16.544424057006836 (16.544424057006836) + | > loss_duration: 1.6531250476837158 (1.6531250476837158) + | > loss_1: 29.14387321472168 (29.14387321472168) + + --> STEP: 1 + | > loss_disc: 2.526553153991699 (2.526553153991699) + | > loss_disc_real_0: 0.11632740497589111 (0.11632740497589111) + | > loss_disc_real_1: 0.27459707856178284 (0.27459707856178284) + | > loss_disc_real_2: 0.23205135762691498 (0.23205135762691498) + | > loss_disc_real_3: 0.2620756924152374 (0.2620756924152374) + | > loss_disc_real_4: 0.2295399010181427 (0.2295399010181427) + | > loss_disc_real_5: 0.25949397683143616 (0.25949397683143616) + | > loss_0: 2.526553153991699 (2.526553153991699) + | > loss_gen: 2.3105220794677734 (2.3105220794677734) + | > loss_kl: 2.479639768600464 (2.479639768600464) + | > loss_feat: 6.823824405670166 (6.823824405670166) + | > loss_mel: 19.065189361572266 (19.065189361572266) + | > loss_duration: 1.656583309173584 (1.656583309173584) + | > loss_1: 32.335758209228516 (32.335758209228516) + + --> STEP: 2 + | > loss_disc: 2.3630294799804688 (2.444791316986084) + | > loss_disc_real_0: 0.151179239153862 (0.13375332206487656) + | > loss_disc_real_1: 0.22755979001522064 (0.25107843428850174) + | > loss_disc_real_2: 0.20538878440856934 (0.21872007101774216) + | > loss_disc_real_3: 0.2134818434715271 (0.23777876794338226) + | > loss_disc_real_4: 0.18529003858566284 (0.20741496980190277) + | > loss_disc_real_5: 0.22279807925224304 (0.2411460280418396) + | > loss_0: 2.3630294799804688 (2.444791316986084) + | > loss_gen: 2.3673529624938965 (2.338937520980835) + | > loss_kl: 2.0326504707336426 (2.2561451196670532) + | > loss_feat: 7.158977508544922 (6.991400957107544) + | > loss_mel: 15.945771217346191 (17.50548028945923) + | > loss_duration: 1.612208366394043 (1.6343958377838135) + | > loss_1: 29.116958618164062 (30.72635841369629) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0030460357666015625 (-0.0015243291854858398) + | > avg_loss_disc: 2.444791316986084 (-0.10934829711914062) + | > avg_loss_disc_real_0: 0.13375332206487656 (+0.007744926959276199) + | > avg_loss_disc_real_1: 0.25107843428850174 (+0.04653646796941757) + | > avg_loss_disc_real_2: 0.21872007101774216 (-0.004315383732318878) + | > avg_loss_disc_real_3: 0.23777876794338226 (-0.007295884191989899) + | > avg_loss_disc_real_4: 0.20741496980190277 (-0.02336333692073822) + | > avg_loss_disc_real_5: 0.2411460280418396 (+0.026287250220775604) + | > avg_loss_0: 2.444791316986084 (-0.10934829711914062) + | > avg_loss_gen: 2.338937520980835 (+0.2845114469528198) + | > avg_loss_kl: 2.2561451196670532 (-0.14577114582061768) + | > avg_loss_feat: 6.991400957107544 (+0.028683900833129883) + | > avg_loss_mel: 17.50548028945923 (-0.16188573837280273) + | > avg_loss_duration: 1.6343958377838135 (-0.009510219097137451) + | > avg_loss_1: 30.72635841369629 (-0.0039730072021484375) + + + > EPOCH: 123/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:01:36)  + + --> TIME: 2025-04-19 01:01:38 -- STEP: 2/142 -- GLOBAL_STEP: 396750 + | > loss_disc: 2.29905366897583 (2.4165107011795044) + | > loss_disc_real_0: 0.1401626616716385 (0.1883140578866005) + | > loss_disc_real_1: 0.1977173238992691 (0.2163783684372902) + | > loss_disc_real_2: 0.22401833534240723 (0.2518220841884613) + | > loss_disc_real_3: 0.2437831312417984 (0.24587522447109222) + | > loss_disc_real_4: 0.24554400146007538 (0.26877550035715103) + | > loss_disc_real_5: 0.21523135900497437 (0.2325054258108139) + | > loss_0: 2.29905366897583 (2.4165107011795044) + | > grad_norm_0: tensor(21.9974, device='cuda:0') (tensor(27.1962, device='cuda:0')) + | > loss_gen: 2.4550774097442627 (2.460225462913513) + | > loss_kl: 1.942664384841919 (1.8589768409729004) + | > loss_feat: 8.378823280334473 (7.344048500061035) + | > loss_mel: 19.047687530517578 (17.400014400482178) + | > loss_duration: 1.4729852676391602 (1.440895140171051) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.297237396240234 (30.50416088104248) + | > grad_norm_1: tensor(516.7238, device='cuda:0') (tensor(337.2619, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.3296 (0.32986605167388916) + | > loader_time: 0.0037 (0.0037147998809814453) + + + --> TIME: 2025-04-19 01:01:46 -- STEP: 27/142 -- GLOBAL_STEP: 396775 + | > loss_disc: 2.531391143798828 (2.4469273620181613) + | > loss_disc_real_0: 0.07495596259832382 (0.1302489747327787) + | > loss_disc_real_1: 0.2477659285068512 (0.20285695018591704) + | > loss_disc_real_2: 0.2025175392627716 (0.22591049362111976) + | > loss_disc_real_3: 0.19873377680778503 (0.22922136220667097) + | > loss_disc_real_4: 0.20910830795764923 (0.22661931757573728) + | > loss_disc_real_5: 0.20654834806919098 (0.2288875033458074) + | > loss_0: 2.531391143798828 (2.4469273620181613) + | > grad_norm_0: tensor(19.0212, device='cuda:0') (tensor(22.8136, device='cuda:0')) + | > loss_gen: 2.199396848678589 (2.4131487210591636) + | > loss_kl: 1.9308388233184814 (1.8913603756162856) + | > loss_feat: 7.266166687011719 (7.246145054146096) + | > loss_mel: 16.805221557617188 (17.81729687584771) + | > loss_duration: 1.4722480773925781 (1.454369999744274) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.673871994018555 (30.82232115003798) + | > grad_norm_1: tensor(455.6033, device='cuda:0') (tensor(399.9804, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.353 (0.3395330817611129) + | > loader_time: 0.0053 (0.004700854972556784) + + + --> TIME: 2025-04-19 01:01:55 -- STEP: 52/142 -- GLOBAL_STEP: 396800 + | > loss_disc: 2.3431880474090576 (2.4519193768501286) + | > loss_disc_real_0: 0.09573356062173843 (0.1283222746390562) + | > loss_disc_real_1: 0.24108673632144928 (0.20372093153687623) + | > loss_disc_real_2: 0.22062785923480988 (0.2243575195853527) + | > loss_disc_real_3: 0.19838544726371765 (0.22847049511395967) + | > loss_disc_real_4: 0.21693675220012665 (0.22702403567158258) + | > loss_disc_real_5: 0.24494364857673645 (0.23228880017995834) + | > loss_0: 2.3431880474090576 (2.4519193768501286) + | > grad_norm_0: tensor(10.0286, device='cuda:0') (tensor(17.8525, device='cuda:0')) + | > loss_gen: 2.4435839653015137 (2.372170329093933) + | > loss_kl: 1.85307776927948 (1.898080289363861) + | > loss_feat: 7.020925521850586 (7.168771266937256) + | > loss_mel: 18.758516311645508 (17.809909838896527) + | > loss_duration: 1.5349363088607788 (1.4626041971720183) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.611040115356445 (30.711536150712234) + | > grad_norm_1: tensor(212.7982, device='cuda:0') (tensor(325.3587, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.3645 (0.349096967623784) + | > loader_time: 0.0056 (0.005041374610020563) + + + --> TIME: 2025-04-19 01:02:05 -- STEP: 77/142 -- GLOBAL_STEP: 396825 + | > loss_disc: 2.425139904022217 (2.4473155969149114) + | > loss_disc_real_0: 0.13877597451210022 (0.12575747798402584) + | > loss_disc_real_1: 0.1904488354921341 (0.2032163988847237) + | > loss_disc_real_2: 0.22943642735481262 (0.22468111306041866) + | > loss_disc_real_3: 0.22267739474773407 (0.22899938519898946) + | > loss_disc_real_4: 0.203838512301445 (0.22686406331402914) + | > loss_disc_real_5: 0.21490445733070374 (0.2322427634294931) + | > loss_0: 2.425139904022217 (2.4473155969149114) + | > grad_norm_0: tensor(7.8554, device='cuda:0') (tensor(14.3396, device='cuda:0')) + | > loss_gen: 2.275451898574829 (2.363784511367996) + | > loss_kl: 1.9510492086410522 (1.9035996988222197) + | > loss_feat: 6.696927547454834 (7.137068624620314) + | > loss_mel: 15.504169464111328 (17.73189000959519) + | > loss_duration: 1.4836969375610352 (1.474028870656893) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.911293029785156 (30.610371800212118) + | > grad_norm_1: tensor(74.2426, device='cuda:0') (tensor(268.7333, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.3902 (0.3603496675367479) + | > loader_time: 0.0063 (0.005407822596562372) + + + --> TIME: 2025-04-19 01:02:16 -- STEP: 102/142 -- GLOBAL_STEP: 396850 + | > loss_disc: 2.6003761291503906 (2.4532780530406) + | > loss_disc_real_0: 0.12399017810821533 (0.12655922828936098) + | > loss_disc_real_1: 0.2595171332359314 (0.20483149164447598) + | > loss_disc_real_2: 0.2647884488105774 (0.22464225748006036) + | > loss_disc_real_3: 0.2506689429283142 (0.22847741199474708) + | > loss_disc_real_4: 0.290426641702652 (0.22799058404623293) + | > loss_disc_real_5: 0.2421201467514038 (0.23320583005746207) + | > loss_0: 2.6003761291503906 (2.4532780530406) + | > grad_norm_0: tensor(4.2756, device='cuda:0') (tensor(12.4144, device='cuda:0')) + | > loss_gen: 2.2409348487854004 (2.348587788787542) + | > loss_kl: 1.9668787717819214 (1.9148170480541153) + | > loss_feat: 6.69423246383667 (7.055133557787128) + | > loss_mel: 18.42811393737793 (17.736293558980897) + | > loss_duration: 1.5034537315368652 (1.4856957536117703) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.833614349365234 (30.54052781123741) + | > grad_norm_1: tensor(45.2735, device='cuda:0') (tensor(227.4421, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.4188 (0.37063951118319644) + | > loader_time: 0.0069 (0.005692269287857355) + + + --> TIME: 2025-04-19 01:02:26 -- STEP: 127/142 -- GLOBAL_STEP: 396875 + | > loss_disc: 2.499816417694092 (2.4610057440329745) + | > loss_disc_real_0: 0.1394059956073761 (0.1284579105499222) + | > loss_disc_real_1: 0.22085027396678925 (0.20513198237250171) + | > loss_disc_real_2: 0.21601375937461853 (0.22518072236241318) + | > loss_disc_real_3: 0.19382329285144806 (0.22865553364509672) + | > loss_disc_real_4: 0.22083896398544312 (0.2284312542733245) + | > loss_disc_real_5: 0.22771815955638885 (0.23379831720055558) + | > loss_0: 2.499816417694092 (2.4610057440329745) + | > grad_norm_0: tensor(11.8065, device='cuda:0') (tensor(11.8403, device='cuda:0')) + | > loss_gen: 2.3717377185821533 (2.3391286575888075) + | > loss_kl: 1.823628544807434 (1.9153381867671575) + | > loss_feat: 7.114171981811523 (6.99205023097241) + | > loss_mel: 17.33217430114746 (17.7309008395578) + | > loss_duration: 1.521362066268921 (1.4929728320264442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.163074493408203 (30.47039084547148) + | > grad_norm_1: tensor(391.4021, device='cuda:0') (tensor(218.2544, device='cuda:0')) + | > current_lr_0: 0.00019694832910564775 + | > current_lr_1: 0.00019694832910564775 + | > step_time: 0.4414 (0.38198847845783385) + | > loader_time: 0.0081 (0.005946726310910202) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.465952157974243 (2.465952157974243) + | > loss_disc_real_0: 0.14307096600532532 (0.14307096600532532) + | > loss_disc_real_1: 0.17679336667060852 (0.17679336667060852) + | > loss_disc_real_2: 0.26063257455825806 (0.26063257455825806) + | > loss_disc_real_3: 0.2653732895851135 (0.2653732895851135) + | > loss_disc_real_4: 0.27364686131477356 (0.27364686131477356) + | > loss_disc_real_5: 0.2442006915807724 (0.2442006915807724) + | > loss_0: 2.465952157974243 (2.465952157974243) + | > loss_gen: 2.428650379180908 (2.428650379180908) + | > loss_kl: 2.3661561012268066 (2.3661561012268066) + | > loss_feat: 7.125604629516602 (7.125604629516602) + | > loss_mel: 18.882160186767578 (18.882160186767578) + | > loss_duration: 1.6368426084518433 (1.6368426084518433) + | > loss_1: 32.43941116333008 (32.43941116333008) + + --> STEP: 1 + | > loss_disc: 2.5311903953552246 (2.5311903953552246) + | > loss_disc_real_0: 0.18769529461860657 (0.18769529461860657) + | > loss_disc_real_1: 0.23859845101833344 (0.23859845101833344) + | > loss_disc_real_2: 0.26198992133140564 (0.26198992133140564) + | > loss_disc_real_3: 0.21559573709964752 (0.21559573709964752) + | > loss_disc_real_4: 0.2326231151819229 (0.2326231151819229) + | > loss_disc_real_5: 0.20989282429218292 (0.20989282429218292) + | > loss_0: 2.5311903953552246 (2.5311903953552246) + | > loss_gen: 2.2648539543151855 (2.2648539543151855) + | > loss_kl: 2.558936595916748 (2.558936595916748) + | > loss_feat: 6.6695451736450195 (6.6695451736450195) + | > loss_mel: 16.958755493164062 (16.958755493164062) + | > loss_duration: 1.646815538406372 (1.646815538406372) + | > loss_1: 30.098907470703125 (30.098907470703125) + + --> STEP: 2 + | > loss_disc: 2.466182231903076 (2.4986863136291504) + | > loss_disc_real_0: 0.14394772052764893 (0.16582150757312775) + | > loss_disc_real_1: 0.23546795547008514 (0.2370332032442093) + | > loss_disc_real_2: 0.24143283069133759 (0.2517113760113716) + | > loss_disc_real_3: 0.22514976561069489 (0.2203727513551712) + | > loss_disc_real_4: 0.25585073232650757 (0.24423692375421524) + | > loss_disc_real_5: 0.22803416848182678 (0.21896349638700485) + | > loss_0: 2.466182231903076 (2.4986863136291504) + | > loss_gen: 2.2529406547546387 (2.258897304534912) + | > loss_kl: 2.181199550628662 (2.370068073272705) + | > loss_feat: 6.3090362548828125 (6.489290714263916) + | > loss_mel: 18.100425720214844 (17.529590606689453) + | > loss_duration: 1.6220654249191284 (1.6344404816627502) + | > loss_1: 30.465667724609375 (30.28228759765625) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003425121307373047 (+0.0003790855407714844) + | > avg_loss_disc: 2.4986863136291504 (+0.053894996643066406) + | > avg_loss_disc_real_0: 0.16582150757312775 (+0.03206818550825119) + | > avg_loss_disc_real_1: 0.2370332032442093 (-0.01404523104429245) + | > avg_loss_disc_real_2: 0.2517113760113716 (+0.032991304993629456) + | > avg_loss_disc_real_3: 0.2203727513551712 (-0.01740601658821106) + | > avg_loss_disc_real_4: 0.24423692375421524 (+0.03682195395231247) + | > avg_loss_disc_real_5: 0.21896349638700485 (-0.022182531654834747) + | > avg_loss_0: 2.4986863136291504 (+0.053894996643066406) + | > avg_loss_gen: 2.258897304534912 (-0.08004021644592285) + | > avg_loss_kl: 2.370068073272705 (+0.11392295360565186) + | > avg_loss_feat: 6.489290714263916 (-0.5021102428436279) + | > avg_loss_mel: 17.529590606689453 (+0.02411031723022461) + | > avg_loss_duration: 1.6344404816627502 (+4.464387893676758e-05) + | > avg_loss_1: 30.28228759765625 (-0.44407081604003906) + + + > EPOCH: 124/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:02:35)  + + --> TIME: 2025-04-19 01:02:39 -- STEP: 10/142 -- GLOBAL_STEP: 396900 + | > loss_disc: 2.457218647003174 (2.432177448272705) + | > loss_disc_real_0: 0.12869910895824432 (0.12443414479494094) + | > loss_disc_real_1: 0.18226687610149384 (0.19469202160835267) + | > loss_disc_real_2: 0.261741578578949 (0.23347289711236954) + | > loss_disc_real_3: 0.22027404606342316 (0.22602086663246154) + | > loss_disc_real_4: 0.22787857055664062 (0.22579682171344756) + | > loss_disc_real_5: 0.2203758955001831 (0.2207702562212944) + | > loss_0: 2.457218647003174 (2.432177448272705) + | > grad_norm_0: tensor(6.8112, device='cuda:0') (tensor(8.1154, device='cuda:0')) + | > loss_gen: 2.4192798137664795 (2.3657154560089113) + | > loss_kl: 1.9916062355041504 (1.8752689242362977) + | > loss_feat: 7.380354404449463 (7.345047807693481) + | > loss_mel: 17.873159408569336 (17.845063400268554) + | > loss_duration: 1.4109563827514648 (1.4434639930725097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.075355529785156 (30.874559211730958) + | > grad_norm_1: tensor(268.1704, device='cuda:0') (tensor(167.5310, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.3349 (0.32885518074035647) + | > loader_time: 0.0052 (0.00440986156463623) + + + --> TIME: 2025-04-19 01:02:48 -- STEP: 35/142 -- GLOBAL_STEP: 396925 + | > loss_disc: 2.396730661392212 (2.4622556754520963) + | > loss_disc_real_0: 0.1255342811346054 (0.12608096301555632) + | > loss_disc_real_1: 0.19918511807918549 (0.2073664039373398) + | > loss_disc_real_2: 0.21345096826553345 (0.2281579694577626) + | > loss_disc_real_3: 0.23147258162498474 (0.23108931141240255) + | > loss_disc_real_4: 0.19607092440128326 (0.2271316145147596) + | > loss_disc_real_5: 0.220895916223526 (0.23144713725362506) + | > loss_0: 2.396730661392212 (2.4622556754520963) + | > grad_norm_0: tensor(6.7331, device='cuda:0') (tensor(8.9081, device='cuda:0')) + | > loss_gen: 2.407745599746704 (2.3391218866620744) + | > loss_kl: 1.9836006164550781 (1.8768756832395281) + | > loss_feat: 7.39651346206665 (7.119394520350865) + | > loss_mel: 18.485746383666992 (17.708400290352955) + | > loss_duration: 1.483100175857544 (1.457520682471139) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.75670623779297 (30.501313182285855) + | > grad_norm_1: tensor(278.5659, device='cuda:0') (tensor(208.7107, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.3628 (0.3472116879054478) + | > loader_time: 0.0058 (0.005125774656023298) + + + --> TIME: 2025-04-19 01:02:57 -- STEP: 60/142 -- GLOBAL_STEP: 396950 + | > loss_disc: 2.524901866912842 (2.467978938420614) + | > loss_disc_real_0: 0.13193292915821075 (0.1280201933036248) + | > loss_disc_real_1: 0.19035175442695618 (0.2078412801027298) + | > loss_disc_real_2: 0.19469182193279266 (0.22625110720594724) + | > loss_disc_real_3: 0.2464146763086319 (0.23099438920617105) + | > loss_disc_real_4: 0.20789122581481934 (0.22817285160223644) + | > loss_disc_real_5: 0.21056310832500458 (0.23275923232237497) + | > loss_0: 2.524901866912842 (2.467978938420614) + | > grad_norm_0: tensor(7.4902, device='cuda:0') (tensor(9.4441, device='cuda:0')) + | > loss_gen: 2.1592211723327637 (2.3330973664919537) + | > loss_kl: 2.0002787113189697 (1.8999506950378418) + | > loss_feat: 6.994388580322266 (7.065989017486572) + | > loss_mel: 17.06269645690918 (17.798001782099405) + | > loss_duration: 1.5183088779449463 (1.469729737440745) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.734893798828125 (30.56676867802938) + | > grad_norm_1: tensor(140.4926, device='cuda:0') (tensor(220.5867, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.3892 (0.36005751291910804) + | > loader_time: 0.0064 (0.005470379193623861) + + + --> TIME: 2025-04-19 01:03:08 -- STEP: 85/142 -- GLOBAL_STEP: 396975 + | > loss_disc: 2.507155656814575 (2.4778918041902434) + | > loss_disc_real_0: 0.12970465421676636 (0.12848707121961264) + | > loss_disc_real_1: 0.2542974352836609 (0.2090949900010053) + | > loss_disc_real_2: 0.24465735256671906 (0.22740749260958504) + | > loss_disc_real_3: 0.25742051005363464 (0.229821739301962) + | > loss_disc_real_4: 0.24297936260700226 (0.22915258144631107) + | > loss_disc_real_5: 0.23461900651454926 (0.236610909595209) + | > loss_0: 2.507155656814575 (2.4778918041902434) + | > grad_norm_0: tensor(22.2315, device='cuda:0') (tensor(10.3291, device='cuda:0')) + | > loss_gen: 2.298931360244751 (2.3400667723487407) + | > loss_kl: 2.0135135650634766 (1.921971120553858) + | > loss_feat: 7.339379787445068 (7.046710878259995) + | > loss_mel: 19.946378707885742 (17.853003423354203) + | > loss_duration: 1.5179860591888428 (1.4826101050657383) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.116188049316406 (30.644362348668714) + | > grad_norm_1: tensor(411.9880, device='cuda:0') (tensor(243.8608, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.3961 (0.3733104789958282) + | > loader_time: 0.0065 (0.005780881993910847) + + + --> TIME: 2025-04-19 01:03:18 -- STEP: 110/142 -- GLOBAL_STEP: 397000 + | > loss_disc: 2.412426233291626 (2.4793199062347413) + | > loss_disc_real_0: 0.06871123611927032 (0.12879842567172922) + | > loss_disc_real_1: 0.19446192681789398 (0.20859371762384069) + | > loss_disc_real_2: 0.24040643870830536 (0.2278680530461398) + | > loss_disc_real_3: 0.220642551779747 (0.2286476365544579) + | > loss_disc_real_4: 0.21994391083717346 (0.22925500625913794) + | > loss_disc_real_5: 0.22983334958553314 (0.2344037355347113) + | > loss_0: 2.412426233291626 (2.4793199062347413) + | > grad_norm_0: tensor(19.3728, device='cuda:0') (tensor(12.2091, device='cuda:0')) + | > loss_gen: 2.6793177127838135 (2.3451867515390576) + | > loss_kl: 2.052324056625366 (1.9294750495390458) + | > loss_feat: 7.341613292694092 (7.010105822303078) + | > loss_mel: 18.886137008666992 (17.823675251007074) + | > loss_duration: 1.5084428787231445 (1.489743283661929) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.46783447265625 (30.59818614612926) + | > grad_norm_1: tensor(469.8388, device='cuda:0') (tensor(267.3490, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.4275 (0.38141522841020076) + | > loader_time: 0.0072 (0.006022143363952638) + + + --> TIME: 2025-04-19 01:03:29 -- STEP: 135/142 -- GLOBAL_STEP: 397025 + | > loss_disc: 2.4707224369049072 (2.4865482383304167) + | > loss_disc_real_0: 0.13823659718036652 (0.13181602938307657) + | > loss_disc_real_1: 0.2502438724040985 (0.2090043680535423) + | > loss_disc_real_2: 0.22958749532699585 (0.22712565042354443) + | > loss_disc_real_3: 0.1930019110441208 (0.22848582322950717) + | > loss_disc_real_4: 0.21989010274410248 (0.229306075308058) + | > loss_disc_real_5: 0.23710165917873383 (0.23483762917695222) + | > loss_0: 2.4707224369049072 (2.4865482383304167) + | > grad_norm_0: tensor(15.9070, device='cuda:0') (tensor(14.3831, device='cuda:0')) + | > loss_gen: 2.480924367904663 (2.3416239526536735) + | > loss_kl: 1.8447598218917847 (1.9329540773674294) + | > loss_feat: 6.823397159576416 (6.960858443931297) + | > loss_mel: 18.096101760864258 (17.775853171171967) + | > loss_duration: 1.4936232566833496 (1.4949081314934625) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.738807678222656 (30.506197795161494) + | > grad_norm_1: tensor(77.9087, device='cuda:0') (tensor(284.2453, device='cuda:0')) + | > current_lr_0: 0.00019692371056450955 + | > current_lr_1: 0.00019692371056450955 + | > step_time: 0.4586 (0.39196539454989976) + | > loader_time: 0.0079 (0.006283661171242043) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.539041757583618 (2.539041757583618) + | > loss_disc_real_0: 0.1958811730146408 (0.1958811730146408) + | > loss_disc_real_1: 0.2393336445093155 (0.2393336445093155) + | > loss_disc_real_2: 0.2183651626110077 (0.2183651626110077) + | > loss_disc_real_3: 0.23120631277561188 (0.23120631277561188) + | > loss_disc_real_4: 0.21160675585269928 (0.21160675585269928) + | > loss_disc_real_5: 0.2691241204738617 (0.2691241204738617) + | > loss_0: 2.539041757583618 (2.539041757583618) + | > loss_gen: 2.415649175643921 (2.415649175643921) + | > loss_kl: 2.328456163406372 (2.328456163406372) + | > loss_feat: 7.422427654266357 (7.422427654266357) + | > loss_mel: 19.2901611328125 (19.2901611328125) + | > loss_duration: 1.6291511058807373 (1.6291511058807373) + | > loss_1: 33.085845947265625 (33.085845947265625) + + --> STEP: 1 + | > loss_disc: 2.546511173248291 (2.546511173248291) + | > loss_disc_real_0: 0.22985291481018066 (0.22985291481018066) + | > loss_disc_real_1: 0.2236899733543396 (0.2236899733543396) + | > loss_disc_real_2: 0.18537934124469757 (0.18537934124469757) + | > loss_disc_real_3: 0.1886122077703476 (0.1886122077703476) + | > loss_disc_real_4: 0.19522757828235626 (0.19522757828235626) + | > loss_disc_real_5: 0.1689528524875641 (0.1689528524875641) + | > loss_0: 2.546511173248291 (2.546511173248291) + | > loss_gen: 2.1179890632629395 (2.1179890632629395) + | > loss_kl: 2.5839645862579346 (2.5839645862579346) + | > loss_feat: 6.785099506378174 (6.785099506378174) + | > loss_mel: 17.957481384277344 (17.957481384277344) + | > loss_duration: 1.6519593000411987 (1.6519593000411987) + | > loss_1: 31.096492767333984 (31.096492767333984) + + --> STEP: 2 + | > loss_disc: 2.5058095455169678 (2.5261603593826294) + | > loss_disc_real_0: 0.19355979561805725 (0.21170635521411896) + | > loss_disc_real_1: 0.2448810636997223 (0.23428551852703094) + | > loss_disc_real_2: 0.20722819864749908 (0.19630376994609833) + | > loss_disc_real_3: 0.21226398646831512 (0.20043809711933136) + | > loss_disc_real_4: 0.23055513203144073 (0.2128913551568985) + | > loss_disc_real_5: 0.2301490306854248 (0.19955094158649445) + | > loss_0: 2.5058095455169678 (2.5261603593826294) + | > loss_gen: 2.42697811126709 (2.2724835872650146) + | > loss_kl: 2.168168306350708 (2.3760664463043213) + | > loss_feat: 7.352512359619141 (7.068805932998657) + | > loss_mel: 18.451139450073242 (18.204310417175293) + | > loss_duration: 1.626741647720337 (1.6393504738807678) + | > loss_1: 32.025543212890625 (31.561017990112305) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033913850784301758 (-3.3736228942871094e-05) + | > avg_loss_disc: 2.5261603593826294 (+0.027474045753479004) + | > avg_loss_disc_real_0: 0.21170635521411896 (+0.04588484764099121) + | > avg_loss_disc_real_1: 0.23428551852703094 (-0.0027476847171783447) + | > avg_loss_disc_real_2: 0.19630376994609833 (-0.055407606065273285) + | > avg_loss_disc_real_3: 0.20043809711933136 (-0.019934654235839844) + | > avg_loss_disc_real_4: 0.2128913551568985 (-0.03134556859731674) + | > avg_loss_disc_real_5: 0.19955094158649445 (-0.019412554800510406) + | > avg_loss_0: 2.5261603593826294 (+0.027474045753479004) + | > avg_loss_gen: 2.2724835872650146 (+0.013586282730102539) + | > avg_loss_kl: 2.3760664463043213 (+0.005998373031616211) + | > avg_loss_feat: 7.068805932998657 (+0.5795152187347412) + | > avg_loss_mel: 18.204310417175293 (+0.6747198104858398) + | > avg_loss_duration: 1.6393504738807678 (+0.004909992218017578) + | > avg_loss_1: 31.561017990112305 (+1.2787303924560547) + + + > EPOCH: 125/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:03:34)  + + --> TIME: 2025-04-19 01:03:41 -- STEP: 18/142 -- GLOBAL_STEP: 397050 + | > loss_disc: 2.389488935470581 (2.4359140263663397) + | > loss_disc_real_0: 0.12235398590564728 (0.12023305230670506) + | > loss_disc_real_1: 0.1985083371400833 (0.20730576167503992) + | > loss_disc_real_2: 0.20421475172042847 (0.22491798632674748) + | > loss_disc_real_3: 0.23534166812896729 (0.22840783993403116) + | > loss_disc_real_4: 0.2046446055173874 (0.22343692266278797) + | > loss_disc_real_5: 0.2286442369222641 (0.2360970386200481) + | > loss_0: 2.389488935470581 (2.4359140263663397) + | > grad_norm_0: tensor(6.6676, device='cuda:0') (tensor(6.5022, device='cuda:0')) + | > loss_gen: 2.388692617416382 (2.3498482439253063) + | > loss_kl: 1.8320910930633545 (1.9081853032112122) + | > loss_feat: 7.112673759460449 (7.230666425493029) + | > loss_mel: 18.19668960571289 (17.810440169440376) + | > loss_duration: 1.4362337589263916 (1.4486872090233698) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.966381072998047 (30.74782700008816) + | > grad_norm_1: tensor(146.9138, device='cuda:0') (tensor(147.4709, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.3394 (0.3295378022723728) + | > loader_time: 0.0047 (0.004574828677707248) + + + --> TIME: 2025-04-19 01:03:50 -- STEP: 43/142 -- GLOBAL_STEP: 397075 + | > loss_disc: 2.484257698059082 (2.4396316894265113) + | > loss_disc_real_0: 0.1507846862077713 (0.12312436467686365) + | > loss_disc_real_1: 0.21262602508068085 (0.20592513784419664) + | > loss_disc_real_2: 0.23676897585391998 (0.22675640881061554) + | > loss_disc_real_3: 0.23012124001979828 (0.22501772087673808) + | > loss_disc_real_4: 0.2621980607509613 (0.22353415918904682) + | > loss_disc_real_5: 0.23499490320682526 (0.2333411279805871) + | > loss_0: 2.484257698059082 (2.4396316894265113) + | > grad_norm_0: tensor(7.0354, device='cuda:0') (tensor(6.0069, device='cuda:0')) + | > loss_gen: 2.4143457412719727 (2.3442812187727102) + | > loss_kl: 1.8999323844909668 (1.8860065299411153) + | > loss_feat: 6.933751106262207 (7.111908458000006) + | > loss_mel: 18.801332473754883 (17.872464956239213) + | > loss_duration: 1.4655518531799316 (1.459319558254508) + | > amp_scaler: 512.0 (381.0232558139535) + | > loss_1: 31.51491355895996 (30.6739804911059) + | > grad_norm_1: tensor(121.4986, device='cuda:0') (tensor(128.1348, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.3632 (0.3423271400983943) + | > loader_time: 0.0055 (0.0050294454707655815) + + + --> TIME: 2025-04-19 01:03:59 -- STEP: 68/142 -- GLOBAL_STEP: 397100 + | > loss_disc: 2.5328617095947266 (2.454141224131864) + | > loss_disc_real_0: 0.1296219825744629 (0.12573015941854784) + | > loss_disc_real_1: 0.16921931505203247 (0.20611598693272648) + | > loss_disc_real_2: 0.23804721236228943 (0.22632064034833627) + | > loss_disc_real_3: 0.2349126636981964 (0.2262625455418054) + | > loss_disc_real_4: 0.19890834391117096 (0.22516563052640243) + | > loss_disc_real_5: 0.24348361790180206 (0.2333890061606379) + | > loss_0: 2.5328617095947266 (2.454141224131864) + | > grad_norm_0: tensor(4.0987, device='cuda:0') (tensor(6.1605, device='cuda:0')) + | > loss_gen: 2.33756947517395 (2.339794726932749) + | > loss_kl: 1.8869330883026123 (1.891475549515556) + | > loss_feat: 7.4587721824646 (7.068753123283386) + | > loss_mel: 17.95977210998535 (17.87536138646743) + | > loss_duration: 1.4960633516311646 (1.471055400722167) + | > amp_scaler: 512.0 (429.1764705882353) + | > loss_1: 31.139110565185547 (30.646440085242777) + | > grad_norm_1: tensor(67.5003, device='cuda:0') (tensor(113.4076, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.3909 (0.3544869107358596) + | > loader_time: 0.0061 (0.00540291561799891) + + + --> TIME: 2025-04-19 01:04:10 -- STEP: 93/142 -- GLOBAL_STEP: 397125 + | > loss_disc: 2.545227527618408 (2.472628526790168) + | > loss_disc_real_0: 0.14358238875865936 (0.1274070251052098) + | > loss_disc_real_1: 0.22914572060108185 (0.20746696796468508) + | > loss_disc_real_2: 0.26218825578689575 (0.22867511789644918) + | > loss_disc_real_3: 0.24845847487449646 (0.22745178399547453) + | > loss_disc_real_4: 0.24470819532871246 (0.22912609705360987) + | > loss_disc_real_5: 0.2425258308649063 (0.23399104017724273) + | > loss_0: 2.545227527618408 (2.472628526790168) + | > grad_norm_0: tensor(7.0736, device='cuda:0') (tensor(6.3919, device='cuda:0')) + | > loss_gen: 2.374438524246216 (2.3289948304494215) + | > loss_kl: 1.8288310766220093 (1.895372434328961) + | > loss_feat: 6.389721393585205 (7.001877087418751) + | > loss_mel: 17.051868438720703 (17.846128094580862) + | > loss_duration: 1.4947493076324463 (1.483907126611279) + | > amp_scaler: 512.0 (451.44086021505376) + | > loss_1: 29.13960838317871 (30.55627941828902) + | > grad_norm_1: tensor(44.3290, device='cuda:0') (tensor(117.4795, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.3971 (0.36550001431536944) + | > loader_time: 0.0068 (0.005681945431617) + + + --> TIME: 2025-04-19 01:04:20 -- STEP: 118/142 -- GLOBAL_STEP: 397150 + | > loss_disc: 2.517209768295288 (2.479293637356516) + | > loss_disc_real_0: 0.14188207685947418 (0.1277804848248676) + | > loss_disc_real_1: 0.195476233959198 (0.2087941974148912) + | > loss_disc_real_2: 0.2308758795261383 (0.22778540496098793) + | > loss_disc_real_3: 0.2256084531545639 (0.2286129341287128) + | > loss_disc_real_4: 0.23672904074192047 (0.23017086656922) + | > loss_disc_real_5: 0.2373768836259842 (0.2353836070683043) + | > loss_0: 2.517209768295288 (2.479293637356516) + | > grad_norm_0: tensor(10.0693, device='cuda:0') (tensor(6.3919, device='cuda:0')) + | > loss_gen: 2.207103967666626 (2.318932761580256) + | > loss_kl: 1.9130337238311768 (1.8988424640590862) + | > loss_feat: 6.448488235473633 (6.938736406423278) + | > loss_mel: 17.80559730529785 (17.842430211729916) + | > loss_duration: 1.4860508441925049 (1.489852775961666) + | > amp_scaler: 512.0 (464.271186440678) + | > loss_1: 29.860273361206055 (30.48879448842194) + | > grad_norm_1: tensor(162.6234, device='cuda:0') (tensor(116.9166, device='cuda:0')) + | > current_lr_0: 0.000196899095100689 + | > current_lr_1: 0.000196899095100689 + | > step_time: 0.4427 (0.37650816723451785) + | > loader_time: 0.0069 (0.00594946287446103) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.565295457839966 (2.565295457839966) + | > loss_disc_real_0: 0.27711325883865356 (0.27711325883865356) + | > loss_disc_real_1: 0.2536766827106476 (0.2536766827106476) + | > loss_disc_real_2: 0.2175307720899582 (0.2175307720899582) + | > loss_disc_real_3: 0.21501290798187256 (0.21501290798187256) + | > loss_disc_real_4: 0.1970062404870987 (0.1970062404870987) + | > loss_disc_real_5: 0.25361600518226624 (0.25361600518226624) + | > loss_0: 2.565295457839966 (2.565295457839966) + | > loss_gen: 2.4560413360595703 (2.4560413360595703) + | > loss_kl: 2.4485042095184326 (2.4485042095184326) + | > loss_feat: 6.266602993011475 (6.266602993011475) + | > loss_mel: 17.101776123046875 (17.101776123046875) + | > loss_duration: 1.6469829082489014 (1.6469829082489014) + | > loss_1: 29.919906616210938 (29.919906616210938) + + --> STEP: 1 + | > loss_disc: 2.545772075653076 (2.545772075653076) + | > loss_disc_real_0: 0.2549138069152832 (0.2549138069152832) + | > loss_disc_real_1: 0.24825741350650787 (0.24825741350650787) + | > loss_disc_real_2: 0.2316829115152359 (0.2316829115152359) + | > loss_disc_real_3: 0.23035511374473572 (0.23035511374473572) + | > loss_disc_real_4: 0.22736255824565887 (0.22736255824565887) + | > loss_disc_real_5: 0.22586727142333984 (0.22586727142333984) + | > loss_0: 2.545772075653076 (2.545772075653076) + | > loss_gen: 2.637890577316284 (2.637890577316284) + | > loss_kl: 2.58945894241333 (2.58945894241333) + | > loss_feat: 7.437911510467529 (7.437911510467529) + | > loss_mel: 17.83608055114746 (17.83608055114746) + | > loss_duration: 1.6813405752182007 (1.6813405752182007) + | > loss_1: 32.182682037353516 (32.182682037353516) + + --> STEP: 2 + | > loss_disc: 2.4287657737731934 (2.4872689247131348) + | > loss_disc_real_0: 0.2143285572528839 (0.23462118208408356) + | > loss_disc_real_1: 0.2648576498031616 (0.25655753165483475) + | > loss_disc_real_2: 0.22159259021282196 (0.22663775086402893) + | > loss_disc_real_3: 0.2164219170808792 (0.22338851541280746) + | > loss_disc_real_4: 0.24567879736423492 (0.2365206778049469) + | > loss_disc_real_5: 0.18931728601455688 (0.20759227871894836) + | > loss_0: 2.4287657737731934 (2.4872689247131348) + | > loss_gen: 2.6218652725219727 (2.6298779249191284) + | > loss_kl: 1.9935462474822998 (2.291502594947815) + | > loss_feat: 7.020604610443115 (7.229258060455322) + | > loss_mel: 17.120546340942383 (17.478313446044922) + | > loss_duration: 1.6250323057174683 (1.6531864404678345) + | > loss_1: 30.381593704223633 (31.282137870788574) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034400224685668945 (+4.863739013671875e-05) + | > avg_loss_disc: 2.4872689247131348 (-0.03889143466949463) + | > avg_loss_disc_real_0: 0.23462118208408356 (+0.0229148268699646) + | > avg_loss_disc_real_1: 0.25655753165483475 (+0.022272013127803802) + | > avg_loss_disc_real_2: 0.22663775086402893 (+0.030333980917930603) + | > avg_loss_disc_real_3: 0.22338851541280746 (+0.022950418293476105) + | > avg_loss_disc_real_4: 0.2365206778049469 (+0.0236293226480484) + | > avg_loss_disc_real_5: 0.20759227871894836 (+0.008041337132453918) + | > avg_loss_0: 2.4872689247131348 (-0.03889143466949463) + | > avg_loss_gen: 2.6298779249191284 (+0.35739433765411377) + | > avg_loss_kl: 2.291502594947815 (-0.08456385135650635) + | > avg_loss_feat: 7.229258060455322 (+0.16045212745666504) + | > avg_loss_mel: 17.478313446044922 (-0.7259969711303711) + | > avg_loss_duration: 1.6531864404678345 (+0.01383596658706665) + | > avg_loss_1: 31.282137870788574 (-0.27888011932373047) + + + > EPOCH: 126/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:04:33)  + + --> TIME: 2025-04-19 01:04:34 -- STEP: 1/142 -- GLOBAL_STEP: 397175 + | > loss_disc: 2.3471925258636475 (2.3471925258636475) + | > loss_disc_real_0: 0.15047763288021088 (0.15047763288021088) + | > loss_disc_real_1: 0.22363512217998505 (0.22363512217998505) + | > loss_disc_real_2: 0.2258036732673645 (0.2258036732673645) + | > loss_disc_real_3: 0.22198565304279327 (0.22198565304279327) + | > loss_disc_real_4: 0.20903265476226807 (0.20903265476226807) + | > loss_disc_real_5: 0.23337502777576447 (0.23337502777576447) + | > loss_0: 2.3471925258636475 (2.3471925258636475) + | > grad_norm_0: tensor(6.9782, device='cuda:0') (tensor(6.9782, device='cuda:0')) + | > loss_gen: 2.4002983570098877 (2.4002983570098877) + | > loss_kl: 1.8176201581954956 (1.8176201581954956) + | > loss_feat: 8.384329795837402 (8.384329795837402) + | > loss_mel: 18.659421920776367 (18.659421920776367) + | > loss_duration: 1.4115805625915527 (1.4115805625915527) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.67325210571289 (32.67325210571289) + | > grad_norm_1: tensor(192.2195, device='cuda:0') (tensor(192.2195, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.3242 (0.32422924041748047) + | > loader_time: 0.0039 (0.003937482833862305) + + + --> TIME: 2025-04-19 01:04:42 -- STEP: 26/142 -- GLOBAL_STEP: 397200 + | > loss_disc: 2.5098018646240234 (2.4564100137123694) + | > loss_disc_real_0: 0.10765164345502853 (0.1195689024260411) + | > loss_disc_real_1: 0.232440784573555 (0.20443127762812835) + | > loss_disc_real_2: 0.2103745937347412 (0.22763378688922295) + | > loss_disc_real_3: 0.22025975584983826 (0.23070780703654656) + | > loss_disc_real_4: 0.2129686027765274 (0.23180252084365258) + | > loss_disc_real_5: 0.24707333743572235 (0.23474407711854348) + | > loss_0: 2.5098018646240234 (2.4564100137123694) + | > grad_norm_0: tensor(6.2954, device='cuda:0') (tensor(7.7307, device='cuda:0')) + | > loss_gen: 2.268435478210449 (2.3791342882009645) + | > loss_kl: 1.7369142770767212 (1.8544180301519542) + | > loss_feat: 6.714463710784912 (7.3475216168623705) + | > loss_mel: 16.490436553955078 (18.066651784456692) + | > loss_duration: 1.4518505334854126 (1.4477304816246033) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.66210174560547 (31.095456490149864) + | > grad_norm_1: tensor(77.5327, device='cuda:0') (tensor(147.3616, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.359 (0.335877932035006) + | > loader_time: 0.0055 (0.00490732376392071) + + + --> TIME: 2025-04-19 01:04:52 -- STEP: 51/142 -- GLOBAL_STEP: 397225 + | > loss_disc: 2.419933319091797 (2.4445302065681007) + | > loss_disc_real_0: 0.08829564601182938 (0.11960455585344165) + | > loss_disc_real_1: 0.22369082272052765 (0.20432386094448612) + | > loss_disc_real_2: 0.23263663053512573 (0.22620890976167193) + | > loss_disc_real_3: 0.21075929701328278 (0.22840230196130043) + | > loss_disc_real_4: 0.21036143600940704 (0.2281918294873892) + | > loss_disc_real_5: 0.19554485380649567 (0.23021888645256267) + | > loss_0: 2.419933319091797 (2.4445302065681007) + | > grad_norm_0: tensor(10.0842, device='cuda:0') (tensor(8.2020, device='cuda:0')) + | > loss_gen: 2.55635666847229 (2.376286501977956) + | > loss_kl: 1.8807673454284668 (1.8770729326734357) + | > loss_feat: 7.480737686157227 (7.241821878096637) + | > loss_mel: 16.888771057128906 (17.90670738968195) + | > loss_duration: 1.4479317665100098 (1.4573539729211844) + | > amp_scaler: 256.0 (501.96078431372547) + | > loss_1: 30.25456428527832 (30.85924290675743) + | > grad_norm_1: tensor(501.0994, device='cuda:0') (tensor(182.2858, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.3741 (0.3485984007517497) + | > loader_time: 0.0054 (0.005355339424282896) + + + --> TIME: 2025-04-19 01:05:01 -- STEP: 76/142 -- GLOBAL_STEP: 397250 + | > loss_disc: 2.478149175643921 (2.455852279537602) + | > loss_disc_real_0: 0.140584334731102 (0.12328266028903033) + | > loss_disc_real_1: 0.203604593873024 (0.20649654045701027) + | > loss_disc_real_2: 0.25198766589164734 (0.22826028791697403) + | > loss_disc_real_3: 0.2185157686471939 (0.2288829603869664) + | > loss_disc_real_4: 0.21039952337741852 (0.22818500881916598) + | > loss_disc_real_5: 0.23746377229690552 (0.23075744960653155) + | > loss_0: 2.478149175643921 (2.455852279537602) + | > grad_norm_0: tensor(15.7349, device='cuda:0') (tensor(10.9138, device='cuda:0')) + | > loss_gen: 2.3381717205047607 (2.376732562717637) + | > loss_kl: 1.8729221820831299 (1.8894540855759068) + | > loss_feat: 7.155828952789307 (7.174004479458458) + | > loss_mel: 18.753387451171875 (17.850219714014155) + | > loss_duration: 1.521575927734375 (1.4710608203160136) + | > amp_scaler: 256.0 (421.05263157894734) + | > loss_1: 31.64188575744629 (30.7614719240289) + | > grad_norm_1: tensor(279.6971, device='cuda:0') (tensor(214.7038, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.3924 (0.36035467762696116) + | > loader_time: 0.0062 (0.0056847741729334775) + + + --> TIME: 2025-04-19 01:05:12 -- STEP: 101/142 -- GLOBAL_STEP: 397275 + | > loss_disc: 2.529237747192383 (2.46552862979398) + | > loss_disc_real_0: 0.09972292929887772 (0.12391698212906865) + | > loss_disc_real_1: 0.1864265650510788 (0.20781931561408656) + | > loss_disc_real_2: 0.23742547631263733 (0.22868279035728756) + | > loss_disc_real_3: 0.2403581291437149 (0.22954600621568094) + | > loss_disc_real_4: 0.22114859521389008 (0.22946016578981193) + | > loss_disc_real_5: 0.27987751364707947 (0.23207152494699648) + | > loss_0: 2.529237747192383 (2.46552862979398) + | > grad_norm_0: tensor(20.6203, device='cuda:0') (tensor(11.8873, device='cuda:0')) + | > loss_gen: 2.202134609222412 (2.366703852568521) + | > loss_kl: 2.0541112422943115 (1.8999286878227006) + | > loss_feat: 6.452897548675537 (7.100842990497552) + | > loss_mel: 17.520401000976562 (17.86070071116533) + | > loss_duration: 1.5453872680664062 (1.4821923864950048) + | > amp_scaler: 256.0 (380.1980198019802) + | > loss_1: 29.774930953979492 (30.7103687701839) + | > grad_norm_1: tensor(232.9351, device='cuda:0') (tensor(235.7386, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.4149 (0.3704391139568669) + | > loader_time: 0.0067 (0.005922027153543906) + + + --> TIME: 2025-04-19 01:05:23 -- STEP: 126/142 -- GLOBAL_STEP: 397300 + | > loss_disc: 2.4527816772460938 (2.4632811792313114) + | > loss_disc_real_0: 0.09560845047235489 (0.12377127928156702) + | > loss_disc_real_1: 0.18468154966831207 (0.20634367872798254) + | > loss_disc_real_2: 0.2514309883117676 (0.22728258607879517) + | > loss_disc_real_3: 0.2236952930688858 (0.22926242010934011) + | > loss_disc_real_4: 0.19786487519741058 (0.2295253864592976) + | > loss_disc_real_5: 0.24134716391563416 (0.23358260899309127) + | > loss_0: 2.4527816772460938 (2.4632811792313114) + | > grad_norm_0: tensor(3.9722, device='cuda:0') (tensor(11.5095, device='cuda:0')) + | > loss_gen: 2.3300650119781494 (2.358574229573445) + | > loss_kl: 1.8517987728118896 (1.9070445166693792) + | > loss_feat: 7.6728901863098145 (7.081343431321401) + | > loss_mel: 18.42302703857422 (17.86870135201348) + | > loss_duration: 1.5169634819030762 (1.4890753248381237) + | > amp_scaler: 256.0 (355.55555555555554) + | > loss_1: 31.79474449157715 (30.704738949972487) + | > grad_norm_1: tensor(185.0240, device='cuda:0') (tensor(234.6739, device='cuda:0')) + | > current_lr_0: 0.0001968744827138014 + | > current_lr_1: 0.0001968744827138014 + | > step_time: 0.4377 (0.3819165475784786) + | > loader_time: 0.0081 (0.006178731010073706) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.375659227371216 (2.375659227371216) + | > loss_disc_real_0: 0.10332417488098145 (0.10332417488098145) + | > loss_disc_real_1: 0.2018263041973114 (0.2018263041973114) + | > loss_disc_real_2: 0.20492056012153625 (0.20492056012153625) + | > loss_disc_real_3: 0.22250671684741974 (0.22250671684741974) + | > loss_disc_real_4: 0.20740875601768494 (0.20740875601768494) + | > loss_disc_real_5: 0.20370472967624664 (0.20370472967624664) + | > loss_0: 2.375659227371216 (2.375659227371216) + | > loss_gen: 2.278740644454956 (2.278740644454956) + | > loss_kl: 2.1954898834228516 (2.1954898834228516) + | > loss_feat: 7.191961765289307 (7.191961765289307) + | > loss_mel: 18.48993492126465 (18.48993492126465) + | > loss_duration: 1.6391724348068237 (1.6391724348068237) + | > loss_1: 31.795297622680664 (31.795297622680664) + + --> STEP: 1 + | > loss_disc: 2.461268424987793 (2.461268424987793) + | > loss_disc_real_0: 0.13901236653327942 (0.13901236653327942) + | > loss_disc_real_1: 0.19764509797096252 (0.19764509797096252) + | > loss_disc_real_2: 0.18710698187351227 (0.18710698187351227) + | > loss_disc_real_3: 0.21927443146705627 (0.21927443146705627) + | > loss_disc_real_4: 0.23801571130752563 (0.23801571130752563) + | > loss_disc_real_5: 0.1764201521873474 (0.1764201521873474) + | > loss_0: 2.461268424987793 (2.461268424987793) + | > loss_gen: 2.1938042640686035 (2.1938042640686035) + | > loss_kl: 2.4695727825164795 (2.4695727825164795) + | > loss_feat: 7.385366916656494 (7.385366916656494) + | > loss_mel: 19.300119400024414 (19.300119400024414) + | > loss_duration: 1.6736257076263428 (1.6736257076263428) + | > loss_1: 33.02248764038086 (33.02248764038086) + + --> STEP: 2 + | > loss_disc: 2.539933681488037 (2.500601053237915) + | > loss_disc_real_0: 0.10480281710624695 (0.12190759181976318) + | > loss_disc_real_1: 0.20843228697776794 (0.20303869247436523) + | > loss_disc_real_2: 0.2208895981311798 (0.20399829000234604) + | > loss_disc_real_3: 0.2321552336215973 (0.22571483254432678) + | > loss_disc_real_4: 0.2531440854072571 (0.24557989835739136) + | > loss_disc_real_5: 0.22328633069992065 (0.19985324144363403) + | > loss_0: 2.539933681488037 (2.500601053237915) + | > loss_gen: 2.1276516914367676 (2.1607279777526855) + | > loss_kl: 1.9956763982772827 (2.232624590396881) + | > loss_feat: 6.997654914855957 (7.191510915756226) + | > loss_mel: 16.954856872558594 (18.127488136291504) + | > loss_duration: 1.6227798461914062 (1.6482027769088745) + | > loss_1: 29.698619842529297 (31.360553741455078) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032519102096557617 (-0.0001881122589111328) + | > avg_loss_disc: 2.500601053237915 (+0.013332128524780273) + | > avg_loss_disc_real_0: 0.12190759181976318 (-0.11271359026432037) + | > avg_loss_disc_real_1: 0.20303869247436523 (-0.05351883918046951) + | > avg_loss_disc_real_2: 0.20399829000234604 (-0.022639460861682892) + | > avg_loss_disc_real_3: 0.22571483254432678 (+0.0023263171315193176) + | > avg_loss_disc_real_4: 0.24557989835739136 (+0.009059220552444458) + | > avg_loss_disc_real_5: 0.19985324144363403 (-0.007739037275314331) + | > avg_loss_0: 2.500601053237915 (+0.013332128524780273) + | > avg_loss_gen: 2.1607279777526855 (-0.46914994716644287) + | > avg_loss_kl: 2.232624590396881 (-0.05887800455093384) + | > avg_loss_feat: 7.191510915756226 (-0.03774714469909668) + | > avg_loss_mel: 18.127488136291504 (+0.649174690246582) + | > avg_loss_duration: 1.6482027769088745 (-0.004983663558959961) + | > avg_loss_1: 31.360553741455078 (+0.0784158706665039) + + + > EPOCH: 127/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:05:31)  + + --> TIME: 2025-04-19 01:05:35 -- STEP: 9/142 -- GLOBAL_STEP: 397325 + | > loss_disc: 2.4715607166290283 (2.4524011611938477) + | > loss_disc_real_0: 0.10343761742115021 (0.12036272717846765) + | > loss_disc_real_1: 0.21231237053871155 (0.19934974114100137) + | > loss_disc_real_2: 0.20912769436836243 (0.22215121653344896) + | > loss_disc_real_3: 0.2419610321521759 (0.22321845756636727) + | > loss_disc_real_4: 0.22834332287311554 (0.2296787334813012) + | > loss_disc_real_5: 0.23132243752479553 (0.23929760191175672) + | > loss_0: 2.4715607166290283 (2.4524011611938477) + | > grad_norm_0: tensor(6.4263, device='cuda:0') (tensor(7.0454, device='cuda:0')) + | > loss_gen: 2.383035182952881 (2.3752521673838296) + | > loss_kl: 1.7315901517868042 (1.838565746943156) + | > loss_feat: 7.032089710235596 (7.090828153822157) + | > loss_mel: 16.87204360961914 (17.742344114515518) + | > loss_duration: 1.4188787937164307 (1.4422382381227281) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.437637329101562 (30.48922856648763) + | > grad_norm_1: tensor(59.0563, device='cuda:0') (tensor(87.2053, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.3257 (0.32497284147474503) + | > loader_time: 0.005 (0.004385497834947374) + + + --> TIME: 2025-04-19 01:05:44 -- STEP: 34/142 -- GLOBAL_STEP: 397350 + | > loss_disc: 2.464214563369751 (2.4718609417186066) + | > loss_disc_real_0: 0.10389550030231476 (0.12265948885503937) + | > loss_disc_real_1: 0.1670372188091278 (0.20980959999210694) + | > loss_disc_real_2: 0.22364291548728943 (0.2320590492557077) + | > loss_disc_real_3: 0.21263360977172852 (0.2262262127855245) + | > loss_disc_real_4: 0.2375730276107788 (0.23021888250813766) + | > loss_disc_real_5: 0.2371109277009964 (0.23202034085988998) + | > loss_0: 2.464214563369751 (2.4718609417186066) + | > grad_norm_0: tensor(6.7746, device='cuda:0') (tensor(7.8372, device='cuda:0')) + | > loss_gen: 2.2458746433258057 (2.3378551216686483) + | > loss_kl: 1.9779109954833984 (1.8654661949943094) + | > loss_feat: 6.824586868286133 (7.039258466047399) + | > loss_mel: 17.61315155029297 (17.888109487645764) + | > loss_duration: 1.4646164178848267 (1.452704692588133) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.126140594482422 (30.583394106696634) + | > grad_norm_1: tensor(97.7381, device='cuda:0') (tensor(128.5564, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.3557 (0.33823611455805164) + | > loader_time: 0.0056 (0.005142331123352051) + + + --> TIME: 2025-04-19 01:05:53 -- STEP: 59/142 -- GLOBAL_STEP: 397375 + | > loss_disc: 2.399984359741211 (2.4678779294935334) + | > loss_disc_real_0: 0.13516956567764282 (0.12514656062348414) + | > loss_disc_real_1: 0.19752153754234314 (0.20887310626143116) + | > loss_disc_real_2: 0.2023095339536667 (0.23009586334228516) + | > loss_disc_real_3: 0.20439597964286804 (0.22637073922965487) + | > loss_disc_real_4: 0.2144017368555069 (0.23019848713430308) + | > loss_disc_real_5: 0.23563764989376068 (0.2329478948298147) + | > loss_0: 2.399984359741211 (2.4678779294935334) + | > grad_norm_0: tensor(16.1004, device='cuda:0') (tensor(8.6799, device='cuda:0')) + | > loss_gen: 2.401691198348999 (2.3417172957274874) + | > loss_kl: 1.874926209449768 (1.8816929995003393) + | > loss_feat: 7.673723220825195 (7.035900164458711) + | > loss_mel: 18.412147521972656 (17.79065990447998) + | > loss_duration: 1.47160804271698 (1.4633344472464869) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.834095001220703 (30.51330492051981) + | > grad_norm_1: tensor(227.4089, device='cuda:0') (tensor(170.6843, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.3764 (0.35236207913544226) + | > loader_time: 0.0061 (0.0054257441375215164) + + + --> TIME: 2025-04-19 01:06:03 -- STEP: 84/142 -- GLOBAL_STEP: 397400 + | > loss_disc: 2.3988263607025146 (2.4702131379218337) + | > loss_disc_real_0: 0.10686733573675156 (0.12563032896390988) + | > loss_disc_real_1: 0.2064536064863205 (0.20864619545283772) + | > loss_disc_real_2: 0.21456052362918854 (0.22878195717930794) + | > loss_disc_real_3: 0.2427295446395874 (0.227128041464658) + | > loss_disc_real_4: 0.21173016726970673 (0.23110544947641237) + | > loss_disc_real_5: 0.22236408293247223 (0.23218448903589023) + | > loss_0: 2.3988263607025146 (2.4702131379218337) + | > grad_norm_0: tensor(16.2993, device='cuda:0') (tensor(9.3488, device='cuda:0')) + | > loss_gen: 2.293484687805176 (2.3371907586143132) + | > loss_kl: 1.949053406715393 (1.9019130425793784) + | > loss_feat: 6.804360866546631 (6.994903076262701) + | > loss_mel: 17.707365036010742 (17.820381471088957) + | > loss_duration: 1.5434849262237549 (1.4760835213320596) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.29775047302246 (30.530472028823127) + | > grad_norm_1: tensor(385.6539, device='cuda:0') (tensor(194.2066, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.3949 (0.36406202259517856) + | > loader_time: 0.0068 (0.005678946063632054) + + + --> TIME: 2025-04-19 01:06:14 -- STEP: 109/142 -- GLOBAL_STEP: 397425 + | > loss_disc: 2.5157535076141357 (2.475317021028712) + | > loss_disc_real_0: 0.07997860759496689 (0.1262238798748463) + | > loss_disc_real_1: 0.20227323472499847 (0.20899760805138753) + | > loss_disc_real_2: 0.25465983152389526 (0.228155937900237) + | > loss_disc_real_3: 0.24705345928668976 (0.22870119804636055) + | > loss_disc_real_4: 0.24110476672649384 (0.23061983437713132) + | > loss_disc_real_5: 0.2315547913312912 (0.23336769230322008) + | > loss_0: 2.5157535076141357 (2.475317021028712) + | > grad_norm_0: tensor(16.4885, device='cuda:0') (tensor(11.0993, device='cuda:0')) + | > loss_gen: 2.312004804611206 (2.336984262553924) + | > loss_kl: 1.9654970169067383 (1.9127057921995811) + | > loss_feat: 6.872747898101807 (6.961267755665911) + | > loss_mel: 18.338972091674805 (17.827688934606154) + | > loss_duration: 1.521627426147461 (1.4859149018558888) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.010848999023438 (30.524561855771125) + | > grad_norm_1: tensor(269.7027, device='cuda:0') (tensor(219.0289, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.4113 (0.37414198840430024) + | > loader_time: 0.0071 (0.005916048627380929) + + + --> TIME: 2025-04-19 01:06:25 -- STEP: 134/142 -- GLOBAL_STEP: 397450 + | > loss_disc: 2.4322361946105957 (2.4765925265070226) + | > loss_disc_real_0: 0.12673059105873108 (0.12762780020486067) + | > loss_disc_real_1: 0.20112808048725128 (0.2087767525617756) + | > loss_disc_real_2: 0.23612722754478455 (0.2277548707910438) + | > loss_disc_real_3: 0.20057152211666107 (0.228457714973101) + | > loss_disc_real_4: 0.20925161242485046 (0.23012202010670704) + | > loss_disc_real_5: 0.18463364243507385 (0.23245577069360818) + | > loss_0: 2.4322361946105957 (2.4765925265070226) + | > grad_norm_0: tensor(8.2851, device='cuda:0') (tensor(11.2992, device='cuda:0')) + | > loss_gen: 2.2848165035247803 (2.333322788352397) + | > loss_kl: 2.083789825439453 (1.9202383516439752) + | > loss_feat: 7.6949920654296875 (6.930788150474207) + | > loss_mel: 18.56856918334961 (17.83126937809276) + | > loss_duration: 1.5398640632629395 (1.4920067395736922) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.17203140258789 (30.507625636769763) + | > grad_norm_1: tensor(369.5366, device='cuda:0') (tensor(227.4350, device='cuda:0')) + | > current_lr_0: 0.00019684987340346216 + | > current_lr_1: 0.00019684987340346216 + | > step_time: 0.4409 (0.3858316051426218) + | > loader_time: 0.0071 (0.006160816149925116) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.542907238006592 (2.542907238006592) + | > loss_disc_real_0: 0.14961352944374084 (0.14961352944374084) + | > loss_disc_real_1: 0.21736609935760498 (0.21736609935760498) + | > loss_disc_real_2: 0.26841771602630615 (0.26841771602630615) + | > loss_disc_real_3: 0.253680020570755 (0.253680020570755) + | > loss_disc_real_4: 0.2428206205368042 (0.2428206205368042) + | > loss_disc_real_5: 0.2487141340970993 (0.2487141340970993) + | > loss_0: 2.542907238006592 (2.542907238006592) + | > loss_gen: 2.298135757446289 (2.298135757446289) + | > loss_kl: 2.134168863296509 (2.134168863296509) + | > loss_feat: 6.70460319519043 (6.70460319519043) + | > loss_mel: 18.269878387451172 (18.269878387451172) + | > loss_duration: 1.649294137954712 (1.649294137954712) + | > loss_1: 31.056079864501953 (31.056079864501953) + + --> STEP: 1 + | > loss_disc: 2.4451563358306885 (2.4451563358306885) + | > loss_disc_real_0: 0.10582931339740753 (0.10582931339740753) + | > loss_disc_real_1: 0.18751348555088043 (0.18751348555088043) + | > loss_disc_real_2: 0.2151302546262741 (0.2151302546262741) + | > loss_disc_real_3: 0.23987606167793274 (0.23987606167793274) + | > loss_disc_real_4: 0.1958703249692917 (0.1958703249692917) + | > loss_disc_real_5: 0.2152460664510727 (0.2152460664510727) + | > loss_0: 2.4451563358306885 (2.4451563358306885) + | > loss_gen: 2.2043397426605225 (2.2043397426605225) + | > loss_kl: 2.7170095443725586 (2.7170095443725586) + | > loss_feat: 7.701720237731934 (7.701720237731934) + | > loss_mel: 19.490066528320312 (19.490066528320312) + | > loss_duration: 1.6810595989227295 (1.6810595989227295) + | > loss_1: 33.79419708251953 (33.79419708251953) + + --> STEP: 2 + | > loss_disc: 2.6761465072631836 (2.560651421546936) + | > loss_disc_real_0: 0.18543091416358948 (0.1456301137804985) + | > loss_disc_real_1: 0.23708316683769226 (0.21229832619428635) + | > loss_disc_real_2: 0.2636731266975403 (0.2394016906619072) + | > loss_disc_real_3: 0.2885619103908539 (0.2642189860343933) + | > loss_disc_real_4: 0.26497983932495117 (0.23042508214712143) + | > loss_disc_real_5: 0.23233191668987274 (0.22378899157047272) + | > loss_0: 2.6761465072631836 (2.560651421546936) + | > loss_gen: 2.241612195968628 (2.222975969314575) + | > loss_kl: 2.3621699810028076 (2.539589762687683) + | > loss_feat: 6.223135948181152 (6.962428092956543) + | > loss_mel: 17.413862228393555 (18.451964378356934) + | > loss_duration: 1.6183686256408691 (1.6497141122817993) + | > loss_1: 29.859148025512695 (31.826672554016113) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037118196487426758 (+0.00045990943908691406) + | > avg_loss_disc: 2.560651421546936 (+0.060050368309020996) + | > avg_loss_disc_real_0: 0.1456301137804985 (+0.02372252196073532) + | > avg_loss_disc_real_1: 0.21229832619428635 (+0.009259633719921112) + | > avg_loss_disc_real_2: 0.2394016906619072 (+0.03540340065956116) + | > avg_loss_disc_real_3: 0.2642189860343933 (+0.03850415349006653) + | > avg_loss_disc_real_4: 0.23042508214712143 (-0.015154816210269928) + | > avg_loss_disc_real_5: 0.22378899157047272 (+0.023935750126838684) + | > avg_loss_0: 2.560651421546936 (+0.060050368309020996) + | > avg_loss_gen: 2.222975969314575 (+0.06224799156188965) + | > avg_loss_kl: 2.539589762687683 (+0.306965172290802) + | > avg_loss_feat: 6.962428092956543 (-0.22908282279968262) + | > avg_loss_mel: 18.451964378356934 (+0.3244762420654297) + | > avg_loss_duration: 1.6497141122817993 (+0.0015113353729248047) + | > avg_loss_1: 31.826672554016113 (+0.46611881256103516) + + + > EPOCH: 128/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:06:30)  + + --> TIME: 2025-04-19 01:06:36 -- STEP: 17/142 -- GLOBAL_STEP: 397475 + | > loss_disc: 2.4239675998687744 (2.4800767197328457) + | > loss_disc_real_0: 0.1562853455543518 (0.1276081349043285) + | > loss_disc_real_1: 0.21527941524982452 (0.20666732332285712) + | > loss_disc_real_2: 0.2197066843509674 (0.2270247261313831) + | > loss_disc_real_3: 0.23001240193843842 (0.23654195140389836) + | > loss_disc_real_4: 0.16792385280132294 (0.23565959842766032) + | > loss_disc_real_5: 0.27161750197410583 (0.24495651967385235) + | > loss_0: 2.4239675998687744 (2.4800767197328457) + | > grad_norm_0: tensor(12.1888, device='cuda:0') (tensor(14.6290, device='cuda:0')) + | > loss_gen: 2.4420413970947266 (2.4036437343148624) + | > loss_kl: 1.8101972341537476 (1.816483820185942) + | > loss_feat: 7.063878059387207 (7.3427494273466225) + | > loss_mel: 17.44471549987793 (17.877131798688104) + | > loss_duration: 1.4228098392486572 (1.4430392139098223) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.18364143371582 (30.883048001457663) + | > grad_norm_1: tensor(378.9148, device='cuda:0') (tensor(308.4187, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.3375 (0.32848791515125947) + | > loader_time: 0.0049 (0.004751920700073242) + + + --> TIME: 2025-04-19 01:06:45 -- STEP: 42/142 -- GLOBAL_STEP: 397500 + | > loss_disc: 2.504096746444702 (2.4701751867930093) + | > loss_disc_real_0: 0.1735977679491043 (0.12638609767669728) + | > loss_disc_real_1: 0.20707584917545319 (0.2057384259644009) + | > loss_disc_real_2: 0.20167367160320282 (0.22561377677179517) + | > loss_disc_real_3: 0.2279261350631714 (0.230991940058413) + | > loss_disc_real_4: 0.2183251529932022 (0.2328492889092082) + | > loss_disc_real_5: 0.21171261370182037 (0.23721238757882798) + | > loss_0: 2.504096746444702 (2.4701751867930093) + | > grad_norm_0: tensor(23.2072, device='cuda:0') (tensor(14.0092, device='cuda:0')) + | > loss_gen: 2.2846410274505615 (2.3707635459445773) + | > loss_kl: 2.0049197673797607 (1.8518647665069217) + | > loss_feat: 6.387456893920898 (7.137483960106259) + | > loss_mel: 17.681657791137695 (17.81680213837396) + | > loss_duration: 1.5065662860870361 (1.4570607656524295) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.865243911743164 (30.633975119817826) + | > grad_norm_1: tensor(283.4792, device='cuda:0') (tensor(296.4232, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.362 (0.34271223772139775) + | > loader_time: 0.006 (0.00519833110627674) + + + --> TIME: 2025-04-19 01:06:55 -- STEP: 67/142 -- GLOBAL_STEP: 397525 + | > loss_disc: 2.3255417346954346 (2.474911259181464) + | > loss_disc_real_0: 0.11934735625982285 (0.12946832458029933) + | > loss_disc_real_1: 0.20447894930839539 (0.2088535897322555) + | > loss_disc_real_2: 0.21057574450969696 (0.22462698305720713) + | > loss_disc_real_3: 0.21313394606113434 (0.23043357970109626) + | > loss_disc_real_4: 0.18085342645645142 (0.23071537266916303) + | > loss_disc_real_5: 0.2091161012649536 (0.2345618721264512) + | > loss_0: 2.3255417346954346 (2.474911259181464) + | > grad_norm_0: tensor(11.4330, device='cuda:0') (tensor(12.5501, device='cuda:0')) + | > loss_gen: 2.312364339828491 (2.3514478847162046) + | > loss_kl: 1.9120973348617554 (1.8760003687730475) + | > loss_feat: 7.717170238494873 (7.026434570995729) + | > loss_mel: 18.25489616394043 (17.759716304380504) + | > loss_duration: 1.4307618141174316 (1.4689112189990372) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.627288818359375 (30.48251029626647) + | > grad_norm_1: tensor(254.9718, device='cuda:0') (tensor(264.5123, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.3855 (0.35554822879051096) + | > loader_time: 0.0064 (0.005501490920337278) + + + --> TIME: 2025-04-19 01:07:05 -- STEP: 92/142 -- GLOBAL_STEP: 397550 + | > loss_disc: 2.3848319053649902 (2.4840090844942178) + | > loss_disc_real_0: 0.10008823871612549 (0.13025119997884915) + | > loss_disc_real_1: 0.1500081717967987 (0.2089144551883573) + | > loss_disc_real_2: 0.23786912858486176 (0.22626643935623375) + | > loss_disc_real_3: 0.24570466578006744 (0.23084328939085422) + | > loss_disc_real_4: 0.2285066694021225 (0.23133656058622443) + | > loss_disc_real_5: 0.21146053075790405 (0.23367030513675316) + | > loss_0: 2.3848319053649902 (2.4840090844942178) + | > grad_norm_0: tensor(5.1331, device='cuda:0') (tensor(10.9779, device='cuda:0')) + | > loss_gen: 2.351728677749634 (2.337784629801045) + | > loss_kl: 2.0397846698760986 (1.8960853724376014) + | > loss_feat: 7.690320014953613 (6.9939099342926685) + | > loss_mel: 17.45050048828125 (17.826883284941967) + | > loss_duration: 1.471092939376831 (1.4815788670726442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.00342559814453 (30.536242028941277) + | > grad_norm_1: tensor(241.8881, device='cuda:0') (tensor(230.6183, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.4106 (0.3670553228129511) + | > loader_time: 0.0068 (0.005760027014690897) + + + --> TIME: 2025-04-19 01:07:16 -- STEP: 117/142 -- GLOBAL_STEP: 397575 + | > loss_disc: 2.56331729888916 (2.487635728640435) + | > loss_disc_real_0: 0.16579681634902954 (0.13158539357857826) + | > loss_disc_real_1: 0.17674168944358826 (0.20869952720454615) + | > loss_disc_real_2: 0.25109773874282837 (0.22716369791927501) + | > loss_disc_real_3: 0.23824317753314972 (0.23120321307936284) + | > loss_disc_real_4: 0.24675416946411133 (0.23169079015397617) + | > loss_disc_real_5: 0.24768777191638947 (0.23375461384271964) + | > loss_0: 2.56331729888916 (2.487635728640435) + | > grad_norm_0: tensor(8.9969, device='cuda:0') (tensor(10.5086, device='cuda:0')) + | > loss_gen: 2.1788318157196045 (2.3285756661341734) + | > loss_kl: 1.7930142879486084 (1.8973682446357532) + | > loss_feat: 5.573192596435547 (6.946716210780999) + | > loss_mel: 15.747203826904297 (17.757653725452908) + | > loss_duration: 1.507227897644043 (1.4881513903283663) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.799468994140625 (30.41846519861466) + | > grad_norm_1: tensor(144.8616, device='cuda:0') (tensor(214.5399, device='cuda:0')) + | > current_lr_0: 0.00019682526716928672 + | > current_lr_1: 0.00019682526716928672 + | > step_time: 0.4359 (0.377351748637664) + | > loader_time: 0.0074 (0.00604196287627913) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.430546760559082 (2.430546760559082) + | > loss_disc_real_0: 0.20104406774044037 (0.20104406774044037) + | > loss_disc_real_1: 0.23244816064834595 (0.23244816064834595) + | > loss_disc_real_2: 0.27469345927238464 (0.27469345927238464) + | > loss_disc_real_3: 0.28550824522972107 (0.28550824522972107) + | > loss_disc_real_4: 0.21490976214408875 (0.21490976214408875) + | > loss_disc_real_5: 0.1902158111333847 (0.1902158111333847) + | > loss_0: 2.430546760559082 (2.430546760559082) + | > loss_gen: 2.582479953765869 (2.582479953765869) + | > loss_kl: 2.3526504039764404 (2.3526504039764404) + | > loss_feat: 7.31648063659668 (7.31648063659668) + | > loss_mel: 17.447288513183594 (17.447288513183594) + | > loss_duration: 1.6801303625106812 (1.6801303625106812) + | > loss_1: 31.379030227661133 (31.379030227661133) + + --> STEP: 1 + | > loss_disc: 2.495598316192627 (2.495598316192627) + | > loss_disc_real_0: 0.1629449874162674 (0.1629449874162674) + | > loss_disc_real_1: 0.21658499538898468 (0.21658499538898468) + | > loss_disc_real_2: 0.23120349645614624 (0.23120349645614624) + | > loss_disc_real_3: 0.2964831590652466 (0.2964831590652466) + | > loss_disc_real_4: 0.2602553069591522 (0.2602553069591522) + | > loss_disc_real_5: 0.2255886197090149 (0.2255886197090149) + | > loss_0: 2.495598316192627 (2.495598316192627) + | > loss_gen: 2.459861993789673 (2.459861993789673) + | > loss_kl: 2.459169864654541 (2.459169864654541) + | > loss_feat: 6.8837504386901855 (6.8837504386901855) + | > loss_mel: 18.307748794555664 (18.307748794555664) + | > loss_duration: 1.6742253303527832 (1.6742253303527832) + | > loss_1: 31.78475570678711 (31.78475570678711) + + --> STEP: 2 + | > loss_disc: 2.2719297409057617 (2.3837640285491943) + | > loss_disc_real_0: 0.13204325735569 (0.1474941223859787) + | > loss_disc_real_1: 0.1624518781900406 (0.18951843678951263) + | > loss_disc_real_2: 0.2043175846338272 (0.21776054054498672) + | > loss_disc_real_3: 0.22549276053905487 (0.2609879598021507) + | > loss_disc_real_4: 0.2087116837501526 (0.2344834953546524) + | > loss_disc_real_5: 0.23247312009334564 (0.22903086990118027) + | > loss_0: 2.2719297409057617 (2.3837640285491943) + | > loss_gen: 2.4426865577697754 (2.451274275779724) + | > loss_kl: 2.065256357192993 (2.262213110923767) + | > loss_feat: 8.431238174438477 (7.657494306564331) + | > loss_mel: 19.25794219970703 (18.782845497131348) + | > loss_duration: 1.6216121912002563 (1.6479187607765198) + | > loss_1: 33.8187370300293 (32.8017463684082) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003566741943359375 (-0.00014507770538330078) + | > avg_loss_disc: 2.3837640285491943 (-0.1768873929977417) + | > avg_loss_disc_real_0: 0.1474941223859787 (+0.001864008605480194) + | > avg_loss_disc_real_1: 0.18951843678951263 (-0.022779889404773712) + | > avg_loss_disc_real_2: 0.21776054054498672 (-0.02164115011692047) + | > avg_loss_disc_real_3: 0.2609879598021507 (-0.0032310262322425842) + | > avg_loss_disc_real_4: 0.2344834953546524 (+0.004058413207530975) + | > avg_loss_disc_real_5: 0.22903086990118027 (+0.00524187833070755) + | > avg_loss_0: 2.3837640285491943 (-0.1768873929977417) + | > avg_loss_gen: 2.451274275779724 (+0.22829830646514893) + | > avg_loss_kl: 2.262213110923767 (-0.277376651763916) + | > avg_loss_feat: 7.657494306564331 (+0.6950662136077881) + | > avg_loss_mel: 18.782845497131348 (+0.33088111877441406) + | > avg_loss_duration: 1.6479187607765198 (-0.001795351505279541) + | > avg_loss_1: 32.8017463684082 (+0.9750738143920898) + + + > EPOCH: 129/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:07:29)  + + --> TIME: 2025-04-19 01:07:29 -- STEP: 0/142 -- GLOBAL_STEP: 397600 + | > loss_disc: 2.57190203666687 (2.57190203666687) + | > loss_disc_real_0: 0.1508421152830124 (0.1508421152830124) + | > loss_disc_real_1: 0.2281692773103714 (0.2281692773103714) + | > loss_disc_real_2: 0.2515430748462677 (0.2515430748462677) + | > loss_disc_real_3: 0.29786741733551025 (0.29786741733551025) + | > loss_disc_real_4: 0.24792927503585815 (0.24792927503585815) + | > loss_disc_real_5: 0.23422646522521973 (0.23422646522521973) + | > loss_0: 2.57190203666687 (2.57190203666687) + | > grad_norm_0: tensor(12.1455, device='cuda:0') (tensor(12.1455, device='cuda:0')) + | > loss_gen: 2.386117935180664 (2.386117935180664) + | > loss_kl: 1.8707547187805176 (1.8707547187805176) + | > loss_feat: 6.960979461669922 (6.960979461669922) + | > loss_mel: 18.08059310913086 (18.08059310913086) + | > loss_duration: 1.469747543334961 (1.469747543334961) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.768192291259766 (30.768192291259766) + | > grad_norm_1: tensor(299.9147, device='cuda:0') (tensor(299.9147, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.3598 (0.35976290702819824) + | > loader_time: 0.3216 (0.3215787410736084) + + + --> TIME: 2025-04-19 01:07:38 -- STEP: 25/142 -- GLOBAL_STEP: 397625 + | > loss_disc: 2.416940450668335 (2.457219247817993) + | > loss_disc_real_0: 0.14852993190288544 (0.1221286916732788) + | > loss_disc_real_1: 0.20219750702381134 (0.2031945776939392) + | > loss_disc_real_2: 0.2103661745786667 (0.21963642060756683) + | > loss_disc_real_3: 0.22726695239543915 (0.22903909683227539) + | > loss_disc_real_4: 0.2225060760974884 (0.2301242995262146) + | > loss_disc_real_5: 0.24410460889339447 (0.2387287873029709) + | > loss_0: 2.416940450668335 (2.457219247817993) + | > grad_norm_0: tensor(6.3653, device='cuda:0') (tensor(9.5493, device='cuda:0')) + | > loss_gen: 2.3170559406280518 (2.340711736679077) + | > loss_kl: 1.9107664823532104 (1.8871807050704956) + | > loss_feat: 7.038300514221191 (7.087588996887207) + | > loss_mel: 16.89691162109375 (17.805555725097655) + | > loss_duration: 1.489540696144104 (1.4523326253890991) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65257453918457 (30.573369674682617) + | > grad_norm_1: tensor(150.6107, device='cuda:0') (tensor(232.9714, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.3449 (0.3350376605987549) + | > loader_time: 0.0055 (0.005005340576171875) + + + --> TIME: 2025-04-19 01:07:47 -- STEP: 50/142 -- GLOBAL_STEP: 397650 + | > loss_disc: 2.4999337196350098 (2.4490815067291254) + | > loss_disc_real_0: 0.07671937346458435 (0.12100460439920425) + | > loss_disc_real_1: 0.23474963009357452 (0.20585246831178666) + | > loss_disc_real_2: 0.24445074796676636 (0.2205346953868866) + | > loss_disc_real_3: 0.21451468765735626 (0.22772726893424988) + | > loss_disc_real_4: 0.23869431018829346 (0.22776195347309114) + | > loss_disc_real_5: 0.227052241563797 (0.2337796464562416) + | > loss_0: 2.4999337196350098 (2.4490815067291254) + | > grad_norm_0: tensor(17.9178, device='cuda:0') (tensor(9.7238, device='cuda:0')) + | > loss_gen: 2.3702797889709473 (2.341037912368775) + | > loss_kl: 1.9205137491226196 (1.8970770406723023) + | > loss_feat: 7.288118362426758 (7.044863643646241) + | > loss_mel: 19.040319442749023 (17.78693021774292) + | > loss_duration: 1.4961028099060059 (1.4604874277114868) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.115333557128906 (30.53039608001709) + | > grad_norm_1: tensor(415.5794, device='cuda:0') (tensor(254.0359, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.3699 (0.3468994760513306) + | > loader_time: 0.0059 (0.0053235721588134775) + + + --> TIME: 2025-04-19 01:07:57 -- STEP: 75/142 -- GLOBAL_STEP: 397675 + | > loss_disc: 2.4870760440826416 (2.450387007395426) + | > loss_disc_real_0: 0.09466752409934998 (0.12264088690280914) + | > loss_disc_real_1: 0.22426332533359528 (0.20603189210096995) + | > loss_disc_real_2: 0.22485892474651337 (0.2230066994825999) + | > loss_disc_real_3: 0.25832799077033997 (0.2294288718700409) + | > loss_disc_real_4: 0.22783635556697845 (0.22760466039180755) + | > loss_disc_real_5: 0.2232540398836136 (0.23183550079663595) + | > loss_0: 2.4870760440826416 (2.450387007395426) + | > grad_norm_0: tensor(13.3376, device='cuda:0') (tensor(10.9411, device='cuda:0')) + | > loss_gen: 2.4266865253448486 (2.351392749150594) + | > loss_kl: 1.9364020824432373 (1.907535564104716) + | > loss_feat: 7.5175461769104 (7.050760008494059) + | > loss_mel: 18.128841400146484 (17.828864631652834) + | > loss_duration: 1.465104341506958 (1.4733279959360759) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.47458267211914 (30.611880900065103) + | > grad_norm_1: tensor(465.9916, device='cuda:0') (tensor(266.4183, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.3964 (0.35860655148824055) + | > loader_time: 0.0068 (0.005685262680053712) + + + --> TIME: 2025-04-19 01:08:07 -- STEP: 100/142 -- GLOBAL_STEP: 397700 + | > loss_disc: 2.5666751861572266 (2.4567073345184327) + | > loss_disc_real_0: 0.12699489295482635 (0.125522338822484) + | > loss_disc_real_1: 0.21661026775836945 (0.20615462452173233) + | > loss_disc_real_2: 0.21756179630756378 (0.2237080992758274) + | > loss_disc_real_3: 0.24808035790920258 (0.22959151223301888) + | > loss_disc_real_4: 0.24357262253761292 (0.22903882548213006) + | > loss_disc_real_5: 0.22872261703014374 (0.2302936254441738) + | > loss_0: 2.5666751861572266 (2.4567073345184327) + | > grad_norm_0: tensor(13.6878, device='cuda:0') (tensor(12.1240, device='cuda:0')) + | > loss_gen: 2.3557307720184326 (2.352467300891877) + | > loss_kl: 1.9559954404830933 (1.9236496043205262) + | > loss_feat: 6.667071342468262 (7.000560464859009) + | > loss_mel: 17.315311431884766 (17.78961365699769) + | > loss_duration: 1.490103006362915 (1.4840291023254395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.784210205078125 (30.550320072174074) + | > grad_norm_1: tensor(447.7415, device='cuda:0') (tensor(278.5245, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.4125 (0.3690329670906067) + | > loader_time: 0.0069 (0.0059150242805480965) + + + --> TIME: 2025-04-19 01:08:18 -- STEP: 125/142 -- GLOBAL_STEP: 397725 + | > loss_disc: 2.485156536102295 (2.465173667907715) + | > loss_disc_real_0: 0.18412916362285614 (0.12702395761013036) + | > loss_disc_real_1: 0.20742157101631165 (0.20817589378356935) + | > loss_disc_real_2: 0.2708595395088196 (0.22572726941108703) + | > loss_disc_real_3: 0.2448657751083374 (0.22980040776729585) + | > loss_disc_real_4: 0.24261906743049622 (0.22862193500995637) + | > loss_disc_real_5: 0.189716637134552 (0.22986695885658265) + | > loss_0: 2.485156536102295 (2.465173667907715) + | > grad_norm_0: tensor(12.6699, device='cuda:0') (tensor(12.5838, device='cuda:0')) + | > loss_gen: 2.2998948097229004 (2.3493438282012944) + | > loss_kl: 1.9197075366973877 (1.9277313861846923) + | > loss_feat: 6.279229640960693 (6.96107999420166) + | > loss_mel: 17.02654266357422 (17.805063659667976) + | > loss_duration: 1.5570247173309326 (1.4918691520690919) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.0823974609375 (30.535087966918944) + | > grad_norm_1: tensor(334.5481, device='cuda:0') (tensor(292.5713, device='cuda:0')) + | > current_lr_0: 0.00019680066401089056 + | > current_lr_1: 0.00019680066401089056 + | > step_time: 0.4355 (0.3805185089111328) + | > loader_time: 0.0077 (0.006161975860595704) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5106418132781982 (2.5106418132781982) + | > loss_disc_real_0: 0.19284503161907196 (0.19284503161907196) + | > loss_disc_real_1: 0.2536816895008087 (0.2536816895008087) + | > loss_disc_real_2: 0.259848028421402 (0.259848028421402) + | > loss_disc_real_3: 0.2626156210899353 (0.2626156210899353) + | > loss_disc_real_4: 0.2604075074195862 (0.2604075074195862) + | > loss_disc_real_5: 0.2813020348548889 (0.2813020348548889) + | > loss_0: 2.5106418132781982 (2.5106418132781982) + | > loss_gen: 2.787613868713379 (2.787613868713379) + | > loss_kl: 2.2644386291503906 (2.2644386291503906) + | > loss_feat: 7.139663219451904 (7.139663219451904) + | > loss_mel: 19.10807228088379 (19.10807228088379) + | > loss_duration: 1.6243541240692139 (1.6243541240692139) + | > loss_1: 32.92414474487305 (32.92414474487305) + + --> STEP: 1 + | > loss_disc: 2.4717142581939697 (2.4717142581939697) + | > loss_disc_real_0: 0.2230304777622223 (0.2230304777622223) + | > loss_disc_real_1: 0.26252609491348267 (0.26252609491348267) + | > loss_disc_real_2: 0.2268940806388855 (0.2268940806388855) + | > loss_disc_real_3: 0.25027599930763245 (0.25027599930763245) + | > loss_disc_real_4: 0.23762956261634827 (0.23762956261634827) + | > loss_disc_real_5: 0.20475216209888458 (0.20475216209888458) + | > loss_0: 2.4717142581939697 (2.4717142581939697) + | > loss_gen: 2.482137680053711 (2.482137680053711) + | > loss_kl: 2.5043253898620605 (2.5043253898620605) + | > loss_feat: 6.045039653778076 (6.045039653778076) + | > loss_mel: 16.35194969177246 (16.35194969177246) + | > loss_duration: 1.6784870624542236 (1.6784870624542236) + | > loss_1: 29.061939239501953 (29.061939239501953) + + --> STEP: 2 + | > loss_disc: 2.6216187477111816 (2.5466665029525757) + | > loss_disc_real_0: 0.23278677463531494 (0.22790862619876862) + | > loss_disc_real_1: 0.30557212233543396 (0.2840491086244583) + | > loss_disc_real_2: 0.2282094806432724 (0.22755178064107895) + | > loss_disc_real_3: 0.24736568331718445 (0.24882084131240845) + | > loss_disc_real_4: 0.258836954832077 (0.24823325872421265) + | > loss_disc_real_5: 0.2382044494152069 (0.22147830575704575) + | > loss_0: 2.6216187477111816 (2.5466665029525757) + | > loss_gen: 2.420172929763794 (2.4511553049087524) + | > loss_kl: 1.9827576875686646 (2.2435415387153625) + | > loss_feat: 5.260746955871582 (5.652893304824829) + | > loss_mel: 16.177350997924805 (16.264650344848633) + | > loss_duration: 1.6472344398498535 (1.6628607511520386) + | > loss_1: 27.488264083862305 (28.27510166168213) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035332441329956055 (-3.349781036376953e-05) + | > avg_loss_disc: 2.5466665029525757 (+0.16290247440338135) + | > avg_loss_disc_real_0: 0.22790862619876862 (+0.08041450381278992) + | > avg_loss_disc_real_1: 0.2840491086244583 (+0.09453067183494568) + | > avg_loss_disc_real_2: 0.22755178064107895 (+0.009791240096092224) + | > avg_loss_disc_real_3: 0.24882084131240845 (-0.012167118489742279) + | > avg_loss_disc_real_4: 0.24823325872421265 (+0.013749763369560242) + | > avg_loss_disc_real_5: 0.22147830575704575 (-0.0075525641441345215) + | > avg_loss_0: 2.5466665029525757 (+0.16290247440338135) + | > avg_loss_gen: 2.4511553049087524 (-0.00011897087097167969) + | > avg_loss_kl: 2.2435415387153625 (-0.01867157220840454) + | > avg_loss_feat: 5.652893304824829 (-2.004601001739502) + | > avg_loss_mel: 16.264650344848633 (-2.518195152282715) + | > avg_loss_duration: 1.6628607511520386 (+0.014941990375518799) + | > avg_loss_1: 28.27510166168213 (-4.526644706726074) + + + > EPOCH: 130/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:08:27)  + + --> TIME: 2025-04-19 01:08:31 -- STEP: 8/142 -- GLOBAL_STEP: 397750 + | > loss_disc: 2.4124629497528076 (2.4525839388370514) + | > loss_disc_real_0: 0.09637077897787094 (0.11003639083355665) + | > loss_disc_real_1: 0.20121023058891296 (0.21430963091552258) + | > loss_disc_real_2: 0.21075834333896637 (0.22986734844744205) + | > loss_disc_real_3: 0.2113088071346283 (0.22781105153262615) + | > loss_disc_real_4: 0.19336865842342377 (0.2383713349699974) + | > loss_disc_real_5: 0.19162794947624207 (0.24358054250478745) + | > loss_0: 2.4124629497528076 (2.4525839388370514) + | > grad_norm_0: tensor(8.2823, device='cuda:0') (tensor(6.7006, device='cuda:0')) + | > loss_gen: 2.3264851570129395 (2.3555799424648285) + | > loss_kl: 1.9472894668579102 (1.8554579317569733) + | > loss_feat: 7.444876670837402 (7.217753231525421) + | > loss_mel: 18.453886032104492 (17.778600931167603) + | > loss_duration: 1.4640271663665771 (1.4495528638362885) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.636564254760742 (30.65694499015808) + | > grad_norm_1: tensor(150.9575, device='cuda:0') (tensor(193.6996, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.3313 (0.3310375213623047) + | > loader_time: 0.005 (0.004393398761749268) + + + --> TIME: 2025-04-19 01:08:39 -- STEP: 33/142 -- GLOBAL_STEP: 397775 + | > loss_disc: 2.3861780166625977 (2.460677334756562) + | > loss_disc_real_0: 0.1145491749048233 (0.119879247564258) + | > loss_disc_real_1: 0.2181427925825119 (0.207616235722195) + | > loss_disc_real_2: 0.24306505918502808 (0.22671442772402908) + | > loss_disc_real_3: 0.2212739735841751 (0.22646230775298495) + | > loss_disc_real_4: 0.19796232879161835 (0.22716792605139993) + | > loss_disc_real_5: 0.23053030669689178 (0.24027159719756155) + | > loss_0: 2.3861780166625977 (2.460677334756562) + | > grad_norm_0: tensor(5.7548, device='cuda:0') (tensor(7.0033, device='cuda:0')) + | > loss_gen: 2.3977103233337402 (2.326133207841354) + | > loss_kl: 1.8049044609069824 (1.9131410772150212) + | > loss_feat: 6.9678473472595215 (7.117929978804155) + | > loss_mel: 19.160354614257812 (18.017000082767378) + | > loss_duration: 1.4584665298461914 (1.4557244127446956) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.789283752441406 (30.82992865822532) + | > grad_norm_1: tensor(189.5004, device='cuda:0') (tensor(138.4214, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.3557 (0.34194500518567633) + | > loader_time: 0.0056 (0.005077477657433712) + + + --> TIME: 2025-04-19 01:08:49 -- STEP: 58/142 -- GLOBAL_STEP: 397800 + | > loss_disc: 2.4123740196228027 (2.4713610122943748) + | > loss_disc_real_0: 0.11339902877807617 (0.1236952042271351) + | > loss_disc_real_1: 0.23246684670448303 (0.20832681039284015) + | > loss_disc_real_2: 0.2270897775888443 (0.22675358349906988) + | > loss_disc_real_3: 0.19927376508712769 (0.22720362734178018) + | > loss_disc_real_4: 0.2068885713815689 (0.2310792500602788) + | > loss_disc_real_5: 0.1950429081916809 (0.2383212991829576) + | > loss_0: 2.4123740196228027 (2.4713610122943748) + | > grad_norm_0: tensor(8.7087, device='cuda:0') (tensor(7.8679, device='cuda:0')) + | > loss_gen: 2.2192976474761963 (2.335019666573097) + | > loss_kl: 2.101552724838257 (1.9138924726124467) + | > loss_feat: 7.166276454925537 (7.056828465955011) + | > loss_mel: 18.604692459106445 (17.922300568942365) + | > loss_duration: 1.4931919574737549 (1.4668724680769034) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.585010528564453 (30.694913568167852) + | > grad_norm_1: tensor(260.1697, device='cuda:0') (tensor(152.9531, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.377 (0.3525344996616758) + | > loader_time: 0.0061 (0.00545755748091073) + + + --> TIME: 2025-04-19 01:08:59 -- STEP: 83/142 -- GLOBAL_STEP: 397825 + | > loss_disc: 2.4462668895721436 (2.471842969756528) + | > loss_disc_real_0: 0.139851912856102 (0.12408157922776349) + | > loss_disc_real_1: 0.1792478710412979 (0.20820298719118877) + | > loss_disc_real_2: 0.21917249262332916 (0.22551989591265298) + | > loss_disc_real_3: 0.23354986310005188 (0.2266765233263912) + | > loss_disc_real_4: 0.25767019391059875 (0.23003338187573905) + | > loss_disc_real_5: 0.2006576508283615 (0.23883018095091166) + | > loss_0: 2.4462668895721436 (2.471842969756528) + | > grad_norm_0: tensor(3.9091, device='cuda:0') (tensor(7.9741, device='cuda:0')) + | > loss_gen: 2.303673505783081 (2.3357423242316186) + | > loss_kl: 1.8070058822631836 (1.9237159548035587) + | > loss_feat: 6.992637634277344 (7.039466059351542) + | > loss_mel: 18.200618743896484 (17.8882729863546) + | > loss_duration: 1.5820746421813965 (1.4789777462740978) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.886011123657227 (30.666175037981517) + | > grad_norm_1: tensor(95.3452, device='cuda:0') (tensor(165.6479, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.3974 (0.3639746459133654) + | > loader_time: 0.0068 (0.005740898201264532) + + + --> TIME: 2025-04-19 01:09:09 -- STEP: 108/142 -- GLOBAL_STEP: 397850 + | > loss_disc: 2.478428602218628 (2.476802823720155) + | > loss_disc_real_0: 0.1450914889574051 (0.12494067012987754) + | > loss_disc_real_1: 0.20271074771881104 (0.2074190735541008) + | > loss_disc_real_2: 0.23187248408794403 (0.22734781175299926) + | > loss_disc_real_3: 0.2525309920310974 (0.22818295312700448) + | > loss_disc_real_4: 0.2139967530965805 (0.23108738605622892) + | > loss_disc_real_5: 0.21683533489704132 (0.2381172207770524) + | > loss_0: 2.478428602218628 (2.476802823720155) + | > grad_norm_0: tensor(7.5628, device='cuda:0') (tensor(8.9949, device='cuda:0')) + | > loss_gen: 2.1574044227600098 (2.3299674943641384) + | > loss_kl: 1.986778974533081 (1.9257398611969418) + | > loss_feat: 6.037132740020752 (6.989402391292431) + | > loss_mel: 17.07178497314453 (17.891283882988823) + | > loss_duration: 1.5107574462890625 (1.4872420198387568) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.763856887817383 (30.623635627605296) + | > grad_norm_1: tensor(321.1406, device='cuda:0') (tensor(198.6955, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.4213 (0.3741064800156487) + | > loader_time: 0.0071 (0.006014044638033268) + + + --> TIME: 2025-04-19 01:09:21 -- STEP: 133/142 -- GLOBAL_STEP: 397875 + | > loss_disc: 2.455568552017212 (2.4825627499056937) + | > loss_disc_real_0: 0.14207378029823303 (0.12606281764749297) + | > loss_disc_real_1: 0.19938433170318604 (0.20738337317803748) + | > loss_disc_real_2: 0.2269301563501358 (0.2278856482720913) + | > loss_disc_real_3: 0.22805987298488617 (0.22891849626723984) + | > loss_disc_real_4: 0.23801374435424805 (0.23106404802853003) + | > loss_disc_real_5: 0.231223002076149 (0.238417987536667) + | > loss_0: 2.455568552017212 (2.4825627499056937) + | > grad_norm_0: tensor(7.2883, device='cuda:0') (tensor(9.2432, device='cuda:0')) + | > loss_gen: 2.181701183319092 (2.3219027949455096) + | > loss_kl: 1.9939783811569214 (1.921192693531065) + | > loss_feat: 6.315622806549072 (6.945483591323509) + | > loss_mel: 17.117050170898438 (17.820785823621247) + | > loss_duration: 1.579326868057251 (1.4927977950949416) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.187679290771484 (30.502162718235102) + | > grad_norm_1: tensor(74.4849, device='cuda:0') (tensor(203.1812, device='cuda:0')) + | > current_lr_0: 0.00019677606392788917 + | > current_lr_1: 0.00019677606392788917 + | > step_time: 0.4368 (0.386897978029753) + | > loader_time: 0.007 (0.006266631578144276) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6237969398498535 (2.6237969398498535) + | > loss_disc_real_0: 0.21037565171718597 (0.21037565171718597) + | > loss_disc_real_1: 0.22289340198040009 (0.22289340198040009) + | > loss_disc_real_2: 0.2547183930873871 (0.2547183930873871) + | > loss_disc_real_3: 0.20067176222801208 (0.20067176222801208) + | > loss_disc_real_4: 0.24479500949382782 (0.24479500949382782) + | > loss_disc_real_5: 0.2065080851316452 (0.2065080851316452) + | > loss_0: 2.6237969398498535 (2.6237969398498535) + | > loss_gen: 2.3048996925354004 (2.3048996925354004) + | > loss_kl: 2.211902379989624 (2.211902379989624) + | > loss_feat: 6.456787586212158 (6.456787586212158) + | > loss_mel: 17.779621124267578 (17.779621124267578) + | > loss_duration: 1.6373286247253418 (1.6373286247253418) + | > loss_1: 30.390539169311523 (30.390539169311523) + + --> STEP: 1 + | > loss_disc: 2.6506271362304688 (2.6506271362304688) + | > loss_disc_real_0: 0.263629674911499 (0.263629674911499) + | > loss_disc_real_1: 0.23657286167144775 (0.23657286167144775) + | > loss_disc_real_2: 0.2424294352531433 (0.2424294352531433) + | > loss_disc_real_3: 0.20725487172603607 (0.20725487172603607) + | > loss_disc_real_4: 0.24711786210536957 (0.24711786210536957) + | > loss_disc_real_5: 0.20961059629917145 (0.20961059629917145) + | > loss_0: 2.6506271362304688 (2.6506271362304688) + | > loss_gen: 2.3159193992614746 (2.3159193992614746) + | > loss_kl: 2.426246166229248 (2.426246166229248) + | > loss_feat: 6.161721229553223 (6.161721229553223) + | > loss_mel: 17.10713005065918 (17.10713005065918) + | > loss_duration: 1.6937129497528076 (1.6937129497528076) + | > loss_1: 29.704729080200195 (29.704729080200195) + + --> STEP: 2 + | > loss_disc: 2.5478756427764893 (2.599251389503479) + | > loss_disc_real_0: 0.26572534441947937 (0.2646775096654892) + | > loss_disc_real_1: 0.21075919270515442 (0.2236660271883011) + | > loss_disc_real_2: 0.2279881238937378 (0.23520877957344055) + | > loss_disc_real_3: 0.1979222148656845 (0.2025885432958603) + | > loss_disc_real_4: 0.21478986740112305 (0.2309538647532463) + | > loss_disc_real_5: 0.24423494935035706 (0.22692277282476425) + | > loss_0: 2.5478756427764893 (2.599251389503479) + | > loss_gen: 2.456578016281128 (2.3862487077713013) + | > loss_kl: 2.0591037273406982 (2.242674946784973) + | > loss_feat: 7.216153621673584 (6.688937425613403) + | > loss_mel: 19.11480140686035 (18.110965728759766) + | > loss_duration: 1.6378626823425293 (1.6657878160476685) + | > loss_1: 32.4844970703125 (31.094613075256348) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003629922866821289 (+9.66787338256836e-05) + | > avg_loss_disc: 2.599251389503479 (+0.05258488655090332) + | > avg_loss_disc_real_0: 0.2646775096654892 (+0.03676888346672058) + | > avg_loss_disc_real_1: 0.2236660271883011 (-0.06038308143615723) + | > avg_loss_disc_real_2: 0.23520877957344055 (+0.007656998932361603) + | > avg_loss_disc_real_3: 0.2025885432958603 (-0.04623229801654816) + | > avg_loss_disc_real_4: 0.2309538647532463 (-0.01727939397096634) + | > avg_loss_disc_real_5: 0.22692277282476425 (+0.005444467067718506) + | > avg_loss_0: 2.599251389503479 (+0.05258488655090332) + | > avg_loss_gen: 2.3862487077713013 (-0.06490659713745117) + | > avg_loss_kl: 2.242674946784973 (-0.0008665919303894043) + | > avg_loss_feat: 6.688937425613403 (+1.0360441207885742) + | > avg_loss_mel: 18.110965728759766 (+1.8463153839111328) + | > avg_loss_duration: 1.6657878160476685 (+0.002927064895629883) + | > avg_loss_1: 31.094613075256348 (+2.8195114135742188) + + + > EPOCH: 131/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:09:26)  + + --> TIME: 2025-04-19 01:09:32 -- STEP: 16/142 -- GLOBAL_STEP: 397900 + | > loss_disc: 2.528397798538208 (2.484802633523941) + | > loss_disc_real_0: 0.19166281819343567 (0.12292772065848112) + | > loss_disc_real_1: 0.21109148859977722 (0.20910388603806496) + | > loss_disc_real_2: 0.23003891110420227 (0.2276030769571662) + | > loss_disc_real_3: 0.22957579791545868 (0.23414126317948103) + | > loss_disc_real_4: 0.2336881458759308 (0.2305167494341731) + | > loss_disc_real_5: 0.2564096450805664 (0.22713173273950815) + | > loss_0: 2.528397798538208 (2.484802633523941) + | > grad_norm_0: tensor(33.1202, device='cuda:0') (tensor(16.8763, device='cuda:0')) + | > loss_gen: 2.3497707843780518 (2.3571745455265045) + | > loss_kl: 1.8182462453842163 (1.8094431310892105) + | > loss_feat: 6.676455974578857 (6.967477738857269) + | > loss_mel: 17.484689712524414 (17.794246196746826) + | > loss_duration: 1.4351423978805542 (1.441170834004879) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.764307022094727 (30.369512677192688) + | > grad_norm_1: tensor(343.2746, device='cuda:0') (tensor(281.7590, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.3394 (0.3306736350059509) + | > loader_time: 0.0051 (0.004526674747467041) + + + --> TIME: 2025-04-19 01:09:41 -- STEP: 41/142 -- GLOBAL_STEP: 397925 + | > loss_disc: 2.445950508117676 (2.47215100614036) + | > loss_disc_real_0: 0.1057126522064209 (0.12388009269063066) + | > loss_disc_real_1: 0.23376673460006714 (0.208438581446322) + | > loss_disc_real_2: 0.23037780821323395 (0.23018617455552265) + | > loss_disc_real_3: 0.2314494401216507 (0.22931808851114133) + | > loss_disc_real_4: 0.26951512694358826 (0.2286632282704842) + | > loss_disc_real_5: 0.21643877029418945 (0.23087374139122846) + | > loss_0: 2.445950508117676 (2.47215100614036) + | > grad_norm_0: tensor(5.8052, device='cuda:0') (tensor(15.7798, device='cuda:0')) + | > loss_gen: 2.4344825744628906 (2.379917586722025) + | > loss_kl: 1.6888444423675537 (1.826964384172021) + | > loss_feat: 7.015122890472412 (6.989165445653404) + | > loss_mel: 17.496448516845703 (17.78234812108482) + | > loss_duration: 1.4339232444763184 (1.4561871784489329) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.068822860717773 (30.434582966129955) + | > grad_norm_1: tensor(304.4414, device='cuda:0') (tensor(298.1489, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.3684 (0.34300264498082605) + | > loader_time: 0.0056 (0.005057166262370784) + + + --> TIME: 2025-04-19 01:09:51 -- STEP: 66/142 -- GLOBAL_STEP: 397950 + | > loss_disc: 2.716731548309326 (2.466905521624016) + | > loss_disc_real_0: 0.10470221191644669 (0.12532939061974036) + | > loss_disc_real_1: 0.2209274172782898 (0.20814509305990103) + | > loss_disc_real_2: 0.18291997909545898 (0.2271740838433757) + | > loss_disc_real_3: 0.23101699352264404 (0.227363476924824) + | > loss_disc_real_4: 0.2201048880815506 (0.22625544554356372) + | > loss_disc_real_5: 0.22959350049495697 (0.23092344945127313) + | > loss_0: 2.716731548309326 (2.466905521624016) + | > grad_norm_0: tensor(12.5555, device='cuda:0') (tensor(13.7262, device='cuda:0')) + | > loss_gen: 2.1432111263275146 (2.359006571047234) + | > loss_kl: 1.8639706373214722 (1.83611261844635) + | > loss_feat: 5.833241939544678 (6.963195923602942) + | > loss_mel: 15.708772659301758 (17.80814416480786) + | > loss_duration: 1.466544270515442 (1.4703408281008403) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.0157413482666 (30.436800147547867) + | > grad_norm_1: tensor(113.8845, device='cuda:0') (tensor(270.0735, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.3843 (0.3552062005707712) + | > loader_time: 0.0061 (0.005362402309070933) + + + --> TIME: 2025-04-19 01:10:01 -- STEP: 91/142 -- GLOBAL_STEP: 397975 + | > loss_disc: 2.506916046142578 (2.479023202435) + | > loss_disc_real_0: 0.15919934213161469 (0.1274594691592258) + | > loss_disc_real_1: 0.21245242655277252 (0.20873873492518624) + | > loss_disc_real_2: 0.228087916970253 (0.22721339418337896) + | > loss_disc_real_3: 0.23539762198925018 (0.22877627365536743) + | > loss_disc_real_4: 0.22967205941677094 (0.2283216960482545) + | > loss_disc_real_5: 0.207734152674675 (0.2335269041768797) + | > loss_0: 2.506916046142578 (2.479023202435) + | > grad_norm_0: tensor(13.2757, device='cuda:0') (tensor(12.7616, device='cuda:0')) + | > loss_gen: 2.3556761741638184 (2.336796110802953) + | > loss_kl: 1.7513526678085327 (1.8519715555421599) + | > loss_feat: 6.718015193939209 (6.903707504272461) + | > loss_mel: 18.504581451416016 (17.837040974543633) + | > loss_duration: 1.5004018545150757 (1.4819357788169778) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.830028533935547 (30.41145201043768) + | > grad_norm_1: tensor(342.8909, device='cuda:0') (tensor(260.9453, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.4025 (0.3663546253036667) + | > loader_time: 0.0063 (0.005613754083822064) + + + --> TIME: 2025-04-19 01:10:11 -- STEP: 116/142 -- GLOBAL_STEP: 398000 + | > loss_disc: 2.3577680587768555 (2.4713401876646897) + | > loss_disc_real_0: 0.09150644391775131 (0.12676688990202442) + | > loss_disc_real_1: 0.1962951123714447 (0.20848354363235935) + | > loss_disc_real_2: 0.21871605515480042 (0.22646450469719953) + | > loss_disc_real_3: 0.2345643788576126 (0.22887406639497856) + | > loss_disc_real_4: 0.25251704454421997 (0.22874168293743297) + | > loss_disc_real_5: 0.23146149516105652 (0.2327793781356565) + | > loss_0: 2.3577680587768555 (2.4713401876646897) + | > grad_norm_0: tensor(19.1831, device='cuda:0') (tensor(12.0507, device='cuda:0')) + | > loss_gen: 2.480478286743164 (2.333084143441298) + | > loss_kl: 1.9447747468948364 (1.8659481416488517) + | > loss_feat: 7.830358028411865 (6.909726027784677) + | > loss_mel: 18.391489028930664 (17.804167739276217) + | > loss_duration: 1.472693681716919 (1.488545444504968) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.11979293823242 (30.401471581952325) + | > grad_norm_1: tensor(588.5898, device='cuda:0') (tensor(260.4350, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.4292 (0.3768264815725129) + | > loader_time: 0.0069 (0.0058609103334361125) + + + --> TIME: 2025-04-19 01:10:23 -- STEP: 141/142 -- GLOBAL_STEP: 398025 + | > loss_disc: 2.7618119716644287 (2.4763931551723615) + | > loss_disc_real_0: 0.21404710412025452 (0.12730835404590507) + | > loss_disc_real_1: 0.23665925860404968 (0.209268719063583) + | > loss_disc_real_2: 0.21688371896743774 (0.22631168777638294) + | > loss_disc_real_3: 0.2027330994606018 (0.22833694073748081) + | > loss_disc_real_4: 0.1665688157081604 (0.22915943833530372) + | > loss_disc_real_5: 0.22903326153755188 (0.2340294467219224) + | > loss_0: 2.7618119716644287 (2.4763931551723615) + | > grad_norm_0: tensor(30.3594, device='cuda:0') (tensor(12.1189, device='cuda:0')) + | > loss_gen: 1.876037836074829 (2.3311606315856275) + | > loss_kl: 1.6314666271209717 (1.8727320390390165) + | > loss_feat: 5.1254143714904785 (6.897273334205574) + | > loss_mel: 12.777803421020508 (17.774878785965278) + | > loss_duration: 1.3489704132080078 (1.493632216825553) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 22.75969123840332 (30.369677090475744) + | > grad_norm_1: tensor(64.5215, device='cuda:0') (tensor(262.0904, device='cuda:0')) + | > current_lr_0: 0.00019675146691989817 + | > current_lr_1: 0.00019675146691989817 + | > step_time: 0.2205 (0.38719809478056355) + | > loader_time: 0.0026 (0.0060908337856860895) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.479661703109741 (2.479661703109741) + | > loss_disc_real_0: 0.18497054278850555 (0.18497054278850555) + | > loss_disc_real_1: 0.17954538762569427 (0.17954538762569427) + | > loss_disc_real_2: 0.2549797594547272 (0.2549797594547272) + | > loss_disc_real_3: 0.23360951244831085 (0.23360951244831085) + | > loss_disc_real_4: 0.20722350478172302 (0.20722350478172302) + | > loss_disc_real_5: 0.23547743260860443 (0.23547743260860443) + | > loss_0: 2.479661703109741 (2.479661703109741) + | > loss_gen: 2.3979134559631348 (2.3979134559631348) + | > loss_kl: 2.290621757507324 (2.290621757507324) + | > loss_feat: 6.854494571685791 (6.854494571685791) + | > loss_mel: 18.53070640563965 (18.53070640563965) + | > loss_duration: 1.6452547311782837 (1.6452547311782837) + | > loss_1: 31.718990325927734 (31.718990325927734) + + --> STEP: 1 + | > loss_disc: 2.66568922996521 (2.66568922996521) + | > loss_disc_real_0: 0.1637309342622757 (0.1637309342622757) + | > loss_disc_real_1: 0.1955961436033249 (0.1955961436033249) + | > loss_disc_real_2: 0.2576463520526886 (0.2576463520526886) + | > loss_disc_real_3: 0.2581118047237396 (0.2581118047237396) + | > loss_disc_real_4: 0.24290859699249268 (0.24290859699249268) + | > loss_disc_real_5: 0.24435536563396454 (0.24435536563396454) + | > loss_0: 2.66568922996521 (2.66568922996521) + | > loss_gen: 2.07519793510437 (2.07519793510437) + | > loss_kl: 2.3137319087982178 (2.3137319087982178) + | > loss_feat: 4.949457168579102 (4.949457168579102) + | > loss_mel: 15.255836486816406 (15.255836486816406) + | > loss_duration: 1.6854588985443115 (1.6854588985443115) + | > loss_1: 26.279682159423828 (26.279682159423828) + + --> STEP: 2 + | > loss_disc: 2.5463078022003174 (2.6059985160827637) + | > loss_disc_real_0: 0.19412200152873993 (0.1789264678955078) + | > loss_disc_real_1: 0.19656170904636383 (0.19607892632484436) + | > loss_disc_real_2: 0.2275659590959549 (0.24260615557432175) + | > loss_disc_real_3: 0.2014417052268982 (0.2297767549753189) + | > loss_disc_real_4: 0.2054286152124405 (0.22416860610246658) + | > loss_disc_real_5: 0.19234277307987213 (0.21834906935691833) + | > loss_0: 2.5463078022003174 (2.6059985160827637) + | > loss_gen: 2.1558287143707275 (2.115513324737549) + | > loss_kl: 2.287102222442627 (2.3004170656204224) + | > loss_feat: 7.105036735534668 (6.027246952056885) + | > loss_mel: 19.34086036682129 (17.298348426818848) + | > loss_duration: 1.645843505859375 (1.6656512022018433) + | > loss_1: 32.534671783447266 (29.407176971435547) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036020278930664062 (-2.7894973754882812e-05) + | > avg_loss_disc: 2.6059985160827637 (+0.006747126579284668) + | > avg_loss_disc_real_0: 0.1789264678955078 (-0.08575104176998138) + | > avg_loss_disc_real_1: 0.19607892632484436 (-0.027587100863456726) + | > avg_loss_disc_real_2: 0.24260615557432175 (+0.007397376000881195) + | > avg_loss_disc_real_3: 0.2297767549753189 (+0.027188211679458618) + | > avg_loss_disc_real_4: 0.22416860610246658 (-0.006785258650779724) + | > avg_loss_disc_real_5: 0.21834906935691833 (-0.008573703467845917) + | > avg_loss_0: 2.6059985160827637 (+0.006747126579284668) + | > avg_loss_gen: 2.115513324737549 (-0.27073538303375244) + | > avg_loss_kl: 2.3004170656204224 (+0.05774211883544922) + | > avg_loss_feat: 6.027246952056885 (-0.6616904735565186) + | > avg_loss_mel: 17.298348426818848 (-0.812617301940918) + | > avg_loss_duration: 1.6656512022018433 (-0.0001366138458251953) + | > avg_loss_1: 29.407176971435547 (-1.6874361038208008) + + + > EPOCH: 132/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:10:25)  + + --> TIME: 2025-04-19 01:10:34 -- STEP: 24/142 -- GLOBAL_STEP: 398050 + | > loss_disc: 2.4056997299194336 (2.4246325890223184) + | > loss_disc_real_0: 0.11547251045703888 (0.12041332107037306) + | > loss_disc_real_1: 0.22399082779884338 (0.19769212044775486) + | > loss_disc_real_2: 0.19366680085659027 (0.22293419080475965) + | > loss_disc_real_3: 0.20272715389728546 (0.22719306436677775) + | > loss_disc_real_4: 0.24722547829151154 (0.22712658221522966) + | > loss_disc_real_5: 0.24256819486618042 (0.23075646410385767) + | > loss_0: 2.4056997299194336 (2.4246325890223184) + | > grad_norm_0: tensor(11.6759, device='cuda:0') (tensor(13.1661, device='cuda:0')) + | > loss_gen: 2.4290266036987305 (2.387215375900268) + | > loss_kl: 1.9654757976531982 (1.832481433947881) + | > loss_feat: 6.762645244598389 (7.263778845469157) + | > loss_mel: 17.833139419555664 (17.919960618019104) + | > loss_duration: 1.4919371604919434 (1.449713294704755) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.48222541809082 (30.853149970372517) + | > grad_norm_1: tensor(363.7144, device='cuda:0') (tensor(260.7649, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.3433 (0.3330323298772176) + | > loader_time: 0.0049 (0.00461496909459432) + + + --> TIME: 2025-04-19 01:10:43 -- STEP: 49/142 -- GLOBAL_STEP: 398075 + | > loss_disc: 2.450080394744873 (2.447567404532919) + | > loss_disc_real_0: 0.12119000405073166 (0.12146398242639035) + | > loss_disc_real_1: 0.20047049224376678 (0.20406080174202823) + | > loss_disc_real_2: 0.23844169080257416 (0.2266708557703057) + | > loss_disc_real_3: 0.26087525486946106 (0.22838125788435645) + | > loss_disc_real_4: 0.22106750309467316 (0.22661763490462788) + | > loss_disc_real_5: 0.2741595506668091 (0.23341805290202705) + | > loss_0: 2.450080394744873 (2.447567404532919) + | > grad_norm_0: tensor(5.8457, device='cuda:0') (tensor(12.7847, device='cuda:0')) + | > loss_gen: 2.137838363647461 (2.3760775157383502) + | > loss_kl: 1.8965275287628174 (1.8486527593768374) + | > loss_feat: 6.567161560058594 (7.172113126637984) + | > loss_mel: 17.632335662841797 (17.883111506092302) + | > loss_duration: 1.4793437719345093 (1.4596022367477417) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.713205337524414 (30.739557344086315) + | > grad_norm_1: tensor(69.9113, device='cuda:0') (tensor(271.2235, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.3709 (0.34565349501006465) + | > loader_time: 0.0061 (0.0050542841152269) + + + --> TIME: 2025-04-19 01:10:53 -- STEP: 74/142 -- GLOBAL_STEP: 398100 + | > loss_disc: 2.4613020420074463 (2.4640329915124015) + | > loss_disc_real_0: 0.13470733165740967 (0.12426154013421084) + | > loss_disc_real_1: 0.2179703712463379 (0.20580918825155978) + | > loss_disc_real_2: 0.2262117713689804 (0.2267398187840307) + | > loss_disc_real_3: 0.23982587456703186 (0.22781312224027273) + | > loss_disc_real_4: 0.24101604521274567 (0.22825830329108882) + | > loss_disc_real_5: 0.22578202188014984 (0.23389717775422172) + | > loss_0: 2.4613020420074463 (2.4640329915124015) + | > grad_norm_0: tensor(6.6624, device='cuda:0') (tensor(11.8159, device='cuda:0')) + | > loss_gen: 2.3229947090148926 (2.350351684802287) + | > loss_kl: 1.8229777812957764 (1.8628556663925584) + | > loss_feat: 6.495754718780518 (7.054269416912182) + | > loss_mel: 17.730924606323242 (17.807736216364678) + | > loss_duration: 1.5008574724197388 (1.4725166961953449) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.87350845336914 (30.547729853037243) + | > grad_norm_1: tensor(50.8888, device='cuda:0') (tensor(244.6229, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.3907 (0.35808118613990586) + | > loader_time: 0.0062 (0.0053742640727275125) + + + --> TIME: 2025-04-19 01:11:03 -- STEP: 99/142 -- GLOBAL_STEP: 398125 + | > loss_disc: 2.498279094696045 (2.4633501154003716) + | > loss_disc_real_0: 0.11317434161901474 (0.125932232448549) + | > loss_disc_real_1: 0.2483130544424057 (0.20624700339153557) + | > loss_disc_real_2: 0.22643113136291504 (0.22591999940799945) + | > loss_disc_real_3: 0.24063003063201904 (0.22758166401675253) + | > loss_disc_real_4: 0.23310691118240356 (0.228001539905866) + | > loss_disc_real_5: 0.27161458134651184 (0.2338478366533915) + | > loss_0: 2.498279094696045 (2.4633501154003716) + | > grad_norm_0: tensor(9.4274, device='cuda:0') (tensor(11.0872, device='cuda:0')) + | > loss_gen: 2.4813246726989746 (2.3440509661279547) + | > loss_kl: 1.9680458307266235 (1.8813080968278828) + | > loss_feat: 7.153740406036377 (7.0164118824583115) + | > loss_mel: 19.33917808532715 (17.800314200044884) + | > loss_duration: 1.516953706741333 (1.4834471170348353) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.45924377441406 (30.525532423847853) + | > grad_norm_1: tensor(393.5364, device='cuda:0') (tensor(237.5487, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.4115 (0.37045647399594095) + | > loader_time: 0.0069 (0.005667575682052457) + + + --> TIME: 2025-04-19 01:11:14 -- STEP: 124/142 -- GLOBAL_STEP: 398150 + | > loss_disc: 2.445214033126831 (2.4680039748068774) + | > loss_disc_real_0: 0.11237968504428864 (0.127389027646953) + | > loss_disc_real_1: 0.22478623688220978 (0.20540293918982627) + | > loss_disc_real_2: 0.2746500074863434 (0.2265089625312436) + | > loss_disc_real_3: 0.259124755859375 (0.2280975812144818) + | > loss_disc_real_4: 0.26231440901756287 (0.22903304474969063) + | > loss_disc_real_5: 0.2378564178943634 (0.23368182093385728) + | > loss_0: 2.445214033126831 (2.4680039748068774) + | > grad_norm_0: tensor(7.2576, device='cuda:0') (tensor(10.5565, device='cuda:0')) + | > loss_gen: 2.5636065006256104 (2.339439255575981) + | > loss_kl: 1.9925638437271118 (1.8941475854765983) + | > loss_feat: 6.86525821685791 (6.963993964656707) + | > loss_mel: 18.4395694732666 (17.77502554462802) + | > loss_duration: 1.5367515087127686 (1.4903537804080595) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.397748947143555 (30.462960320134318) + | > grad_norm_1: tensor(181.8772, device='cuda:0') (tensor(223.4434, device='cuda:0')) + | > current_lr_0: 0.00019672687298653317 + | > current_lr_1: 0.00019672687298653317 + | > step_time: 0.4381 (0.38251065631066605) + | > loader_time: 0.0071 (0.005988347914911085) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.536048412322998 (2.536048412322998) + | > loss_disc_real_0: 0.08983311802148819 (0.08983311802148819) + | > loss_disc_real_1: 0.18918567895889282 (0.18918567895889282) + | > loss_disc_real_2: 0.2130352109670639 (0.2130352109670639) + | > loss_disc_real_3: 0.21445399522781372 (0.21445399522781372) + | > loss_disc_real_4: 0.23741626739501953 (0.23741626739501953) + | > loss_disc_real_5: 0.248317152261734 (0.248317152261734) + | > loss_0: 2.536048412322998 (2.536048412322998) + | > loss_gen: 2.005805492401123 (2.005805492401123) + | > loss_kl: 2.3383193016052246 (2.3383193016052246) + | > loss_feat: 6.794368743896484 (6.794368743896484) + | > loss_mel: 17.743101119995117 (17.743101119995117) + | > loss_duration: 1.6469752788543701 (1.6469752788543701) + | > loss_1: 30.5285701751709 (30.5285701751709) + + --> STEP: 1 + | > loss_disc: 2.5697743892669678 (2.5697743892669678) + | > loss_disc_real_0: 0.09883502870798111 (0.09883502870798111) + | > loss_disc_real_1: 0.21822156012058258 (0.21822156012058258) + | > loss_disc_real_2: 0.23665201663970947 (0.23665201663970947) + | > loss_disc_real_3: 0.25602468848228455 (0.25602468848228455) + | > loss_disc_real_4: 0.26368361711502075 (0.26368361711502075) + | > loss_disc_real_5: 0.27030178904533386 (0.27030178904533386) + | > loss_0: 2.5697743892669678 (2.5697743892669678) + | > loss_gen: 2.1511778831481934 (2.1511778831481934) + | > loss_kl: 2.5196099281311035 (2.5196099281311035) + | > loss_feat: 7.020196437835693 (7.020196437835693) + | > loss_mel: 19.128620147705078 (19.128620147705078) + | > loss_duration: 1.6715009212493896 (1.6715009212493896) + | > loss_1: 32.49110412597656 (32.49110412597656) + + --> STEP: 2 + | > loss_disc: 2.5538597106933594 (2.5618170499801636) + | > loss_disc_real_0: 0.1366882622241974 (0.11776164546608925) + | > loss_disc_real_1: 0.1813153326511383 (0.19976844638586044) + | > loss_disc_real_2: 0.2238738238811493 (0.23026292026042938) + | > loss_disc_real_3: 0.19750186800956726 (0.2267632782459259) + | > loss_disc_real_4: 0.2121741622686386 (0.23792888969182968) + | > loss_disc_real_5: 0.2585579752922058 (0.26442988216876984) + | > loss_0: 2.5538597106933594 (2.5618170499801636) + | > loss_gen: 1.9560540914535522 (2.053615987300873) + | > loss_kl: 1.9942922592163086 (2.256951093673706) + | > loss_feat: 6.507846832275391 (6.764021635055542) + | > loss_mel: 15.558389663696289 (17.343504905700684) + | > loss_duration: 1.6060714721679688 (1.6387861967086792) + | > loss_1: 27.62265396118164 (30.0568790435791) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003450632095336914 (-0.0001513957977294922) + | > avg_loss_disc: 2.5618170499801636 (-0.0441814661026001) + | > avg_loss_disc_real_0: 0.11776164546608925 (-0.061164822429418564) + | > avg_loss_disc_real_1: 0.19976844638586044 (+0.0036895200610160828) + | > avg_loss_disc_real_2: 0.23026292026042938 (-0.012343235313892365) + | > avg_loss_disc_real_3: 0.2267632782459259 (-0.0030134767293930054) + | > avg_loss_disc_real_4: 0.23792888969182968 (+0.013760283589363098) + | > avg_loss_disc_real_5: 0.26442988216876984 (+0.0460808128118515) + | > avg_loss_0: 2.5618170499801636 (-0.0441814661026001) + | > avg_loss_gen: 2.053615987300873 (-0.061897337436676025) + | > avg_loss_kl: 2.256951093673706 (-0.04346597194671631) + | > avg_loss_feat: 6.764021635055542 (+0.7367746829986572) + | > avg_loss_mel: 17.343504905700684 (+0.04515647888183594) + | > avg_loss_duration: 1.6387861967086792 (-0.026865005493164062) + | > avg_loss_1: 30.0568790435791 (+0.6497020721435547) + + + > EPOCH: 133/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:11:24)  + + --> TIME: 2025-04-19 01:11:27 -- STEP: 7/142 -- GLOBAL_STEP: 398175 + | > loss_disc: 2.487821578979492 (2.47703971181597) + | > loss_disc_real_0: 0.1813320368528366 (0.136285129402365) + | > loss_disc_real_1: 0.2511119246482849 (0.2143268883228302) + | > loss_disc_real_2: 0.21637599170207977 (0.23573968240192958) + | > loss_disc_real_3: 0.2201458364725113 (0.2225445189646312) + | > loss_disc_real_4: 0.24096255004405975 (0.22594031265803746) + | > loss_disc_real_5: 0.2281828075647354 (0.2421427901302065) + | > loss_0: 2.487821578979492 (2.47703971181597) + | > grad_norm_0: tensor(4.7170, device='cuda:0') (tensor(9.2607, device='cuda:0')) + | > loss_gen: 2.2639808654785156 (2.348241056714739) + | > loss_kl: 1.7520997524261475 (1.833023020199367) + | > loss_feat: 6.806858062744141 (6.922358512878418) + | > loss_mel: 17.095670700073242 (17.589150020054408) + | > loss_duration: 1.4328193664550781 (1.450445260320391) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.351428985595703 (30.143218176705496) + | > grad_norm_1: tensor(119.6149, device='cuda:0') (tensor(212.8722, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.3245 (0.3249451773507254) + | > loader_time: 0.004 (0.00410008430480957) + + + --> TIME: 2025-04-19 01:11:35 -- STEP: 32/142 -- GLOBAL_STEP: 398200 + | > loss_disc: 2.481870651245117 (2.4517802521586414) + | > loss_disc_real_0: 0.13702556490898132 (0.12666645715944466) + | > loss_disc_real_1: 0.18520353734493256 (0.20724866166710854) + | > loss_disc_real_2: 0.2145872861146927 (0.22733590425923467) + | > loss_disc_real_3: 0.21857665479183197 (0.2232868834398687) + | > loss_disc_real_4: 0.22735342383384705 (0.2286786469630897) + | > loss_disc_real_5: 0.24376767873764038 (0.2321401215158403) + | > loss_0: 2.481870651245117 (2.4517802521586414) + | > grad_norm_0: tensor(9.7879, device='cuda:0') (tensor(8.1731, device='cuda:0')) + | > loss_gen: 2.3010919094085693 (2.3549867421388626) + | > loss_kl: 1.8172787427902222 (1.8557598404586315) + | > loss_feat: 7.216101169586182 (7.0758475959300995) + | > loss_mel: 17.9375057220459 (17.845331013202667) + | > loss_duration: 1.4520299434661865 (1.4552101977169514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.724008560180664 (30.587135553359985) + | > grad_norm_1: tensor(236.6440, device='cuda:0') (tensor(162.6449, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.3545 (0.33692681789398193) + | > loader_time: 0.0051 (0.004903234541416167) + + + --> TIME: 2025-04-19 01:11:45 -- STEP: 57/142 -- GLOBAL_STEP: 398225 + | > loss_disc: 2.502106189727783 (2.45813120457164) + | > loss_disc_real_0: 0.10958651453256607 (0.12914925839817315) + | > loss_disc_real_1: 0.2145383059978485 (0.20722367261585437) + | > loss_disc_real_2: 0.24305762350559235 (0.2266135542539128) + | > loss_disc_real_3: 0.2421269714832306 (0.22676719123857064) + | > loss_disc_real_4: 0.2445264756679535 (0.22690667133582265) + | > loss_disc_real_5: 0.20702442526817322 (0.23172049433515782) + | > loss_0: 2.502106189727783 (2.45813120457164) + | > grad_norm_0: tensor(19.9587, device='cuda:0') (tensor(10.3226, device='cuda:0')) + | > loss_gen: 2.19516921043396 (2.350569720853839) + | > loss_kl: 1.9041622877120972 (1.8729314072090282) + | > loss_feat: 5.9261555671691895 (7.035092755367882) + | > loss_mel: 15.080986022949219 (17.77251715409128) + | > loss_duration: 1.5168461799621582 (1.4652933781607105) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.623319625854492 (30.496404580902635) + | > grad_norm_1: tensor(376.6129, device='cuda:0') (tensor(204.0250, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.3732 (0.3490063014783357) + | > loader_time: 0.0063 (0.005265089503505774) + + + --> TIME: 2025-04-19 01:11:55 -- STEP: 82/142 -- GLOBAL_STEP: 398250 + | > loss_disc: 2.5544724464416504 (2.475431355034433) + | > loss_disc_real_0: 0.22561708092689514 (0.13412859776943195) + | > loss_disc_real_1: 0.22634747624397278 (0.2083220783530212) + | > loss_disc_real_2: 0.22685515880584717 (0.2259565657231866) + | > loss_disc_real_3: 0.2532274127006531 (0.22683654071354284) + | > loss_disc_real_4: 0.23990494012832642 (0.228506030105963) + | > loss_disc_real_5: 0.21618686616420746 (0.22991812138295756) + | > loss_0: 2.5544724464416504 (2.475431355034433) + | > grad_norm_0: tensor(36.8363, device='cuda:0') (tensor(13.4980, device='cuda:0')) + | > loss_gen: 2.47009539604187 (2.3588590185816694) + | > loss_kl: 1.9674021005630493 (1.8913694794585065) + | > loss_feat: 6.468703746795654 (7.004437062798477) + | > loss_mel: 17.439298629760742 (17.725784057524145) + | > loss_duration: 1.5645561218261719 (1.4776288503553805) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.91005516052246 (30.45807854722186) + | > grad_norm_1: tensor(60.0536, device='cuda:0') (tensor(244.2245, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.3919 (0.3612325976534588) + | > loader_time: 0.0069 (0.005581675506219632) + + + --> TIME: 2025-04-19 01:12:05 -- STEP: 107/142 -- GLOBAL_STEP: 398275 + | > loss_disc: 2.451359510421753 (2.4861822774477083) + | > loss_disc_real_0: 0.13320675492286682 (0.13300930569383593) + | > loss_disc_real_1: 0.20687085390090942 (0.20965282187283596) + | > loss_disc_real_2: 0.20442628860473633 (0.22761891350567898) + | > loss_disc_real_3: 0.2013654261827469 (0.22888903166646155) + | > loss_disc_real_4: 0.19425800442695618 (0.23094461064472377) + | > loss_disc_real_5: 0.22681619226932526 (0.23318365285886783) + | > loss_0: 2.451359510421753 (2.4861822774477083) + | > grad_norm_0: tensor(15.4072, device='cuda:0') (tensor(14.7350, device='cuda:0')) + | > loss_gen: 2.2695860862731934 (2.3584240619267254) + | > loss_kl: 1.8437272310256958 (1.8987047761400169) + | > loss_feat: 6.961840629577637 (6.95571520172547) + | > loss_mel: 17.36026382446289 (17.752287098180474) + | > loss_duration: 1.498049020767212 (1.4866745416249068) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.933467864990234 (30.451805863424998) + | > grad_norm_1: tensor(320.1740, device='cuda:0') (tensor(266.8775, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.4212 (0.37265186443507115) + | > loader_time: 0.0068 (0.005862195915150866) + + + --> TIME: 2025-04-19 01:12:16 -- STEP: 132/142 -- GLOBAL_STEP: 398300 + | > loss_disc: 2.6407904624938965 (2.4846303607478277) + | > loss_disc_real_0: 0.2043464481830597 (0.13242883901252897) + | > loss_disc_real_1: 0.19012024998664856 (0.20959435787164804) + | > loss_disc_real_2: 0.2629605233669281 (0.22736111588098787) + | > loss_disc_real_3: 0.22502976655960083 (0.22894259519649274) + | > loss_disc_real_4: 0.2520136535167694 (0.2309852218763395) + | > loss_disc_real_5: 0.25138649344444275 (0.2330353313097448) + | > loss_0: 2.6407904624938965 (2.4846303607478277) + | > grad_norm_0: tensor(27.5418, device='cuda:0') (tensor(14.0539, device='cuda:0')) + | > loss_gen: 2.114638328552246 (2.3441543506853506) + | > loss_kl: 1.98752760887146 (1.8999247921235634) + | > loss_feat: 4.860647678375244 (6.884268298293605) + | > loss_mel: 14.034676551818848 (17.59731355580417) + | > loss_duration: 1.4867801666259766 (1.492263053402756) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 24.484270095825195 (30.217924146941215) + | > grad_norm_1: tensor(178.6574, device='cuda:0') (tensor(261.5978, device='cuda:0')) + | > current_lr_0: 0.00019670228212740986 + | > current_lr_1: 0.00019670228212740986 + | > step_time: 0.4444 (0.38468714374484436) + | > loader_time: 0.0076 (0.006142849271947688) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3550808429718018 (2.3550808429718018) + | > loss_disc_real_0: 0.19374603033065796 (0.19374603033065796) + | > loss_disc_real_1: 0.24855954945087433 (0.24855954945087433) + | > loss_disc_real_2: 0.21590840816497803 (0.21590840816497803) + | > loss_disc_real_3: 0.2291264533996582 (0.2291264533996582) + | > loss_disc_real_4: 0.2179080843925476 (0.2179080843925476) + | > loss_disc_real_5: 0.24688105285167694 (0.24688105285167694) + | > loss_0: 2.3550808429718018 (2.3550808429718018) + | > loss_gen: 2.744410276412964 (2.744410276412964) + | > loss_kl: 2.4254508018493652 (2.4254508018493652) + | > loss_feat: 7.334115982055664 (7.334115982055664) + | > loss_mel: 19.145906448364258 (19.145906448364258) + | > loss_duration: 1.6416983604431152 (1.6416983604431152) + | > loss_1: 33.29158020019531 (33.29158020019531) + + --> STEP: 1 + | > loss_disc: 2.603834390640259 (2.603834390640259) + | > loss_disc_real_0: 0.22208969295024872 (0.22208969295024872) + | > loss_disc_real_1: 0.28505706787109375 (0.28505706787109375) + | > loss_disc_real_2: 0.252256840467453 (0.252256840467453) + | > loss_disc_real_3: 0.25105950236320496 (0.25105950236320496) + | > loss_disc_real_4: 0.2632943093776703 (0.2632943093776703) + | > loss_disc_real_5: 0.2574795186519623 (0.2574795186519623) + | > loss_0: 2.603834390640259 (2.603834390640259) + | > loss_gen: 2.488534688949585 (2.488534688949585) + | > loss_kl: 2.521652936935425 (2.521652936935425) + | > loss_feat: 6.277061462402344 (6.277061462402344) + | > loss_mel: 16.312063217163086 (16.312063217163086) + | > loss_duration: 1.6828933954238892 (1.6828933954238892) + | > loss_1: 29.282207489013672 (29.282207489013672) + + --> STEP: 2 + | > loss_disc: 2.5189285278320312 (2.561381459236145) + | > loss_disc_real_0: 0.21824528276920319 (0.22016748785972595) + | > loss_disc_real_1: 0.2575142979621887 (0.27128568291664124) + | > loss_disc_real_2: 0.23430660367012024 (0.24328172206878662) + | > loss_disc_real_3: 0.22247624397277832 (0.23676787316799164) + | > loss_disc_real_4: 0.23052005469799042 (0.24690718203783035) + | > loss_disc_real_5: 0.27453044056892395 (0.2660049796104431) + | > loss_0: 2.5189285278320312 (2.561381459236145) + | > loss_gen: 2.4651918411254883 (2.4768632650375366) + | > loss_kl: 2.2089996337890625 (2.3653262853622437) + | > loss_feat: 5.339280128479004 (5.808170795440674) + | > loss_mel: 15.898168563842773 (16.10511589050293) + | > loss_duration: 1.6398680210113525 (1.6613807082176208) + | > loss_1: 27.5515079498291 (28.416857719421387) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035539865493774414 (+0.00010335445404052734) + | > avg_loss_disc: 2.561381459236145 (-0.0004355907440185547) + | > avg_loss_disc_real_0: 0.22016748785972595 (+0.1024058423936367) + | > avg_loss_disc_real_1: 0.27128568291664124 (+0.07151723653078079) + | > avg_loss_disc_real_2: 0.24328172206878662 (+0.013018801808357239) + | > avg_loss_disc_real_3: 0.23676787316799164 (+0.010004594922065735) + | > avg_loss_disc_real_4: 0.24690718203783035 (+0.008978292346000671) + | > avg_loss_disc_real_5: 0.2660049796104431 (+0.0015750974416732788) + | > avg_loss_0: 2.561381459236145 (-0.0004355907440185547) + | > avg_loss_gen: 2.4768632650375366 (+0.4232472777366638) + | > avg_loss_kl: 2.3653262853622437 (+0.1083751916885376) + | > avg_loss_feat: 5.808170795440674 (-0.9558508396148682) + | > avg_loss_mel: 16.10511589050293 (-1.238389015197754) + | > avg_loss_duration: 1.6613807082176208 (+0.02259451150894165) + | > avg_loss_1: 28.416857719421387 (-1.6400213241577148) + + + > EPOCH: 134/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:12:22)  + + --> TIME: 2025-04-19 01:12:28 -- STEP: 15/142 -- GLOBAL_STEP: 398325 + | > loss_disc: 2.624931812286377 (2.4576543807983398) + | > loss_disc_real_0: 0.17152179777622223 (0.12536304742097854) + | > loss_disc_real_1: 0.21290051937103271 (0.19831123054027558) + | > loss_disc_real_2: 0.2364526093006134 (0.22425745328267416) + | > loss_disc_real_3: 0.24222393333911896 (0.22758891781171162) + | > loss_disc_real_4: 0.23885226249694824 (0.22852506438891093) + | > loss_disc_real_5: 0.25794389843940735 (0.2318664898475011) + | > loss_0: 2.624931812286377 (2.4576543807983398) + | > grad_norm_0: tensor(11.9725, device='cuda:0') (tensor(7.0870, device='cuda:0')) + | > loss_gen: 2.241046667098999 (2.3207515239715577) + | > loss_kl: 1.7706458568572998 (1.8685504118601481) + | > loss_feat: 6.79072904586792 (7.107924938201904) + | > loss_mel: 17.319355010986328 (17.794302241007486) + | > loss_duration: 1.4364516735076904 (1.4462497393290201) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.5582275390625 (30.537779235839842) + | > grad_norm_1: tensor(114.8766, device='cuda:0') (tensor(112.1839, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.3345 (0.32795146306355794) + | > loader_time: 0.0051 (0.004641898473103841) + + + --> TIME: 2025-04-19 01:12:37 -- STEP: 40/142 -- GLOBAL_STEP: 398350 + | > loss_disc: 2.3996219635009766 (2.449899172782898) + | > loss_disc_real_0: 0.13965736329555511 (0.12105186711996793) + | > loss_disc_real_1: 0.17867588996887207 (0.2019894912838936) + | > loss_disc_real_2: 0.20624767243862152 (0.22451723031699658) + | > loss_disc_real_3: 0.22028087079524994 (0.228520692512393) + | > loss_disc_real_4: 0.20654946565628052 (0.23009726777672768) + | > loss_disc_real_5: 0.24370957911014557 (0.23359881564974785) + | > loss_0: 2.3996219635009766 (2.449899172782898) + | > grad_norm_0: tensor(6.8577, device='cuda:0') (tensor(6.2446, device='cuda:0')) + | > loss_gen: 2.4040870666503906 (2.3315181732177734) + | > loss_kl: 1.7678693532943726 (1.8521519303321838) + | > loss_feat: 7.15786075592041 (7.055989503860474) + | > loss_mel: 18.44388771057129 (17.825745582580563) + | > loss_duration: 1.4295878410339355 (1.457662034034729) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.203292846679688 (30.5230673789978) + | > grad_norm_1: tensor(96.1252, device='cuda:0') (tensor(109.3990, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.3583 (0.34121503829956057) + | > loader_time: 0.0058 (0.0050736904144287115) + + + --> TIME: 2025-04-19 01:12:47 -- STEP: 65/142 -- GLOBAL_STEP: 398375 + | > loss_disc: 2.4925336837768555 (2.452817230958205) + | > loss_disc_real_0: 0.12259884178638458 (0.1253517060325696) + | > loss_disc_real_1: 0.22442671656608582 (0.20297540793052085) + | > loss_disc_real_2: 0.21637225151062012 (0.2226823022732368) + | > loss_disc_real_3: 0.22901776432991028 (0.22630693385234246) + | > loss_disc_real_4: 0.22452734410762787 (0.2289102519933994) + | > loss_disc_real_5: 0.24230428040027618 (0.234402237030176) + | > loss_0: 2.4925336837768555 (2.452817230958205) + | > grad_norm_0: tensor(7.9428, device='cuda:0') (tensor(6.3708, device='cuda:0')) + | > loss_gen: 2.362675905227661 (2.3333868686969463) + | > loss_kl: 1.8553075790405273 (1.865772685637841) + | > loss_feat: 6.978227615356445 (7.035255512824425) + | > loss_mel: 17.894039154052734 (17.77536241091214) + | > loss_duration: 1.4731311798095703 (1.4701867268635676) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.56338119506836 (30.47996435899001) + | > grad_norm_1: tensor(97.2731, device='cuda:0') (tensor(106.6791, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.3929 (0.353884194447444) + | > loader_time: 0.0063 (0.005448124958918645) + + + --> TIME: 2025-04-19 01:12:57 -- STEP: 90/142 -- GLOBAL_STEP: 398400 + | > loss_disc: 2.611342668533325 (2.462484396828545) + | > loss_disc_real_0: 0.1574924886226654 (0.1270264894598061) + | > loss_disc_real_1: 0.1949339359998703 (0.2046645366483264) + | > loss_disc_real_2: 0.23833228647708893 (0.22292294634713067) + | > loss_disc_real_3: 0.2266944795846939 (0.22836262434720994) + | > loss_disc_real_4: 0.21956562995910645 (0.2297396524084939) + | > loss_disc_real_5: 0.23285488784313202 (0.23456461115015878) + | > loss_0: 2.611342668533325 (2.462484396828545) + | > grad_norm_0: tensor(4.7482, device='cuda:0') (tensor(6.5971, device='cuda:0')) + | > loss_gen: 2.0426511764526367 (2.3232667075263134) + | > loss_kl: 1.799727201461792 (1.8770042286978827) + | > loss_feat: 5.128437042236328 (6.9580769909752735) + | > loss_mel: 15.230137825012207 (17.70771185557047) + | > loss_duration: 1.522655963897705 (1.4812220599916246) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.723609924316406 (30.347281879848904) + | > grad_norm_1: tensor(117.9933, device='cuda:0') (tensor(122.3707, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.4081 (0.3665418227513631) + | > loader_time: 0.0064 (0.005737259652879503) + + + --> TIME: 2025-04-19 01:13:08 -- STEP: 115/142 -- GLOBAL_STEP: 398425 + | > loss_disc: 2.393198013305664 (2.4685077211131214) + | > loss_disc_real_0: 0.11280937492847443 (0.12868492266406187) + | > loss_disc_real_1: 0.1797512024641037 (0.20505676049253213) + | > loss_disc_real_2: 0.16656042635440826 (0.22293801048527592) + | > loss_disc_real_3: 0.2093566507101059 (0.22893786482189013) + | > loss_disc_real_4: 0.19695419073104858 (0.2295779579359552) + | > loss_disc_real_5: 0.18625786900520325 (0.2348041692505712) + | > loss_0: 2.393198013305664 (2.4685077211131214) + | > grad_norm_0: tensor(12.4757, device='cuda:0') (tensor(6.9889, device='cuda:0')) + | > loss_gen: 2.3614161014556885 (2.3152430016061536) + | > loss_kl: 1.8791743516921997 (1.888178042743517) + | > loss_feat: 7.416699409484863 (6.896177942856498) + | > loss_mel: 17.68402862548828 (17.633239978292714) + | > loss_duration: 1.5233774185180664 (1.4877324892126995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.864696502685547 (30.220571501358695) + | > grad_norm_1: tensor(51.4353, device='cuda:0') (tensor(138.0511, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.4432 (0.37878291917883833) + | > loader_time: 0.0074 (0.006032495913298234) + + + --> TIME: 2025-04-19 01:13:19 -- STEP: 140/142 -- GLOBAL_STEP: 398450 + | > loss_disc: 2.495504856109619 (2.47080169234957) + | > loss_disc_real_0: 0.16075026988983154 (0.12945356805409705) + | > loss_disc_real_1: 0.20505860447883606 (0.20534161978534288) + | > loss_disc_real_2: 0.2325337529182434 (0.22416465356945992) + | > loss_disc_real_3: 0.20906949043273926 (0.22927133132304464) + | > loss_disc_real_4: 0.2235340178012848 (0.22985648353184973) + | > loss_disc_real_5: 0.19766154885292053 (0.23481573260256222) + | > loss_0: 2.495504856109619 (2.47080169234957) + | > grad_norm_0: tensor(6.9254, device='cuda:0') (tensor(7.0351, device='cuda:0')) + | > loss_gen: 2.279157876968384 (2.318656281062535) + | > loss_kl: 1.796810507774353 (1.8922717962946212) + | > loss_feat: 7.60098123550415 (6.9152356692722865) + | > loss_mel: 17.723012924194336 (17.63671020780291) + | > loss_duration: 1.5599894523620605 (1.4934966632298061) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.959951400756836 (30.25637068067278) + | > grad_norm_1: tensor(226.5211, device='cuda:0') (tensor(138.7936, device='cuda:0')) + | > current_lr_0: 0.00019667769434214392 + | > current_lr_1: 0.00019667769434214392 + | > step_time: 0.4576 (0.39120367595127653) + | > loader_time: 0.0074 (0.006299115930284773) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5878751277923584 (2.5878751277923584) + | > loss_disc_real_0: 0.17326627671718597 (0.17326627671718597) + | > loss_disc_real_1: 0.2332434058189392 (0.2332434058189392) + | > loss_disc_real_2: 0.2890741527080536 (0.2890741527080536) + | > loss_disc_real_3: 0.228383406996727 (0.228383406996727) + | > loss_disc_real_4: 0.27078795433044434 (0.27078795433044434) + | > loss_disc_real_5: 0.24388155341148376 (0.24388155341148376) + | > loss_0: 2.5878751277923584 (2.5878751277923584) + | > loss_gen: 2.3522846698760986 (2.3522846698760986) + | > loss_kl: 2.4615042209625244 (2.4615042209625244) + | > loss_feat: 5.99294900894165 (5.99294900894165) + | > loss_mel: 17.289854049682617 (17.289854049682617) + | > loss_duration: 1.6619386672973633 (1.6619386672973633) + | > loss_1: 29.758533477783203 (29.758533477783203) + + --> STEP: 1 + | > loss_disc: 2.7628912925720215 (2.7628912925720215) + | > loss_disc_real_0: 0.16557356715202332 (0.16557356715202332) + | > loss_disc_real_1: 0.2722036838531494 (0.2722036838531494) + | > loss_disc_real_2: 0.274016797542572 (0.274016797542572) + | > loss_disc_real_3: 0.23921290040016174 (0.23921290040016174) + | > loss_disc_real_4: 0.3115246593952179 (0.3115246593952179) + | > loss_disc_real_5: 0.2973942458629608 (0.2973942458629608) + | > loss_0: 2.7628912925720215 (2.7628912925720215) + | > loss_gen: 2.3203134536743164 (2.3203134536743164) + | > loss_kl: 2.5864548683166504 (2.5864548683166504) + | > loss_feat: 6.9844183921813965 (6.9844183921813965) + | > loss_mel: 17.996448516845703 (17.996448516845703) + | > loss_duration: 1.6775178909301758 (1.6775178909301758) + | > loss_1: 31.56515121459961 (31.56515121459961) + + --> STEP: 2 + | > loss_disc: 2.6153690814971924 (2.689130187034607) + | > loss_disc_real_0: 0.15395796298980713 (0.15976576507091522) + | > loss_disc_real_1: 0.28174513578414917 (0.2769744098186493) + | > loss_disc_real_2: 0.2557836174964905 (0.26490020751953125) + | > loss_disc_real_3: 0.24538084864616394 (0.24229687452316284) + | > loss_disc_real_4: 0.2684441804885864 (0.28998441994190216) + | > loss_disc_real_5: 0.24399232864379883 (0.2706932872533798) + | > loss_0: 2.6153690814971924 (2.689130187034607) + | > loss_gen: 2.356759786605835 (2.3385366201400757) + | > loss_kl: 2.111363172531128 (2.348909020423889) + | > loss_feat: 5.977046012878418 (6.480732202529907) + | > loss_mel: 16.233774185180664 (17.115111351013184) + | > loss_duration: 1.624733805656433 (1.6511258482933044) + | > loss_1: 28.30367660522461 (29.93441390991211) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034216642379760742 (-0.0001323223114013672) + | > avg_loss_disc: 2.689130187034607 (+0.12774872779846191) + | > avg_loss_disc_real_0: 0.15976576507091522 (-0.06040172278881073) + | > avg_loss_disc_real_1: 0.2769744098186493 (+0.005688726902008057) + | > avg_loss_disc_real_2: 0.26490020751953125 (+0.02161848545074463) + | > avg_loss_disc_real_3: 0.24229687452316284 (+0.005529001355171204) + | > avg_loss_disc_real_4: 0.28998441994190216 (+0.04307723790407181) + | > avg_loss_disc_real_5: 0.2706932872533798 (+0.0046883076429367065) + | > avg_loss_0: 2.689130187034607 (+0.12774872779846191) + | > avg_loss_gen: 2.3385366201400757 (-0.13832664489746094) + | > avg_loss_kl: 2.348909020423889 (-0.016417264938354492) + | > avg_loss_feat: 6.480732202529907 (+0.6725614070892334) + | > avg_loss_mel: 17.115111351013184 (+1.009995460510254) + | > avg_loss_duration: 1.6511258482933044 (-0.010254859924316406) + | > avg_loss_1: 29.93441390991211 (+1.5175561904907227) + + + > EPOCH: 135/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:13:21)  + + --> TIME: 2025-04-19 01:13:30 -- STEP: 23/142 -- GLOBAL_STEP: 398475 + | > loss_disc: 2.3464951515197754 (2.455834699713665) + | > loss_disc_real_0: 0.09695515781641006 (0.12341063890768134) + | > loss_disc_real_1: 0.2074591964483261 (0.20846259010874707) + | > loss_disc_real_2: 0.23022806644439697 (0.23059473802214084) + | > loss_disc_real_3: 0.2235874980688095 (0.22808642102324445) + | > loss_disc_real_4: 0.22442515194416046 (0.23654026894465738) + | > loss_disc_real_5: 0.2381116896867752 (0.22826166515764984) + | > loss_0: 2.3464951515197754 (2.455834699713665) + | > grad_norm_0: tensor(5.6415, device='cuda:0') (tensor(8.1289, device='cuda:0')) + | > loss_gen: 2.492070198059082 (2.3733436853989303) + | > loss_kl: 1.7724274396896362 (1.8509366356808206) + | > loss_feat: 7.270970344543457 (7.2432996086452315) + | > loss_mel: 18.659378051757812 (17.956384161244266) + | > loss_duration: 1.4937515258789062 (1.4485516962797746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.6885986328125 (30.872515719869863) + | > grad_norm_1: tensor(142.6783, device='cuda:0') (tensor(174.6577, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.3513 (0.33387365548507025) + | > loader_time: 0.0051 (0.004810457644255265) + + + --> TIME: 2025-04-19 01:13:39 -- STEP: 48/142 -- GLOBAL_STEP: 398500 + | > loss_disc: 2.407057046890259 (2.4679045925537753) + | > loss_disc_real_0: 0.112260602414608 (0.12695756492515406) + | > loss_disc_real_1: 0.18703697621822357 (0.20786285772919655) + | > loss_disc_real_2: 0.1955597996711731 (0.2270786485945185) + | > loss_disc_real_3: 0.21899327635765076 (0.22913361402849355) + | > loss_disc_real_4: 0.20672647655010223 (0.23162131197750568) + | > loss_disc_real_5: 0.21958373486995697 (0.23342807466785112) + | > loss_0: 2.407057046890259 (2.4679045925537753) + | > grad_norm_0: tensor(6.2566, device='cuda:0') (tensor(7.7098, device='cuda:0')) + | > loss_gen: 2.210418462753296 (2.3539703836043677) + | > loss_kl: 2.007887363433838 (1.862836557130019) + | > loss_feat: 7.106040954589844 (7.133454571167628) + | > loss_mel: 18.70279884338379 (17.956548889478047) + | > loss_duration: 1.4699130058288574 (1.4594166005651157) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.497058868408203 (30.76622684796651) + | > grad_norm_1: tensor(117.4756, device='cuda:0') (tensor(148.9851, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.3727 (0.34897506237030035) + | > loader_time: 0.0058 (0.005208879709243774) + + + --> TIME: 2025-04-19 01:13:49 -- STEP: 73/142 -- GLOBAL_STEP: 398525 + | > loss_disc: 2.549943685531616 (2.4886742389365417) + | > loss_disc_real_0: 0.10224858671426773 (0.12875577991139403) + | > loss_disc_real_1: 0.20738931000232697 (0.20832617948316548) + | > loss_disc_real_2: 0.20598141849040985 (0.22866111299762987) + | > loss_disc_real_3: 0.20486906170845032 (0.22980477262849677) + | > loss_disc_real_4: 0.22023184597492218 (0.23126763656531293) + | > loss_disc_real_5: 0.22652731835842133 (0.2356748436003515) + | > loss_0: 2.549943685531616 (2.4886742389365417) + | > grad_norm_0: tensor(15.5699, device='cuda:0') (tensor(7.7593, device='cuda:0')) + | > loss_gen: 2.2278101444244385 (2.3349312168278105) + | > loss_kl: 1.943649411201477 (1.8828358731857717) + | > loss_feat: 7.22614049911499 (7.04139427289571) + | > loss_mel: 17.549911499023438 (17.91675646011143) + | > loss_duration: 1.5316966772079468 (1.4724311991913681) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.47920799255371 (30.648348951992922) + | > grad_norm_1: tensor(114.2249, device='cuda:0') (tensor(139.4542, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.4081 (0.36302508393379107) + | > loader_time: 0.0067 (0.005616201113348138) + + + --> TIME: 2025-04-19 01:14:00 -- STEP: 98/142 -- GLOBAL_STEP: 398550 + | > loss_disc: 2.560471773147583 (2.4912466297344302) + | > loss_disc_real_0: 0.11103606224060059 (0.13039051452461559) + | > loss_disc_real_1: 0.21193163096904755 (0.20798594565415868) + | > loss_disc_real_2: 0.2278163731098175 (0.22739763360242454) + | > loss_disc_real_3: 0.19448329508304596 (0.22995413535711717) + | > loss_disc_real_4: 0.24185387790203094 (0.23101324603265647) + | > loss_disc_real_5: 0.20385952293872833 (0.23721338276352202) + | > loss_0: 2.560471773147583 (2.4912466297344302) + | > grad_norm_0: tensor(11.2292, device='cuda:0') (tensor(9.2404, device='cuda:0')) + | > loss_gen: 2.431983470916748 (2.3303201490518997) + | > loss_kl: 1.8389071226119995 (1.8951035737991333) + | > loss_feat: 7.640664577484131 (6.937116481819933) + | > loss_mel: 18.755781173706055 (17.747862251437436) + | > loss_duration: 1.4767745733261108 (1.4829155498621418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.14411163330078 (30.39331800110486) + | > grad_norm_1: tensor(379.4179, device='cuda:0') (tensor(178.1968, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.4262 (0.3755709443773542) + | > loader_time: 0.0081 (0.005928406910020478) + + + --> TIME: 2025-04-19 01:14:11 -- STEP: 123/142 -- GLOBAL_STEP: 398575 + | > loss_disc: 2.4293599128723145 (2.491968401079257) + | > loss_disc_real_0: 0.12882529199123383 (0.13145917502602916) + | > loss_disc_real_1: 0.21012704074382782 (0.2091428550278268) + | > loss_disc_real_2: 0.2135433405637741 (0.2272928535211377) + | > loss_disc_real_3: 0.22748835384845734 (0.2295809831561112) + | > loss_disc_real_4: 0.20154324173927307 (0.23116289793960448) + | > loss_disc_real_5: 0.18126727640628815 (0.23596425087955908) + | > loss_0: 2.4293599128723145 (2.491968401079257) + | > grad_norm_0: tensor(20.5488, device='cuda:0') (tensor(10.0514, device='cuda:0')) + | > loss_gen: 2.210289716720581 (2.323007135856442) + | > loss_kl: 1.980002522468567 (1.9031640455974796) + | > loss_feat: 7.1298508644104 (6.912068188675052) + | > loss_mel: 18.44601058959961 (17.694523160050554) + | > loss_duration: 1.507808804512024 (1.4899254639943442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.273962020874023 (30.32268806395492) + | > grad_norm_1: tensor(331.2092, device='cuda:0') (tensor(195.6363, device='cuda:0')) + | > current_lr_0: 0.00019665310963035113 + | > current_lr_1: 0.00019665310963035113 + | > step_time: 0.4696 (0.38835327218218546) + | > loader_time: 0.0077 (0.006172641506039999) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.496873378753662 (2.496873378753662) + | > loss_disc_real_0: 0.17756371200084686 (0.17756371200084686) + | > loss_disc_real_1: 0.26820698380470276 (0.26820698380470276) + | > loss_disc_real_2: 0.23702548444271088 (0.23702548444271088) + | > loss_disc_real_3: 0.26015421748161316 (0.26015421748161316) + | > loss_disc_real_4: 0.24172714352607727 (0.24172714352607727) + | > loss_disc_real_5: 0.2819210886955261 (0.2819210886955261) + | > loss_0: 2.496873378753662 (2.496873378753662) + | > loss_gen: 2.5589346885681152 (2.5589346885681152) + | > loss_kl: 2.469420909881592 (2.469420909881592) + | > loss_feat: 7.484982490539551 (7.484982490539551) + | > loss_mel: 19.409473419189453 (19.409473419189453) + | > loss_duration: 1.646165132522583 (1.646165132522583) + | > loss_1: 33.56897735595703 (33.56897735595703) + + --> STEP: 1 + | > loss_disc: 2.537687063217163 (2.537687063217163) + | > loss_disc_real_0: 0.1784786581993103 (0.1784786581993103) + | > loss_disc_real_1: 0.21474333107471466 (0.21474333107471466) + | > loss_disc_real_2: 0.26149749755859375 (0.26149749755859375) + | > loss_disc_real_3: 0.2883714437484741 (0.2883714437484741) + | > loss_disc_real_4: 0.25967225432395935 (0.25967225432395935) + | > loss_disc_real_5: 0.2875189185142517 (0.2875189185142517) + | > loss_0: 2.537687063217163 (2.537687063217163) + | > loss_gen: 2.6453769207000732 (2.6453769207000732) + | > loss_kl: 2.542694091796875 (2.542694091796875) + | > loss_feat: 6.806052207946777 (6.806052207946777) + | > loss_mel: 19.21230697631836 (19.21230697631836) + | > loss_duration: 1.663761019706726 (1.663761019706726) + | > loss_1: 32.87018966674805 (32.87018966674805) + + --> STEP: 2 + | > loss_disc: 2.4920706748962402 (2.5148788690567017) + | > loss_disc_real_0: 0.16634167730808258 (0.17241016775369644) + | > loss_disc_real_1: 0.21079149842262268 (0.21276741474866867) + | > loss_disc_real_2: 0.2929247319698334 (0.27721111476421356) + | > loss_disc_real_3: 0.24850915372371674 (0.26844029873609543) + | > loss_disc_real_4: 0.2551441192626953 (0.25740818679332733) + | > loss_disc_real_5: 0.23519308865070343 (0.26135600358247757) + | > loss_0: 2.4920706748962402 (2.5148788690567017) + | > loss_gen: 2.502159357070923 (2.573768138885498) + | > loss_kl: 2.063660144805908 (2.3031771183013916) + | > loss_feat: 8.053056716918945 (7.429554462432861) + | > loss_mel: 19.97159194946289 (19.591949462890625) + | > loss_duration: 1.6249558925628662 (1.6443584561347961) + | > loss_1: 34.215423583984375 (33.54280662536621) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00357663631439209 (+0.00015497207641601562) + | > avg_loss_disc: 2.5148788690567017 (-0.17425131797790527) + | > avg_loss_disc_real_0: 0.17241016775369644 (+0.01264440268278122) + | > avg_loss_disc_real_1: 0.21276741474866867 (-0.06420699506998062) + | > avg_loss_disc_real_2: 0.27721111476421356 (+0.012310907244682312) + | > avg_loss_disc_real_3: 0.26844029873609543 (+0.026143424212932587) + | > avg_loss_disc_real_4: 0.25740818679332733 (-0.03257623314857483) + | > avg_loss_disc_real_5: 0.26135600358247757 (-0.009337283670902252) + | > avg_loss_0: 2.5148788690567017 (-0.17425131797790527) + | > avg_loss_gen: 2.573768138885498 (+0.23523151874542236) + | > avg_loss_kl: 2.3031771183013916 (-0.04573190212249756) + | > avg_loss_feat: 7.429554462432861 (+0.9488222599029541) + | > avg_loss_mel: 19.591949462890625 (+2.4768381118774414) + | > avg_loss_duration: 1.6443584561347961 (-0.006767392158508301) + | > avg_loss_1: 33.54280662536621 (+3.6083927154541016) + + + > EPOCH: 136/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:14:21)  + + --> TIME: 2025-04-19 01:14:24 -- STEP: 6/142 -- GLOBAL_STEP: 398600 + | > loss_disc: 2.409705638885498 (2.413866957028707) + | > loss_disc_real_0: 0.10649716854095459 (0.10395095249017079) + | > loss_disc_real_1: 0.20170998573303223 (0.20828584084908167) + | > loss_disc_real_2: 0.24921882152557373 (0.23141580075025558) + | > loss_disc_real_3: 0.19621959328651428 (0.2201353684067726) + | > loss_disc_real_4: 0.19409745931625366 (0.22965629895528158) + | > loss_disc_real_5: 0.2095663994550705 (0.23223193486531576) + | > loss_0: 2.409705638885498 (2.413866957028707) + | > grad_norm_0: tensor(4.0129, device='cuda:0') (tensor(11.9303, device='cuda:0')) + | > loss_gen: 2.3776063919067383 (2.424832820892334) + | > loss_kl: 1.7195535898208618 (1.7966849207878113) + | > loss_feat: 7.41892671585083 (7.250900030136108) + | > loss_mel: 19.135684967041016 (18.593489011128742) + | > loss_duration: 1.4592926502227783 (1.4502610961596172) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.11106491088867 (31.516167958577473) + | > grad_norm_1: tensor(325.7914, device='cuda:0') (tensor(354.0838, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.3334 (0.32776423295338947) + | > loader_time: 0.0061 (0.004646579424540202) + + + --> TIME: 2025-04-19 01:14:33 -- STEP: 31/142 -- GLOBAL_STEP: 398625 + | > loss_disc: 2.488757610321045 (2.4552881717681885) + | > loss_disc_real_0: 0.12412827461957932 (0.11875462892555422) + | > loss_disc_real_1: 0.21982182562351227 (0.2087125648413935) + | > loss_disc_real_2: 0.23975735902786255 (0.22605764673602197) + | > loss_disc_real_3: 0.22383210062980652 (0.2272594662443284) + | > loss_disc_real_4: 0.236570805311203 (0.2303378317625292) + | > loss_disc_real_5: 0.24452009797096252 (0.22917789365014724) + | > loss_0: 2.488757610321045 (2.4552881717681885) + | > grad_norm_0: tensor(11.8803, device='cuda:0') (tensor(11.8125, device='cuda:0')) + | > loss_gen: 2.373806953430176 (2.3747143822331584) + | > loss_kl: 1.8637683391571045 (1.8501627599039385) + | > loss_feat: 7.470648765563965 (7.18473300626201) + | > loss_mel: 17.924047470092773 (18.034091518771266) + | > loss_duration: 1.4472689628601074 (1.4539996231755903) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.079540252685547 (30.897701201900357) + | > grad_norm_1: tensor(433.8042, device='cuda:0') (tensor(291.9635, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.3523 (0.33698934124362084) + | > loader_time: 0.0054 (0.004999583767306419) + + + --> TIME: 2025-04-19 01:14:42 -- STEP: 56/142 -- GLOBAL_STEP: 398650 + | > loss_disc: 2.441668748855591 (2.473847325359072) + | > loss_disc_real_0: 0.13820435106754303 (0.12456710969230955) + | > loss_disc_real_1: 0.22594839334487915 (0.21174369805625506) + | > loss_disc_real_2: 0.24955841898918152 (0.22660729608365468) + | > loss_disc_real_3: 0.24999845027923584 (0.22929789977414267) + | > loss_disc_real_4: 0.21166300773620605 (0.2285026740282774) + | > loss_disc_real_5: 0.2687962055206299 (0.23227934379662787) + | > loss_0: 2.441668748855591 (2.473847325359072) + | > grad_norm_0: tensor(10.3422, device='cuda:0') (tensor(10.9873, device='cuda:0')) + | > loss_gen: 2.381934881210327 (2.3478358260222856) + | > loss_kl: 1.9316155910491943 (1.868509258542742) + | > loss_feat: 6.615952014923096 (7.050845376082829) + | > loss_mel: 18.14147186279297 (17.856467553547457) + | > loss_duration: 1.4909029006958008 (1.4636775531939097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.561878204345703 (30.587335552488053) + | > grad_norm_1: tensor(291.0253, device='cuda:0') (tensor(257.4228, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.3723 (0.34882801345416475) + | > loader_time: 0.0059 (0.00524224979536874) + + + --> TIME: 2025-04-19 01:14:52 -- STEP: 81/142 -- GLOBAL_STEP: 398675 + | > loss_disc: 2.531604766845703 (2.4798528323938815) + | > loss_disc_real_0: 0.10426656156778336 (0.12583883502233173) + | > loss_disc_real_1: 0.25474146008491516 (0.2111563789255825) + | > loss_disc_real_2: 0.23891431093215942 (0.22729383372230294) + | > loss_disc_real_3: 0.2267865240573883 (0.22954324899632253) + | > loss_disc_real_4: 0.21853113174438477 (0.22959590896412177) + | > loss_disc_real_5: 0.24897916615009308 (0.23333918459621472) + | > loss_0: 2.531604766845703 (2.4798528323938815) + | > grad_norm_0: tensor(12.0904, device='cuda:0') (tensor(10.7759, device='cuda:0')) + | > loss_gen: 2.2588202953338623 (2.3347087848333667) + | > loss_kl: 1.8213095664978027 (1.880413317386015) + | > loss_feat: 6.731940746307373 (7.012393845452203) + | > loss_mel: 17.407299041748047 (17.884991775324316) + | > loss_duration: 1.492645502090454 (1.4744563912167958) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.712013244628906 (30.586964006777162) + | > grad_norm_1: tensor(197.3239, device='cuda:0') (tensor(236.9105, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.3935 (0.36051053176691505) + | > loader_time: 0.0069 (0.0055477265958432794) + + + --> TIME: 2025-04-19 01:15:02 -- STEP: 106/142 -- GLOBAL_STEP: 398700 + | > loss_disc: 2.463977098464966 (2.490064378054636) + | > loss_disc_real_0: 0.12457295507192612 (0.13008862178561817) + | > loss_disc_real_1: 0.23854754865169525 (0.20993624107455308) + | > loss_disc_real_2: 0.21320614218711853 (0.22783460510226916) + | > loss_disc_real_3: 0.24701383709907532 (0.22979925572872162) + | > loss_disc_real_4: 0.2313428819179535 (0.2312502255136112) + | > loss_disc_real_5: 0.24669425189495087 (0.2341896644178426) + | > loss_0: 2.463977098464966 (2.490064378054636) + | > grad_norm_0: tensor(11.4825, device='cuda:0') (tensor(10.5279, device='cuda:0')) + | > loss_gen: 2.377584934234619 (2.3182163328494676) + | > loss_kl: 1.953552007675171 (1.8944319374156449) + | > loss_feat: 7.139236927032471 (6.927719017244735) + | > loss_mel: 17.901206970214844 (17.86239884934335) + | > loss_duration: 1.5165815353393555 (1.4854678579096519) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.888160705566406 (30.488233872179716) + | > grad_norm_1: tensor(342.9741, device='cuda:0') (tensor(230.9123, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.4101 (0.3709638591082591) + | > loader_time: 0.0073 (0.005827049039444831) + + + --> TIME: 2025-04-19 01:15:13 -- STEP: 131/142 -- GLOBAL_STEP: 398725 + | > loss_disc: 2.489375352859497 (2.4918213491221413) + | > loss_disc_real_0: 0.11344480514526367 (0.13196866276837485) + | > loss_disc_real_1: 0.1964736133813858 (0.20936791632921642) + | > loss_disc_real_2: 0.2281436175107956 (0.22692798988054727) + | > loss_disc_real_3: 0.20880736410617828 (0.22954356204007417) + | > loss_disc_real_4: 0.2122742384672165 (0.2318048288348977) + | > loss_disc_real_5: 0.21984735131263733 (0.23341250146618323) + | > loss_0: 2.489375352859497 (2.4918213491221413) + | > grad_norm_0: tensor(16.3664, device='cuda:0') (tensor(10.5204, device='cuda:0')) + | > loss_gen: 2.44209623336792 (2.3115718528514613) + | > loss_kl: 1.9670734405517578 (1.8973025802437586) + | > loss_feat: 7.374542236328125 (6.904542504376128) + | > loss_mel: 17.914894104003906 (17.827667498406562) + | > loss_duration: 1.5184065103530884 (1.4909951677759181) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.217012405395508 (30.432079460784678) + | > grad_norm_1: tensor(217.5545, device='cuda:0') (tensor(220.8088, device='cuda:0')) + | > current_lr_0: 0.00019662852799164733 + | > current_lr_1: 0.00019662852799164733 + | > step_time: 0.4625 (0.3844602508399323) + | > loader_time: 0.0074 (0.006108234856874887) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.470731019973755 (2.470731019973755) + | > loss_disc_real_0: 0.1749039888381958 (0.1749039888381958) + | > loss_disc_real_1: 0.2202806919813156 (0.2202806919813156) + | > loss_disc_real_2: 0.2336268126964569 (0.2336268126964569) + | > loss_disc_real_3: 0.26257771253585815 (0.26257771253585815) + | > loss_disc_real_4: 0.24115462601184845 (0.24115462601184845) + | > loss_disc_real_5: 0.2714020609855652 (0.2714020609855652) + | > loss_0: 2.470731019973755 (2.470731019973755) + | > loss_gen: 2.486046314239502 (2.486046314239502) + | > loss_kl: 2.153334140777588 (2.153334140777588) + | > loss_feat: 7.653344631195068 (7.653344631195068) + | > loss_mel: 19.561079025268555 (19.561079025268555) + | > loss_duration: 1.6538734436035156 (1.6538734436035156) + | > loss_1: 33.50767517089844 (33.50767517089844) + + --> STEP: 1 + | > loss_disc: 2.5116446018218994 (2.5116446018218994) + | > loss_disc_real_0: 0.18939705193042755 (0.18939705193042755) + | > loss_disc_real_1: 0.2309124767780304 (0.2309124767780304) + | > loss_disc_real_2: 0.2558102011680603 (0.2558102011680603) + | > loss_disc_real_3: 0.25143498182296753 (0.25143498182296753) + | > loss_disc_real_4: 0.24011065065860748 (0.24011065065860748) + | > loss_disc_real_5: 0.26772645115852356 (0.26772645115852356) + | > loss_0: 2.5116446018218994 (2.5116446018218994) + | > loss_gen: 2.4549331665039062 (2.4549331665039062) + | > loss_kl: 2.484485149383545 (2.484485149383545) + | > loss_feat: 6.309745788574219 (6.309745788574219) + | > loss_mel: 18.211196899414062 (18.211196899414062) + | > loss_duration: 1.68850839138031 (1.68850839138031) + | > loss_1: 31.14887046813965 (31.14887046813965) + + --> STEP: 2 + | > loss_disc: 2.6518056392669678 (2.5817251205444336) + | > loss_disc_real_0: 0.23003587126731873 (0.20971646159887314) + | > loss_disc_real_1: 0.18876518309116364 (0.20983882993459702) + | > loss_disc_real_2: 0.22005443274974823 (0.23793231695890427) + | > loss_disc_real_3: 0.25113537907600403 (0.2512851804494858) + | > loss_disc_real_4: 0.2403966337442398 (0.24025364220142365) + | > loss_disc_real_5: 0.2960920035839081 (0.2819092273712158) + | > loss_0: 2.6518056392669678 (2.5817251205444336) + | > loss_gen: 2.2371926307678223 (2.3460628986358643) + | > loss_kl: 2.159575939178467 (2.322030544281006) + | > loss_feat: 6.678430557250977 (6.494088172912598) + | > loss_mel: 16.884748458862305 (17.547972679138184) + | > loss_duration: 1.6040916442871094 (1.6463000178337097) + | > loss_1: 29.56403923034668 (30.356454849243164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033212900161743164 (-0.00025534629821777344) + | > avg_loss_disc: 2.5817251205444336 (+0.06684625148773193) + | > avg_loss_disc_real_0: 0.20971646159887314 (+0.0373062938451767) + | > avg_loss_disc_real_1: 0.20983882993459702 (-0.0029285848140716553) + | > avg_loss_disc_real_2: 0.23793231695890427 (-0.039278797805309296) + | > avg_loss_disc_real_3: 0.2512851804494858 (-0.01715511828660965) + | > avg_loss_disc_real_4: 0.24025364220142365 (-0.017154544591903687) + | > avg_loss_disc_real_5: 0.2819092273712158 (+0.02055322378873825) + | > avg_loss_0: 2.5817251205444336 (+0.06684625148773193) + | > avg_loss_gen: 2.3460628986358643 (-0.2277052402496338) + | > avg_loss_kl: 2.322030544281006 (+0.018853425979614258) + | > avg_loss_feat: 6.494088172912598 (-0.9354662895202637) + | > avg_loss_mel: 17.547972679138184 (-2.0439767837524414) + | > avg_loss_duration: 1.6463000178337097 (+0.0019415616989135742) + | > avg_loss_1: 30.356454849243164 (-3.186351776123047) + + + > EPOCH: 137/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:15:20)  + + --> TIME: 2025-04-19 01:15:26 -- STEP: 14/142 -- GLOBAL_STEP: 398750 + | > loss_disc: 2.439460515975952 (2.4570796489715576) + | > loss_disc_real_0: 0.11769374459981918 (0.12158384759511266) + | > loss_disc_real_1: 0.2009800523519516 (0.20708106564623968) + | > loss_disc_real_2: 0.23202155530452728 (0.22829755076340266) + | > loss_disc_real_3: 0.20452314615249634 (0.23034501395055226) + | > loss_disc_real_4: 0.23190896213054657 (0.22201127878257207) + | > loss_disc_real_5: 0.24844099581241608 (0.24028190970420837) + | > loss_0: 2.439460515975952 (2.4570796489715576) + | > grad_norm_0: tensor(12.1882, device='cuda:0') (tensor(13.4034, device='cuda:0')) + | > loss_gen: 2.3575499057769775 (2.3379794699805125) + | > loss_kl: 1.8094922304153442 (1.8095600605010986) + | > loss_feat: 7.19092321395874 (7.111808947154453) + | > loss_mel: 17.501358032226562 (17.684979506901332) + | > loss_duration: 1.4480156898498535 (1.4408700381006514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.30733871459961 (30.385198048182897) + | > grad_norm_1: tensor(404.0056, device='cuda:0') (tensor(292.7439, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.3383 (0.33107740538460867) + | > loader_time: 0.0046 (0.004521659442356655) + + + --> TIME: 2025-04-19 01:15:34 -- STEP: 39/142 -- GLOBAL_STEP: 398775 + | > loss_disc: 2.3503522872924805 (2.4074471302521534) + | > loss_disc_real_0: 0.12408142536878586 (0.11493615404917644) + | > loss_disc_real_1: 0.1995917111635208 (0.20377403306655398) + | > loss_disc_real_2: 0.22631609439849854 (0.22570206377750787) + | > loss_disc_real_3: 0.26037612557411194 (0.22587858178676704) + | > loss_disc_real_4: 0.2904738485813141 (0.2233537263595141) + | > loss_disc_real_5: 0.29205521941185 (0.23156487827117628) + | > loss_0: 2.3503522872924805 (2.4074471302521534) + | > grad_norm_0: tensor(4.1228, device='cuda:0') (tensor(11.5547, device='cuda:0')) + | > loss_gen: 2.576390027999878 (2.386287200145232) + | > loss_kl: 2.0290260314941406 (1.8799205284852247) + | > loss_feat: 7.520291328430176 (7.243086680387839) + | > loss_mel: 18.204402923583984 (17.898342205927925) + | > loss_duration: 1.4811365604400635 (1.4553262300980396) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.81124496459961 (30.862962820591072) + | > grad_norm_1: tensor(335.5736, device='cuda:0') (tensor(298.6257, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.359 (0.34193809215839094) + | > loader_time: 0.0058 (0.005141044274354592) + + + --> TIME: 2025-04-19 01:15:44 -- STEP: 64/142 -- GLOBAL_STEP: 398800 + | > loss_disc: 2.473971128463745 (2.440969668328762) + | > loss_disc_real_0: 0.18499764800071716 (0.12072980008088052) + | > loss_disc_real_1: 0.21590672433376312 (0.20704339700751007) + | > loss_disc_real_2: 0.20934392511844635 (0.2247450682334602) + | > loss_disc_real_3: 0.25180116295814514 (0.22686659637838602) + | > loss_disc_real_4: 0.22530023753643036 (0.22482627420686185) + | > loss_disc_real_5: 0.2244599461555481 (0.2338839320000261) + | > loss_0: 2.473971128463745 (2.440969668328762) + | > grad_norm_0: tensor(11.1722, device='cuda:0') (tensor(11.0936, device='cuda:0')) + | > loss_gen: 2.227296829223633 (2.3546875342726716) + | > loss_kl: 1.9472993612289429 (1.8991152085363865) + | > loss_feat: 6.26506233215332 (7.082192748785019) + | > loss_mel: 17.395761489868164 (17.79030722379685) + | > loss_duration: 1.512547254562378 (1.4674122333526611) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.34796714782715 (30.59371480345726) + | > grad_norm_1: tensor(244.9908, device='cuda:0') (tensor(273.6906, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.3865 (0.3567661978304386) + | > loader_time: 0.0064 (0.005518652498722078) + + + --> TIME: 2025-04-19 01:15:54 -- STEP: 89/142 -- GLOBAL_STEP: 398825 + | > loss_disc: 2.430652141571045 (2.437758767203) + | > loss_disc_real_0: 0.1449037492275238 (0.12222468325596177) + | > loss_disc_real_1: 0.24734553694725037 (0.2058476788944073) + | > loss_disc_real_2: 0.22064143419265747 (0.22383299336004792) + | > loss_disc_real_3: 0.18101763725280762 (0.2259679294369194) + | > loss_disc_real_4: 0.2446904331445694 (0.226229892018136) + | > loss_disc_real_5: 0.20999674499034882 (0.23126913088091303) + | > loss_0: 2.430652141571045 (2.437758767203) + | > grad_norm_0: tensor(15.3604, device='cuda:0') (tensor(11.4075, device='cuda:0')) + | > loss_gen: 2.3442461490631104 (2.3578548565339514) + | > loss_kl: 2.014528512954712 (1.9084146183528257) + | > loss_feat: 6.968430519104004 (7.086436437756828) + | > loss_mel: 17.627994537353516 (17.770699897508948) + | > loss_duration: 1.4875760078430176 (1.4787392897552316) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.44277572631836 (30.602144969983048) + | > grad_norm_1: tensor(402.6564, device='cuda:0') (tensor(284.4970, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.4021 (0.3672179961472415) + | > loader_time: 0.007 (0.005783943647748969) + + + --> TIME: 2025-04-19 01:16:05 -- STEP: 114/142 -- GLOBAL_STEP: 398850 + | > loss_disc: 2.435285806655884 (2.453505871588724) + | > loss_disc_real_0: 0.08352148532867432 (0.12318495497630354) + | > loss_disc_real_1: 0.20119203627109528 (0.20643347619395508) + | > loss_disc_real_2: 0.23943062126636505 (0.2263080728681464) + | > loss_disc_real_3: 0.21668750047683716 (0.22806726147731146) + | > loss_disc_real_4: 0.2399226576089859 (0.22760564742381112) + | > loss_disc_real_5: 0.2522985339164734 (0.23261743195747075) + | > loss_0: 2.435285806655884 (2.453505871588724) + | > grad_norm_0: tensor(8.3794, device='cuda:0') (tensor(11.5243, device='cuda:0')) + | > loss_gen: 2.3566324710845947 (2.352278293224803) + | > loss_kl: 1.9581040143966675 (1.9182392254210354) + | > loss_feat: 7.086509704589844 (7.0376180180332115) + | > loss_mel: 18.020389556884766 (17.810028059440743) + | > loss_duration: 1.5077629089355469 (1.4864343749849418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.929397583007812 (30.60459782784445) + | > grad_norm_1: tensor(77.2697, device='cuda:0') (tensor(277.8071, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.4215 (0.3773800113744903) + | > loader_time: 0.0073 (0.006065092588725843) + + + --> TIME: 2025-04-19 01:16:16 -- STEP: 139/142 -- GLOBAL_STEP: 398875 + | > loss_disc: 2.5099503993988037 (2.455338140185789) + | > loss_disc_real_0: 0.08938684314489365 (0.12391137664052222) + | > loss_disc_real_1: 0.21723444759845734 (0.20717660557451867) + | > loss_disc_real_2: 0.23383285105228424 (0.22645113558220348) + | > loss_disc_real_3: 0.22485004365444183 (0.22808079871771147) + | > loss_disc_real_4: 0.2486456036567688 (0.22758762013140343) + | > loss_disc_real_5: 0.23330058157444 (0.23241576895439367) + | > loss_0: 2.5099503993988037 (2.455338140185789) + | > grad_norm_0: tensor(6.3701, device='cuda:0') (tensor(11.4530, device='cuda:0')) + | > loss_gen: 2.3756518363952637 (2.3482201648273056) + | > loss_kl: 1.9249236583709717 (1.92136937336956) + | > loss_feat: 6.914777755737305 (7.018558903563794) + | > loss_mel: 17.544282913208008 (17.78106401292539) + | > loss_duration: 1.52924382686615 (1.4924408506146434) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.28887939453125 (30.56165317837283) + | > grad_norm_1: tensor(249.1196, device='cuda:0') (tensor(276.1335, device='cuda:0')) + | > current_lr_0: 0.00019660394942564837 + | > current_lr_1: 0.00019660394942564837 + | > step_time: 0.4538 (0.38912316878064923) + | > loader_time: 0.0077 (0.006330423217883213) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.542417526245117 (2.542417526245117) + | > loss_disc_real_0: 0.22179964184761047 (0.22179964184761047) + | > loss_disc_real_1: 0.2804892361164093 (0.2804892361164093) + | > loss_disc_real_2: 0.2411184310913086 (0.2411184310913086) + | > loss_disc_real_3: 0.2215367704629898 (0.2215367704629898) + | > loss_disc_real_4: 0.2269524484872818 (0.2269524484872818) + | > loss_disc_real_5: 0.1999456286430359 (0.1999456286430359) + | > loss_0: 2.542417526245117 (2.542417526245117) + | > loss_gen: 2.2952239513397217 (2.2952239513397217) + | > loss_kl: 2.1144163608551025 (2.1144163608551025) + | > loss_feat: 5.044621467590332 (5.044621467590332) + | > loss_mel: 16.06744384765625 (16.06744384765625) + | > loss_duration: 1.6333932876586914 (1.6333932876586914) + | > loss_1: 27.15509796142578 (27.15509796142578) + + --> STEP: 1 + | > loss_disc: 2.386930465698242 (2.386930465698242) + | > loss_disc_real_0: 0.1447209268808365 (0.1447209268808365) + | > loss_disc_real_1: 0.2758646607398987 (0.2758646607398987) + | > loss_disc_real_2: 0.2319657802581787 (0.2319657802581787) + | > loss_disc_real_3: 0.2309207171201706 (0.2309207171201706) + | > loss_disc_real_4: 0.24008023738861084 (0.24008023738861084) + | > loss_disc_real_5: 0.19663605093955994 (0.19663605093955994) + | > loss_0: 2.386930465698242 (2.386930465698242) + | > loss_gen: 2.4851200580596924 (2.4851200580596924) + | > loss_kl: 2.691049098968506 (2.691049098968506) + | > loss_feat: 6.553728103637695 (6.553728103637695) + | > loss_mel: 16.159648895263672 (16.159648895263672) + | > loss_duration: 1.6545801162719727 (1.6545801162719727) + | > loss_1: 29.54412841796875 (29.54412841796875) + + --> STEP: 2 + | > loss_disc: 2.618656873703003 (2.5027936697006226) + | > loss_disc_real_0: 0.1835574209690094 (0.16413917392492294) + | > loss_disc_real_1: 0.28234797716140747 (0.2791063189506531) + | > loss_disc_real_2: 0.23659676313400269 (0.2342812716960907) + | > loss_disc_real_3: 0.2379322201013565 (0.23442646861076355) + | > loss_disc_real_4: 0.23073457181453705 (0.23540740460157394) + | > loss_disc_real_5: 0.22868840396404266 (0.2126622274518013) + | > loss_0: 2.618656873703003 (2.5027936697006226) + | > loss_gen: 2.2678399085998535 (2.376479983329773) + | > loss_kl: 2.020066738128662 (2.355557918548584) + | > loss_feat: 6.108052730560303 (6.330890417098999) + | > loss_mel: 16.55084228515625 (16.35524559020996) + | > loss_duration: 1.6273608207702637 (1.6409704685211182) + | > loss_1: 28.57416343688965 (29.0591459274292) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003978848457336426 (+0.0006575584411621094) + | > avg_loss_disc: 2.5027936697006226 (-0.07893145084381104) + | > avg_loss_disc_real_0: 0.16413917392492294 (-0.045577287673950195) + | > avg_loss_disc_real_1: 0.2791063189506531 (+0.06926748901605606) + | > avg_loss_disc_real_2: 0.2342812716960907 (-0.003651045262813568) + | > avg_loss_disc_real_3: 0.23442646861076355 (-0.01685871183872223) + | > avg_loss_disc_real_4: 0.23540740460157394 (-0.004846237599849701) + | > avg_loss_disc_real_5: 0.2126622274518013 (-0.06924699991941452) + | > avg_loss_0: 2.5027936697006226 (-0.07893145084381104) + | > avg_loss_gen: 2.376479983329773 (+0.03041708469390869) + | > avg_loss_kl: 2.355557918548584 (+0.033527374267578125) + | > avg_loss_feat: 6.330890417098999 (-0.16319775581359863) + | > avg_loss_mel: 16.35524559020996 (-1.1927270889282227) + | > avg_loss_duration: 1.6409704685211182 (-0.005329549312591553) + | > avg_loss_1: 29.0591459274292 (-1.2973089218139648) + + + > EPOCH: 138/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:16:19)  + + --> TIME: 2025-04-19 01:16:27 -- STEP: 22/142 -- GLOBAL_STEP: 398900 + | > loss_disc: 2.45084810256958 (2.4371043226935645) + | > loss_disc_real_0: 0.17967602610588074 (0.11873330243609169) + | > loss_disc_real_1: 0.1749068647623062 (0.21158582852645355) + | > loss_disc_real_2: 0.20527416467666626 (0.22310792112892325) + | > loss_disc_real_3: 0.2544056177139282 (0.22938975285400043) + | > loss_disc_real_4: 0.26772114634513855 (0.23004748265851627) + | > loss_disc_real_5: 0.22242708504199982 (0.22465479373931885) + | > loss_0: 2.45084810256958 (2.4371043226935645) + | > grad_norm_0: tensor(18.4436, device='cuda:0') (tensor(13.3497, device='cuda:0')) + | > loss_gen: 2.542818307876587 (2.3966893499547783) + | > loss_kl: 1.9670839309692383 (1.8506274006583474) + | > loss_feat: 7.470510005950928 (7.215886159376665) + | > loss_mel: 17.921154022216797 (17.733163963664662) + | > loss_duration: 1.4063129425048828 (1.4445048299702732) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.307878494262695 (30.640871568159625) + | > grad_norm_1: tensor(563.8236, device='cuda:0') (tensor(304.3786, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.3421 (0.3315126354044134) + | > loader_time: 0.0056 (0.004803408275951038) + + + --> TIME: 2025-04-19 01:16:36 -- STEP: 47/142 -- GLOBAL_STEP: 398925 + | > loss_disc: 2.3870773315429688 (2.4527982397282377) + | > loss_disc_real_0: 0.1422000676393509 (0.12336975811643804) + | > loss_disc_real_1: 0.20536012947559357 (0.2103104829154116) + | > loss_disc_real_2: 0.23508748412132263 (0.22412885662089002) + | > loss_disc_real_3: 0.20975568890571594 (0.22739550883465626) + | > loss_disc_real_4: 0.2240981161594391 (0.225632203386185) + | > loss_disc_real_5: 0.23086291551589966 (0.22919988315156165) + | > loss_0: 2.3870773315429688 (2.4527982397282377) + | > grad_norm_0: tensor(13.2883, device='cuda:0') (tensor(12.1005, device='cuda:0')) + | > loss_gen: 2.474783420562744 (2.3656212522628466) + | > loss_kl: 1.9256272315979004 (1.858245776054707) + | > loss_feat: 7.4677886962890625 (7.048195534564079) + | > loss_mel: 17.82340431213379 (17.627162973931497) + | > loss_duration: 1.438694953918457 (1.457562213248395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.130298614501953 (30.35678745837922) + | > grad_norm_1: tensor(354.9960, device='cuda:0') (tensor(251.0308, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.3634 (0.34463840849856114) + | > loader_time: 0.0061 (0.005242033207670171) + + + --> TIME: 2025-04-19 01:16:46 -- STEP: 72/142 -- GLOBAL_STEP: 398950 + | > loss_disc: 2.3512139320373535 (2.454644021060732) + | > loss_disc_real_0: 0.10459176450967789 (0.1243275800305936) + | > loss_disc_real_1: 0.15990687906742096 (0.20964119604064357) + | > loss_disc_real_2: 0.2467958778142929 (0.22485612643261751) + | > loss_disc_real_3: 0.24204716086387634 (0.2275819710145394) + | > loss_disc_real_4: 0.20412449538707733 (0.2278654682967398) + | > loss_disc_real_5: 0.2017175257205963 (0.22857687270475757) + | > loss_0: 2.3512139320373535 (2.454644021060732) + | > grad_norm_0: tensor(9.2588, device='cuda:0') (tensor(10.7903, device='cuda:0')) + | > loss_gen: 2.345715284347534 (2.3529459999667277) + | > loss_kl: 1.8018269538879395 (1.8719579809241824) + | > loss_feat: 6.834372520446777 (6.9998778104782104) + | > loss_mel: 16.757129669189453 (17.696862273746067) + | > loss_duration: 1.466112732887268 (1.4690719644228618) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.205156326293945 (30.39071578449673) + | > grad_norm_1: tensor(275.8562, device='cuda:0') (tensor(233.9893, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.3946 (0.35671626528104144) + | > loader_time: 0.0063 (0.005567858616511027) + + + --> TIME: 2025-04-19 01:16:56 -- STEP: 97/142 -- GLOBAL_STEP: 398975 + | > loss_disc: 2.503777027130127 (2.465027622340881) + | > loss_disc_real_0: 0.15250526368618011 (0.12715834540497392) + | > loss_disc_real_1: 0.19443608820438385 (0.2088290142644312) + | > loss_disc_real_2: 0.23905174434185028 (0.22610932035544484) + | > loss_disc_real_3: 0.23038411140441895 (0.22882482424839257) + | > loss_disc_real_4: 0.23120509088039398 (0.22970088763335317) + | > loss_disc_real_5: 0.23779228329658508 (0.22923456516462504) + | > loss_0: 2.503777027130127 (2.465027622340881) + | > grad_norm_0: tensor(11.6800, device='cuda:0') (tensor(10.6562, device='cuda:0')) + | > loss_gen: 2.308212995529175 (2.3443283690619716) + | > loss_kl: 1.8776569366455078 (1.8836301921569194) + | > loss_feat: 7.415060043334961 (6.976168022942297) + | > loss_mel: 18.16895294189453 (17.651741362109618) + | > loss_duration: 1.5115034580230713 (1.4811819730345739) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.28138542175293 (30.337049739876974) + | > grad_norm_1: tensor(147.9578, device='cuda:0') (tensor(235.1024, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.4087 (0.3673615062359682) + | > loader_time: 0.0069 (0.005823022311495752) + + + --> TIME: 2025-04-19 01:17:07 -- STEP: 122/142 -- GLOBAL_STEP: 399000 + | > loss_disc: 2.4794206619262695 (2.4635315586308963) + | > loss_disc_real_0: 0.11676927655935287 (0.12658477330305537) + | > loss_disc_real_1: 0.23441579937934875 (0.20820053019484536) + | > loss_disc_real_2: 0.19131255149841309 (0.2258151449629518) + | > loss_disc_real_3: 0.21303808689117432 (0.22827076716501205) + | > loss_disc_real_4: 0.19878257811069489 (0.23008914646066603) + | > loss_disc_real_5: 0.2141924500465393 (0.2294285439076971) + | > loss_0: 2.4794206619262695 (2.4635315586308963) + | > grad_norm_0: tensor(8.9140, device='cuda:0') (tensor(10.5876, device='cuda:0')) + | > loss_gen: 2.3600621223449707 (2.340672391359923) + | > loss_kl: 1.9077214002609253 (1.8919737729869905) + | > loss_feat: 6.863319396972656 (6.931929756383427) + | > loss_mel: 16.747861862182617 (17.591709332387946) + | > loss_duration: 1.496752142906189 (1.4874280974513197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.375715255737305 (30.24371314439617) + | > grad_norm_1: tensor(226.9890, device='cuda:0') (tensor(231.3921, device='cuda:0')) + | > current_lr_0: 0.00019657937393197016 + | > current_lr_1: 0.00019657937393197016 + | > step_time: 0.4519 (0.3786759786918515) + | > loader_time: 0.0073 (0.006079914139919594) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.375359058380127 (2.375359058380127) + | > loss_disc_real_0: 0.16170254349708557 (0.16170254349708557) + | > loss_disc_real_1: 0.21724876761436462 (0.21724876761436462) + | > loss_disc_real_2: 0.23221364617347717 (0.23221364617347717) + | > loss_disc_real_3: 0.24615536630153656 (0.24615536630153656) + | > loss_disc_real_4: 0.2226312905550003 (0.2226312905550003) + | > loss_disc_real_5: 0.2596110701560974 (0.2596110701560974) + | > loss_0: 2.375359058380127 (2.375359058380127) + | > loss_gen: 2.537172794342041 (2.537172794342041) + | > loss_kl: 2.2926478385925293 (2.2926478385925293) + | > loss_feat: 7.372880935668945 (7.372880935668945) + | > loss_mel: 19.654579162597656 (19.654579162597656) + | > loss_duration: 1.6461946964263916 (1.6461946964263916) + | > loss_1: 33.503475189208984 (33.503475189208984) + + --> STEP: 1 + | > loss_disc: 2.388270854949951 (2.388270854949951) + | > loss_disc_real_0: 0.13359488546848297 (0.13359488546848297) + | > loss_disc_real_1: 0.20590999722480774 (0.20590999722480774) + | > loss_disc_real_2: 0.25996261835098267 (0.25996261835098267) + | > loss_disc_real_3: 0.23479175567626953 (0.23479175567626953) + | > loss_disc_real_4: 0.2723667025566101 (0.2723667025566101) + | > loss_disc_real_5: 0.22419847548007965 (0.22419847548007965) + | > loss_0: 2.388270854949951 (2.388270854949951) + | > loss_gen: 2.5236902236938477 (2.5236902236938477) + | > loss_kl: 2.5431509017944336 (2.5431509017944336) + | > loss_feat: 7.067322731018066 (7.067322731018066) + | > loss_mel: 17.750375747680664 (17.750375747680664) + | > loss_duration: 1.670870304107666 (1.670870304107666) + | > loss_1: 31.555410385131836 (31.555410385131836) + + --> STEP: 2 + | > loss_disc: 2.4443552494049072 (2.416313052177429) + | > loss_disc_real_0: 0.18263885378837585 (0.1581168696284294) + | > loss_disc_real_1: 0.22553399205207825 (0.215721994638443) + | > loss_disc_real_2: 0.22859203815460205 (0.24427732825279236) + | > loss_disc_real_3: 0.22000733017921448 (0.227399542927742) + | > loss_disc_real_4: 0.2625734210014343 (0.2674700617790222) + | > loss_disc_real_5: 0.2438797950744629 (0.23403913527727127) + | > loss_0: 2.4443552494049072 (2.416313052177429) + | > loss_gen: 2.4279263019561768 (2.475808262825012) + | > loss_kl: 2.1456735134124756 (2.3444122076034546) + | > loss_feat: 6.726855278015137 (6.897089004516602) + | > loss_mel: 18.701223373413086 (18.225799560546875) + | > loss_duration: 1.6012338399887085 (1.6360520720481873) + | > loss_1: 31.602914810180664 (31.57916259765625) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034078359603881836 (-0.0005710124969482422) + | > avg_loss_disc: 2.416313052177429 (-0.08648061752319336) + | > avg_loss_disc_real_0: 0.1581168696284294 (-0.00602230429649353) + | > avg_loss_disc_real_1: 0.215721994638443 (-0.06338432431221008) + | > avg_loss_disc_real_2: 0.24427732825279236 (+0.00999605655670166) + | > avg_loss_disc_real_3: 0.227399542927742 (-0.007026925683021545) + | > avg_loss_disc_real_4: 0.2674700617790222 (+0.03206265717744827) + | > avg_loss_disc_real_5: 0.23403913527727127 (+0.02137690782546997) + | > avg_loss_0: 2.416313052177429 (-0.08648061752319336) + | > avg_loss_gen: 2.475808262825012 (+0.09932827949523926) + | > avg_loss_kl: 2.3444122076034546 (-0.011145710945129395) + | > avg_loss_feat: 6.897089004516602 (+0.5661985874176025) + | > avg_loss_mel: 18.225799560546875 (+1.870553970336914) + | > avg_loss_duration: 1.6360520720481873 (-0.004918396472930908) + | > avg_loss_1: 31.57916259765625 (+2.520016670227051) + + + > EPOCH: 139/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:17:18)  + + --> TIME: 2025-04-19 01:17:20 -- STEP: 5/142 -- GLOBAL_STEP: 399025 + | > loss_disc: 2.358870506286621 (2.4124186992645265) + | > loss_disc_real_0: 0.1413179337978363 (0.1186819389462471) + | > loss_disc_real_1: 0.188407763838768 (0.19994715452194214) + | > loss_disc_real_2: 0.2200743705034256 (0.2226597100496292) + | > loss_disc_real_3: 0.1923982799053192 (0.224225053191185) + | > loss_disc_real_4: 0.21402692794799805 (0.23350712954998015) + | > loss_disc_real_5: 0.19932296872138977 (0.2335401237010956) + | > loss_0: 2.358870506286621 (2.4124186992645265) + | > grad_norm_0: tensor(3.9379, device='cuda:0') (tensor(7.7417, device='cuda:0')) + | > loss_gen: 2.5195090770721436 (2.406963014602661) + | > loss_kl: 1.9293506145477295 (1.9339376926422118) + | > loss_feat: 7.277341365814209 (7.205988025665283) + | > loss_mel: 17.726886749267578 (17.760618209838867) + | > loss_duration: 1.443914771080017 (1.449734139442444) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.897001266479492 (30.757240295410156) + | > grad_norm_1: tensor(277.2966, device='cuda:0') (tensor(218.1081, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.3292 (0.33327469825744627) + | > loader_time: 0.0045 (0.004398965835571289) + + + --> TIME: 2025-04-19 01:17:29 -- STEP: 30/142 -- GLOBAL_STEP: 399050 + | > loss_disc: 2.381588935852051 (2.4192507823308316) + | > loss_disc_real_0: 0.08043214678764343 (0.12162516911824545) + | > loss_disc_real_1: 0.2014288604259491 (0.202184563378493) + | > loss_disc_real_2: 0.19058328866958618 (0.22391837735970815) + | > loss_disc_real_3: 0.20734377205371857 (0.22257102479537327) + | > loss_disc_real_4: 0.18991419672966003 (0.22838174154361088) + | > loss_disc_real_5: 0.22609785199165344 (0.2285440370440483) + | > loss_0: 2.381588935852051 (2.4192507823308316) + | > grad_norm_0: tensor(7.7467, device='cuda:0') (tensor(10.7025, device='cuda:0')) + | > loss_gen: 2.326617479324341 (2.3794741153717043) + | > loss_kl: 1.8508490324020386 (1.8667985916137695) + | > loss_feat: 8.148100852966309 (7.181722180048625) + | > loss_mel: 18.892953872680664 (17.64807351430257) + | > loss_duration: 1.4534268379211426 (1.453949248790741) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.67194747924805 (30.53001740773519) + | > grad_norm_1: tensor(199.5702, device='cuda:0') (tensor(252.9913, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.3513 (0.3418586572011312) + | > loader_time: 0.006 (0.0050827900568644205) + + + --> TIME: 2025-04-19 01:17:38 -- STEP: 55/142 -- GLOBAL_STEP: 399075 + | > loss_disc: 2.539933681488037 (2.4389241478659893) + | > loss_disc_real_0: 0.15799887478351593 (0.12620821215889672) + | > loss_disc_real_1: 0.20968425273895264 (0.2051341631195762) + | > loss_disc_real_2: 0.265312522649765 (0.2242536571892825) + | > loss_disc_real_3: 0.2533431053161621 (0.22565541836348446) + | > loss_disc_real_4: 0.2519227862358093 (0.2277415849945762) + | > loss_disc_real_5: 0.307302862405777 (0.22862480472434651) + | > loss_0: 2.539933681488037 (2.4389241478659893) + | > grad_norm_0: tensor(8.4462, device='cuda:0') (tensor(10.8043, device='cuda:0')) + | > loss_gen: 2.391098737716675 (2.376370651071722) + | > loss_kl: 1.9559003114700317 (1.8906124916943636) + | > loss_feat: 6.863095760345459 (7.157282274419611) + | > loss_mel: 18.35374641418457 (17.735300514914773) + | > loss_duration: 1.5312987565994263 (1.4614996324885976) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.09514045715332 (30.621065417203035) + | > grad_norm_1: tensor(107.1638, device='cuda:0') (tensor(257.9209, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.3752 (0.3516754453832453) + | > loader_time: 0.0062 (0.005412110415371982) + + + --> TIME: 2025-04-19 01:17:48 -- STEP: 80/142 -- GLOBAL_STEP: 399100 + | > loss_disc: 2.2884879112243652 (2.456462541222573) + | > loss_disc_real_0: 0.09811515361070633 (0.12754325047135356) + | > loss_disc_real_1: 0.18320780992507935 (0.20532168671488762) + | > loss_disc_real_2: 0.17510586977005005 (0.22560656890273095) + | > loss_disc_real_3: 0.2138594686985016 (0.22634539771825074) + | > loss_disc_real_4: 0.22561697661876678 (0.23001616578549147) + | > loss_disc_real_5: 0.17102256417274475 (0.22938938904553652) + | > loss_0: 2.2884879112243652 (2.456462541222573) + | > grad_norm_0: tensor(13.6335, device='cuda:0') (tensor(11.1627, device='cuda:0')) + | > loss_gen: 2.421337127685547 (2.3602668851614004) + | > loss_kl: 2.0092179775238037 (1.8982154205441475) + | > loss_feat: 7.565479278564453 (7.12605852484703) + | > loss_mel: 17.658428192138672 (17.796804106235502) + | > loss_duration: 1.5074284076690674 (1.4733749344944957) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.16189193725586 (30.654719829559326) + | > grad_norm_1: tensor(359.6350, device='cuda:0') (tensor(259.2331, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.4073 (0.3630400687456131) + | > loader_time: 0.0064 (0.005710190534591675) + + + --> TIME: 2025-04-19 01:17:58 -- STEP: 105/142 -- GLOBAL_STEP: 399125 + | > loss_disc: 2.5391881465911865 (2.4647607576279422) + | > loss_disc_real_0: 0.10082997381687164 (0.1294531351753645) + | > loss_disc_real_1: 0.2206861823797226 (0.20519594791389648) + | > loss_disc_real_2: 0.23111841082572937 (0.22539375254086086) + | > loss_disc_real_3: 0.2689626216888428 (0.2279235433964502) + | > loss_disc_real_4: 0.26017433404922485 (0.22933623975231535) + | > loss_disc_real_5: 0.24438466131687164 (0.23126104261193955) + | > loss_0: 2.5391881465911865 (2.4647607576279422) + | > grad_norm_0: tensor(10.1296, device='cuda:0') (tensor(11.2073, device='cuda:0')) + | > loss_gen: 2.2529349327087402 (2.3546903678349076) + | > loss_kl: 1.9429709911346436 (1.9131857531411307) + | > loss_feat: 6.623901844024658 (7.047816539946057) + | > loss_mel: 17.25820541381836 (17.76672493162609) + | > loss_duration: 1.5296231508255005 (1.483471756889707) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.607635498046875 (30.565889322190056) + | > grad_norm_1: tensor(191.7302, device='cuda:0') (tensor(255.9200, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.416 (0.37280367669605075) + | > loader_time: 0.0069 (0.005969535736810593) + + + --> TIME: 2025-04-19 01:18:09 -- STEP: 130/142 -- GLOBAL_STEP: 399150 + | > loss_disc: 2.4556615352630615 (2.472043273999142) + | > loss_disc_real_0: 0.1530544012784958 (0.13126375176585647) + | > loss_disc_real_1: 0.21856454014778137 (0.2067908158669105) + | > loss_disc_real_2: 0.20682400465011597 (0.22523819368619186) + | > loss_disc_real_3: 0.23181700706481934 (0.22802532143317736) + | > loss_disc_real_4: 0.2234664261341095 (0.23009004501196054) + | > loss_disc_real_5: 0.23608210682868958 (0.23133134658520038) + | > loss_0: 2.4556615352630615 (2.472043273999142) + | > grad_norm_0: tensor(10.6097, device='cuda:0') (tensor(11.2743, device='cuda:0')) + | > loss_gen: 2.459167957305908 (2.34788214793572) + | > loss_kl: 1.8246371746063232 (1.920174292417673) + | > loss_feat: 6.755931377410889 (7.007382957751934) + | > loss_mel: 17.2686767578125 (17.75453957044161) + | > loss_duration: 1.4904346466064453 (1.4893968829741846) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.798847198486328 (30.519375757070687) + | > grad_norm_1: tensor(259.0371, device='cuda:0') (tensor(249.8899, device='cuda:0')) + | > current_lr_0: 0.00019655480151022865 + | > current_lr_1: 0.00019655480151022865 + | > step_time: 0.4484 (0.38418447971343994) + | > loader_time: 0.0078 (0.006243982681861291) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.7318873405456543 (2.7318873405456543) + | > loss_disc_real_0: 0.18909989297389984 (0.18909989297389984) + | > loss_disc_real_1: 0.2830343246459961 (0.2830343246459961) + | > loss_disc_real_2: 0.2708531618118286 (0.2708531618118286) + | > loss_disc_real_3: 0.24901384115219116 (0.24901384115219116) + | > loss_disc_real_4: 0.24070243537425995 (0.24070243537425995) + | > loss_disc_real_5: 0.30198240280151367 (0.30198240280151367) + | > loss_0: 2.7318873405456543 (2.7318873405456543) + | > loss_gen: 2.204509973526001 (2.204509973526001) + | > loss_kl: 2.1392664909362793 (2.1392664909362793) + | > loss_feat: 5.897500991821289 (5.897500991821289) + | > loss_mel: 17.064340591430664 (17.064340591430664) + | > loss_duration: 1.6500632762908936 (1.6500632762908936) + | > loss_1: 28.95568084716797 (28.95568084716797) + + --> STEP: 1 + | > loss_disc: 2.5400028228759766 (2.5400028228759766) + | > loss_disc_real_0: 0.1115599200129509 (0.1115599200129509) + | > loss_disc_real_1: 0.2047344595193863 (0.2047344595193863) + | > loss_disc_real_2: 0.22989577054977417 (0.22989577054977417) + | > loss_disc_real_3: 0.23424097895622253 (0.23424097895622253) + | > loss_disc_real_4: 0.23575104773044586 (0.23575104773044586) + | > loss_disc_real_5: 0.29466357827186584 (0.29466357827186584) + | > loss_0: 2.5400028228759766 (2.5400028228759766) + | > loss_gen: 2.1756205558776855 (2.1756205558776855) + | > loss_kl: 2.482607841491699 (2.482607841491699) + | > loss_feat: 7.366421699523926 (7.366421699523926) + | > loss_mel: 18.144058227539062 (18.144058227539062) + | > loss_duration: 1.6742883920669556 (1.6742883920669556) + | > loss_1: 31.84299659729004 (31.84299659729004) + + --> STEP: 2 + | > loss_disc: 2.4736580848693848 (2.5068304538726807) + | > loss_disc_real_0: 0.12162100523710251 (0.1165904626250267) + | > loss_disc_real_1: 0.23742826282978058 (0.22108136117458344) + | > loss_disc_real_2: 0.24357688426971436 (0.23673632740974426) + | > loss_disc_real_3: 0.22631241381168365 (0.2302766963839531) + | > loss_disc_real_4: 0.2565142810344696 (0.24613266438245773) + | > loss_disc_real_5: 0.21432305872440338 (0.2544933184981346) + | > loss_0: 2.4736580848693848 (2.5068304538726807) + | > loss_gen: 2.2367827892303467 (2.206201672554016) + | > loss_kl: 2.301347017288208 (2.3919774293899536) + | > loss_feat: 6.425965309143066 (6.896193504333496) + | > loss_mel: 16.95479393005371 (17.549426078796387) + | > loss_duration: 1.6176714897155762 (1.6459799408912659) + | > loss_1: 29.53656005859375 (30.689778327941895) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037320852279663086 (+0.000324249267578125) + | > avg_loss_disc: 2.5068304538726807 (+0.09051740169525146) + | > avg_loss_disc_real_0: 0.1165904626250267 (-0.04152640700340271) + | > avg_loss_disc_real_1: 0.22108136117458344 (+0.005359366536140442) + | > avg_loss_disc_real_2: 0.23673632740974426 (-0.007541000843048096) + | > avg_loss_disc_real_3: 0.2302766963839531 (+0.00287715345621109) + | > avg_loss_disc_real_4: 0.24613266438245773 (-0.021337397396564484) + | > avg_loss_disc_real_5: 0.2544933184981346 (+0.020454183220863342) + | > avg_loss_0: 2.5068304538726807 (+0.09051740169525146) + | > avg_loss_gen: 2.206201672554016 (-0.2696065902709961) + | > avg_loss_kl: 2.3919774293899536 (+0.04756522178649902) + | > avg_loss_feat: 6.896193504333496 (-0.0008955001831054688) + | > avg_loss_mel: 17.549426078796387 (-0.6763734817504883) + | > avg_loss_duration: 1.6459799408912659 (+0.009927868843078613) + | > avg_loss_1: 30.689778327941895 (-0.8893842697143555) + + + > EPOCH: 140/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:18:16)  + + --> TIME: 2025-04-19 01:18:21 -- STEP: 13/142 -- GLOBAL_STEP: 399175 + | > loss_disc: 2.412837505340576 (2.436318745979896) + | > loss_disc_real_0: 0.1206008791923523 (0.12001857448082703) + | > loss_disc_real_1: 0.2235633134841919 (0.20131345551747543) + | > loss_disc_real_2: 0.1995215117931366 (0.2297721688564007) + | > loss_disc_real_3: 0.21405506134033203 (0.22566043642851022) + | > loss_disc_real_4: 0.2296113222837448 (0.2351902975485875) + | > loss_disc_real_5: 0.23506219685077667 (0.22820167587353632) + | > loss_0: 2.412837505340576 (2.436318745979896) + | > grad_norm_0: tensor(12.3111, device='cuda:0') (tensor(8.8810, device='cuda:0')) + | > loss_gen: 2.284270763397217 (2.339252600303063) + | > loss_kl: 1.8651305437088013 (1.8562589517006507) + | > loss_feat: 7.2981133460998535 (6.977487600766695) + | > loss_mel: 16.35433006286621 (17.562011351952183) + | > loss_duration: 1.4540627002716064 (1.440404360110943) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.255908966064453 (30.17541474562425) + | > grad_norm_1: tensor(333.3804, device='cuda:0') (tensor(211.4443, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.3312 (0.3451207967904898) + | > loader_time: 0.0051 (0.004694846960214469) + + + --> TIME: 2025-04-19 01:18:30 -- STEP: 38/142 -- GLOBAL_STEP: 399200 + | > loss_disc: 2.430631399154663 (2.449716360945451) + | > loss_disc_real_0: 0.10326162725687027 (0.1206478739255353) + | > loss_disc_real_1: 0.21401356160640717 (0.2043982555991725) + | > loss_disc_real_2: 0.224452406167984 (0.22854061032596387) + | > loss_disc_real_3: 0.20910410583019257 (0.22733955830335617) + | > loss_disc_real_4: 0.21337267756462097 (0.23016849160194397) + | > loss_disc_real_5: 0.22018371522426605 (0.22975239667453265) + | > loss_0: 2.430631399154663 (2.449716360945451) + | > grad_norm_0: tensor(6.2703, device='cuda:0') (tensor(7.5514, device='cuda:0')) + | > loss_gen: 2.2946369647979736 (2.3368894614671403) + | > loss_kl: 1.8511220216751099 (1.8486464996086924) + | > loss_feat: 6.446211338043213 (6.963742833388479) + | > loss_mel: 17.32490348815918 (17.588879610362806) + | > loss_duration: 1.4733514785766602 (1.4558456853816384) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.390228271484375 (30.194004259611432) + | > grad_norm_1: tensor(79.7916, device='cuda:0') (tensor(160.2072, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.3537 (0.3468207560087506) + | > loader_time: 0.0057 (0.005081565756546824) + + + --> TIME: 2025-04-19 01:18:40 -- STEP: 63/142 -- GLOBAL_STEP: 399225 + | > loss_disc: 2.482280731201172 (2.4699121354118225) + | > loss_disc_real_0: 0.10770060867071152 (0.12477566372780574) + | > loss_disc_real_1: 0.18280604481697083 (0.2046620117293464) + | > loss_disc_real_2: 0.19345365464687347 (0.22766696342400142) + | > loss_disc_real_3: 0.2578129768371582 (0.22887963624227614) + | > loss_disc_real_4: 0.26577192544937134 (0.23171227342552608) + | > loss_disc_real_5: 0.24368253350257874 (0.23435346971428583) + | > loss_0: 2.482280731201172 (2.4699121354118225) + | > grad_norm_0: tensor(7.8938, device='cuda:0') (tensor(8.5780, device='cuda:0')) + | > loss_gen: 2.2094969749450684 (2.3267005814446344) + | > loss_kl: 1.9675191640853882 (1.859060974348159) + | > loss_feat: 6.762180328369141 (6.916376060909695) + | > loss_mel: 18.74461555480957 (17.518479498605878) + | > loss_duration: 1.5332510471343994 (1.4686813184193201) + | > amp_scaler: 512.0 (264.12698412698415) + | > loss_1: 31.217063903808594 (30.089298702421644) + | > grad_norm_1: tensor(211.2895, device='cuda:0') (tensor(191.3152, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.3902 (0.3568590330699134) + | > loader_time: 0.0063 (0.005395230792817615) + + + --> TIME: 2025-04-19 01:18:50 -- STEP: 88/142 -- GLOBAL_STEP: 399250 + | > loss_disc: 2.5394253730773926 (2.4720467112281117) + | > loss_disc_real_0: 0.1597273349761963 (0.12601365309885956) + | > loss_disc_real_1: 0.24303852021694183 (0.205488623035225) + | > loss_disc_real_2: 0.2210383415222168 (0.2281927429139614) + | > loss_disc_real_3: 0.2455107420682907 (0.2290821928869594) + | > loss_disc_real_4: 0.20789054036140442 (0.2302787982943383) + | > loss_disc_real_5: 0.2337859869003296 (0.23358512669801712) + | > loss_0: 2.5394253730773926 (2.4720467112281117) + | > grad_norm_0: tensor(9.2682, device='cuda:0') (tensor(8.6475, device='cuda:0')) + | > loss_gen: 2.3630917072296143 (2.3233499174768273) + | > loss_kl: 2.0202677249908447 (1.8743631636554545) + | > loss_feat: 5.641066074371338 (6.866792911832983) + | > loss_mel: 15.854169845581055 (17.517011404037472) + | > loss_duration: 1.5556268692016602 (1.4800785116174004) + | > amp_scaler: 512.0 (334.54545454545445) + | > loss_1: 27.434223175048828 (30.061596025120128) + | > grad_norm_1: tensor(271.1850, device='cuda:0') (tensor(195.2840, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.398 (0.36806547641754156) + | > loader_time: 0.0067 (0.005744901570406827) + + + --> TIME: 2025-04-19 01:19:01 -- STEP: 113/142 -- GLOBAL_STEP: 399275 + | > loss_disc: 2.4081273078918457 (2.4758802325324694) + | > loss_disc_real_0: 0.09037696570158005 (0.12686400016588453) + | > loss_disc_real_1: 0.23072528839111328 (0.20632327684258991) + | > loss_disc_real_2: 0.21375706791877747 (0.2273712377105139) + | > loss_disc_real_3: 0.19163855910301208 (0.22952383176415367) + | > loss_disc_real_4: 0.19126616418361664 (0.23065097880574453) + | > loss_disc_real_5: 0.24172668159008026 (0.23418221853475654) + | > loss_0: 2.4081273078918457 (2.4758802325324694) + | > grad_norm_0: tensor(6.2153, device='cuda:0') (tensor(10.1564, device='cuda:0')) + | > loss_gen: 2.5156657695770264 (2.3377499158403503) + | > loss_kl: 1.9376364946365356 (1.8886562606929678) + | > loss_feat: 6.860269069671631 (6.901324356551719) + | > loss_mel: 17.580326080322266 (17.587823496455634) + | > loss_duration: 1.5116987228393555 (1.486402382892845) + | > amp_scaler: 256.0 (321.69911504424766) + | > loss_1: 30.405593872070312 (30.201956436697362) + | > grad_norm_1: tensor(231.8788, device='cuda:0') (tensor(213.0335, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.4284 (0.3778627868247244) + | > loader_time: 0.0069 (0.005998797121301161) + + + --> TIME: 2025-04-19 01:19:12 -- STEP: 138/142 -- GLOBAL_STEP: 399300 + | > loss_disc: 2.463688611984253 (2.4794478813807186) + | > loss_disc_real_0: 0.1778523027896881 (0.12784717264382742) + | > loss_disc_real_1: 0.22228071093559265 (0.20626845664304233) + | > loss_disc_real_2: 0.23184558749198914 (0.227881226202716) + | > loss_disc_real_3: 0.22999967634677887 (0.22978373152622275) + | > loss_disc_real_4: 0.20391389727592468 (0.23124121388663416) + | > loss_disc_real_5: 0.21954165399074554 (0.23412689275067786) + | > loss_0: 2.463688611984253 (2.4794478813807186) + | > grad_norm_0: tensor(28.1229, device='cuda:0') (tensor(11.5707, device='cuda:0')) + | > loss_gen: 2.3066246509552 (2.3432295754335923) + | > loss_kl: 1.947465419769287 (1.9022548466488935) + | > loss_feat: 6.027907848358154 (6.894589285919632) + | > loss_mel: 17.568084716796875 (17.639963550844055) + | > loss_duration: 1.5674177408218384 (1.492936008218406) + | > amp_scaler: 256.0 (309.79710144927526) + | > loss_1: 29.417499542236328 (30.27297336467798) + | > grad_norm_1: tensor(65.5279, device='cuda:0') (tensor(233.5013, device='cuda:0')) + | > current_lr_0: 0.00019653023216003985 + | > current_lr_1: 0.00019653023216003985 + | > step_time: 0.451 (0.3893052875131801) + | > loader_time: 0.0081 (0.0062591010245724) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.461508274078369 (2.461508274078369) + | > loss_disc_real_0: 0.10666461288928986 (0.10666461288928986) + | > loss_disc_real_1: 0.19318997859954834 (0.19318997859954834) + | > loss_disc_real_2: 0.2528180181980133 (0.2528180181980133) + | > loss_disc_real_3: 0.219844788312912 (0.219844788312912) + | > loss_disc_real_4: 0.22689202427864075 (0.22689202427864075) + | > loss_disc_real_5: 0.2624151110649109 (0.2624151110649109) + | > loss_0: 2.461508274078369 (2.461508274078369) + | > loss_gen: 2.221640110015869 (2.221640110015869) + | > loss_kl: 2.2778613567352295 (2.2778613567352295) + | > loss_feat: 7.207429885864258 (7.207429885864258) + | > loss_mel: 18.56726837158203 (18.56726837158203) + | > loss_duration: 1.6404180526733398 (1.6404180526733398) + | > loss_1: 31.914615631103516 (31.914615631103516) + + --> STEP: 1 + | > loss_disc: 2.5376648902893066 (2.5376648902893066) + | > loss_disc_real_0: 0.09488378465175629 (0.09488378465175629) + | > loss_disc_real_1: 0.2558632493019104 (0.2558632493019104) + | > loss_disc_real_2: 0.28115350008010864 (0.28115350008010864) + | > loss_disc_real_3: 0.24501191079616547 (0.24501191079616547) + | > loss_disc_real_4: 0.21841543912887573 (0.21841543912887573) + | > loss_disc_real_5: 0.25443020462989807 (0.25443020462989807) + | > loss_0: 2.5376648902893066 (2.5376648902893066) + | > loss_gen: 2.1779236793518066 (2.1779236793518066) + | > loss_kl: 2.2789151668548584 (2.2789151668548584) + | > loss_feat: 6.8141326904296875 (6.8141326904296875) + | > loss_mel: 17.88947105407715 (17.88947105407715) + | > loss_duration: 1.6974117755889893 (1.6974117755889893) + | > loss_1: 30.85785484313965 (30.85785484313965) + + --> STEP: 2 + | > loss_disc: 2.5834403038024902 (2.5605525970458984) + | > loss_disc_real_0: 0.1624067723751068 (0.12864527851343155) + | > loss_disc_real_1: 0.21252426505088806 (0.23419375717639923) + | > loss_disc_real_2: 0.24662405252456665 (0.26388877630233765) + | > loss_disc_real_3: 0.22945930063724518 (0.23723560571670532) + | > loss_disc_real_4: 0.19555577635765076 (0.20698560774326324) + | > loss_disc_real_5: 0.23574824631214142 (0.24508922547101974) + | > loss_0: 2.5834403038024902 (2.5605525970458984) + | > loss_gen: 2.0065455436706543 (2.0922346115112305) + | > loss_kl: 1.9204654693603516 (2.099690318107605) + | > loss_feat: 5.539793014526367 (6.176962852478027) + | > loss_mel: 16.633529663085938 (17.261500358581543) + | > loss_duration: 1.6166117191314697 (1.6570117473602295) + | > loss_1: 27.71694564819336 (29.287400245666504) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003410816192626953 (-0.00032126903533935547) + | > avg_loss_disc: 2.5605525970458984 (+0.05372214317321777) + | > avg_loss_disc_real_0: 0.12864527851343155 (+0.012054815888404846) + | > avg_loss_disc_real_1: 0.23419375717639923 (+0.013112396001815796) + | > avg_loss_disc_real_2: 0.26388877630233765 (+0.027152448892593384) + | > avg_loss_disc_real_3: 0.23723560571670532 (+0.006958909332752228) + | > avg_loss_disc_real_4: 0.20698560774326324 (-0.03914705663919449) + | > avg_loss_disc_real_5: 0.24508922547101974 (-0.009404093027114868) + | > avg_loss_0: 2.5605525970458984 (+0.05372214317321777) + | > avg_loss_gen: 2.0922346115112305 (-0.11396706104278564) + | > avg_loss_kl: 2.099690318107605 (-0.29228711128234863) + | > avg_loss_feat: 6.176962852478027 (-0.7192306518554688) + | > avg_loss_mel: 17.261500358581543 (-0.28792572021484375) + | > avg_loss_duration: 1.6570117473602295 (+0.011031806468963623) + | > avg_loss_1: 29.287400245666504 (-1.4023780822753906) + + + > EPOCH: 141/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:19:15)  + + --> TIME: 2025-04-19 01:19:23 -- STEP: 21/142 -- GLOBAL_STEP: 399325 + | > loss_disc: 2.579831838607788 (2.468552816481817) + | > loss_disc_real_0: 0.14433178305625916 (0.12767103509533972) + | > loss_disc_real_1: 0.2296951413154602 (0.20983034869035086) + | > loss_disc_real_2: 0.23167668282985687 (0.2271788368622462) + | > loss_disc_real_3: 0.25148820877075195 (0.22757225377219065) + | > loss_disc_real_4: 0.2647823393344879 (0.22815750468344914) + | > loss_disc_real_5: 0.2756819725036621 (0.2350521343094962) + | > loss_0: 2.579831838607788 (2.468552816481817) + | > grad_norm_0: tensor(13.0109, device='cuda:0') (tensor(11.6661, device='cuda:0')) + | > loss_gen: 2.23866868019104 (2.370260579245431) + | > loss_kl: 1.8074404001235962 (1.8218913816270375) + | > loss_feat: 6.850504398345947 (7.178371633802142) + | > loss_mel: 18.555381774902344 (17.93866693405878) + | > loss_duration: 1.4390783309936523 (1.4481635831651234) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.891071319580078 (30.757354009719123) + | > grad_norm_1: tensor(304.4860, device='cuda:0') (tensor(233.1460, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.3417 (0.33129186857314336) + | > loader_time: 0.0054 (0.004865124112083798) + + + --> TIME: 2025-04-19 01:19:32 -- STEP: 46/142 -- GLOBAL_STEP: 399350 + | > loss_disc: 2.3253777027130127 (2.4528153771939487) + | > loss_disc_real_0: 0.09235558658838272 (0.12152733840048313) + | > loss_disc_real_1: 0.1827719658613205 (0.2075236391114152) + | > loss_disc_real_2: 0.18992038071155548 (0.2243333523688109) + | > loss_disc_real_3: 0.21364565193653107 (0.22614760275768198) + | > loss_disc_real_4: 0.20320527255535126 (0.22615522372981775) + | > loss_disc_real_5: 0.2301306128501892 (0.23217673729295316) + | > loss_0: 2.3253777027130127 (2.4528153771939487) + | > grad_norm_0: tensor(8.6942, device='cuda:0') (tensor(10.2026, device='cuda:0')) + | > loss_gen: 2.494943141937256 (2.345237768214681) + | > loss_kl: 1.9263509511947632 (1.8521676219027976) + | > loss_feat: 7.278831958770752 (7.122102809988934) + | > loss_mel: 18.79276466369629 (17.865063667297367) + | > loss_duration: 1.4646636247634888 (1.456395149230957) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.957555770874023 (30.64096691297448) + | > grad_norm_1: tensor(359.6436, device='cuda:0') (tensor(218.6088, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.3707 (0.3463792334432187) + | > loader_time: 0.0063 (0.005392774291660474) + + + --> TIME: 2025-04-19 01:19:42 -- STEP: 71/142 -- GLOBAL_STEP: 399375 + | > loss_disc: 2.395326852798462 (2.4598141320994205) + | > loss_disc_real_0: 0.10731389373540878 (0.12557962898847083) + | > loss_disc_real_1: 0.17363087832927704 (0.2065526822083433) + | > loss_disc_real_2: 0.236732617020607 (0.226149610860247) + | > loss_disc_real_3: 0.2268194556236267 (0.22737973928451538) + | > loss_disc_real_4: 0.25940650701522827 (0.22891509113177447) + | > loss_disc_real_5: 0.25214987993240356 (0.23298225919125784) + | > loss_0: 2.395326852798462 (2.4598141320994205) + | > grad_norm_0: tensor(4.8102, device='cuda:0') (tensor(9.8908, device='cuda:0')) + | > loss_gen: 2.4111709594726562 (2.3474464718724626) + | > loss_kl: 1.944622278213501 (1.872788560222572) + | > loss_feat: 7.414477348327637 (7.097261684041627) + | > loss_mel: 17.90279769897461 (17.836919395016952) + | > loss_duration: 1.4931682348251343 (1.467039081412302) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.166236877441406 (30.621455179133886) + | > grad_norm_1: tensor(97.2649, device='cuda:0') (tensor(203.8114, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.3838 (0.35766674431276996) + | > loader_time: 0.0065 (0.005612662140752228) + + + --> TIME: 2025-04-19 01:19:52 -- STEP: 96/142 -- GLOBAL_STEP: 399400 + | > loss_disc: 2.5870566368103027 (2.469637972613176) + | > loss_disc_real_0: 0.10511898994445801 (0.12652200355660168) + | > loss_disc_real_1: 0.17598138749599457 (0.20820626135294637) + | > loss_disc_real_2: 0.22160932421684265 (0.2271440386151274) + | > loss_disc_real_3: 0.23475123941898346 (0.22811625991016626) + | > loss_disc_real_4: 0.2583409547805786 (0.22962510849659643) + | > loss_disc_real_5: 0.2562141716480255 (0.2328939256258309) + | > loss_0: 2.5870566368103027 (2.469637972613176) + | > grad_norm_0: tensor(5.1639, device='cuda:0') (tensor(9.1855, device='cuda:0')) + | > loss_gen: 2.1272454261779785 (2.340305420259634) + | > loss_kl: 1.8791660070419312 (1.88911688576142) + | > loss_feat: 6.571433067321777 (7.0511664897203445) + | > loss_mel: 18.512466430664062 (17.895514269669853) + | > loss_duration: 1.4967114925384521 (1.4788441285490992) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.58702278137207 (30.654947181542713) + | > grad_norm_1: tensor(85.9153, device='cuda:0') (tensor(185.6541, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.4047 (0.3680239245295526) + | > loader_time: 0.0068 (0.005876983205477397) + + + --> TIME: 2025-04-19 01:20:03 -- STEP: 121/142 -- GLOBAL_STEP: 399425 + | > loss_disc: 2.432187795639038 (2.4721590172160757) + | > loss_disc_real_0: 0.15863364934921265 (0.12894453191437008) + | > loss_disc_real_1: 0.21053217351436615 (0.2081292942782079) + | > loss_disc_real_2: 0.21036340296268463 (0.22728366755749568) + | > loss_disc_real_3: 0.19663873314857483 (0.22863664410331033) + | > loss_disc_real_4: 0.1670188009738922 (0.22851452563912417) + | > loss_disc_real_5: 0.23397164046764374 (0.2324912209402431) + | > loss_0: 2.432187795639038 (2.4721590172160757) + | > grad_norm_0: tensor(14.9310, device='cuda:0') (tensor(9.4742, device='cuda:0')) + | > loss_gen: 2.2296383380889893 (2.3334241208951334) + | > loss_kl: 2.0555365085601807 (1.8978330103819034) + | > loss_feat: 6.944278717041016 (6.989565565566386) + | > loss_mel: 18.497608184814453 (17.831051061961276) + | > loss_duration: 1.550016164779663 (1.4861483810361755) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.27707862854004 (30.538022128018465) + | > grad_norm_1: tensor(282.5921, device='cuda:0') (tensor(190.8634, device='cuda:0')) + | > current_lr_0: 0.00019650566588101984 + | > current_lr_1: 0.00019650566588101984 + | > step_time: 0.4406 (0.3788787116689131) + | > loader_time: 0.0072 (0.006126053077130278) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3963515758514404 (2.3963515758514404) + | > loss_disc_real_0: 0.13246531784534454 (0.13246531784534454) + | > loss_disc_real_1: 0.24344971776008606 (0.24344971776008606) + | > loss_disc_real_2: 0.22996507585048676 (0.22996507585048676) + | > loss_disc_real_3: 0.2264643758535385 (0.2264643758535385) + | > loss_disc_real_4: 0.22766251862049103 (0.22766251862049103) + | > loss_disc_real_5: 0.23860593140125275 (0.23860593140125275) + | > loss_0: 2.3963515758514404 (2.3963515758514404) + | > loss_gen: 2.4014523029327393 (2.4014523029327393) + | > loss_kl: 2.331470251083374 (2.331470251083374) + | > loss_feat: 7.518714904785156 (7.518714904785156) + | > loss_mel: 19.319583892822266 (19.319583892822266) + | > loss_duration: 1.6567747592926025 (1.6567747592926025) + | > loss_1: 33.227996826171875 (33.227996826171875) + + --> STEP: 1 + | > loss_disc: 2.498871088027954 (2.498871088027954) + | > loss_disc_real_0: 0.1376965492963791 (0.1376965492963791) + | > loss_disc_real_1: 0.2496376633644104 (0.2496376633644104) + | > loss_disc_real_2: 0.24433201551437378 (0.24433201551437378) + | > loss_disc_real_3: 0.2403223216533661 (0.2403223216533661) + | > loss_disc_real_4: 0.26677805185317993 (0.26677805185317993) + | > loss_disc_real_5: 0.22045160830020905 (0.22045160830020905) + | > loss_0: 2.498871088027954 (2.498871088027954) + | > loss_gen: 2.3258371353149414 (2.3258371353149414) + | > loss_kl: 2.3982038497924805 (2.3982038497924805) + | > loss_feat: 7.312740802764893 (7.312740802764893) + | > loss_mel: 19.14842987060547 (19.14842987060547) + | > loss_duration: 1.6942098140716553 (1.6942098140716553) + | > loss_1: 32.87942123413086 (32.87942123413086) + + --> STEP: 2 + | > loss_disc: 2.335334539413452 (2.417102813720703) + | > loss_disc_real_0: 0.15073326230049133 (0.1442149057984352) + | > loss_disc_real_1: 0.23355281352996826 (0.24159523844718933) + | > loss_disc_real_2: 0.18737703561782837 (0.21585452556610107) + | > loss_disc_real_3: 0.18613234162330627 (0.21322733163833618) + | > loss_disc_real_4: 0.22613097727298737 (0.24645451456308365) + | > loss_disc_real_5: 0.21558889746665955 (0.2180202528834343) + | > loss_0: 2.335334539413452 (2.417102813720703) + | > loss_gen: 2.3762316703796387 (2.35103440284729) + | > loss_kl: 2.1206417083740234 (2.259422779083252) + | > loss_feat: 7.755718231201172 (7.534229516983032) + | > loss_mel: 18.43270492553711 (18.79056739807129) + | > loss_duration: 1.6487994194030762 (1.6715046167373657) + | > loss_1: 32.3340950012207 (32.60675811767578) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003636956214904785 (+0.00022614002227783203) + | > avg_loss_disc: 2.417102813720703 (-0.1434497833251953) + | > avg_loss_disc_real_0: 0.1442149057984352 (+0.015569627285003662) + | > avg_loss_disc_real_1: 0.24159523844718933 (+0.0074014812707901) + | > avg_loss_disc_real_2: 0.21585452556610107 (-0.04803425073623657) + | > avg_loss_disc_real_3: 0.21322733163833618 (-0.02400827407836914) + | > avg_loss_disc_real_4: 0.24645451456308365 (+0.039468906819820404) + | > avg_loss_disc_real_5: 0.2180202528834343 (-0.02706897258758545) + | > avg_loss_0: 2.417102813720703 (-0.1434497833251953) + | > avg_loss_gen: 2.35103440284729 (+0.25879979133605957) + | > avg_loss_kl: 2.259422779083252 (+0.15973246097564697) + | > avg_loss_feat: 7.534229516983032 (+1.3572666645050049) + | > avg_loss_mel: 18.79056739807129 (+1.529067039489746) + | > avg_loss_duration: 1.6715046167373657 (+0.01449286937713623) + | > avg_loss_1: 32.60675811767578 (+3.3193578720092773) + + + > EPOCH: 142/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:20:14)  + + --> TIME: 2025-04-19 01:20:16 -- STEP: 4/142 -- GLOBAL_STEP: 399450 + | > loss_disc: 2.357830762863159 (2.383648157119751) + | > loss_disc_real_0: 0.10045073181390762 (0.11204042099416256) + | > loss_disc_real_1: 0.17372551560401917 (0.20471757277846336) + | > loss_disc_real_2: 0.2224322110414505 (0.21596794202923775) + | > loss_disc_real_3: 0.22097280621528625 (0.22370890900492668) + | > loss_disc_real_4: 0.2027186006307602 (0.23326880857348442) + | > loss_disc_real_5: 0.22601287066936493 (0.234708733856678) + | > loss_0: 2.357830762863159 (2.383648157119751) + | > grad_norm_0: tensor(7.6303, device='cuda:0') (tensor(6.7651, device='cuda:0')) + | > loss_gen: 2.411637306213379 (2.387436091899872) + | > loss_kl: 1.8959147930145264 (1.8444401621818542) + | > loss_feat: 7.491147994995117 (7.250198602676392) + | > loss_mel: 17.627302169799805 (17.793089866638184) + | > loss_duration: 1.4722073078155518 (1.4516286551952362) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.898210525512695 (30.72679376602173) + | > grad_norm_1: tensor(293.2612, device='cuda:0') (tensor(191.0724, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.3215 (0.3234066963195801) + | > loader_time: 0.0051 (0.0045850276947021484) + + + --> TIME: 2025-04-19 01:20:24 -- STEP: 29/142 -- GLOBAL_STEP: 399475 + | > loss_disc: 2.4727816581726074 (2.444539020801413) + | > loss_disc_real_0: 0.09435775876045227 (0.1212342594718111) + | > loss_disc_real_1: 0.21274776756763458 (0.21136272621565852) + | > loss_disc_real_2: 0.22318881750106812 (0.22365068875510116) + | > loss_disc_real_3: 0.23975013196468353 (0.22738933563232422) + | > loss_disc_real_4: 0.25843167304992676 (0.2286900338427774) + | > loss_disc_real_5: 0.26782557368278503 (0.22850142522104855) + | > loss_0: 2.4727816581726074 (2.444539020801413) + | > grad_norm_0: tensor(7.3999, device='cuda:0') (tensor(9.4949, device='cuda:0')) + | > loss_gen: 2.2910218238830566 (2.3691761658109476) + | > loss_kl: 1.825547695159912 (1.830074565163974) + | > loss_feat: 6.929476261138916 (7.147730597134294) + | > loss_mel: 16.719697952270508 (17.873552848552837) + | > loss_duration: 1.4513264894485474 (1.4499274615583748) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.217069625854492 (30.670461588892444) + | > grad_norm_1: tensor(234.4551, device='cuda:0') (tensor(202.3574, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.3464 (0.33632530015090417) + | > loader_time: 0.0054 (0.004907566925574993) + + + --> TIME: 2025-04-19 01:20:34 -- STEP: 54/142 -- GLOBAL_STEP: 399500 + | > loss_disc: 2.5480387210845947 (2.457501274568063) + | > loss_disc_real_0: 0.20625589787960052 (0.12662792467960604) + | > loss_disc_real_1: 0.21338056027889252 (0.21085254102945328) + | > loss_disc_real_2: 0.22275182604789734 (0.22384116412312896) + | > loss_disc_real_3: 0.226898193359375 (0.22683484245229651) + | > loss_disc_real_4: 0.225688636302948 (0.22630651460753548) + | > loss_disc_real_5: 0.2391231209039688 (0.22992125346704764) + | > loss_0: 2.5480387210845947 (2.457501274568063) + | > grad_norm_0: tensor(12.8043, device='cuda:0') (tensor(9.2188, device='cuda:0')) + | > loss_gen: 2.301544666290283 (2.350274951369675) + | > loss_kl: 2.0174992084503174 (1.8589341640472412) + | > loss_feat: 6.785898685455322 (7.038262446721395) + | > loss_mel: 16.80549430847168 (17.79914691713121) + | > loss_duration: 1.4777497053146362 (1.4575508126506098) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.388185501098633 (30.50416914621989) + | > grad_norm_1: tensor(208.8019, device='cuda:0') (tensor(204.1354, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.3689 (0.3482890349847299) + | > loader_time: 0.0058 (0.00527955867626049) + + + --> TIME: 2025-04-19 01:20:44 -- STEP: 79/142 -- GLOBAL_STEP: 399525 + | > loss_disc: 2.339153289794922 (2.4705396724652635) + | > loss_disc_real_0: 0.09116585552692413 (0.12785092062210737) + | > loss_disc_real_1: 0.21361428499221802 (0.21083644843554195) + | > loss_disc_real_2: 0.24039344489574432 (0.22527994309799582) + | > loss_disc_real_3: 0.2285500317811966 (0.22810978018030337) + | > loss_disc_real_4: 0.2279534935951233 (0.22929840223698678) + | > loss_disc_real_5: 0.18740259110927582 (0.22997255340407166) + | > loss_0: 2.339153289794922 (2.4705396724652635) + | > grad_norm_0: tensor(14.7492, device='cuda:0') (tensor(10.3441, device='cuda:0')) + | > loss_gen: 2.4263203144073486 (2.344883604894711) + | > loss_kl: 1.8176716566085815 (1.8635793094393573) + | > loss_feat: 7.269068717956543 (7.023217563387714) + | > loss_mel: 19.765567779541016 (17.821773951566673) + | > loss_duration: 1.5402019023895264 (1.4692231914665128) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.81883239746094 (30.522677650934533) + | > grad_norm_1: tensor(429.5896, device='cuda:0') (tensor(228.6550, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.3861 (0.3603585309620145) + | > loader_time: 0.0067 (0.005615910397300236) + + + --> TIME: 2025-04-19 01:20:54 -- STEP: 104/142 -- GLOBAL_STEP: 399550 + | > loss_disc: 2.511446237564087 (2.4729281549270334) + | > loss_disc_real_0: 0.1337323933839798 (0.12789515699618137) + | > loss_disc_real_1: 0.2131674885749817 (0.21020373014303353) + | > loss_disc_real_2: 0.2072744518518448 (0.22656944752312624) + | > loss_disc_real_3: 0.23442158102989197 (0.22846146429387423) + | > loss_disc_real_4: 0.22259151935577393 (0.22994793435701957) + | > loss_disc_real_5: 0.23171789944171906 (0.2302686023310973) + | > loss_0: 2.511446237564087 (2.4729281549270334) + | > grad_norm_0: tensor(4.5304, device='cuda:0') (tensor(11.1605, device='cuda:0')) + | > loss_gen: 2.284904718399048 (2.3428220886450557) + | > loss_kl: 1.9621926546096802 (1.882367458481055) + | > loss_feat: 6.514142036437988 (6.98900998555697) + | > loss_mel: 17.414648056030273 (17.84667816528907) + | > loss_duration: 1.5346968173980713 (1.4805554506870418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.71058464050293 (30.54143316929157) + | > grad_norm_1: tensor(171.5411, device='cuda:0') (tensor(249.9039, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.4075 (0.37087002167334926) + | > loader_time: 0.0072 (0.005841502776512731) + + + --> TIME: 2025-04-19 01:21:05 -- STEP: 129/142 -- GLOBAL_STEP: 399575 + | > loss_disc: 2.4203591346740723 (2.4757595228594402) + | > loss_disc_real_0: 0.11276626586914062 (0.12984485514173214) + | > loss_disc_real_1: 0.19574123620986938 (0.2095205385324567) + | > loss_disc_real_2: 0.21780554950237274 (0.22695800973925478) + | > loss_disc_real_3: 0.19699729979038239 (0.22834930851940036) + | > loss_disc_real_4: 0.2232132852077484 (0.22936958351800607) + | > loss_disc_real_5: 0.22144542634487152 (0.23088042692158572) + | > loss_0: 2.4203591346740723 (2.4757595228594402) + | > grad_norm_0: tensor(25.3398, device='cuda:0') (tensor(12.9130, device='cuda:0')) + | > loss_gen: 2.406662702560425 (2.3495728895645747) + | > loss_kl: 1.9506665468215942 (1.8865248399187429) + | > loss_feat: 7.911944389343262 (6.984359792960706) + | > loss_mel: 17.89698028564453 (17.852293635523598) + | > loss_duration: 1.490048885345459 (1.4874802375024603) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.656301498413086 (30.560231349264928) + | > grad_norm_1: tensor(549.5668, device='cuda:0') (tensor(264.4687, device='cuda:0')) + | > current_lr_0: 0.0001964811026727847 + | > current_lr_1: 0.0001964811026727847 + | > step_time: 0.4456 (0.38236566107402475) + | > loader_time: 0.0072 (0.006107226822727408) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.680455207824707 (2.680455207824707) + | > loss_disc_real_0: 0.10269536823034286 (0.10269536823034286) + | > loss_disc_real_1: 0.21088336408138275 (0.21088336408138275) + | > loss_disc_real_2: 0.2886750102043152 (0.2886750102043152) + | > loss_disc_real_3: 0.2774873673915863 (0.2774873673915863) + | > loss_disc_real_4: 0.25089535117149353 (0.25089535117149353) + | > loss_disc_real_5: 0.27132928371429443 (0.27132928371429443) + | > loss_0: 2.680455207824707 (2.680455207824707) + | > loss_gen: 1.9627339839935303 (1.9627339839935303) + | > loss_kl: 2.455317974090576 (2.455317974090576) + | > loss_feat: 4.650547027587891 (4.650547027587891) + | > loss_mel: 14.820371627807617 (14.820371627807617) + | > loss_duration: 1.6405837535858154 (1.6405837535858154) + | > loss_1: 25.52955436706543 (25.52955436706543) + + --> STEP: 1 + | > loss_disc: 2.5380635261535645 (2.5380635261535645) + | > loss_disc_real_0: 0.06972560286521912 (0.06972560286521912) + | > loss_disc_real_1: 0.2059587985277176 (0.2059587985277176) + | > loss_disc_real_2: 0.2514515817165375 (0.2514515817165375) + | > loss_disc_real_3: 0.2099866271018982 (0.2099866271018982) + | > loss_disc_real_4: 0.22789046168327332 (0.22789046168327332) + | > loss_disc_real_5: 0.21770715713500977 (0.21770715713500977) + | > loss_0: 2.5380635261535645 (2.5380635261535645) + | > loss_gen: 2.075784921646118 (2.075784921646118) + | > loss_kl: 2.5304980278015137 (2.5304980278015137) + | > loss_feat: 7.336065292358398 (7.336065292358398) + | > loss_mel: 18.261077880859375 (18.261077880859375) + | > loss_duration: 1.6870203018188477 (1.6870203018188477) + | > loss_1: 31.890445709228516 (31.890445709228516) + + --> STEP: 2 + | > loss_disc: 2.506749153137207 (2.5224063396453857) + | > loss_disc_real_0: 0.0579402856528759 (0.06383294425904751) + | > loss_disc_real_1: 0.21757127344608307 (0.21176503598690033) + | > loss_disc_real_2: 0.2666065990924835 (0.2590290904045105) + | > loss_disc_real_3: 0.25603723526000977 (0.23301193118095398) + | > loss_disc_real_4: 0.23371048271656036 (0.23080047219991684) + | > loss_disc_real_5: 0.22323940694332123 (0.2204732820391655) + | > loss_0: 2.506749153137207 (2.5224063396453857) + | > loss_gen: 2.073946475982666 (2.074865698814392) + | > loss_kl: 2.140369176864624 (2.335433602333069) + | > loss_feat: 7.3267645835876465 (7.3314149379730225) + | > loss_mel: 18.04785919189453 (18.154468536376953) + | > loss_duration: 1.6382503509521484 (1.662635326385498) + | > loss_1: 31.227188110351562 (31.55881690979004) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038558244705200195 (+0.00021886825561523438) + | > avg_loss_disc: 2.5224063396453857 (+0.10530352592468262) + | > avg_loss_disc_real_0: 0.06383294425904751 (-0.0803819615393877) + | > avg_loss_disc_real_1: 0.21176503598690033 (-0.029830202460289) + | > avg_loss_disc_real_2: 0.2590290904045105 (+0.043174564838409424) + | > avg_loss_disc_real_3: 0.23301193118095398 (+0.019784599542617798) + | > avg_loss_disc_real_4: 0.23080047219991684 (-0.01565404236316681) + | > avg_loss_disc_real_5: 0.2204732820391655 (+0.002453029155731201) + | > avg_loss_0: 2.5224063396453857 (+0.10530352592468262) + | > avg_loss_gen: 2.074865698814392 (-0.27616870403289795) + | > avg_loss_kl: 2.335433602333069 (+0.0760108232498169) + | > avg_loss_feat: 7.3314149379730225 (-0.20281457901000977) + | > avg_loss_mel: 18.154468536376953 (-0.6360988616943359) + | > avg_loss_duration: 1.662635326385498 (-0.008869290351867676) + | > avg_loss_1: 31.55881690979004 (-1.0479412078857422) + + + > EPOCH: 143/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:21:12)  + + --> TIME: 2025-04-19 01:21:17 -- STEP: 12/142 -- GLOBAL_STEP: 399600 + | > loss_disc: 2.339542865753174 (2.4473394751548767) + | > loss_disc_real_0: 0.07363670319318771 (0.12803078225503364) + | > loss_disc_real_1: 0.1594768464565277 (0.2038076768318812) + | > loss_disc_real_2: 0.224085733294487 (0.2196325920522213) + | > loss_disc_real_3: 0.24121323227882385 (0.22590825955073038) + | > loss_disc_real_4: 0.23660071194171906 (0.23753155022859573) + | > loss_disc_real_5: 0.21211281418800354 (0.23370934650301933) + | > loss_0: 2.339542865753174 (2.4473394751548767) + | > grad_norm_0: tensor(26.0851, device='cuda:0') (tensor(19.1518, device='cuda:0')) + | > loss_gen: 2.6255671977996826 (2.426046888033549) + | > loss_kl: 1.8076810836791992 (1.8886731266975403) + | > loss_feat: 8.190550804138184 (7.120126843452454) + | > loss_mel: 18.959325790405273 (17.49491270383199) + | > loss_duration: 1.4619810581207275 (1.4379130601882935) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.04510498046875 (30.36767292022705) + | > grad_norm_1: tensor(582.1013, device='cuda:0') (tensor(328.4041, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.3309 (0.33144764105478924) + | > loader_time: 0.0053 (0.0046271880467732745) + + + --> TIME: 2025-04-19 01:21:26 -- STEP: 37/142 -- GLOBAL_STEP: 399625 + | > loss_disc: 2.3944733142852783 (2.4561360720041643) + | > loss_disc_real_0: 0.10229511559009552 (0.122779126505594) + | > loss_disc_real_1: 0.216158926486969 (0.2042372488492244) + | > loss_disc_real_2: 0.2475096881389618 (0.227580973425427) + | > loss_disc_real_3: 0.2231779396533966 (0.23070444005566673) + | > loss_disc_real_4: 0.22478917241096497 (0.2315159452122611) + | > loss_disc_real_5: 0.19620749354362488 (0.23146114397693324) + | > loss_0: 2.3944733142852783 (2.4561360720041643) + | > grad_norm_0: tensor(11.4999, device='cuda:0') (tensor(16.0591, device='cuda:0')) + | > loss_gen: 2.3793342113494873 (2.380832130844529) + | > loss_kl: 1.8045037984848022 (1.892681588997712) + | > loss_feat: 7.5533246994018555 (7.066321050798571) + | > loss_mel: 18.534976959228516 (17.781700417802142) + | > loss_duration: 1.472611427307129 (1.4498879522890658) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.7447509765625 (30.571423040853965) + | > grad_norm_1: tensor(142.3957, device='cuda:0') (tensor(318.0500, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.3612 (0.34255982089687037) + | > loader_time: 0.0054 (0.0051080665072879275) + + + --> TIME: 2025-04-19 01:21:35 -- STEP: 62/142 -- GLOBAL_STEP: 399650 + | > loss_disc: 2.3713250160217285 (2.4605202136501196) + | > loss_disc_real_0: 0.08393586426973343 (0.12141687747451567) + | > loss_disc_real_1: 0.20909930765628815 (0.2069049261270031) + | > loss_disc_real_2: 0.22567448019981384 (0.22787116948635347) + | > loss_disc_real_3: 0.2320670187473297 (0.23057612248005405) + | > loss_disc_real_4: 0.21497443318367004 (0.23141466105176556) + | > loss_disc_real_5: 0.23943659663200378 (0.23343389505340206) + | > loss_0: 2.3713250160217285 (2.4605202136501196) + | > grad_norm_0: tensor(4.9985, device='cuda:0') (tensor(13.2565, device='cuda:0')) + | > loss_gen: 2.418137550354004 (2.3554946145703726) + | > loss_kl: 2.010256767272949 (1.8973496325554386) + | > loss_feat: 7.6874895095825195 (7.082376880030478) + | > loss_mel: 19.055662155151367 (17.897819995880127) + | > loss_duration: 1.5206193923950195 (1.4635659744662624) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.69216537475586 (30.69660706673899) + | > grad_norm_1: tensor(232.5807, device='cuda:0') (tensor(286.6230, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.3843 (0.355024476205149) + | > loader_time: 0.0065 (0.005452090694058326) + + + --> TIME: 2025-04-19 01:21:45 -- STEP: 87/142 -- GLOBAL_STEP: 399675 + | > loss_disc: 2.4642603397369385 (2.4612924696385177) + | > loss_disc_real_0: 0.17307458817958832 (0.1233342289068233) + | > loss_disc_real_1: 0.1869320273399353 (0.20638826917642836) + | > loss_disc_real_2: 0.24165762960910797 (0.22711153705229706) + | > loss_disc_real_3: 0.2247568964958191 (0.23033394433301072) + | > loss_disc_real_4: 0.2630162835121155 (0.23079513253151684) + | > loss_disc_real_5: 0.22749869525432587 (0.23181732769670158) + | > loss_0: 2.4642603397369385 (2.4612924696385177) + | > grad_norm_0: tensor(10.1917, device='cuda:0') (tensor(11.8327, device='cuda:0')) + | > loss_gen: 2.358531951904297 (2.3470870790810423) + | > loss_kl: 1.9818435907363892 (1.9024746116550488) + | > loss_feat: 7.480893611907959 (7.018384653946447) + | > loss_mel: 18.45648193359375 (17.851511494866735) + | > loss_duration: 1.532336711883545 (1.4768558836531365) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.810089111328125 (30.596313761568616) + | > grad_norm_1: tensor(196.9775, device='cuda:0') (tensor(248.7706, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.4039 (0.3663666796410221) + | > loader_time: 0.0067 (0.005779249914761247) + + + --> TIME: 2025-04-19 01:21:56 -- STEP: 112/142 -- GLOBAL_STEP: 399700 + | > loss_disc: 2.4187071323394775 (2.4629927128553395) + | > loss_disc_real_0: 0.1541563719511032 (0.12447442593319076) + | > loss_disc_real_1: 0.1981671303510666 (0.20689484995922872) + | > loss_disc_real_2: 0.22259044647216797 (0.22691543120890856) + | > loss_disc_real_3: 0.24931509792804718 (0.22967657194073712) + | > loss_disc_real_4: 0.19893518090248108 (0.2296522484560098) + | > loss_disc_real_5: 0.212692990899086 (0.23260679401989495) + | > loss_0: 2.4187071323394775 (2.4629927128553395) + | > grad_norm_0: tensor(5.9281, device='cuda:0') (tensor(10.5565, device='cuda:0')) + | > loss_gen: 2.2696778774261475 (2.3374176280839114) + | > loss_kl: 1.8207062482833862 (1.9155053113188063) + | > loss_feat: 6.48587703704834 (6.980438973222459) + | > loss_mel: 15.82010555267334 (17.86682561465671) + | > loss_duration: 1.4893558025360107 (1.4844905936292239) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.885723114013672 (30.584678087915695) + | > grad_norm_1: tensor(37.8396, device='cuda:0') (tensor(211.4334, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.4275 (0.37626634751047405) + | > loader_time: 0.0073 (0.006046205759048462) + + + --> TIME: 2025-04-19 01:22:07 -- STEP: 137/142 -- GLOBAL_STEP: 399725 + | > loss_disc: 2.464627742767334 (2.4646121811692727) + | > loss_disc_real_0: 0.12362008541822433 (0.12575341500069973) + | > loss_disc_real_1: 0.24398957192897797 (0.20703422675167557) + | > loss_disc_real_2: 0.21252824366092682 (0.22724928005333364) + | > loss_disc_real_3: 0.23780737817287445 (0.22909926650297902) + | > loss_disc_real_4: 0.2394053339958191 (0.22963326288400776) + | > loss_disc_real_5: 0.23835094273090363 (0.23243502097843338) + | > loss_0: 2.464627742767334 (2.4646121811692727) + | > grad_norm_0: tensor(15.5419, device='cuda:0') (tensor(10.1026, device='cuda:0')) + | > loss_gen: 2.519235372543335 (2.332961524490023) + | > loss_kl: 1.9737166166305542 (1.9195103079733187) + | > loss_feat: 6.509077548980713 (6.944144255923528) + | > loss_mel: 18.190521240234375 (17.811798471603943) + | > loss_duration: 1.5130778551101685 (1.4899408147282844) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.705629348754883 (30.498355308588405) + | > grad_norm_1: tensor(287.1396, device='cuda:0') (tensor(205.2981, device='cuda:0')) + | > current_lr_0: 0.00019645654253495058 + | > current_lr_1: 0.00019645654253495058 + | > step_time: 0.4552 (0.3879761869889976) + | > loader_time: 0.008 (0.006293143669184107) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.496955394744873 (2.496955394744873) + | > loss_disc_real_0: 0.1467796117067337 (0.1467796117067337) + | > loss_disc_real_1: 0.2474532425403595 (0.2474532425403595) + | > loss_disc_real_2: 0.24961483478546143 (0.24961483478546143) + | > loss_disc_real_3: 0.23755455017089844 (0.23755455017089844) + | > loss_disc_real_4: 0.21402877569198608 (0.21402877569198608) + | > loss_disc_real_5: 0.29091277718544006 (0.29091277718544006) + | > loss_0: 2.496955394744873 (2.496955394744873) + | > loss_gen: 2.445641040802002 (2.445641040802002) + | > loss_kl: 2.3736956119537354 (2.3736956119537354) + | > loss_feat: 6.667985439300537 (6.667985439300537) + | > loss_mel: 17.917560577392578 (17.917560577392578) + | > loss_duration: 1.6646997928619385 (1.6646997928619385) + | > loss_1: 31.069581985473633 (31.069581985473633) + + --> STEP: 1 + | > loss_disc: 2.5788016319274902 (2.5788016319274902) + | > loss_disc_real_0: 0.20319247245788574 (0.20319247245788574) + | > loss_disc_real_1: 0.19936539232730865 (0.19936539232730865) + | > loss_disc_real_2: 0.2652306854724884 (0.2652306854724884) + | > loss_disc_real_3: 0.2506527900695801 (0.2506527900695801) + | > loss_disc_real_4: 0.24861934781074524 (0.24861934781074524) + | > loss_disc_real_5: 0.24831664562225342 (0.24831664562225342) + | > loss_0: 2.5788016319274902 (2.5788016319274902) + | > loss_gen: 2.4393627643585205 (2.4393627643585205) + | > loss_kl: 2.4858686923980713 (2.4858686923980713) + | > loss_feat: 6.41618013381958 (6.41618013381958) + | > loss_mel: 19.40921974182129 (19.40921974182129) + | > loss_duration: 1.6776485443115234 (1.6776485443115234) + | > loss_1: 32.428279876708984 (32.428279876708984) + + --> STEP: 2 + | > loss_disc: 2.4692742824554443 (2.5240379571914673) + | > loss_disc_real_0: 0.2052803784608841 (0.20423642545938492) + | > loss_disc_real_1: 0.23774904012680054 (0.2185572162270546) + | > loss_disc_real_2: 0.24411658942699432 (0.25467363744974136) + | > loss_disc_real_3: 0.23343072831630707 (0.24204175919294357) + | > loss_disc_real_4: 0.22693423926830292 (0.23777679353952408) + | > loss_disc_real_5: 0.2367817759513855 (0.24254921078681946) + | > loss_0: 2.4692742824554443 (2.5240379571914673) + | > loss_gen: 2.5339181423187256 (2.486640453338623) + | > loss_kl: 2.2442803382873535 (2.3650745153427124) + | > loss_feat: 7.163017749786377 (6.7895989418029785) + | > loss_mel: 19.159326553344727 (19.284273147583008) + | > loss_duration: 1.6423999071121216 (1.6600242257118225) + | > loss_1: 32.742942810058594 (32.58561134338379) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003931403160095215 (+7.557868957519531e-05) + | > avg_loss_disc: 2.5240379571914673 (+0.001631617546081543) + | > avg_loss_disc_real_0: 0.20423642545938492 (+0.1404034812003374) + | > avg_loss_disc_real_1: 0.2185572162270546 (+0.006792180240154266) + | > avg_loss_disc_real_2: 0.25467363744974136 (-0.0043554529547691345) + | > avg_loss_disc_real_3: 0.24204175919294357 (+0.009029828011989594) + | > avg_loss_disc_real_4: 0.23777679353952408 (+0.006976321339607239) + | > avg_loss_disc_real_5: 0.24254921078681946 (+0.02207592874765396) + | > avg_loss_0: 2.5240379571914673 (+0.001631617546081543) + | > avg_loss_gen: 2.486640453338623 (+0.41177475452423096) + | > avg_loss_kl: 2.3650745153427124 (+0.029640913009643555) + | > avg_loss_feat: 6.7895989418029785 (-0.541815996170044) + | > avg_loss_mel: 19.284273147583008 (+1.1298046112060547) + | > avg_loss_duration: 1.6600242257118225 (-0.002611100673675537) + | > avg_loss_1: 32.58561134338379 (+1.02679443359375) + + + > EPOCH: 144/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:22:11)  + + --> TIME: 2025-04-19 01:22:19 -- STEP: 20/142 -- GLOBAL_STEP: 399750 + | > loss_disc: 2.4296083450317383 (2.439676356315613) + | > loss_disc_real_0: 0.0850372463464737 (0.13146149199455973) + | > loss_disc_real_1: 0.21945969760417938 (0.20718772187829018) + | > loss_disc_real_2: 0.23934966325759888 (0.22509054765105246) + | > loss_disc_real_3: 0.23464339971542358 (0.2262567885220051) + | > loss_disc_real_4: 0.20913873612880707 (0.22480999305844307) + | > loss_disc_real_5: 0.18755128979682922 (0.22746634259819984) + | > loss_0: 2.4296083450317383 (2.439676356315613) + | > grad_norm_0: tensor(12.9827, device='cuda:0') (tensor(12.0969, device='cuda:0')) + | > loss_gen: 2.337113857269287 (2.3665651679039) + | > loss_kl: 1.9983822107315063 (1.8324363112449646) + | > loss_feat: 7.344873428344727 (7.182540464401245) + | > loss_mel: 17.642431259155273 (18.12040433883667) + | > loss_duration: 1.5145263671875 (1.4402916371822356) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.83732795715332 (30.94223804473877) + | > grad_norm_1: tensor(294.7159, device='cuda:0') (tensor(236.9283, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.3423 (0.33249515295028687) + | > loader_time: 0.0053 (0.0046399950981140135) + + + --> TIME: 2025-04-19 01:22:28 -- STEP: 45/142 -- GLOBAL_STEP: 399775 + | > loss_disc: 2.4710278511047363 (2.4454755677117235) + | > loss_disc_real_0: 0.1062830314040184 (0.12475528029931916) + | > loss_disc_real_1: 0.17561616003513336 (0.20793420175711314) + | > loss_disc_real_2: 0.22097165882587433 (0.22625093691878848) + | > loss_disc_real_3: 0.22522790729999542 (0.22958090603351594) + | > loss_disc_real_4: 0.2445695549249649 (0.22660285433133442) + | > loss_disc_real_5: 0.21773995459079742 (0.22747846278879377) + | > loss_0: 2.4710278511047363 (2.4454755677117235) + | > grad_norm_0: tensor(13.1865, device='cuda:0') (tensor(9.9427, device='cuda:0')) + | > loss_gen: 2.202524185180664 (2.3668128437466103) + | > loss_kl: 1.9008007049560547 (1.8662833372751872) + | > loss_feat: 6.600447654724121 (7.130152395036485) + | > loss_mel: 17.94091033935547 (17.871052190992568) + | > loss_duration: 1.4890809059143066 (1.4546799845165677) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.13376235961914 (30.688980695936415) + | > grad_norm_1: tensor(278.7573, device='cuda:0') (tensor(212.4177, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.3671 (0.34444473584492996) + | > loader_time: 0.0057 (0.004963239034016927) + + + --> TIME: 2025-04-19 01:22:37 -- STEP: 70/142 -- GLOBAL_STEP: 399800 + | > loss_disc: 2.292426586151123 (2.4560228552137096) + | > loss_disc_real_0: 0.14165471494197845 (0.12487217736031328) + | > loss_disc_real_1: 0.20535458624362946 (0.2083892006959234) + | > loss_disc_real_2: 0.1906445473432541 (0.22495193651744297) + | > loss_disc_real_3: 0.233743354678154 (0.22982792896883827) + | > loss_disc_real_4: 0.19620224833488464 (0.22703018529074534) + | > loss_disc_real_5: 0.2454790472984314 (0.23206925583737237) + | > loss_0: 2.292426586151123 (2.4560228552137096) + | > grad_norm_0: tensor(12.4626, device='cuda:0') (tensor(10.1644, device='cuda:0')) + | > loss_gen: 2.5788285732269287 (2.3635705777576996) + | > loss_kl: 1.968957781791687 (1.8838061298642839) + | > loss_feat: 7.944901943206787 (7.085329832349505) + | > loss_mel: 19.558584213256836 (17.778015913282122) + | > loss_duration: 1.4942872524261475 (1.467176503794534) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.545555114746094 (30.57789887019566) + | > grad_norm_1: tensor(311.1747, device='cuda:0') (tensor(222.6430, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.391 (0.35606983389173236) + | > loader_time: 0.0063 (0.005271502903529576) + + + --> TIME: 2025-04-19 01:22:47 -- STEP: 95/142 -- GLOBAL_STEP: 399825 + | > loss_disc: 2.5075838565826416 (2.460400019193949) + | > loss_disc_real_0: 0.1277143806219101 (0.12492706575676014) + | > loss_disc_real_1: 0.1939157396554947 (0.2082296898490504) + | > loss_disc_real_2: 0.22292475402355194 (0.22598175547624888) + | > loss_disc_real_3: 0.22102558612823486 (0.2294718784721274) + | > loss_disc_real_4: 0.2147534191608429 (0.22895340684213136) + | > loss_disc_real_5: 0.22296670079231262 (0.23127091072107617) + | > loss_0: 2.5075838565826416 (2.460400019193949) + | > grad_norm_0: tensor(4.5815, device='cuda:0') (tensor(10.9029, device='cuda:0')) + | > loss_gen: 2.078778028488159 (2.36445857600162) + | > loss_kl: 1.8814159631729126 (1.9043367222735756) + | > loss_feat: 6.662119388580322 (7.080969604692961) + | > loss_mel: 18.236953735351562 (17.81766832251298) + | > loss_duration: 1.523377537727356 (1.4796271800994873) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.382644653320312 (30.647060313977693) + | > grad_norm_1: tensor(35.2465, device='cuda:0') (tensor(236.1109, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.4125 (0.3678649576086748) + | > loader_time: 0.0068 (0.005597363020244395) + + + --> TIME: 2025-04-19 01:22:58 -- STEP: 120/142 -- GLOBAL_STEP: 399850 + | > loss_disc: 2.4213027954101562 (2.468045632044473) + | > loss_disc_real_0: 0.12295603007078171 (0.1252325910143554) + | > loss_disc_real_1: 0.18182875216007233 (0.20891163448492686) + | > loss_disc_real_2: 0.23861300945281982 (0.2264493703842163) + | > loss_disc_real_3: 0.2406044751405716 (0.23026659240325292) + | > loss_disc_real_4: 0.2454403191804886 (0.23052503528694313) + | > loss_disc_real_5: 0.2521437406539917 (0.23123066735764344) + | > loss_0: 2.4213027954101562 (2.468045632044473) + | > grad_norm_0: tensor(3.6441, device='cuda:0') (tensor(10.8091, device='cuda:0')) + | > loss_gen: 2.3981966972351074 (2.3503715296586365) + | > loss_kl: 1.7923394441604614 (1.9051674286524454) + | > loss_feat: 6.701221466064453 (6.994420067469279) + | > loss_mel: 18.34845733642578 (17.690630586942042) + | > loss_duration: 1.5295360088348389 (1.4863573918739954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.769750595092773 (30.426946878433228) + | > grad_norm_1: tensor(64.3466, device='cuda:0') (tensor(241.7686, device='cuda:0')) + | > current_lr_0: 0.0001964319854671337 + | > current_lr_1: 0.0001964319854671337 + | > step_time: 0.4376 (0.3795010308424633) + | > loader_time: 0.0075 (0.00591505169868469) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4726927280426025 (2.4726927280426025) + | > loss_disc_real_0: 0.2391318380832672 (0.2391318380832672) + | > loss_disc_real_1: 0.20611979067325592 (0.20611979067325592) + | > loss_disc_real_2: 0.23045466840267181 (0.23045466840267181) + | > loss_disc_real_3: 0.23783718049526215 (0.23783718049526215) + | > loss_disc_real_4: 0.22686800360679626 (0.22686800360679626) + | > loss_disc_real_5: 0.234432652592659 (0.234432652592659) + | > loss_0: 2.4726927280426025 (2.4726927280426025) + | > loss_gen: 2.5102431774139404 (2.5102431774139404) + | > loss_kl: 2.4565865993499756 (2.4565865993499756) + | > loss_feat: 6.563752174377441 (6.563752174377441) + | > loss_mel: 17.568517684936523 (17.568517684936523) + | > loss_duration: 1.6410622596740723 (1.6410622596740723) + | > loss_1: 30.74015998840332 (30.74015998840332) + + --> STEP: 1 + | > loss_disc: 2.386291027069092 (2.386291027069092) + | > loss_disc_real_0: 0.19816477596759796 (0.19816477596759796) + | > loss_disc_real_1: 0.19106285274028778 (0.19106285274028778) + | > loss_disc_real_2: 0.23698388040065765 (0.23698388040065765) + | > loss_disc_real_3: 0.2511673867702484 (0.2511673867702484) + | > loss_disc_real_4: 0.23394036293029785 (0.23394036293029785) + | > loss_disc_real_5: 0.24307334423065186 (0.24307334423065186) + | > loss_0: 2.386291027069092 (2.386291027069092) + | > loss_gen: 2.627978563308716 (2.627978563308716) + | > loss_kl: 2.5530619621276855 (2.5530619621276855) + | > loss_feat: 7.317337989807129 (7.317337989807129) + | > loss_mel: 18.904495239257812 (18.904495239257812) + | > loss_duration: 1.6837291717529297 (1.6837291717529297) + | > loss_1: 33.08660125732422 (33.08660125732422) + + --> STEP: 2 + | > loss_disc: 2.510474681854248 (2.44838285446167) + | > loss_disc_real_0: 0.19279742240905762 (0.1954810991883278) + | > loss_disc_real_1: 0.20781412720680237 (0.19943848997354507) + | > loss_disc_real_2: 0.23663610219955444 (0.23680999130010605) + | > loss_disc_real_3: 0.23086629807949066 (0.24101684242486954) + | > loss_disc_real_4: 0.2711254060268402 (0.25253288447856903) + | > loss_disc_real_5: 0.28103065490722656 (0.2620519995689392) + | > loss_0: 2.510474681854248 (2.44838285446167) + | > loss_gen: 2.6668295860290527 (2.6474040746688843) + | > loss_kl: 2.3608968257904053 (2.4569793939590454) + | > loss_feat: 5.641490936279297 (6.479414463043213) + | > loss_mel: 17.342819213867188 (18.1236572265625) + | > loss_duration: 1.651808738708496 (1.667768955230713) + | > loss_1: 29.66384506225586 (31.37522315979004) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034519433975219727 (-0.0004794597625732422) + | > avg_loss_disc: 2.44838285446167 (-0.07565510272979736) + | > avg_loss_disc_real_0: 0.1954810991883278 (-0.008755326271057129) + | > avg_loss_disc_real_1: 0.19943848997354507 (-0.01911872625350952) + | > avg_loss_disc_real_2: 0.23680999130010605 (-0.017863646149635315) + | > avg_loss_disc_real_3: 0.24101684242486954 (-0.0010249167680740356) + | > avg_loss_disc_real_4: 0.25253288447856903 (+0.014756090939044952) + | > avg_loss_disc_real_5: 0.2620519995689392 (+0.01950278878211975) + | > avg_loss_0: 2.44838285446167 (-0.07565510272979736) + | > avg_loss_gen: 2.6474040746688843 (+0.16076362133026123) + | > avg_loss_kl: 2.4569793939590454 (+0.09190487861633301) + | > avg_loss_feat: 6.479414463043213 (-0.3101844787597656) + | > avg_loss_mel: 18.1236572265625 (-1.1606159210205078) + | > avg_loss_duration: 1.667768955230713 (+0.007744729518890381) + | > avg_loss_1: 31.37522315979004 (-1.21038818359375) + + + > EPOCH: 145/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:23:10)  + + --> TIME: 2025-04-19 01:23:11 -- STEP: 3/142 -- GLOBAL_STEP: 399875 + | > loss_disc: 2.370944023132324 (2.3808526198069253) + | > loss_disc_real_0: 0.0683509111404419 (0.13450957338015238) + | > loss_disc_real_1: 0.1915261447429657 (0.20472017427285513) + | > loss_disc_real_2: 0.17482833564281464 (0.20416336754957834) + | > loss_disc_real_3: 0.18756528198719025 (0.21804656088352203) + | > loss_disc_real_4: 0.24317222833633423 (0.24138825138409933) + | > loss_disc_real_5: 0.22834791243076324 (0.22476255893707275) + | > loss_0: 2.370944023132324 (2.3808526198069253) + | > grad_norm_0: tensor(19.9457, device='cuda:0') (tensor(11.5522, device='cuda:0')) + | > loss_gen: 2.3555753231048584 (2.3403549194335938) + | > loss_kl: 1.8305546045303345 (1.7955249547958374) + | > loss_feat: 8.00054931640625 (7.668351968129476) + | > loss_mel: 19.50664710998535 (18.089344660441082) + | > loss_duration: 1.4604661464691162 (1.4390480915705364) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.15379333496094 (31.332625071207683) + | > grad_norm_1: tensor(462.5502, device='cuda:0') (tensor(313.4913, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.3214 (0.32300130526224774) + | > loader_time: 0.0037 (0.003935575485229492) + + + --> TIME: 2025-04-19 01:23:20 -- STEP: 28/142 -- GLOBAL_STEP: 399900 + | > loss_disc: 2.4823992252349854 (2.4154828190803523) + | > loss_disc_real_0: 0.16269147396087646 (0.12525053721453464) + | > loss_disc_real_1: 0.23399192094802856 (0.20054559196744645) + | > loss_disc_real_2: 0.2251601666212082 (0.2227355060832841) + | > loss_disc_real_3: 0.23678629100322723 (0.22853300986545427) + | > loss_disc_real_4: 0.2304333746433258 (0.22856532782316208) + | > loss_disc_real_5: 0.24789096415042877 (0.22883362163390433) + | > loss_0: 2.4823992252349854 (2.4154828190803523) + | > grad_norm_0: tensor(13.4409, device='cuda:0') (tensor(11.2079, device='cuda:0')) + | > loss_gen: 2.3892126083374023 (2.3909902572631836) + | > loss_kl: 1.7711988687515259 (1.829289048910141) + | > loss_feat: 6.884802341461182 (7.218251875468662) + | > loss_mel: 18.460412979125977 (17.806632518768314) + | > loss_duration: 1.491037368774414 (1.447068018572671) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.99666404724121 (30.69223185947963) + | > grad_norm_1: tensor(357.3180, device='cuda:0') (tensor(275.1950, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.3448 (0.33511126041412354) + | > loader_time: 0.0051 (0.004867042813982281) + + + --> TIME: 2025-04-19 01:23:29 -- STEP: 53/142 -- GLOBAL_STEP: 399925 + | > loss_disc: 2.5603435039520264 (2.4130694506303323) + | > loss_disc_real_0: 0.09101366251707077 (0.12181888070871245) + | > loss_disc_real_1: 0.19685067236423492 (0.2011272316833712) + | > loss_disc_real_2: 0.2164660096168518 (0.22202464263394195) + | > loss_disc_real_3: 0.22970733046531677 (0.22773594563862062) + | > loss_disc_real_4: 0.2363557070493698 (0.22556498646736145) + | > loss_disc_real_5: 0.25701749324798584 (0.22908375145129437) + | > loss_0: 2.5603435039520264 (2.4130694506303323) + | > grad_norm_0: tensor(13.0509, device='cuda:0') (tensor(10.4934, device='cuda:0')) + | > loss_gen: 2.229254961013794 (2.3950618123108476) + | > loss_kl: 1.8020122051239014 (1.8337069052570272) + | > loss_feat: 6.971031188964844 (7.240954183182627) + | > loss_mel: 17.303089141845703 (17.86710938867533) + | > loss_duration: 1.4590051174163818 (1.4561347916441143) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.764394760131836 (30.792967202528466) + | > grad_norm_1: tensor(72.8876, device='cuda:0') (tensor(266.1103, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.3762 (0.34992826659724385) + | > loader_time: 0.0056 (0.0052879486443861475) + + + --> TIME: 2025-04-19 01:23:39 -- STEP: 78/142 -- GLOBAL_STEP: 399950 + | > loss_disc: 2.4232242107391357 (2.4297302013788467) + | > loss_disc_real_0: 0.10177566856145859 (0.122078103132737) + | > loss_disc_real_1: 0.19523738324642181 (0.20210527552243993) + | > loss_disc_real_2: 0.24677185714244843 (0.22533225344541746) + | > loss_disc_real_3: 0.22573991119861603 (0.22881923195643303) + | > loss_disc_real_4: 0.20811325311660767 (0.22703032864209932) + | > loss_disc_real_5: 0.2378493696451187 (0.22871914410438293) + | > loss_0: 2.4232242107391357 (2.4297302013788467) + | > grad_norm_0: tensor(6.3852, device='cuda:0') (tensor(10.1001, device='cuda:0')) + | > loss_gen: 2.4829630851745605 (2.382187323692517) + | > loss_kl: 1.9659427404403687 (1.8617601471069531) + | > loss_feat: 7.452411651611328 (7.173664245850002) + | > loss_mel: 18.924467086791992 (17.85768916056706) + | > loss_duration: 1.522576093673706 (1.4691296770022464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.34836196899414 (30.744430566445374) + | > grad_norm_1: tensor(235.1731, device='cuda:0') (tensor(249.9662, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.4088 (0.3648558488258948) + | > loader_time: 0.0065 (0.0056308324520404525) + + + --> TIME: 2025-04-19 01:23:50 -- STEP: 103/142 -- GLOBAL_STEP: 399975 + | > loss_disc: 2.560800552368164 (2.4459559477648685) + | > loss_disc_real_0: 0.10759078711271286 (0.1252143287542954) + | > loss_disc_real_1: 0.20558574795722961 (0.2035332951441552) + | > loss_disc_real_2: 0.28906819224357605 (0.22682042506713312) + | > loss_disc_real_3: 0.26846015453338623 (0.22875075519663615) + | > loss_disc_real_4: 0.2758437991142273 (0.22936264100005327) + | > loss_disc_real_5: 0.2818564474582672 (0.2315419780687221) + | > loss_0: 2.560800552368164 (2.4459559477648685) + | > grad_norm_0: tensor(5.4850, device='cuda:0') (tensor(10.8332, device='cuda:0')) + | > loss_gen: 2.2106733322143555 (2.3729627849986237) + | > loss_kl: 1.900830864906311 (1.8769364472731804) + | > loss_feat: 6.409771919250488 (7.099715218960661) + | > loss_mel: 16.586627960205078 (17.835251835943424) + | > loss_duration: 1.544952630996704 (1.480174607443578) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.65285873413086 (30.66504091429479) + | > grad_norm_1: tensor(134.2605, device='cuda:0') (tensor(263.5440, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.4154 (0.37928827295025563) + | > loader_time: 0.0072 (0.00594956897994847) + + + --> TIME: 2025-04-19 01:24:01 -- STEP: 128/142 -- GLOBAL_STEP: 400000 + | > loss_disc: 2.5693745613098145 (2.451760940253734) + | > loss_disc_real_0: 0.15347890555858612 (0.12599480117205525) + | > loss_disc_real_1: 0.24753552675247192 (0.20491568325087428) + | > loss_disc_real_2: 0.22041882574558258 (0.2264745916472748) + | > loss_disc_real_3: 0.23611514270305634 (0.22878619481343776) + | > loss_disc_real_4: 0.22882257401943207 (0.228681965963915) + | > loss_disc_real_5: 0.2255525290966034 (0.23070603283122182) + | > loss_0: 2.5693745613098145 (2.451760940253734) + | > grad_norm_0: tensor(13.2720, device='cuda:0') (tensor(11.2537, device='cuda:0')) + | > loss_gen: 2.2182488441467285 (2.361476521939038) + | > loss_kl: 1.9185727834701538 (1.888180117122829) + | > loss_feat: 6.164635181427002 (7.04640867188573) + | > loss_mel: 17.19441032409668 (17.798401549458504) + | > loss_duration: 1.525193214416504 (1.4864932792261243) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.021060943603516 (30.580960139632225) + | > grad_norm_1: tensor(233.3531, device='cuda:0') (tensor(267.5648, device='cuda:0')) + | > current_lr_0: 0.0001964074314689503 + | > current_lr_1: 0.0001964074314689503 + | > step_time: 0.4376 (0.38914575800299633) + | > loader_time: 0.0073 (0.0061821285635232925) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_400000.pth + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5331356525421143 (2.5331356525421143) + | > loss_disc_real_0: 0.12321759760379791 (0.12321759760379791) + | > loss_disc_real_1: 0.19269363582134247 (0.19269363582134247) + | > loss_disc_real_2: 0.2457934468984604 (0.2457934468984604) + | > loss_disc_real_3: 0.27338218688964844 (0.27338218688964844) + | > loss_disc_real_4: 0.2544306516647339 (0.2544306516647339) + | > loss_disc_real_5: 0.25148555636405945 (0.25148555636405945) + | > loss_0: 2.5331356525421143 (2.5331356525421143) + | > loss_gen: 2.173325538635254 (2.173325538635254) + | > loss_kl: 2.4577341079711914 (2.4577341079711914) + | > loss_feat: 7.144156455993652 (7.144156455993652) + | > loss_mel: 19.081789016723633 (19.081789016723633) + | > loss_duration: 1.6418131589889526 (1.6418131589889526) + | > loss_1: 32.498817443847656 (32.498817443847656) + + --> STEP: 1 + | > loss_disc: 2.465454339981079 (2.465454339981079) + | > loss_disc_real_0: 0.1095559373497963 (0.1095559373497963) + | > loss_disc_real_1: 0.2650938928127289 (0.2650938928127289) + | > loss_disc_real_2: 0.24891437590122223 (0.24891437590122223) + | > loss_disc_real_3: 0.24434871971607208 (0.24434871971607208) + | > loss_disc_real_4: 0.24285008013248444 (0.24285008013248444) + | > loss_disc_real_5: 0.23260366916656494 (0.23260366916656494) + | > loss_0: 2.465454339981079 (2.465454339981079) + | > loss_gen: 2.436356544494629 (2.436356544494629) + | > loss_kl: 2.4749324321746826 (2.4749324321746826) + | > loss_feat: 5.982629299163818 (5.982629299163818) + | > loss_mel: 18.30838394165039 (18.30838394165039) + | > loss_duration: 1.7122972011566162 (1.7122972011566162) + | > loss_1: 30.91459846496582 (30.91459846496582) + + --> STEP: 2 + | > loss_disc: 2.398056983947754 (2.4317556619644165) + | > loss_disc_real_0: 0.1110379546880722 (0.11029694601893425) + | > loss_disc_real_1: 0.16906443238258362 (0.21707916259765625) + | > loss_disc_real_2: 0.24740584194660187 (0.24816010892391205) + | > loss_disc_real_3: 0.23088175058364868 (0.23761523514986038) + | > loss_disc_real_4: 0.19288292527198792 (0.21786650270223618) + | > loss_disc_real_5: 0.23340074717998505 (0.233002208173275) + | > loss_0: 2.398056983947754 (2.4317556619644165) + | > loss_gen: 2.2004048824310303 (2.3183807134628296) + | > loss_kl: 2.179954767227173 (2.3274435997009277) + | > loss_feat: 7.197498798370361 (6.59006404876709) + | > loss_mel: 18.336589813232422 (18.322486877441406) + | > loss_duration: 1.623326301574707 (1.6678117513656616) + | > loss_1: 31.53777313232422 (31.22618579864502) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034558773040771484 (+3.933906555175781e-06) + | > avg_loss_disc: 2.4317556619644165 (-0.016627192497253418) + | > avg_loss_disc_real_0: 0.11029694601893425 (-0.08518415316939354) + | > avg_loss_disc_real_1: 0.21707916259765625 (+0.017640672624111176) + | > avg_loss_disc_real_2: 0.24816010892391205 (+0.011350117623806) + | > avg_loss_disc_real_3: 0.23761523514986038 (-0.0034016072750091553) + | > avg_loss_disc_real_4: 0.21786650270223618 (-0.034666381776332855) + | > avg_loss_disc_real_5: 0.233002208173275 (-0.029049791395664215) + | > avg_loss_0: 2.4317556619644165 (-0.016627192497253418) + | > avg_loss_gen: 2.3183807134628296 (-0.3290233612060547) + | > avg_loss_kl: 2.3274435997009277 (-0.12953579425811768) + | > avg_loss_feat: 6.59006404876709 (+0.11064958572387695) + | > avg_loss_mel: 18.322486877441406 (+0.19882965087890625) + | > avg_loss_duration: 1.6678117513656616 (+4.279613494873047e-05) + | > avg_loss_1: 31.22618579864502 (-0.14903736114501953) + + + > EPOCH: 146/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:24:10)  + + --> TIME: 2025-04-19 01:24:15 -- STEP: 11/142 -- GLOBAL_STEP: 400025 + | > loss_disc: 2.3880157470703125 (2.409557754343206) + | > loss_disc_real_0: 0.12068253010511398 (0.11194695667787032) + | > loss_disc_real_1: 0.20250828564167023 (0.20268667421557687) + | > loss_disc_real_2: 0.22147299349308014 (0.2259332618930123) + | > loss_disc_real_3: 0.2075594961643219 (0.23477923599156467) + | > loss_disc_real_4: 0.21744027733802795 (0.22685780172998254) + | > loss_disc_real_5: 0.21239346265792847 (0.24224519458684055) + | > loss_0: 2.3880157470703125 (2.409557754343206) + | > grad_norm_0: tensor(11.4725, device='cuda:0') (tensor(13.7822, device='cuda:0')) + | > loss_gen: 2.404313325881958 (2.448568257418546) + | > loss_kl: 1.7999229431152344 (1.871619083664634) + | > loss_feat: 7.2777509689331055 (7.492100498893044) + | > loss_mel: 18.791353225708008 (18.345394307916816) + | > loss_duration: 1.4408743381500244 (1.4373329552737149) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.714214324951172 (31.595015265724875) + | > grad_norm_1: tensor(407.5739, device='cuda:0') (tensor(347.3902, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.3285 (0.32521520961414685) + | > loader_time: 0.0052 (0.004784107208251953) + + + --> TIME: 2025-04-19 01:24:24 -- STEP: 36/142 -- GLOBAL_STEP: 400050 + | > loss_disc: 2.2915163040161133 (2.4162232412232294) + | > loss_disc_real_0: 0.10599013417959213 (0.11403584749334389) + | > loss_disc_real_1: 0.24949169158935547 (0.204589883900351) + | > loss_disc_real_2: 0.2408323585987091 (0.22398136307795843) + | > loss_disc_real_3: 0.21318548917770386 (0.2282262241674794) + | > loss_disc_real_4: 0.21019011735916138 (0.2278940168519815) + | > loss_disc_real_5: 0.20996302366256714 (0.23378968031869996) + | > loss_0: 2.2915163040161133 (2.4162232412232294) + | > grad_norm_0: tensor(8.1158, device='cuda:0') (tensor(12.9961, device='cuda:0')) + | > loss_gen: 2.672760248184204 (2.4245014720492892) + | > loss_kl: 1.941282868385315 (1.8793517748514812) + | > loss_feat: 7.954453468322754 (7.375328898429871) + | > loss_mel: 19.3558292388916 (18.065869596269394) + | > loss_duration: 1.460747241973877 (1.4529126584529877) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.385074615478516 (31.19796429740058) + | > grad_norm_1: tensor(421.2163, device='cuda:0') (tensor(322.5987, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.3542 (0.33821544382307267) + | > loader_time: 0.0058 (0.005249453915490044) + + + --> TIME: 2025-04-19 01:24:33 -- STEP: 61/142 -- GLOBAL_STEP: 400075 + | > loss_disc: 2.5422825813293457 (2.4470856189727774) + | > loss_disc_real_0: 0.1474636048078537 (0.11755262168704486) + | > loss_disc_real_1: 0.21831081807613373 (0.20636869747130598) + | > loss_disc_real_2: 0.19247443974018097 (0.22587646299698314) + | > loss_disc_real_3: 0.20853976905345917 (0.23043123425030318) + | > loss_disc_real_4: 0.20394295454025269 (0.231107980990019) + | > loss_disc_real_5: 0.2866436839103699 (0.23492063386518447) + | > loss_0: 2.5422825813293457 (2.4470856189727774) + | > grad_norm_0: tensor(28.9975, device='cuda:0') (tensor(14.5306, device='cuda:0')) + | > loss_gen: 2.1255903244018555 (2.3919166541490404) + | > loss_kl: 1.8417657613754272 (1.8892868210057743) + | > loss_feat: 6.7199835777282715 (7.235052014960617) + | > loss_mel: 17.91930389404297 (18.05797881767398) + | > loss_duration: 1.4841846227645874 (1.4643913116611418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.09082794189453 (31.038625467019003) + | > grad_norm_1: tensor(418.8383, device='cuda:0') (tensor(321.9826, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.3817 (0.3501676106062091) + | > loader_time: 0.0067 (0.005618752026167072) + + + --> TIME: 2025-04-19 01:24:43 -- STEP: 86/142 -- GLOBAL_STEP: 400100 + | > loss_disc: 2.4334323406219482 (2.45286524018576) + | > loss_disc_real_0: 0.08197291195392609 (0.11996507046874179) + | > loss_disc_real_1: 0.20371881127357483 (0.20813400281030078) + | > loss_disc_real_2: 0.21816426515579224 (0.22504288218049115) + | > loss_disc_real_3: 0.23321200907230377 (0.2304913479574891) + | > loss_disc_real_4: 0.23246410489082336 (0.2289114407328672) + | > loss_disc_real_5: 0.24257127940654755 (0.23235326936078626) + | > loss_0: 2.4334323406219482 (2.45286524018576) + | > grad_norm_0: tensor(23.2540, device='cuda:0') (tensor(14.3246, device='cuda:0')) + | > loss_gen: 2.391767740249634 (2.3821701809417375) + | > loss_kl: 1.9738118648529053 (1.9027292672977891) + | > loss_feat: 7.075652599334717 (7.089412838913674) + | > loss_mel: 18.466081619262695 (17.81724761253179) + | > loss_duration: 1.5378423929214478 (1.475422584733298) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.445154190063477 (30.666982251544333) + | > grad_norm_1: tensor(370.5425, device='cuda:0') (tensor(320.9545, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.3983 (0.36185638294663547) + | > loader_time: 0.0065 (0.0058392591254655705) + + + --> TIME: 2025-04-19 01:24:53 -- STEP: 111/142 -- GLOBAL_STEP: 400125 + | > loss_disc: 2.4964277744293213 (2.4629906577032967) + | > loss_disc_real_0: 0.15761493146419525 (0.12250617958672412) + | > loss_disc_real_1: 0.20238295197486877 (0.2087961470221614) + | > loss_disc_real_2: 0.2281816303730011 (0.22611238841000977) + | > loss_disc_real_3: 0.2294699251651764 (0.23042758319291984) + | > loss_disc_real_4: 0.25452473759651184 (0.2287433735422186) + | > loss_disc_real_5: 0.22802716493606567 (0.23309338052530545) + | > loss_0: 2.4964277744293213 (2.4629906577032967) + | > grad_norm_0: tensor(13.4212, device='cuda:0') (tensor(13.8476, device='cuda:0')) + | > loss_gen: 2.3575096130371094 (2.362697137368692) + | > loss_kl: 1.842901349067688 (1.9165030094954345) + | > loss_feat: 6.449959754943848 (7.034661348875578) + | > loss_mel: 17.336027145385742 (17.802488060684887) + | > loss_duration: 1.5372272729873657 (1.4840897536492565) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.523624420166016 (30.6004391747552) + | > grad_norm_1: tensor(94.3525, device='cuda:0') (tensor(306.0779, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.4231 (0.37256801235783216) + | > loader_time: 0.0074 (0.00609290277635729) + + + --> TIME: 2025-04-19 01:25:05 -- STEP: 136/142 -- GLOBAL_STEP: 400150 + | > loss_disc: 2.394803285598755 (2.4652394231627968) + | > loss_disc_real_0: 0.12932975590229034 (0.12428032650667079) + | > loss_disc_real_1: 0.16427260637283325 (0.20804285893545432) + | > loss_disc_real_2: 0.24425195157527924 (0.2257801952826626) + | > loss_disc_real_3: 0.2021259069442749 (0.2296015621327302) + | > loss_disc_real_4: 0.20848867297172546 (0.22957178764045238) + | > loss_disc_real_5: 0.25003379583358765 (0.23369335098301663) + | > loss_0: 2.394803285598755 (2.4652394231627968) + | > grad_norm_0: tensor(21.5510, device='cuda:0') (tensor(13.5473, device='cuda:0')) + | > loss_gen: 2.4317731857299805 (2.3609675589729755) + | > loss_kl: 1.9866281747817993 (1.9217175245285034) + | > loss_feat: 7.39994478225708 (7.012160220566918) + | > loss_mel: 18.905149459838867 (17.7624006060993) + | > loss_duration: 1.514329433441162 (1.4900924388100123) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.237823486328125 (30.547338233274573) + | > grad_norm_1: tensor(478.1517, device='cuda:0') (tensor(298.0564, device='cuda:0')) + | > current_lr_0: 0.00019638288054001668 + | > current_lr_1: 0.00019638288054001668 + | > step_time: 0.451 (0.3845816692885231) + | > loader_time: 0.0077 (0.006363719701766968) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5051352977752686 (2.5051352977752686) + | > loss_disc_real_0: 0.15118207037448883 (0.15118207037448883) + | > loss_disc_real_1: 0.2545568645000458 (0.2545568645000458) + | > loss_disc_real_2: 0.2738591730594635 (0.2738591730594635) + | > loss_disc_real_3: 0.25108614563941956 (0.25108614563941956) + | > loss_disc_real_4: 0.25340890884399414 (0.25340890884399414) + | > loss_disc_real_5: 0.21903786063194275 (0.21903786063194275) + | > loss_0: 2.5051352977752686 (2.5051352977752686) + | > loss_gen: 2.421941041946411 (2.421941041946411) + | > loss_kl: 2.149940252304077 (2.149940252304077) + | > loss_feat: 6.887301921844482 (6.887301921844482) + | > loss_mel: 18.74650001525879 (18.74650001525879) + | > loss_duration: 1.6641509532928467 (1.6641509532928467) + | > loss_1: 31.86983299255371 (31.86983299255371) + + --> STEP: 1 + | > loss_disc: 2.6252269744873047 (2.6252269744873047) + | > loss_disc_real_0: 0.20671673119068146 (0.20671673119068146) + | > loss_disc_real_1: 0.28003254532814026 (0.28003254532814026) + | > loss_disc_real_2: 0.24108660221099854 (0.24108660221099854) + | > loss_disc_real_3: 0.2529531717300415 (0.2529531717300415) + | > loss_disc_real_4: 0.2534208297729492 (0.2534208297729492) + | > loss_disc_real_5: 0.22012503445148468 (0.22012503445148468) + | > loss_0: 2.6252269744873047 (2.6252269744873047) + | > loss_gen: 2.286311388015747 (2.286311388015747) + | > loss_kl: 2.5390450954437256 (2.5390450954437256) + | > loss_feat: 6.448429107666016 (6.448429107666016) + | > loss_mel: 18.423717498779297 (18.423717498779297) + | > loss_duration: 1.6999943256378174 (1.6999943256378174) + | > loss_1: 31.397499084472656 (31.397499084472656) + + --> STEP: 2 + | > loss_disc: 2.526109218597412 (2.5756680965423584) + | > loss_disc_real_0: 0.1692495048046112 (0.18798311799764633) + | > loss_disc_real_1: 0.23742935061454773 (0.258730947971344) + | > loss_disc_real_2: 0.2234094887971878 (0.23224804550409317) + | > loss_disc_real_3: 0.2214473932981491 (0.2372002825140953) + | > loss_disc_real_4: 0.22873005270957947 (0.24107544124126434) + | > loss_disc_real_5: 0.2034354954957962 (0.21178026497364044) + | > loss_0: 2.526109218597412 (2.5756680965423584) + | > loss_gen: 2.33927059173584 (2.3127909898757935) + | > loss_kl: 2.0832271575927734 (2.3111361265182495) + | > loss_feat: 6.484531879425049 (6.466480493545532) + | > loss_mel: 16.52311134338379 (17.473414421081543) + | > loss_duration: 1.6618461608886719 (1.6809202432632446) + | > loss_1: 29.09198760986328 (30.24474334716797) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003334641456604004 (-0.00012123584747314453) + | > avg_loss_disc: 2.5756680965423584 (+0.1439124345779419) + | > avg_loss_disc_real_0: 0.18798311799764633 (+0.07768617197871208) + | > avg_loss_disc_real_1: 0.258730947971344 (+0.041651785373687744) + | > avg_loss_disc_real_2: 0.23224804550409317 (-0.015912063419818878) + | > avg_loss_disc_real_3: 0.2372002825140953 (-0.0004149526357650757) + | > avg_loss_disc_real_4: 0.24107544124126434 (+0.023208938539028168) + | > avg_loss_disc_real_5: 0.21178026497364044 (-0.021221943199634552) + | > avg_loss_0: 2.5756680965423584 (+0.1439124345779419) + | > avg_loss_gen: 2.3127909898757935 (-0.005589723587036133) + | > avg_loss_kl: 2.3111361265182495 (-0.016307473182678223) + | > avg_loss_feat: 6.466480493545532 (-0.12358355522155762) + | > avg_loss_mel: 17.473414421081543 (-0.8490724563598633) + | > avg_loss_duration: 1.6809202432632446 (+0.013108491897583008) + | > avg_loss_1: 30.24474334716797 (-0.9814424514770508) + + + > EPOCH: 147/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:25:09)  + + --> TIME: 2025-04-19 01:25:16 -- STEP: 19/142 -- GLOBAL_STEP: 400175 + | > loss_disc: 2.4045021533966064 (2.460961379502949) + | > loss_disc_real_0: 0.06212253496050835 (0.13639322452639283) + | > loss_disc_real_1: 0.19725428521633148 (0.20902968629410393) + | > loss_disc_real_2: 0.2381695955991745 (0.22959482748257487) + | > loss_disc_real_3: 0.22211861610412598 (0.22157191602807297) + | > loss_disc_real_4: 0.24356628954410553 (0.22423367280709117) + | > loss_disc_real_5: 0.21240243315696716 (0.22692202580602547) + | > loss_0: 2.4045021533966064 (2.460961379502949) + | > grad_norm_0: tensor(18.8072, device='cuda:0') (tensor(18.7040, device='cuda:0')) + | > loss_gen: 2.380286455154419 (2.359352801975451) + | > loss_kl: 1.766910195350647 (1.8452240416878147) + | > loss_feat: 7.411244869232178 (7.135450915286415) + | > loss_mel: 17.147655487060547 (17.763806192498457) + | > loss_duration: 1.434509515762329 (1.4349631573024548) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.140605926513672 (30.53879697699296) + | > grad_norm_1: tensor(395.6646, device='cuda:0') (tensor(291.3387, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.3393 (0.33142347084848506) + | > loader_time: 0.0051 (0.004819669221576892) + + + --> TIME: 2025-04-19 01:25:25 -- STEP: 44/142 -- GLOBAL_STEP: 400200 + | > loss_disc: 2.5293517112731934 (2.4453114921396426) + | > loss_disc_real_0: 0.12445367872714996 (0.12750538734888492) + | > loss_disc_real_1: 0.2503076493740082 (0.2081840837543661) + | > loss_disc_real_2: 0.20785944163799286 (0.22493600608272987) + | > loss_disc_real_3: 0.20813865959644318 (0.2247940278188749) + | > loss_disc_real_4: 0.2359730452299118 (0.22503423826261001) + | > loss_disc_real_5: 0.2681049406528473 (0.23122940652749754) + | > loss_0: 2.5293517112731934 (2.4453114921396426) + | > grad_norm_0: tensor(9.9827, device='cuda:0') (tensor(16.9568, device='cuda:0')) + | > loss_gen: 2.3197638988494873 (2.392582405697216) + | > loss_kl: 1.9527993202209473 (1.8506456261331385) + | > loss_feat: 6.8202290534973145 (7.172900394959883) + | > loss_mel: 19.06136703491211 (17.84831766648726) + | > loss_duration: 1.4901816844940186 (1.45194937153296) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.64434051513672 (30.716395464810457) + | > grad_norm_1: tensor(208.8838, device='cuda:0') (tensor(312.6392, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.3596 (0.34403321417895233) + | > loader_time: 0.0059 (0.005230990323153408) + + + --> TIME: 2025-04-19 01:25:35 -- STEP: 69/142 -- GLOBAL_STEP: 400225 + | > loss_disc: 2.3505306243896484 (2.453784137532331) + | > loss_disc_real_0: 0.09851445257663727 (0.12718453101705812) + | > loss_disc_real_1: 0.24991261959075928 (0.20731610232505243) + | > loss_disc_real_2: 0.2282271832227707 (0.22495229283104773) + | > loss_disc_real_3: 0.21557137370109558 (0.22651848693688711) + | > loss_disc_real_4: 0.21324461698532104 (0.22667923504891602) + | > loss_disc_real_5: 0.21343225240707397 (0.23005975329357645) + | > loss_0: 2.3505306243896484 (2.453784137532331) + | > grad_norm_0: tensor(3.6056, device='cuda:0') (tensor(13.6083, device='cuda:0')) + | > loss_gen: 2.5110316276550293 (2.3740301581396572) + | > loss_kl: 2.0907816886901855 (1.8822703344234522) + | > loss_feat: 7.53680944442749 (7.141986736352893) + | > loss_mel: 19.276517868041992 (17.890080023502957) + | > loss_duration: 1.545788049697876 (1.464136305062667) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.96092987060547 (30.75250360240107) + | > grad_norm_1: tensor(65.8512, device='cuda:0') (tensor(257.1396, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.3955 (0.3561499049698097) + | > loader_time: 0.0062 (0.005572882251463074) + + + --> TIME: 2025-04-19 01:25:45 -- STEP: 94/142 -- GLOBAL_STEP: 400250 + | > loss_disc: 2.439840078353882 (2.4574501387616414) + | > loss_disc_real_0: 0.10734238475561142 (0.12941392451683256) + | > loss_disc_real_1: 0.19287702441215515 (0.20580568006064026) + | > loss_disc_real_2: 0.24548575282096863 (0.22508848775574503) + | > loss_disc_real_3: 0.21535852551460266 (0.22674925672881147) + | > loss_disc_real_4: 0.23904651403427124 (0.22748861097274942) + | > loss_disc_real_5: 0.21987886726856232 (0.23341166941409416) + | > loss_0: 2.439840078353882 (2.4574501387616414) + | > grad_norm_0: tensor(6.9455, device='cuda:0') (tensor(12.0171, device='cuda:0')) + | > loss_gen: 2.372204303741455 (2.3638066028026814) + | > loss_kl: 2.0677874088287354 (1.911106192051096) + | > loss_feat: 6.40090274810791 (7.073475781907427) + | > loss_mel: 17.00711441040039 (17.756910993697794) + | > loss_duration: 1.521583080291748 (1.4759865644130294) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.369590759277344 (30.58128616657663) + | > grad_norm_1: tensor(204.7071, device='cuda:0') (tensor(224.5634, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.3959 (0.3665542982994242) + | > loader_time: 0.0066 (0.005820685244621115) + + + --> TIME: 2025-04-19 01:25:56 -- STEP: 119/142 -- GLOBAL_STEP: 400275 + | > loss_disc: 2.4528725147247314 (2.465909314756634) + | > loss_disc_real_0: 0.11907374858856201 (0.13176418925408553) + | > loss_disc_real_1: 0.21975676715373993 (0.20637482116703224) + | > loss_disc_real_2: 0.20608285069465637 (0.22545889558411447) + | > loss_disc_real_3: 0.20069842040538788 (0.22667791312482177) + | > loss_disc_real_4: 0.2369609922170639 (0.22802500076153698) + | > loss_disc_real_5: 0.23261915147304535 (0.2322951381196495) + | > loss_0: 2.4528725147247314 (2.465909314756634) + | > grad_norm_0: tensor(9.4817, device='cuda:0') (tensor(11.3531, device='cuda:0')) + | > loss_gen: 2.3674399852752686 (2.3482396021610548) + | > loss_kl: 1.9267926216125488 (1.9133155015336365) + | > loss_feat: 6.389073848724365 (7.014345112968893) + | > loss_mel: 16.85077667236328 (17.72568439836263) + | > loss_duration: 1.5698373317718506 (1.4836128429204472) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.103919982910156 (30.485197500020515) + | > grad_norm_1: tensor(233.8431, device='cuda:0') (tensor(209.3350, device='cuda:0')) + | > current_lr_0: 0.00019635833267994917 + | > current_lr_1: 0.00019635833267994917 + | > step_time: 0.4293 (0.37860563622803245) + | > loader_time: 0.0077 (0.0061194495994503755) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.569509506225586 (2.569509506225586) + | > loss_disc_real_0: 0.15277455747127533 (0.15277455747127533) + | > loss_disc_real_1: 0.22334876656532288 (0.22334876656532288) + | > loss_disc_real_2: 0.2638997435569763 (0.2638997435569763) + | > loss_disc_real_3: 0.24237625300884247 (0.24237625300884247) + | > loss_disc_real_4: 0.23498357832431793 (0.23498357832431793) + | > loss_disc_real_5: 0.2412756085395813 (0.2412756085395813) + | > loss_0: 2.569509506225586 (2.569509506225586) + | > loss_gen: 2.256619930267334 (2.256619930267334) + | > loss_kl: 2.240450143814087 (2.240450143814087) + | > loss_feat: 6.2187089920043945 (6.2187089920043945) + | > loss_mel: 18.407764434814453 (18.407764434814453) + | > loss_duration: 1.6985560655593872 (1.6985560655593872) + | > loss_1: 30.822101593017578 (30.822101593017578) + + --> STEP: 1 + | > loss_disc: 2.4877829551696777 (2.4877829551696777) + | > loss_disc_real_0: 0.1334385722875595 (0.1334385722875595) + | > loss_disc_real_1: 0.23813146352767944 (0.23813146352767944) + | > loss_disc_real_2: 0.23790813982486725 (0.23790813982486725) + | > loss_disc_real_3: 0.22595804929733276 (0.22595804929733276) + | > loss_disc_real_4: 0.24357870221138 (0.24357870221138) + | > loss_disc_real_5: 0.22220879793167114 (0.22220879793167114) + | > loss_0: 2.4877829551696777 (2.4877829551696777) + | > loss_gen: 2.274754524230957 (2.274754524230957) + | > loss_kl: 2.4744372367858887 (2.4744372367858887) + | > loss_feat: 6.159054756164551 (6.159054756164551) + | > loss_mel: 17.191055297851562 (17.191055297851562) + | > loss_duration: 1.71537184715271 (1.71537184715271) + | > loss_1: 29.814672470092773 (29.814672470092773) + + --> STEP: 2 + | > loss_disc: 2.5000927448272705 (2.493937849998474) + | > loss_disc_real_0: 0.10265601426362991 (0.11804729327559471) + | > loss_disc_real_1: 0.2507876455783844 (0.24445955455303192) + | > loss_disc_real_2: 0.23951907455921173 (0.2387136071920395) + | > loss_disc_real_3: 0.24077454209327698 (0.23336629569530487) + | > loss_disc_real_4: 0.20253902673721313 (0.22305886447429657) + | > loss_disc_real_5: 0.23384742438793182 (0.22802811115980148) + | > loss_0: 2.5000927448272705 (2.493937849998474) + | > loss_gen: 2.2498552799224854 (2.262304902076721) + | > loss_kl: 2.127939224243164 (2.3011882305145264) + | > loss_feat: 7.334991455078125 (6.747023105621338) + | > loss_mel: 17.7485408782959 (17.46979808807373) + | > loss_duration: 1.630378246307373 (1.6728750467300415) + | > loss_1: 31.091705322265625 (30.4531888961792) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003550887107849121 (+0.0002162456512451172) + | > avg_loss_disc: 2.493937849998474 (-0.08173024654388428) + | > avg_loss_disc_real_0: 0.11804729327559471 (-0.06993582472205162) + | > avg_loss_disc_real_1: 0.24445955455303192 (-0.014271393418312073) + | > avg_loss_disc_real_2: 0.2387136071920395 (+0.00646556168794632) + | > avg_loss_disc_real_3: 0.23336629569530487 (-0.003833986818790436) + | > avg_loss_disc_real_4: 0.22305886447429657 (-0.018016576766967773) + | > avg_loss_disc_real_5: 0.22802811115980148 (+0.01624784618616104) + | > avg_loss_0: 2.493937849998474 (-0.08173024654388428) + | > avg_loss_gen: 2.262304902076721 (-0.050486087799072266) + | > avg_loss_kl: 2.3011882305145264 (-0.009947896003723145) + | > avg_loss_feat: 6.747023105621338 (+0.28054261207580566) + | > avg_loss_mel: 17.46979808807373 (-0.0036163330078125) + | > avg_loss_duration: 1.6728750467300415 (-0.008045196533203125) + | > avg_loss_1: 30.4531888961792 (+0.20844554901123047) + + + > EPOCH: 148/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:26:08)  + + --> TIME: 2025-04-19 01:26:09 -- STEP: 2/142 -- GLOBAL_STEP: 400300 + | > loss_disc: 2.464873790740967 (2.466000199317932) + | > loss_disc_real_0: 0.15838554501533508 (0.13795384392142296) + | > loss_disc_real_1: 0.18748998641967773 (0.20074495673179626) + | > loss_disc_real_2: 0.24833764135837555 (0.23790617287158966) + | > loss_disc_real_3: 0.24150608479976654 (0.23742922395467758) + | > loss_disc_real_4: 0.23637880384922028 (0.23950917273759842) + | > loss_disc_real_5: 0.24675221741199493 (0.23246270418167114) + | > loss_0: 2.464873790740967 (2.466000199317932) + | > grad_norm_0: tensor(7.4872, device='cuda:0') (tensor(7.3273, device='cuda:0')) + | > loss_gen: 2.5018210411071777 (2.4882309436798096) + | > loss_kl: 1.9336462020874023 (1.8570266962051392) + | > loss_feat: 6.94992208480835 (7.2058775424957275) + | > loss_mel: 16.94552230834961 (17.098602294921875) + | > loss_duration: 1.465168833732605 (1.4436002373695374) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.796079635620117 (30.093337059020996) + | > grad_norm_1: tensor(47.1962, device='cuda:0') (tensor(184.5732, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.322 (0.324161171913147) + | > loader_time: 0.0046 (0.004590868949890137) + + + --> TIME: 2025-04-19 01:26:18 -- STEP: 27/142 -- GLOBAL_STEP: 400325 + | > loss_disc: 2.509758710861206 (2.4598241911994085) + | > loss_disc_real_0: 0.10542275756597519 (0.11869911573551319) + | > loss_disc_real_1: 0.2257186323404312 (0.2097007339751279) + | > loss_disc_real_2: 0.2583288550376892 (0.2301044883551421) + | > loss_disc_real_3: 0.2477690428495407 (0.22491347513816976) + | > loss_disc_real_4: 0.22470691800117493 (0.2284306933482488) + | > loss_disc_real_5: 0.22287917137145996 (0.23173160409485852) + | > loss_0: 2.509758710861206 (2.4598241911994085) + | > grad_norm_0: tensor(4.1822, device='cuda:0') (tensor(7.9437, device='cuda:0')) + | > loss_gen: 2.4908862113952637 (2.3788136729487666) + | > loss_kl: 1.7988141775131226 (1.8413261969884236) + | > loss_feat: 7.279004096984863 (7.3394642759252475) + | > loss_mel: 18.792373657226562 (18.123765980755845) + | > loss_duration: 1.4406810998916626 (1.448144206294307) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.801759719848633 (31.131514655219185) + | > grad_norm_1: tensor(335.9016, device='cuda:0') (tensor(249.5168, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.3499 (0.3447372560147886) + | > loader_time: 0.0047 (0.004798580099035193) + + + --> TIME: 2025-04-19 01:26:27 -- STEP: 52/142 -- GLOBAL_STEP: 400350 + | > loss_disc: 2.600536584854126 (2.4515243172645564) + | > loss_disc_real_0: 0.10183916240930557 (0.1216805402476054) + | > loss_disc_real_1: 0.2057368904352188 (0.20849281721390212) + | > loss_disc_real_2: 0.2189406305551529 (0.2261626600072934) + | > loss_disc_real_3: 0.24873581528663635 (0.22717863837113747) + | > loss_disc_real_4: 0.24966125190258026 (0.2282076162787584) + | > loss_disc_real_5: 0.2654041051864624 (0.23274902540903825) + | > loss_0: 2.600536584854126 (2.4515243172645564) + | > grad_norm_0: tensor(11.3310, device='cuda:0') (tensor(9.2551, device='cuda:0')) + | > loss_gen: 2.1775450706481934 (2.381964298395011) + | > loss_kl: 1.9399176836013794 (1.864314615726471) + | > loss_feat: 6.307357311248779 (7.192269453635583) + | > loss_mel: 18.794612884521484 (18.043762280390816) + | > loss_duration: 1.5019958019256592 (1.4593562667186442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.7214298248291 (30.9416671532851) + | > grad_norm_1: tensor(234.7560, device='cuda:0') (tensor(251.4264, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.3668 (0.3526436365567721) + | > loader_time: 0.006 (0.005079393203441913) + + + --> TIME: 2025-04-19 01:26:37 -- STEP: 77/142 -- GLOBAL_STEP: 400375 + | > loss_disc: 2.4351420402526855 (2.465752697610236) + | > loss_disc_real_0: 0.08013606816530228 (0.12525141771350587) + | > loss_disc_real_1: 0.2391626238822937 (0.2088926463545143) + | > loss_disc_real_2: 0.218606099486351 (0.22709183994825785) + | > loss_disc_real_3: 0.2597697377204895 (0.22699465109156325) + | > loss_disc_real_4: 0.2179391086101532 (0.22799074320824114) + | > loss_disc_real_5: 0.21770213544368744 (0.23374635368198543) + | > loss_0: 2.4351420402526855 (2.465752697610236) + | > grad_norm_0: tensor(11.8671, device='cuda:0') (tensor(11.7922, device='cuda:0')) + | > loss_gen: 2.4323861598968506 (2.3776008649305873) + | > loss_kl: 1.9503297805786133 (1.8895936910208169) + | > loss_feat: 7.79973840713501 (7.162768617852942) + | > loss_mel: 17.38440704345703 (18.00070046759271) + | > loss_duration: 1.4927282333374023 (1.4710332511307354) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.059589385986328 (30.90169708450119) + | > grad_norm_1: tensor(311.1729, device='cuda:0') (tensor(267.0604, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.3894 (0.3631815693595193) + | > loader_time: 0.0065 (0.005415767818302303) + + + --> TIME: 2025-04-19 01:26:47 -- STEP: 102/142 -- GLOBAL_STEP: 400400 + | > loss_disc: 2.5117154121398926 (2.4674922578475065) + | > loss_disc_real_0: 0.06883931159973145 (0.12630623250323178) + | > loss_disc_real_1: 0.24896572530269623 (0.2093939727135733) + | > loss_disc_real_2: 0.26270774006843567 (0.2266714770127745) + | > loss_disc_real_3: 0.2174496352672577 (0.22763986491105137) + | > loss_disc_real_4: 0.25343677401542664 (0.22789057475679061) + | > loss_disc_real_5: 0.24021756649017334 (0.2340576844472511) + | > loss_0: 2.5117154121398926 (2.4674922578475065) + | > grad_norm_0: tensor(21.5420, device='cuda:0') (tensor(13.3652, device='cuda:0')) + | > loss_gen: 2.43854022026062 (2.3875682798086437) + | > loss_kl: 1.9684163331985474 (1.9137659855917388) + | > loss_feat: 7.083406448364258 (7.136292967141843) + | > loss_mel: 18.942649841308594 (17.98583288753734) + | > loss_duration: 1.5039196014404297 (1.481878343750448) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.936931610107422 (30.905338661343443) + | > grad_norm_1: tensor(411.1683, device='cuda:0') (tensor(293.3259, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.4173 (0.3727515772277233) + | > loader_time: 0.0072 (0.005706483242558498) + + + --> TIME: 2025-04-19 01:26:58 -- STEP: 127/142 -- GLOBAL_STEP: 400425 + | > loss_disc: 2.511293411254883 (2.477099336038425) + | > loss_disc_real_0: 0.1442212164402008 (0.12890776487316666) + | > loss_disc_real_1: 0.20422305166721344 (0.2093717161360688) + | > loss_disc_real_2: 0.22280022501945496 (0.22644699769695914) + | > loss_disc_real_3: 0.21016845107078552 (0.22812145162285782) + | > loss_disc_real_4: 0.259211003780365 (0.22909406167785012) + | > loss_disc_real_5: 0.24073131382465363 (0.23623869902505648) + | > loss_0: 2.511293411254883 (2.477099336038425) + | > grad_norm_0: tensor(16.6973, device='cuda:0') (tensor(14.2022, device='cuda:0')) + | > loss_gen: 2.2376060485839844 (2.3708556336680746) + | > loss_kl: 1.9962595701217651 (1.916883896654985) + | > loss_feat: 5.320127010345459 (7.023127484509326) + | > loss_mel: 14.84392261505127 (17.861950769199165) + | > loss_duration: 1.522143840789795 (1.4889707377576447) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.920059204101562 (30.66178865507832) + | > grad_norm_1: tensor(102.2100, device='cuda:0') (tensor(298.5367, device='cuda:0')) + | > current_lr_0: 0.00019633378788836418 + | > current_lr_1: 0.00019633378788836418 + | > step_time: 0.4393 (0.3837518973613348) + | > loader_time: 0.0072 (0.006004708958423044) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6112704277038574 (2.6112704277038574) + | > loss_disc_real_0: 0.2066703885793686 (0.2066703885793686) + | > loss_disc_real_1: 0.22905655205249786 (0.22905655205249786) + | > loss_disc_real_2: 0.24820515513420105 (0.24820515513420105) + | > loss_disc_real_3: 0.2889857590198517 (0.2889857590198517) + | > loss_disc_real_4: 0.2660280168056488 (0.2660280168056488) + | > loss_disc_real_5: 0.25984987616539 (0.25984987616539) + | > loss_0: 2.6112704277038574 (2.6112704277038574) + | > loss_gen: 2.652074098587036 (2.652074098587036) + | > loss_kl: 2.1220297813415527 (2.1220297813415527) + | > loss_feat: 7.923717021942139 (7.923717021942139) + | > loss_mel: 20.345775604248047 (20.345775604248047) + | > loss_duration: 1.6508619785308838 (1.6508619785308838) + | > loss_1: 34.6944580078125 (34.6944580078125) + + --> STEP: 1 + | > loss_disc: 2.764263153076172 (2.764263153076172) + | > loss_disc_real_0: 0.23360243439674377 (0.23360243439674377) + | > loss_disc_real_1: 0.2909773588180542 (0.2909773588180542) + | > loss_disc_real_2: 0.23147647082805634 (0.23147647082805634) + | > loss_disc_real_3: 0.23861071467399597 (0.23861071467399597) + | > loss_disc_real_4: 0.24719969928264618 (0.24719969928264618) + | > loss_disc_real_5: 0.26100948452949524 (0.26100948452949524) + | > loss_0: 2.764263153076172 (2.764263153076172) + | > loss_gen: 2.309173822402954 (2.309173822402954) + | > loss_kl: 2.4467999935150146 (2.4467999935150146) + | > loss_feat: 6.495996475219727 (6.495996475219727) + | > loss_mel: 18.373905181884766 (18.373905181884766) + | > loss_duration: 1.7084753513336182 (1.7084753513336182) + | > loss_1: 31.3343505859375 (31.3343505859375) + + --> STEP: 2 + | > loss_disc: 2.5473625659942627 (2.6558128595352173) + | > loss_disc_real_0: 0.16930291056632996 (0.20145267248153687) + | > loss_disc_real_1: 0.24249807000160217 (0.2667377144098282) + | > loss_disc_real_2: 0.2522396445274353 (0.24185805767774582) + | > loss_disc_real_3: 0.2549917697906494 (0.2468012422323227) + | > loss_disc_real_4: 0.2581619620323181 (0.25268083065748215) + | > loss_disc_real_5: 0.30078887939453125 (0.28089918196201324) + | > loss_0: 2.5473625659942627 (2.6558128595352173) + | > loss_gen: 2.4727444648742676 (2.390959143638611) + | > loss_kl: 2.177055597305298 (2.3119277954101562) + | > loss_feat: 6.949530601501465 (6.722763538360596) + | > loss_mel: 17.255414962768555 (17.81466007232666) + | > loss_duration: 1.6447465419769287 (1.6766109466552734) + | > loss_1: 30.49949073791504 (30.91692066192627) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003229856491088867 (-0.0003210306167602539) + | > avg_loss_disc: 2.6558128595352173 (+0.16187500953674316) + | > avg_loss_disc_real_0: 0.20145267248153687 (+0.08340537920594215) + | > avg_loss_disc_real_1: 0.2667377144098282 (+0.022278159856796265) + | > avg_loss_disc_real_2: 0.24185805767774582 (+0.0031444504857063293) + | > avg_loss_disc_real_3: 0.2468012422323227 (+0.013434946537017822) + | > avg_loss_disc_real_4: 0.25268083065748215 (+0.029621966183185577) + | > avg_loss_disc_real_5: 0.28089918196201324 (+0.05287107080221176) + | > avg_loss_0: 2.6558128595352173 (+0.16187500953674316) + | > avg_loss_gen: 2.390959143638611 (+0.12865424156188965) + | > avg_loss_kl: 2.3119277954101562 (+0.010739564895629883) + | > avg_loss_feat: 6.722763538360596 (-0.024259567260742188) + | > avg_loss_mel: 17.81466007232666 (+0.3448619842529297) + | > avg_loss_duration: 1.6766109466552734 (+0.0037358999252319336) + | > avg_loss_1: 30.91692066192627 (+0.4637317657470703) + + + > EPOCH: 149/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:27:06)  + + --> TIME: 2025-04-19 01:27:10 -- STEP: 10/142 -- GLOBAL_STEP: 400450 + | > loss_disc: 2.4938430786132812 (2.521489381790161) + | > loss_disc_real_0: 0.13866844773292542 (0.11892803907394409) + | > loss_disc_real_1: 0.1944727748632431 (0.20710664242506027) + | > loss_disc_real_2: 0.2234010100364685 (0.2406834229826927) + | > loss_disc_real_3: 0.25435882806777954 (0.23951828926801683) + | > loss_disc_real_4: 0.24305318295955658 (0.24342708438634872) + | > loss_disc_real_5: 0.2729376256465912 (0.2509219601750374) + | > loss_0: 2.4938430786132812 (2.521489381790161) + | > grad_norm_0: tensor(8.2690, device='cuda:0') (tensor(6.2650, device='cuda:0')) + | > loss_gen: 2.4640512466430664 (2.33917396068573) + | > loss_kl: 1.8828376531600952 (1.8365971803665162) + | > loss_feat: 7.451731204986572 (7.188487577438354) + | > loss_mel: 18.169414520263672 (18.139242744445802) + | > loss_duration: 1.404526948928833 (1.4374199628829956) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.372560501098633 (30.940921592712403) + | > grad_norm_1: tensor(129.2892, device='cuda:0') (tensor(128.9199, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.3349 (0.32579789161682127) + | > loader_time: 0.0052 (0.0044831991195678714) + + + --> TIME: 2025-04-19 01:27:19 -- STEP: 35/142 -- GLOBAL_STEP: 400475 + | > loss_disc: 2.5208396911621094 (2.4786450931004116) + | > loss_disc_real_0: 0.12636835873126984 (0.12226432455437523) + | > loss_disc_real_1: 0.23380596935749054 (0.2098137255225863) + | > loss_disc_real_2: 0.19579780101776123 (0.2260080648320062) + | > loss_disc_real_3: 0.227655291557312 (0.22930270561150143) + | > loss_disc_real_4: 0.23769065737724304 (0.2313244274684361) + | > loss_disc_real_5: 0.2490713745355606 (0.23794980091708048) + | > loss_0: 2.5208396911621094 (2.4786450931004116) + | > grad_norm_0: tensor(5.0962, device='cuda:0') (tensor(8.4983, device='cuda:0')) + | > loss_gen: 2.3773727416992188 (2.3263149193355024) + | > loss_kl: 1.9043927192687988 (1.8726031848362514) + | > loss_feat: 7.395583152770996 (7.066256373269217) + | > loss_mel: 18.076908111572266 (17.827638244628908) + | > loss_duration: 1.4645476341247559 (1.4496408019747054) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.21880531311035 (30.54245343889509) + | > grad_norm_1: tensor(74.2364, device='cuda:0') (tensor(200.8754, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.3548 (0.3385658264160156) + | > loader_time: 0.0059 (0.005001933234078544) + + + --> TIME: 2025-04-19 01:27:28 -- STEP: 60/142 -- GLOBAL_STEP: 400500 + | > loss_disc: 2.615936756134033 (2.4714155395825705) + | > loss_disc_real_0: 0.10430829972028732 (0.12186518609523773) + | > loss_disc_real_1: 0.1929888278245926 (0.2088818818330765) + | > loss_disc_real_2: 0.22386841475963593 (0.22671886458992957) + | > loss_disc_real_3: 0.21750199794769287 (0.22939178546269734) + | > loss_disc_real_4: 0.2714977562427521 (0.23021446466445922) + | > loss_disc_real_5: 0.22613176703453064 (0.23564512878656388) + | > loss_0: 2.615936756134033 (2.4714155395825705) + | > grad_norm_0: tensor(5.4868, device='cuda:0') (tensor(8.0682, device='cuda:0')) + | > loss_gen: 2.172752618789673 (2.326755738258363) + | > loss_kl: 1.9058479070663452 (1.8801595250765482) + | > loss_feat: 6.7509260177612305 (7.096773473421733) + | > loss_mel: 17.789339065551758 (17.888114643096923) + | > loss_duration: 1.5111452341079712 (1.4609874824682871) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.1300106048584 (30.6527907371521) + | > grad_norm_1: tensor(89.2262, device='cuda:0') (tensor(188.8009, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.3759 (0.35004618565241497) + | > loader_time: 0.0057 (0.005326151847839356) + + + --> TIME: 2025-04-19 01:27:38 -- STEP: 85/142 -- GLOBAL_STEP: 400525 + | > loss_disc: 2.4192214012145996 (2.473791046703563) + | > loss_disc_real_0: 0.09617460519075394 (0.1238801310167593) + | > loss_disc_real_1: 0.20987974107265472 (0.2078359165612389) + | > loss_disc_real_2: 0.2167603075504303 (0.22750830036752365) + | > loss_disc_real_3: 0.23214979469776154 (0.23093864023685456) + | > loss_disc_real_4: 0.2420852780342102 (0.23047606471706839) + | > loss_disc_real_5: 0.2439374327659607 (0.23571250912021188) + | > loss_0: 2.4192214012145996 (2.473791046703563) + | > grad_norm_0: tensor(6.6179, device='cuda:0') (tensor(8.0092, device='cuda:0')) + | > loss_gen: 2.3281373977661133 (2.3261066100176646) + | > loss_kl: 1.9078582525253296 (1.9013723695979399) + | > loss_feat: 6.223012924194336 (7.03563014198752) + | > loss_mel: 17.964677810668945 (17.888197865205647) + | > loss_duration: 1.5112438201904297 (1.4743146615869858) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9349308013916 (30.62562161613913) + | > grad_norm_1: tensor(55.0885, device='cuda:0') (tensor(185.0845, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.3962 (0.36189661306493415) + | > loader_time: 0.0065 (0.005579373415778667) + + + --> TIME: 2025-04-19 01:27:49 -- STEP: 110/142 -- GLOBAL_STEP: 400550 + | > loss_disc: 2.517043113708496 (2.4776269999417395) + | > loss_disc_real_0: 0.16992855072021484 (0.12616098780523646) + | > loss_disc_real_1: 0.18881075084209442 (0.20742993341250854) + | > loss_disc_real_2: 0.21692119538784027 (0.2273630285804922) + | > loss_disc_real_3: 0.21691454946994781 (0.2301258068193089) + | > loss_disc_real_4: 0.21603448688983917 (0.22969978194345128) + | > loss_disc_real_5: 0.2549927532672882 (0.23528695025227286) + | > loss_0: 2.517043113708496 (2.4776269999417395) + | > grad_norm_0: tensor(4.0048, device='cuda:0') (tensor(7.7106, device='cuda:0')) + | > loss_gen: 2.274674892425537 (2.3216272419149235) + | > loss_kl: 2.081263780593872 (1.9085999044505033) + | > loss_feat: 6.831948757171631 (7.009142004359852) + | > loss_mel: 18.184398651123047 (17.88960985703901) + | > loss_duration: 1.5049265623092651 (1.4837189587679773) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.877212524414062 (30.612698017467153) + | > grad_norm_1: tensor(84.9509, device='cuda:0') (tensor(164.8130, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.4308 (0.37355866865678267) + | > loader_time: 0.0071 (0.005831555886702106) + + + --> TIME: 2025-04-19 01:28:00 -- STEP: 135/142 -- GLOBAL_STEP: 400575 + | > loss_disc: 2.460127830505371 (2.4833145848026987) + | > loss_disc_real_0: 0.13566730916500092 (0.12847496938926203) + | > loss_disc_real_1: 0.26551246643066406 (0.2087936309752641) + | > loss_disc_real_2: 0.2142714411020279 (0.22720253346142946) + | > loss_disc_real_3: 0.2234521508216858 (0.22944875138777274) + | > loss_disc_real_4: 0.20590803027153015 (0.22953418106944473) + | > loss_disc_real_5: 0.25103360414505005 (0.23655571981712623) + | > loss_0: 2.460127830505371 (2.4833145848026987) + | > grad_norm_0: tensor(8.4585, device='cuda:0') (tensor(8.4149, device='cuda:0')) + | > loss_gen: 2.446897268295288 (2.3180992779908367) + | > loss_kl: 1.9610233306884766 (1.9098876096584179) + | > loss_feat: 7.40588903427124 (6.946686511569553) + | > loss_mel: 18.520565032958984 (17.83663152765344) + | > loss_duration: 1.4854707717895508 (1.4897028207778928) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.819847106933594 (30.501007772375036) + | > grad_norm_1: tensor(170.0168, device='cuda:0') (tensor(181.3918, device='cuda:0')) + | > current_lr_0: 0.00019630924616487812 + | > current_lr_1: 0.00019630924616487812 + | > step_time: 0.4614 (0.385850786279749) + | > loader_time: 0.0085 (0.006138232902244287) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4471044540405273 (2.4471044540405273) + | > loss_disc_real_0: 0.08311786502599716 (0.08311786502599716) + | > loss_disc_real_1: 0.2518251836299896 (0.2518251836299896) + | > loss_disc_real_2: 0.23498640954494476 (0.23498640954494476) + | > loss_disc_real_3: 0.2294604778289795 (0.2294604778289795) + | > loss_disc_real_4: 0.22168675065040588 (0.22168675065040588) + | > loss_disc_real_5: 0.24773825705051422 (0.24773825705051422) + | > loss_0: 2.4471044540405273 (2.4471044540405273) + | > loss_gen: 2.2799270153045654 (2.2799270153045654) + | > loss_kl: 2.32078218460083 (2.32078218460083) + | > loss_feat: 6.964646339416504 (6.964646339416504) + | > loss_mel: 18.353801727294922 (18.353801727294922) + | > loss_duration: 1.6524097919464111 (1.6524097919464111) + | > loss_1: 31.57156753540039 (31.57156753540039) + + --> STEP: 1 + | > loss_disc: 2.6434226036071777 (2.6434226036071777) + | > loss_disc_real_0: 0.11980648338794708 (0.11980648338794708) + | > loss_disc_real_1: 0.25987911224365234 (0.25987911224365234) + | > loss_disc_real_2: 0.24725282192230225 (0.24725282192230225) + | > loss_disc_real_3: 0.2166922390460968 (0.2166922390460968) + | > loss_disc_real_4: 0.24373793601989746 (0.24373793601989746) + | > loss_disc_real_5: 0.25885725021362305 (0.25885725021362305) + | > loss_0: 2.6434226036071777 (2.6434226036071777) + | > loss_gen: 2.082430362701416 (2.082430362701416) + | > loss_kl: 2.4807240962982178 (2.4807240962982178) + | > loss_feat: 6.119974613189697 (6.119974613189697) + | > loss_mel: 16.93706512451172 (16.93706512451172) + | > loss_duration: 1.6708509922027588 (1.6708509922027588) + | > loss_1: 29.291044235229492 (29.291044235229492) + + --> STEP: 2 + | > loss_disc: 2.5482382774353027 (2.5958304405212402) + | > loss_disc_real_0: 0.09417037665843964 (0.10698843002319336) + | > loss_disc_real_1: 0.25953420996665955 (0.25970666110515594) + | > loss_disc_real_2: 0.22444716095924377 (0.235849991440773) + | > loss_disc_real_3: 0.25982755422592163 (0.23825989663600922) + | > loss_disc_real_4: 0.2355450540781021 (0.2396414950489998) + | > loss_disc_real_5: 0.22752691805362701 (0.24319208413362503) + | > loss_0: 2.5482382774353027 (2.5958304405212402) + | > loss_gen: 2.1843912601470947 (2.1334108114242554) + | > loss_kl: 2.216033697128296 (2.348378896713257) + | > loss_feat: 6.9201860427856445 (6.520080327987671) + | > loss_mel: 18.266794204711914 (17.601929664611816) + | > loss_duration: 1.6332628726959229 (1.6520569324493408) + | > loss_1: 31.22066879272461 (30.25585651397705) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035283565521240234 (+0.00029850006103515625) + | > avg_loss_disc: 2.5958304405212402 (-0.05998241901397705) + | > avg_loss_disc_real_0: 0.10698843002319336 (-0.0944642424583435) + | > avg_loss_disc_real_1: 0.25970666110515594 (-0.007031053304672241) + | > avg_loss_disc_real_2: 0.235849991440773 (-0.006008066236972809) + | > avg_loss_disc_real_3: 0.23825989663600922 (-0.008541345596313477) + | > avg_loss_disc_real_4: 0.2396414950489998 (-0.01303933560848236) + | > avg_loss_disc_real_5: 0.24319208413362503 (-0.037707097828388214) + | > avg_loss_0: 2.5958304405212402 (-0.05998241901397705) + | > avg_loss_gen: 2.1334108114242554 (-0.25754833221435547) + | > avg_loss_kl: 2.348378896713257 (+0.036451101303100586) + | > avg_loss_feat: 6.520080327987671 (-0.2026832103729248) + | > avg_loss_mel: 17.601929664611816 (-0.21273040771484375) + | > avg_loss_duration: 1.6520569324493408 (-0.024554014205932617) + | > avg_loss_1: 30.25585651397705 (-0.6610641479492188) + + + > EPOCH: 150/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:28:05)  + + --> TIME: 2025-04-19 01:28:12 -- STEP: 18/142 -- GLOBAL_STEP: 400600 + | > loss_disc: 2.346071243286133 (2.4474475383758545) + | > loss_disc_real_0: 0.09311162680387497 (0.11685091960761282) + | > loss_disc_real_1: 0.17814406752586365 (0.19994001670016182) + | > loss_disc_real_2: 0.230990469455719 (0.22618927309910455) + | > loss_disc_real_3: 0.16552965342998505 (0.226291347708967) + | > loss_disc_real_4: 0.18686166405677795 (0.22460164378086725) + | > loss_disc_real_5: 0.18440523743629456 (0.23440054224597084) + | > loss_0: 2.346071243286133 (2.4474475383758545) + | > grad_norm_0: tensor(9.7433, device='cuda:0') (tensor(12.2004, device='cuda:0')) + | > loss_gen: 2.4622511863708496 (2.3896822796927557) + | > loss_kl: 1.8247292041778564 (1.8511656125386555) + | > loss_feat: 8.0217924118042 (7.2045114570193824) + | > loss_mel: 19.3740177154541 (18.04633543226454) + | > loss_duration: 1.4571881294250488 (1.437021533648173) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.13998031616211 (30.92871591779921) + | > grad_norm_1: tensor(422.1162, device='cuda:0') (tensor(289.0944, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.3398 (0.33129413922627765) + | > loader_time: 0.0049 (0.004767325189378526) + + + --> TIME: 2025-04-19 01:28:21 -- STEP: 43/142 -- GLOBAL_STEP: 400625 + | > loss_disc: 2.397891044616699 (2.453196198441262) + | > loss_disc_real_0: 0.15911871194839478 (0.11919037651183993) + | > loss_disc_real_1: 0.21346116065979004 (0.20353810004023618) + | > loss_disc_real_2: 0.18902753293514252 (0.22585669749004897) + | > loss_disc_real_3: 0.2279125601053238 (0.23005430781564049) + | > loss_disc_real_4: 0.22768518328666687 (0.22979121048783147) + | > loss_disc_real_5: 0.2186836451292038 (0.23281707597333332) + | > loss_0: 2.397891044616699 (2.453196198441262) + | > grad_norm_0: tensor(10.0935, device='cuda:0') (tensor(11.8220, device='cuda:0')) + | > loss_gen: 2.4607884883880615 (2.3824495881102807) + | > loss_kl: 1.8912986516952515 (1.861348920090254) + | > loss_feat: 7.505557060241699 (7.158656863279121) + | > loss_mel: 19.30252456665039 (17.985027978586597) + | > loss_duration: 1.4761288166046143 (1.4551445495250612) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.636295318603516 (30.84262781364973) + | > grad_norm_1: tensor(311.1239, device='cuda:0') (tensor(281.2913, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.3639 (0.3435553506363269) + | > loader_time: 0.0059 (0.005246068156042763) + + + --> TIME: 2025-04-19 01:28:30 -- STEP: 68/142 -- GLOBAL_STEP: 400650 + | > loss_disc: 2.3570446968078613 (2.468191606156966) + | > loss_disc_real_0: 0.1423245519399643 (0.1218152001280995) + | > loss_disc_real_1: 0.168575718998909 (0.20485256765695178) + | > loss_disc_real_2: 0.2027600109577179 (0.22819115594029427) + | > loss_disc_real_3: 0.21417661011219025 (0.23082239597159274) + | > loss_disc_real_4: 0.1997648924589157 (0.2310435861787375) + | > loss_disc_real_5: 0.23576699197292328 (0.23390119325588732) + | > loss_0: 2.3570446968078613 (2.468191606156966) + | > grad_norm_0: tensor(3.5776, device='cuda:0') (tensor(10.7157, device='cuda:0')) + | > loss_gen: 2.4029574394226074 (2.3581493146279278) + | > loss_kl: 1.8585538864135742 (1.882598142413532) + | > loss_feat: 7.2451558113098145 (7.049037260167739) + | > loss_mel: 16.873443603515625 (17.91482212964226) + | > loss_duration: 1.4687784910202026 (1.4653046937549816) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.84889030456543 (30.669911468730252) + | > grad_norm_1: tensor(102.3045, device='cuda:0') (tensor(246.5623, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.3888 (0.35544709948932424) + | > loader_time: 0.0059 (0.005500095732071821) + + + --> TIME: 2025-04-19 01:28:41 -- STEP: 93/142 -- GLOBAL_STEP: 400675 + | > loss_disc: 2.4591126441955566 (2.464235808259697) + | > loss_disc_real_0: 0.13156484067440033 (0.12351177312353606) + | > loss_disc_real_1: 0.21090397238731384 (0.20391453594289796) + | > loss_disc_real_2: 0.2731662094593048 (0.2281714302878226) + | > loss_disc_real_3: 0.23279789090156555 (0.22875528165730097) + | > loss_disc_real_4: 0.19940537214279175 (0.23036424286903873) + | > loss_disc_real_5: 0.21095925569534302 (0.23360166129886464) + | > loss_0: 2.4591126441955566 (2.464235808259697) + | > grad_norm_0: tensor(13.3611, device='cuda:0') (tensor(10.6603, device='cuda:0')) + | > loss_gen: 2.2368860244750977 (2.354484473505328) + | > loss_kl: 1.8366172313690186 (1.8957727493778351) + | > loss_feat: 6.783989429473877 (6.991035681898876) + | > loss_mel: 17.138105392456055 (17.873525578488586) + | > loss_duration: 1.4929935932159424 (1.478069293883539) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.48859214782715 (30.59288769383584) + | > grad_norm_1: tensor(317.3927, device='cuda:0') (tensor(249.9644, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.4011 (0.36705038111696964) + | > loader_time: 0.0061 (0.005777184681225849) + + + --> TIME: 2025-04-19 01:28:51 -- STEP: 118/142 -- GLOBAL_STEP: 400700 + | > loss_disc: 2.5405964851379395 (2.4701180074174522) + | > loss_disc_real_0: 0.09040342271327972 (0.12515030144634895) + | > loss_disc_real_1: 0.2109191119670868 (0.204654513526771) + | > loss_disc_real_2: 0.21376913785934448 (0.22772381227400343) + | > loss_disc_real_3: 0.22204236686229706 (0.22959370481765876) + | > loss_disc_real_4: 0.24251016974449158 (0.23146575032654454) + | > loss_disc_real_5: 0.2272711992263794 (0.2329716547313383) + | > loss_0: 2.5405964851379395 (2.4701180074174522) + | > grad_norm_0: tensor(24.4852, device='cuda:0') (tensor(11.6504, device='cuda:0')) + | > loss_gen: 2.391921043395996 (2.3523838398820254) + | > loss_kl: 1.8728080987930298 (1.904145448894824) + | > loss_feat: 6.960522174835205 (6.985662092596797) + | > loss_mel: 17.623706817626953 (17.872437865047132) + | > loss_duration: 1.469756007194519 (1.4837609145600916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.31871223449707 (30.59839011046846) + | > grad_norm_1: tensor(433.3775, device='cuda:0') (tensor(266.5355, device='cuda:0')) + | > current_lr_0: 0.0001962847075091075 + | > current_lr_1: 0.0001962847075091075 + | > step_time: 0.4423 (0.37769262871499787) + | > loader_time: 0.0078 (0.006044549457097458) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.458831548690796 (2.458831548690796) + | > loss_disc_real_0: 0.18300336599349976 (0.18300336599349976) + | > loss_disc_real_1: 0.22901394963264465 (0.22901394963264465) + | > loss_disc_real_2: 0.24973152577877045 (0.24973152577877045) + | > loss_disc_real_3: 0.238390251994133 (0.238390251994133) + | > loss_disc_real_4: 0.2093360871076584 (0.2093360871076584) + | > loss_disc_real_5: 0.2602244019508362 (0.2602244019508362) + | > loss_0: 2.458831548690796 (2.458831548690796) + | > loss_gen: 2.4433162212371826 (2.4433162212371826) + | > loss_kl: 2.3605475425720215 (2.3605475425720215) + | > loss_feat: 6.774940013885498 (6.774940013885498) + | > loss_mel: 17.148569107055664 (17.148569107055664) + | > loss_duration: 1.656754732131958 (1.656754732131958) + | > loss_1: 30.38412857055664 (30.38412857055664) + + --> STEP: 1 + | > loss_disc: 2.392380475997925 (2.392380475997925) + | > loss_disc_real_0: 0.14501680433750153 (0.14501680433750153) + | > loss_disc_real_1: 0.199096217751503 (0.199096217751503) + | > loss_disc_real_2: 0.23145197331905365 (0.23145197331905365) + | > loss_disc_real_3: 0.20219048857688904 (0.20219048857688904) + | > loss_disc_real_4: 0.1924503594636917 (0.1924503594636917) + | > loss_disc_real_5: 0.23221156001091003 (0.23221156001091003) + | > loss_0: 2.392380475997925 (2.392380475997925) + | > loss_gen: 2.404202938079834 (2.404202938079834) + | > loss_kl: 2.5388123989105225 (2.5388123989105225) + | > loss_feat: 7.51509952545166 (7.51509952545166) + | > loss_mel: 18.511598587036133 (18.511598587036133) + | > loss_duration: 1.6559168100357056 (1.6559168100357056) + | > loss_1: 32.62562942504883 (32.62562942504883) + + --> STEP: 2 + | > loss_disc: 2.6776859760284424 (2.5350332260131836) + | > loss_disc_real_0: 0.1433679461479187 (0.1441923752427101) + | > loss_disc_real_1: 0.2840668261051178 (0.2415815219283104) + | > loss_disc_real_2: 0.27073851227760315 (0.2510952427983284) + | > loss_disc_real_3: 0.21605820953845978 (0.2091243490576744) + | > loss_disc_real_4: 0.2302381545305252 (0.21134425699710846) + | > loss_disc_real_5: 0.18436354398727417 (0.2082875519990921) + | > loss_0: 2.6776859760284424 (2.5350332260131836) + | > loss_gen: 2.0806190967559814 (2.2424110174179077) + | > loss_kl: 2.057100296020508 (2.297956347465515) + | > loss_feat: 6.535801410675049 (7.0254504680633545) + | > loss_mel: 17.012392044067383 (17.761995315551758) + | > loss_duration: 1.6348026990890503 (1.645359754562378) + | > loss_1: 29.320714950561523 (30.973172187805176) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004056334495544434 (+0.0005279779434204102) + | > avg_loss_disc: 2.5350332260131836 (-0.06079721450805664) + | > avg_loss_disc_real_0: 0.1441923752427101 (+0.037203945219516754) + | > avg_loss_disc_real_1: 0.2415815219283104 (-0.01812513917684555) + | > avg_loss_disc_real_2: 0.2510952427983284 (+0.01524525135755539) + | > avg_loss_disc_real_3: 0.2091243490576744 (-0.02913554757833481) + | > avg_loss_disc_real_4: 0.21134425699710846 (-0.028297238051891327) + | > avg_loss_disc_real_5: 0.2082875519990921 (-0.03490453213453293) + | > avg_loss_0: 2.5350332260131836 (-0.06079721450805664) + | > avg_loss_gen: 2.2424110174179077 (+0.10900020599365234) + | > avg_loss_kl: 2.297956347465515 (-0.0504225492477417) + | > avg_loss_feat: 7.0254504680633545 (+0.5053701400756836) + | > avg_loss_mel: 17.761995315551758 (+0.1600656509399414) + | > avg_loss_duration: 1.645359754562378 (-0.006697177886962891) + | > avg_loss_1: 30.973172187805176 (+0.717315673828125) + + + > EPOCH: 151/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:29:04)  + + --> TIME: 2025-04-19 01:29:05 -- STEP: 1/142 -- GLOBAL_STEP: 400725 + | > loss_disc: 2.338348150253296 (2.338348150253296) + | > loss_disc_real_0: 0.10305414348840714 (0.10305414348840714) + | > loss_disc_real_1: 0.2206818014383316 (0.2206818014383316) + | > loss_disc_real_2: 0.20739825069904327 (0.20739825069904327) + | > loss_disc_real_3: 0.19719058275222778 (0.19719058275222778) + | > loss_disc_real_4: 0.2050752341747284 (0.2050752341747284) + | > loss_disc_real_5: 0.2155270278453827 (0.2155270278453827) + | > loss_0: 2.338348150253296 (2.338348150253296) + | > grad_norm_0: tensor(12.1102, device='cuda:0') (tensor(12.1102, device='cuda:0')) + | > loss_gen: 2.438824415206909 (2.438824415206909) + | > loss_kl: 1.8199228048324585 (1.8199228048324585) + | > loss_feat: 8.210505485534668 (8.210505485534668) + | > loss_mel: 18.91692543029785 (18.91692543029785) + | > loss_duration: 1.4123814105987549 (1.4123814105987549) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.79855728149414 (32.79855728149414) + | > grad_norm_1: tensor(326.7801, device='cuda:0') (tensor(326.7801, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.3245 (0.32447123527526855) + | > loader_time: 0.0039 (0.0039174556732177734) + + + --> TIME: 2025-04-19 01:29:13 -- STEP: 26/142 -- GLOBAL_STEP: 400750 + | > loss_disc: 2.4666759967803955 (2.448981743592482) + | > loss_disc_real_0: 0.09042860567569733 (0.11924168868706776) + | > loss_disc_real_1: 0.2439156472682953 (0.20459500700235367) + | > loss_disc_real_2: 0.2107095867395401 (0.22332889300126296) + | > loss_disc_real_3: 0.23218277096748352 (0.22895062772127298) + | > loss_disc_real_4: 0.25090524554252625 (0.2278032383093467) + | > loss_disc_real_5: 0.2243724763393402 (0.2329672953257194) + | > loss_0: 2.4666759967803955 (2.448981743592482) + | > grad_norm_0: tensor(6.4793, device='cuda:0') (tensor(8.4292, device='cuda:0')) + | > loss_gen: 2.4178237915039062 (2.367449531188378) + | > loss_kl: 1.8156510591506958 (1.8376923616115863) + | > loss_feat: 6.780194282531738 (7.298396550692045) + | > loss_mel: 18.188364028930664 (18.17744753910945) + | > loss_duration: 1.483170509338379 (1.4504253726739147) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.685203552246094 (31.131411479069637) + | > grad_norm_1: tensor(61.9875, device='cuda:0') (tensor(196.0891, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.3497 (0.33449745178222656) + | > loader_time: 0.0057 (0.005000829696655274) + + + --> TIME: 2025-04-19 01:29:22 -- STEP: 51/142 -- GLOBAL_STEP: 400775 + | > loss_disc: 2.4552743434906006 (2.4643138623705094) + | > loss_disc_real_0: 0.08184405416250229 (0.12235566097147324) + | > loss_disc_real_1: 0.22545146942138672 (0.2079186950828515) + | > loss_disc_real_2: 0.21588002145290375 (0.22593311118144616) + | > loss_disc_real_3: 0.2487218677997589 (0.22798469428922616) + | > loss_disc_real_4: 0.27633607387542725 (0.22954594183201885) + | > loss_disc_real_5: 0.26376935839653015 (0.2361537627145356) + | > loss_0: 2.4552743434906006 (2.4643138623705094) + | > grad_norm_0: tensor(16.2289, device='cuda:0') (tensor(8.1453, device='cuda:0')) + | > loss_gen: 2.4297163486480713 (2.3590955453760483) + | > loss_kl: 1.914982795715332 (1.8569401432486141) + | > loss_feat: 7.293161869049072 (7.126240122552011) + | > loss_mel: 18.46613121032715 (17.95223826988071) + | > loss_duration: 1.45072340965271 (1.4578350922640633) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.554716110229492 (30.752349180333756) + | > grad_norm_1: tensor(542.7826, device='cuda:0') (tensor(198.3721, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.3671 (0.3463176185009525) + | > loader_time: 0.0054 (0.005366344077914369) + + + --> TIME: 2025-04-19 01:29:32 -- STEP: 76/142 -- GLOBAL_STEP: 400800 + | > loss_disc: 2.4859113693237305 (2.474441559691178) + | > loss_disc_real_0: 0.12243280559778214 (0.12601138867045703) + | > loss_disc_real_1: 0.21671199798583984 (0.2084513361516752) + | > loss_disc_real_2: 0.2735382616519928 (0.22702145517656677) + | > loss_disc_real_3: 0.23983439803123474 (0.2287606104816261) + | > loss_disc_real_4: 0.23494018614292145 (0.23018105386903412) + | > loss_disc_real_5: 0.2736050486564636 (0.23772550570337395) + | > loss_0: 2.4859113693237305 (2.474441559691178) + | > grad_norm_0: tensor(15.9152, device='cuda:0') (tensor(9.3339, device='cuda:0')) + | > loss_gen: 2.403763771057129 (2.357280706104479) + | > loss_kl: 1.9186131954193115 (1.8810024732037594) + | > loss_feat: 7.0985426902771 (7.107003626070525) + | > loss_mel: 18.49641227722168 (17.97303099381296) + | > loss_duration: 1.5147230625152588 (1.469773336460716) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.43205451965332 (30.788091207805433) + | > grad_norm_1: tensor(323.7887, device='cuda:0') (tensor(224.1780, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.3907 (0.35832518652865764) + | > loader_time: 0.0059 (0.005634747053447522) + + + --> TIME: 2025-04-19 01:29:42 -- STEP: 101/142 -- GLOBAL_STEP: 400825 + | > loss_disc: 2.4749393463134766 (2.4818614851130114) + | > loss_disc_real_0: 0.16125670075416565 (0.1269870948525939) + | > loss_disc_real_1: 0.24893712997436523 (0.20870376621732617) + | > loss_disc_real_2: 0.22370100021362305 (0.22648765338529456) + | > loss_disc_real_3: 0.21160316467285156 (0.22874635841586802) + | > loss_disc_real_4: 0.24285030364990234 (0.2310295845612441) + | > loss_disc_real_5: 0.22659216821193695 (0.23755867543196915) + | > loss_0: 2.4749393463134766 (2.4818614851130114) + | > grad_norm_0: tensor(6.7324, device='cuda:0') (tensor(10.2588, device='cuda:0')) + | > loss_gen: 2.4872655868530273 (2.348684025282906) + | > loss_kl: 2.020131826400757 (1.8928759534760278) + | > loss_feat: 7.390799522399902 (7.051423700729219) + | > loss_mel: 18.114551544189453 (17.97650025622678) + | > loss_duration: 1.5486226081848145 (1.4803855643414037) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.561370849609375 (30.74986950713809) + | > grad_norm_1: tensor(431.3558, device='cuda:0') (tensor(247.6839, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.4149 (0.3687703397014353) + | > loader_time: 0.0071 (0.0059175845420006475) + + + --> TIME: 2025-04-19 01:29:54 -- STEP: 126/142 -- GLOBAL_STEP: 400850 + | > loss_disc: 2.557600975036621 (2.4835815599986484) + | > loss_disc_real_0: 0.18131332099437714 (0.12813931230514775) + | > loss_disc_real_1: 0.19577524065971375 (0.20831090649442066) + | > loss_disc_real_2: 0.23281458020210266 (0.22608935253487694) + | > loss_disc_real_3: 0.24154984951019287 (0.2292958101819432) + | > loss_disc_real_4: 0.24175730347633362 (0.23077199135034804) + | > loss_disc_real_5: 0.22423766553401947 (0.23741010813012955) + | > loss_0: 2.557600975036621 (2.4835815599986484) + | > grad_norm_0: tensor(10.6582, device='cuda:0') (tensor(11.3478, device='cuda:0')) + | > loss_gen: 2.226412773132324 (2.3423871710186903) + | > loss_kl: 1.9017983675003052 (1.8989625478547716) + | > loss_feat: 6.445362567901611 (7.004955602070642) + | > loss_mel: 17.102807998657227 (17.936896876683296) + | > loss_duration: 1.5007871389389038 (1.4870583396109327) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.17717170715332 (30.6702605807592) + | > grad_norm_1: tensor(316.3776, device='cuda:0') (tensor(261.5520, device='cuda:0')) + | > current_lr_0: 0.00019626017192066886 + | > current_lr_1: 0.00019626017192066886 + | > step_time: 0.4387 (0.38175113049764475) + | > loader_time: 0.0073 (0.00615714466761029) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4814913272857666 (2.4814913272857666) + | > loss_disc_real_0: 0.11187460273504257 (0.11187460273504257) + | > loss_disc_real_1: 0.21928885579109192 (0.21928885579109192) + | > loss_disc_real_2: 0.24128024280071259 (0.24128024280071259) + | > loss_disc_real_3: 0.23227284848690033 (0.23227284848690033) + | > loss_disc_real_4: 0.24022924900054932 (0.24022924900054932) + | > loss_disc_real_5: 0.23177829384803772 (0.23177829384803772) + | > loss_0: 2.4814913272857666 (2.4814913272857666) + | > loss_gen: 2.238816976547241 (2.238816976547241) + | > loss_kl: 2.399566411972046 (2.399566411972046) + | > loss_feat: 6.424494743347168 (6.424494743347168) + | > loss_mel: 17.548227310180664 (17.548227310180664) + | > loss_duration: 1.6624171733856201 (1.6624171733856201) + | > loss_1: 30.273521423339844 (30.273521423339844) + + --> STEP: 1 + | > loss_disc: 2.5076849460601807 (2.5076849460601807) + | > loss_disc_real_0: 0.12486252933740616 (0.12486252933740616) + | > loss_disc_real_1: 0.24428731203079224 (0.24428731203079224) + | > loss_disc_real_2: 0.23059968650341034 (0.23059968650341034) + | > loss_disc_real_3: 0.22844064235687256 (0.22844064235687256) + | > loss_disc_real_4: 0.23549117147922516 (0.23549117147922516) + | > loss_disc_real_5: 0.22767072916030884 (0.22767072916030884) + | > loss_0: 2.5076849460601807 (2.5076849460601807) + | > loss_gen: 2.3331587314605713 (2.3331587314605713) + | > loss_kl: 2.7192766666412354 (2.7192766666412354) + | > loss_feat: 6.070669174194336 (6.070669174194336) + | > loss_mel: 17.143657684326172 (17.143657684326172) + | > loss_duration: 1.6898860931396484 (1.6898860931396484) + | > loss_1: 29.956647872924805 (29.956647872924805) + + --> STEP: 2 + | > loss_disc: 2.491100788116455 (2.499392867088318) + | > loss_disc_real_0: 0.11124326288700104 (0.1180528961122036) + | > loss_disc_real_1: 0.20056788623332977 (0.222427599132061) + | > loss_disc_real_2: 0.22097893059253693 (0.22578930854797363) + | > loss_disc_real_3: 0.228678360581398 (0.22855950146913528) + | > loss_disc_real_4: 0.21561259031295776 (0.22555188089609146) + | > loss_disc_real_5: 0.2025410681962967 (0.21510589867830276) + | > loss_0: 2.491100788116455 (2.499392867088318) + | > loss_gen: 2.049053192138672 (2.1911059617996216) + | > loss_kl: 2.041118860244751 (2.380197763442993) + | > loss_feat: 6.198792934417725 (6.13473105430603) + | > loss_mel: 17.035099029541016 (17.089378356933594) + | > loss_duration: 1.6301040649414062 (1.6599950790405273) + | > loss_1: 28.95416831970215 (29.455408096313477) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035049915313720703 (-0.0005513429641723633) + | > avg_loss_disc: 2.499392867088318 (-0.03564035892486572) + | > avg_loss_disc_real_0: 0.1180528961122036 (-0.026139479130506516) + | > avg_loss_disc_real_1: 0.222427599132061 (-0.01915392279624939) + | > avg_loss_disc_real_2: 0.22578930854797363 (-0.025305934250354767) + | > avg_loss_disc_real_3: 0.22855950146913528 (+0.019435152411460876) + | > avg_loss_disc_real_4: 0.22555188089609146 (+0.014207623898983002) + | > avg_loss_disc_real_5: 0.21510589867830276 (+0.006818346679210663) + | > avg_loss_0: 2.499392867088318 (-0.03564035892486572) + | > avg_loss_gen: 2.1911059617996216 (-0.05130505561828613) + | > avg_loss_kl: 2.380197763442993 (+0.08224141597747803) + | > avg_loss_feat: 6.13473105430603 (-0.8907194137573242) + | > avg_loss_mel: 17.089378356933594 (-0.6726169586181641) + | > avg_loss_duration: 1.6599950790405273 (+0.014635324478149414) + | > avg_loss_1: 29.455408096313477 (-1.5177640914916992) + + + > EPOCH: 152/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:30:02)  + + --> TIME: 2025-04-19 01:30:06 -- STEP: 9/142 -- GLOBAL_STEP: 400875 + | > loss_disc: 2.510272979736328 (2.4123783111572266) + | > loss_disc_real_0: 0.1301463097333908 (0.11815547860331005) + | > loss_disc_real_1: 0.2689093053340912 (0.20470102462503645) + | > loss_disc_real_2: 0.25296491384506226 (0.22446337673399183) + | > loss_disc_real_3: 0.21800042688846588 (0.22337393959363303) + | > loss_disc_real_4: 0.25207599997520447 (0.22527010407712725) + | > loss_disc_real_5: 0.20226113498210907 (0.23723818361759186) + | > loss_0: 2.510272979736328 (2.4123783111572266) + | > grad_norm_0: tensor(22.2364, device='cuda:0') (tensor(14.4385, device='cuda:0')) + | > loss_gen: 2.3728766441345215 (2.419491688410441) + | > loss_kl: 1.900831937789917 (1.8552641736136541) + | > loss_feat: 7.293025970458984 (7.1713230344984265) + | > loss_mel: 19.00374412536621 (17.42665735880534) + | > loss_duration: 1.4163024425506592 (1.4398460123274062) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.986780166625977 (30.31258201599121) + | > grad_norm_1: tensor(278.9398, device='cuda:0') (tensor(302.1642, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.334 (0.3317778905232747) + | > loader_time: 0.005 (0.004696051279703776) + + + --> TIME: 2025-04-19 01:30:15 -- STEP: 34/142 -- GLOBAL_STEP: 400900 + | > loss_disc: 2.5615904331207275 (2.430777255226584) + | > loss_disc_real_0: 0.13937212526798248 (0.12326025634127505) + | > loss_disc_real_1: 0.21690325438976288 (0.20565334006267436) + | > loss_disc_real_2: 0.2292889654636383 (0.22685357577660503) + | > loss_disc_real_3: 0.22062626481056213 (0.2235056635211496) + | > loss_disc_real_4: 0.22294145822525024 (0.22308388527701883) + | > loss_disc_real_5: 0.23390036821365356 (0.22702460692209356) + | > loss_0: 2.5615904331207275 (2.430777255226584) + | > grad_norm_0: tensor(10.8943, device='cuda:0') (tensor(11.7038, device='cuda:0')) + | > loss_gen: 2.292034149169922 (2.3886167161604948) + | > loss_kl: 1.9664244651794434 (1.8900553023113924) + | > loss_feat: 7.548695087432861 (7.171529699774349) + | > loss_mel: 18.554014205932617 (17.748783672557153) + | > loss_duration: 1.469415307044983 (1.4477647648138159) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.830583572387695 (30.64675005744485) + | > grad_norm_1: tensor(270.5789, device='cuda:0') (tensor(259.1944, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.357 (0.34271930245792165) + | > loader_time: 0.0051 (0.005104548790875603) + + + --> TIME: 2025-04-19 01:30:24 -- STEP: 59/142 -- GLOBAL_STEP: 400925 + | > loss_disc: 2.461151361465454 (2.44778784250809) + | > loss_disc_real_0: 0.12307269871234894 (0.12613070465750614) + | > loss_disc_real_1: 0.16027270257472992 (0.2067591934385946) + | > loss_disc_real_2: 0.23919515311717987 (0.22523648961115691) + | > loss_disc_real_3: 0.21580153703689575 (0.22597790926189745) + | > loss_disc_real_4: 0.24808751046657562 (0.22668834232677848) + | > loss_disc_real_5: 0.1876508891582489 (0.22997903571290484) + | > loss_0: 2.461151361465454 (2.44778784250809) + | > grad_norm_0: tensor(14.0281, device='cuda:0') (tensor(10.7988, device='cuda:0')) + | > loss_gen: 2.27528977394104 (2.364918462300705) + | > loss_kl: 1.8106764554977417 (1.8995873867455175) + | > loss_feat: 6.809250354766846 (7.08961854546757) + | > loss_mel: 18.186952590942383 (17.95120669219453) + | > loss_duration: 1.4658993482589722 (1.4591726266731655) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.548067092895508 (30.764503802283336) + | > grad_norm_1: tensor(384.4033, device='cuda:0') (tensor(237.5739, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.3818 (0.35271494267350534) + | > loader_time: 0.006 (0.005456956766419492) + + + --> TIME: 2025-04-19 01:30:34 -- STEP: 84/142 -- GLOBAL_STEP: 400950 + | > loss_disc: 2.576181650161743 (2.4612308428401035) + | > loss_disc_real_0: 0.15874338150024414 (0.1269735922770841) + | > loss_disc_real_1: 0.2271430641412735 (0.20819504754174323) + | > loss_disc_real_2: 0.21102768182754517 (0.2253628032548087) + | > loss_disc_real_3: 0.22851113975048065 (0.22870481138428053) + | > loss_disc_real_4: 0.23182412981987 (0.2270572291953223) + | > loss_disc_real_5: 0.262852281332016 (0.23164459814627966) + | > loss_0: 2.576181650161743 (2.4612308428401035) + | > grad_norm_0: tensor(16.6340, device='cuda:0') (tensor(10.7590, device='cuda:0')) + | > loss_gen: 2.1788411140441895 (2.362401190258208) + | > loss_kl: 1.8879663944244385 (1.910062183936437) + | > loss_feat: 6.377335071563721 (7.068766690435863) + | > loss_mel: 16.70890235900879 (17.910195373353503) + | > loss_duration: 1.5382881164550781 (1.4725528245880495) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.691333770751953 (30.72397838320051) + | > grad_norm_1: tensor(85.6158, device='cuda:0') (tensor(237.7330, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.3957 (0.3646651052293324) + | > loader_time: 0.0068 (0.005758606252216159) + + + --> TIME: 2025-04-19 01:30:45 -- STEP: 109/142 -- GLOBAL_STEP: 400975 + | > loss_disc: 2.4683127403259277 (2.470920893030429) + | > loss_disc_real_0: 0.10061222314834595 (0.1293223425745964) + | > loss_disc_real_1: 0.18657074868679047 (0.20853022779893438) + | > loss_disc_real_2: 0.212545245885849 (0.2255840305614909) + | > loss_disc_real_3: 0.24076154828071594 (0.228529231127249) + | > loss_disc_real_4: 0.22964143753051758 (0.2273233430374653) + | > loss_disc_real_5: 0.24179933965206146 (0.23300599833147242) + | > loss_0: 2.4683127403259277 (2.470920893030429) + | > grad_norm_0: tensor(31.5330, device='cuda:0') (tensor(12.8902, device='cuda:0')) + | > loss_gen: 2.216144323348999 (2.3554701805114755) + | > loss_kl: 1.8713579177856445 (1.9201322006523063) + | > loss_feat: 7.358743190765381 (6.98664292501747) + | > loss_mel: 18.118915557861328 (17.82327939829695) + | > loss_duration: 1.5142040252685547 (1.4806298629953232) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.079364776611328 (30.56615465496658) + | > grad_norm_1: tensor(375.6871, device='cuda:0') (tensor(266.3777, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.4139 (0.3747485383934931) + | > loader_time: 0.0073 (0.0060295380583596896) + + + --> TIME: 2025-04-19 01:30:56 -- STEP: 134/142 -- GLOBAL_STEP: 401000 + | > loss_disc: 2.587747573852539 (2.47617161096032) + | > loss_disc_real_0: 0.05768614634871483 (0.13188109246652518) + | > loss_disc_real_1: 0.22273573279380798 (0.20806097639585608) + | > loss_disc_real_2: 0.25720643997192383 (0.2260356651090864) + | > loss_disc_real_3: 0.23004250228405 (0.22843451419873023) + | > loss_disc_real_4: 0.22600018978118896 (0.22758259034868497) + | > loss_disc_real_5: 0.23526078462600708 (0.2323720545243861) + | > loss_0: 2.587747573852539 (2.47617161096032) + | > grad_norm_0: tensor(33.4068, device='cuda:0') (tensor(14.8573, device='cuda:0')) + | > loss_gen: 2.3336825370788574 (2.354664361298975) + | > loss_kl: 1.9979181289672852 (1.9191716446805356) + | > loss_feat: 6.646224021911621 (6.958469892615702) + | > loss_mel: 17.66000747680664 (17.753161295136408) + | > loss_duration: 1.555206060409546 (1.4876830613435208) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.193038940429688 (30.473150352933512) + | > grad_norm_1: tensor(505.2978, device='cuda:0') (tensor(287.4959, device='cuda:0')) + | > current_lr_0: 0.00019623563939917877 + | > current_lr_1: 0.00019623563939917877 + | > step_time: 0.4398 (0.3871778338702757) + | > loader_time: 0.0076 (0.0062933484120155465) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.572226047515869 (2.572226047515869) + | > loss_disc_real_0: 0.16786101460456848 (0.16786101460456848) + | > loss_disc_real_1: 0.18411681056022644 (0.18411681056022644) + | > loss_disc_real_2: 0.2353220134973526 (0.2353220134973526) + | > loss_disc_real_3: 0.2440280020236969 (0.2440280020236969) + | > loss_disc_real_4: 0.27127957344055176 (0.27127957344055176) + | > loss_disc_real_5: 0.25580111145973206 (0.25580111145973206) + | > loss_0: 2.572226047515869 (2.572226047515869) + | > loss_gen: 2.1573915481567383 (2.1573915481567383) + | > loss_kl: 2.345233678817749 (2.345233678817749) + | > loss_feat: 6.112644195556641 (6.112644195556641) + | > loss_mel: 16.58450698852539 (16.58450698852539) + | > loss_duration: 1.662070393562317 (1.662070393562317) + | > loss_1: 28.861846923828125 (28.861846923828125) + + --> STEP: 1 + | > loss_disc: 2.3688554763793945 (2.3688554763793945) + | > loss_disc_real_0: 0.15160781145095825 (0.15160781145095825) + | > loss_disc_real_1: 0.17272405326366425 (0.17272405326366425) + | > loss_disc_real_2: 0.21473997831344604 (0.21473997831344604) + | > loss_disc_real_3: 0.19811150431632996 (0.19811150431632996) + | > loss_disc_real_4: 0.2324114441871643 (0.2324114441871643) + | > loss_disc_real_5: 0.2535788118839264 (0.2535788118839264) + | > loss_0: 2.3688554763793945 (2.3688554763793945) + | > loss_gen: 2.335522174835205 (2.335522174835205) + | > loss_kl: 2.574436664581299 (2.574436664581299) + | > loss_feat: 7.701318740844727 (7.701318740844727) + | > loss_mel: 19.111469268798828 (19.111469268798828) + | > loss_duration: 1.6701252460479736 (1.6701252460479736) + | > loss_1: 33.39287185668945 (33.39287185668945) + + --> STEP: 2 + | > loss_disc: 2.5013604164123535 (2.435107946395874) + | > loss_disc_real_0: 0.17229825258255005 (0.16195303201675415) + | > loss_disc_real_1: 0.18927505612373352 (0.18099955469369888) + | > loss_disc_real_2: 0.2195090353488922 (0.21712450683116913) + | > loss_disc_real_3: 0.233202263712883 (0.21565688401460648) + | > loss_disc_real_4: 0.21763797104358673 (0.22502470761537552) + | > loss_disc_real_5: 0.2572619318962097 (0.25542037189006805) + | > loss_0: 2.5013604164123535 (2.435107946395874) + | > loss_gen: 2.1830291748046875 (2.2592756748199463) + | > loss_kl: 2.230560779571533 (2.402498722076416) + | > loss_feat: 6.719677448272705 (7.210498094558716) + | > loss_mel: 19.05205726623535 (19.08176326751709) + | > loss_duration: 1.660840392112732 (1.6654828190803528) + | > loss_1: 31.846166610717773 (32.61951923370361) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003781437873840332 (+0.0002764463424682617) + | > avg_loss_disc: 2.435107946395874 (-0.06428492069244385) + | > avg_loss_disc_real_0: 0.16195303201675415 (+0.04390013590455055) + | > avg_loss_disc_real_1: 0.18099955469369888 (-0.04142804443836212) + | > avg_loss_disc_real_2: 0.21712450683116913 (-0.008664801716804504) + | > avg_loss_disc_real_3: 0.21565688401460648 (-0.012902617454528809) + | > avg_loss_disc_real_4: 0.22502470761537552 (-0.0005271732807159424) + | > avg_loss_disc_real_5: 0.25542037189006805 (+0.04031447321176529) + | > avg_loss_0: 2.435107946395874 (-0.06428492069244385) + | > avg_loss_gen: 2.2592756748199463 (+0.06816971302032471) + | > avg_loss_kl: 2.402498722076416 (+0.02230095863342285) + | > avg_loss_feat: 7.210498094558716 (+1.0757670402526855) + | > avg_loss_mel: 19.08176326751709 (+1.992384910583496) + | > avg_loss_duration: 1.6654828190803528 (+0.0054877400398254395) + | > avg_loss_1: 32.61951923370361 (+3.1641111373901367) + + + > EPOCH: 153/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:31:01)  + + --> TIME: 2025-04-19 01:31:08 -- STEP: 17/142 -- GLOBAL_STEP: 401025 + | > loss_disc: 2.3865652084350586 (2.404242249096141) + | > loss_disc_real_0: 0.11341498047113419 (0.13168220550698392) + | > loss_disc_real_1: 0.20983842015266418 (0.19852614402770996) + | > loss_disc_real_2: 0.20777200162410736 (0.218941834043054) + | > loss_disc_real_3: 0.18730668723583221 (0.22308441382997177) + | > loss_disc_real_4: 0.2191578894853592 (0.2220557402161991) + | > loss_disc_real_5: 0.1925322711467743 (0.22903311427901774) + | > loss_0: 2.3865652084350586 (2.404242249096141) + | > grad_norm_0: tensor(17.7624, device='cuda:0') (tensor(21.2506, device='cuda:0')) + | > loss_gen: 2.29535174369812 (2.4293167450848747) + | > loss_kl: 1.8277286291122437 (1.8669992825564217) + | > loss_feat: 7.726227760314941 (7.244683321784525) + | > loss_mel: 17.32622528076172 (17.654815898222086) + | > loss_duration: 1.431469202041626 (1.438234146903543) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.60700035095215 (30.634049808277805) + | > grad_norm_1: tensor(464.8658, device='cuda:0') (tensor(361.0887, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.3388 (0.3349553697249469) + | > loader_time: 0.005 (0.004625628976260915) + + + --> TIME: 2025-04-19 01:31:17 -- STEP: 42/142 -- GLOBAL_STEP: 401050 + | > loss_disc: 2.602771282196045 (2.450531562169393) + | > loss_disc_real_0: 0.13938936591148376 (0.12618117877060456) + | > loss_disc_real_1: 0.22972702980041504 (0.20566276354449137) + | > loss_disc_real_2: 0.22037701308727264 (0.22424311261801494) + | > loss_disc_real_3: 0.2164345681667328 (0.2273295209521339) + | > loss_disc_real_4: 0.22206316888332367 (0.2286682114714668) + | > loss_disc_real_5: 0.22019906342029572 (0.230273690252077) + | > loss_0: 2.602771282196045 (2.450531562169393) + | > grad_norm_0: tensor(6.3547, device='cuda:0') (tensor(17.5207, device='cuda:0')) + | > loss_gen: 2.010989189147949 (2.3703994126546952) + | > loss_kl: 1.7767932415008545 (1.853790093035925) + | > loss_feat: 6.050723075866699 (7.02730917930603) + | > loss_mel: 16.022247314453125 (17.572689692179367) + | > loss_duration: 1.5180566310882568 (1.4534522891044617) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.378808975219727 (30.277640933082218) + | > grad_norm_1: tensor(119.0932, device='cuda:0') (tensor(317.5099, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.363 (0.3456070309593563) + | > loader_time: 0.0056 (0.005160808563232423) + + + --> TIME: 2025-04-19 01:31:26 -- STEP: 67/142 -- GLOBAL_STEP: 401075 + | > loss_disc: 2.418342113494873 (2.452634807842881) + | > loss_disc_real_0: 0.08847802132368088 (0.12663996636645122) + | > loss_disc_real_1: 0.19985367357730865 (0.2051874145643035) + | > loss_disc_real_2: 0.22298386693000793 (0.2240246076192429) + | > loss_disc_real_3: 0.18792442977428436 (0.22615558865355023) + | > loss_disc_real_4: 0.19765768945217133 (0.22785071961915315) + | > loss_disc_real_5: 0.19171908497810364 (0.23259383528979857) + | > loss_0: 2.418342113494873 (2.452634807842881) + | > grad_norm_0: tensor(5.0857, device='cuda:0') (tensor(13.7950, device='cuda:0')) + | > loss_gen: 2.220808267593384 (2.347705965611472) + | > loss_kl: 1.8960281610488892 (1.8673034408199254) + | > loss_feat: 7.296294689178467 (6.983233081760691) + | > loss_mel: 17.267967224121094 (17.602302878650267) + | > loss_duration: 1.4271878004074097 (1.4643272463955097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.10828399658203 (30.264872821409313) + | > grad_norm_1: tensor(195.3061, device='cuda:0') (tensor(254.5564, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.3832 (0.3563870778724328) + | > loader_time: 0.0064 (0.005470051694272169) + + + --> TIME: 2025-04-19 01:31:37 -- STEP: 92/142 -- GLOBAL_STEP: 401100 + | > loss_disc: 2.5999953746795654 (2.4589329206425212) + | > loss_disc_real_0: 0.09785886108875275 (0.12748209642165384) + | > loss_disc_real_1: 0.22554732859134674 (0.2072869859959768) + | > loss_disc_real_2: 0.22693227231502533 (0.22408269199988115) + | > loss_disc_real_3: 0.27489563822746277 (0.22772302284188892) + | > loss_disc_real_4: 0.24097226560115814 (0.22836157549982486) + | > loss_disc_real_5: 0.24262723326683044 (0.23219910668938057) + | > loss_0: 2.5999953746795654 (2.4589329206425212) + | > grad_norm_0: tensor(9.6622, device='cuda:0') (tensor(11.9706, device='cuda:0')) + | > loss_gen: 2.2028369903564453 (2.340938951658165) + | > loss_kl: 1.9678856134414673 (1.8917660557705422) + | > loss_feat: 6.700929164886475 (6.979949692021245) + | > loss_mel: 17.641326904296875 (17.728412037310388) + | > loss_duration: 1.4693883657455444 (1.4773951587469683) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.98236846923828 (30.41846215206644) + | > grad_norm_1: tensor(238.6436, device='cuda:0') (tensor(224.8739, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.4094 (0.36719858128091565) + | > loader_time: 0.0077 (0.005746400874593984) + + + --> TIME: 2025-04-19 01:31:47 -- STEP: 117/142 -- GLOBAL_STEP: 401125 + | > loss_disc: 2.4576497077941895 (2.4594852924346924) + | > loss_disc_real_0: 0.11117339134216309 (0.12713537966975805) + | > loss_disc_real_1: 0.20352515578269958 (0.20655621575494099) + | > loss_disc_real_2: 0.22322437167167664 (0.22336415704498944) + | > loss_disc_real_3: 0.2465922236442566 (0.22839413328557953) + | > loss_disc_real_4: 0.23956342041492462 (0.22844471979854453) + | > loss_disc_real_5: 0.26801177859306335 (0.23285782706533742) + | > loss_0: 2.4576497077941895 (2.4594852924346924) + | > grad_norm_0: tensor(9.1449, device='cuda:0') (tensor(11.0954, device='cuda:0')) + | > loss_gen: 2.399733543395996 (2.3358965694394884) + | > loss_kl: 2.0458123683929443 (1.8930062840127537) + | > loss_feat: 6.687278747558594 (6.9617318332704725) + | > loss_mel: 16.5257625579834 (17.68633427579178) + | > loss_duration: 1.4755489826202393 (1.4838715975101178) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.134138107299805 (30.36084083817963) + | > grad_norm_1: tensor(226.8657, device='cuda:0') (tensor(219.1259, device='cuda:0')) + | > current_lr_0: 0.00019621110994425385 + | > current_lr_1: 0.00019621110994425385 + | > step_time: 0.4349 (0.3777611826220129) + | > loader_time: 0.0073 (0.005989347767626119) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5283446311950684 (2.5283446311950684) + | > loss_disc_real_0: 0.1982097178697586 (0.1982097178697586) + | > loss_disc_real_1: 0.26929232478141785 (0.26929232478141785) + | > loss_disc_real_2: 0.27574795484542847 (0.27574795484542847) + | > loss_disc_real_3: 0.2283746302127838 (0.2283746302127838) + | > loss_disc_real_4: 0.2172970026731491 (0.2172970026731491) + | > loss_disc_real_5: 0.24786216020584106 (0.24786216020584106) + | > loss_0: 2.5283446311950684 (2.5283446311950684) + | > loss_gen: 2.69018292427063 (2.69018292427063) + | > loss_kl: 2.2901573181152344 (2.2901573181152344) + | > loss_feat: 7.673421859741211 (7.673421859741211) + | > loss_mel: 20.268733978271484 (20.268733978271484) + | > loss_duration: 1.6750819683074951 (1.6750819683074951) + | > loss_1: 34.59757995605469 (34.59757995605469) + + --> STEP: 1 + | > loss_disc: 2.489750623703003 (2.489750623703003) + | > loss_disc_real_0: 0.1637367457151413 (0.1637367457151413) + | > loss_disc_real_1: 0.2691793739795685 (0.2691793739795685) + | > loss_disc_real_2: 0.3097909986972809 (0.3097909986972809) + | > loss_disc_real_3: 0.2422439604997635 (0.2422439604997635) + | > loss_disc_real_4: 0.2580753564834595 (0.2580753564834595) + | > loss_disc_real_5: 0.2860701084136963 (0.2860701084136963) + | > loss_0: 2.489750623703003 (2.489750623703003) + | > loss_gen: 2.734515428543091 (2.734515428543091) + | > loss_kl: 2.581739902496338 (2.581739902496338) + | > loss_feat: 7.912696361541748 (7.912696361541748) + | > loss_mel: 19.85282325744629 (19.85282325744629) + | > loss_duration: 1.7065320014953613 (1.7065320014953613) + | > loss_1: 34.788307189941406 (34.788307189941406) + + --> STEP: 2 + | > loss_disc: 2.6048786640167236 (2.5473146438598633) + | > loss_disc_real_0: 0.23101326823234558 (0.19737500697374344) + | > loss_disc_real_1: 0.2511041462421417 (0.2601417601108551) + | > loss_disc_real_2: 0.2729617655277252 (0.29137638211250305) + | > loss_disc_real_3: 0.26310521364212036 (0.2526745870709419) + | > loss_disc_real_4: 0.235708087682724 (0.24689172208309174) + | > loss_disc_real_5: 0.2828604280948639 (0.2844652682542801) + | > loss_0: 2.6048786640167236 (2.5473146438598633) + | > loss_gen: 2.607598066329956 (2.6710567474365234) + | > loss_kl: 2.1765077114105225 (2.37912380695343) + | > loss_feat: 7.736097812652588 (7.824397087097168) + | > loss_mel: 16.841808319091797 (18.347315788269043) + | > loss_duration: 1.64645516872406 (1.6764935851097107) + | > loss_1: 31.008466720581055 (32.89838695526123) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004006147384643555 (+0.00022470951080322266) + | > avg_loss_disc: 2.5473146438598633 (+0.11220669746398926) + | > avg_loss_disc_real_0: 0.19737500697374344 (+0.03542197495698929) + | > avg_loss_disc_real_1: 0.2601417601108551 (+0.07914220541715622) + | > avg_loss_disc_real_2: 0.29137638211250305 (+0.07425187528133392) + | > avg_loss_disc_real_3: 0.2526745870709419 (+0.03701770305633545) + | > avg_loss_disc_real_4: 0.24689172208309174 (+0.021867014467716217) + | > avg_loss_disc_real_5: 0.2844652682542801 (+0.029044896364212036) + | > avg_loss_0: 2.5473146438598633 (+0.11220669746398926) + | > avg_loss_gen: 2.6710567474365234 (+0.41178107261657715) + | > avg_loss_kl: 2.37912380695343 (-0.02337491512298584) + | > avg_loss_feat: 7.824397087097168 (+0.6138989925384521) + | > avg_loss_mel: 18.347315788269043 (-0.7344474792480469) + | > avg_loss_duration: 1.6764935851097107 (+0.01101076602935791) + | > avg_loss_1: 32.89838695526123 (+0.2788677215576172) + + + > EPOCH: 154/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:32:00)  + + --> TIME: 2025-04-19 01:32:01 -- STEP: 0/142 -- GLOBAL_STEP: 401150 + | > loss_disc: 2.509796619415283 (2.509796619415283) + | > loss_disc_real_0: 0.1770782768726349 (0.1770782768726349) + | > loss_disc_real_1: 0.28050029277801514 (0.28050029277801514) + | > loss_disc_real_2: 0.29337072372436523 (0.29337072372436523) + | > loss_disc_real_3: 0.25572142004966736 (0.25572142004966736) + | > loss_disc_real_4: 0.2458072304725647 (0.2458072304725647) + | > loss_disc_real_5: 0.2532201111316681 (0.2532201111316681) + | > loss_0: 2.509796619415283 (2.509796619415283) + | > grad_norm_0: tensor(8.0973, device='cuda:0') (tensor(8.0973, device='cuda:0')) + | > loss_gen: 2.5902304649353027 (2.5902304649353027) + | > loss_kl: 1.8447167873382568 (1.8447167873382568) + | > loss_feat: 7.402609348297119 (7.402609348297119) + | > loss_mel: 18.757295608520508 (18.757295608520508) + | > loss_duration: 1.4694154262542725 (1.4694154262542725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.064266204833984 (32.064266204833984) + | > grad_norm_1: tensor(168.3549, device='cuda:0') (tensor(168.3549, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.3772 (0.37718844413757324) + | > loader_time: 0.3234 (0.32343125343322754) + + + --> TIME: 2025-04-19 01:32:09 -- STEP: 25/142 -- GLOBAL_STEP: 401175 + | > loss_disc: 2.421873092651367 (2.4768890190124506) + | > loss_disc_real_0: 0.11469382047653198 (0.12119722932577133) + | > loss_disc_real_1: 0.18270905315876007 (0.20709435522556305) + | > loss_disc_real_2: 0.24000972509384155 (0.22653299629688262) + | > loss_disc_real_3: 0.20572488009929657 (0.23122140526771545) + | > loss_disc_real_4: 0.2284657061100006 (0.2319921863079071) + | > loss_disc_real_5: 0.19220727682113647 (0.23905832350254058) + | > loss_0: 2.421873092651367 (2.4768890190124506) + | > grad_norm_0: tensor(10.7897, device='cuda:0') (tensor(8.2061, device='cuda:0')) + | > loss_gen: 2.2942726612091064 (2.320030269622803) + | > loss_kl: 1.835325002670288 (1.8535438776016235) + | > loss_feat: 6.962993144989014 (6.977881774902344) + | > loss_mel: 17.42723274230957 (17.65105941772461) + | > loss_duration: 1.4790587425231934 (1.4457620191574097) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.998884201049805 (30.24827766418457) + | > grad_norm_1: tensor(239.7210, device='cuda:0') (tensor(185.2474, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.3459 (0.3338038635253906) + | > loader_time: 0.0055 (0.004819755554199219) + + + --> TIME: 2025-04-19 01:32:19 -- STEP: 50/142 -- GLOBAL_STEP: 401200 + | > loss_disc: 2.4870002269744873 (2.462491068840027) + | > loss_disc_real_0: 0.10362255573272705 (0.1213211952149868) + | > loss_disc_real_1: 0.16054153442382812 (0.20779093712568283) + | > loss_disc_real_2: 0.20687849819660187 (0.22645597785711288) + | > loss_disc_real_3: 0.2206641584634781 (0.23046476811170577) + | > loss_disc_real_4: 0.227413609623909 (0.23006198853254317) + | > loss_disc_real_5: 0.25890934467315674 (0.23163287580013275) + | > loss_0: 2.4870002269744873 (2.462491068840027) + | > grad_norm_0: tensor(4.8372, device='cuda:0') (tensor(8.6392, device='cuda:0')) + | > loss_gen: 2.381563186645508 (2.3457772922515865) + | > loss_kl: 2.0009329319000244 (1.8722238588333129) + | > loss_feat: 7.4200263023376465 (7.110570850372315) + | > loss_mel: 18.391054153442383 (17.887336006164553) + | > loss_duration: 1.4889543056488037 (1.455344738960266) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.682531356811523 (30.67125297546387) + | > grad_norm_1: tensor(112.8874, device='cuda:0') (tensor(206.7196, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.3695 (0.34909595012664796) + | > loader_time: 0.0062 (0.005330872535705566) + + + --> TIME: 2025-04-19 01:32:28 -- STEP: 75/142 -- GLOBAL_STEP: 401225 + | > loss_disc: 2.5329012870788574 (2.465157642364502) + | > loss_disc_real_0: 0.07286911457777023 (0.12261139164368312) + | > loss_disc_real_1: 0.1745852828025818 (0.20905008773008985) + | > loss_disc_real_2: 0.20043395459651947 (0.22746476292610168) + | > loss_disc_real_3: 0.21558694541454315 (0.22946882247924805) + | > loss_disc_real_4: 0.23519039154052734 (0.22882115503152212) + | > loss_disc_real_5: 0.23689059913158417 (0.23301842153072358) + | > loss_0: 2.5329012870788574 (2.465157642364502) + | > grad_norm_0: tensor(7.9521, device='cuda:0') (tensor(8.3861, device='cuda:0')) + | > loss_gen: 2.25394868850708 (2.3540509796142577) + | > loss_kl: 1.9626030921936035 (1.8784445556004843) + | > loss_feat: 7.823942184448242 (7.171721693674724) + | > loss_mel: 18.139198303222656 (18.067067082722982) + | > loss_duration: 1.4602463245391846 (1.4682057030995688) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.639938354492188 (30.939490076700846) + | > grad_norm_1: tensor(95.7452, device='cuda:0') (tensor(186.5351, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.3959 (0.3600695514678956) + | > loader_time: 0.0064 (0.005591729482014973) + + + --> TIME: 2025-04-19 01:32:39 -- STEP: 100/142 -- GLOBAL_STEP: 401250 + | > loss_disc: 2.453960418701172 (2.4729724168777487) + | > loss_disc_real_0: 0.12703585624694824 (0.12600561283528808) + | > loss_disc_real_1: 0.1933547556400299 (0.20997969210147857) + | > loss_disc_real_2: 0.21667402982711792 (0.2266872139275074) + | > loss_disc_real_3: 0.20773886144161224 (0.22935949236154557) + | > loss_disc_real_4: 0.23204629123210907 (0.23069353133440018) + | > loss_disc_real_5: 0.20987698435783386 (0.23252871498465538) + | > loss_0: 2.453960418701172 (2.4729724168777487) + | > grad_norm_0: tensor(10.0794, device='cuda:0') (tensor(8.0887, device='cuda:0')) + | > loss_gen: 2.220665216445923 (2.3386674118041997) + | > loss_kl: 2.028165817260742 (1.8978883957862853) + | > loss_feat: 7.013649940490723 (7.101564302444458) + | > loss_mel: 18.23868751525879 (18.025737895965566) + | > loss_duration: 1.4905152320861816 (1.4791278481483454) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.991682052612305 (30.842985877990724) + | > grad_norm_1: tensor(146.7382, device='cuda:0') (tensor(172.9652, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.4124 (0.3703637337684632) + | > loader_time: 0.0084 (0.0059295368194580075) + + + --> TIME: 2025-04-19 01:32:49 -- STEP: 125/142 -- GLOBAL_STEP: 401275 + | > loss_disc: 2.566206932067871 (2.4763370189666762) + | > loss_disc_real_0: 0.12005644291639328 (0.12684426701068885) + | > loss_disc_real_1: 0.21530164778232574 (0.20875532042980194) + | > loss_disc_real_2: 0.30364954471588135 (0.22703200161457063) + | > loss_disc_real_3: 0.2847505807876587 (0.2295847486257553) + | > loss_disc_real_4: 0.27658453583717346 (0.23062444746494293) + | > loss_disc_real_5: 0.2666836977005005 (0.23366553580760957) + | > loss_0: 2.566206932067871 (2.4763370189666762) + | > grad_norm_0: tensor(5.1210, device='cuda:0') (tensor(7.9014, device='cuda:0')) + | > loss_gen: 2.4038681983947754 (2.33013341331482) + | > loss_kl: 1.897072434425354 (1.9068841962814331) + | > loss_feat: 5.824550151824951 (7.031813789367676) + | > loss_mel: 16.33892822265625 (17.914640525817866) + | > loss_duration: 1.5377440452575684 (1.4866338481903072) + | > amp_scaler: 512.0 (303.1039999999999) + | > loss_1: 28.00216293334961 (30.670105758666992) + | > grad_norm_1: tensor(63.1441, device='cuda:0') (tensor(166.9366, device='cuda:0')) + | > current_lr_0: 0.00019618658355551082 + | > current_lr_1: 0.00019618658355551082 + | > step_time: 0.4377 (0.38176872253417976) + | > loader_time: 0.0074 (0.006189838409423828) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.49183988571167 (2.49183988571167) + | > loss_disc_real_0: 0.20303069055080414 (0.20303069055080414) + | > loss_disc_real_1: 0.21050478518009186 (0.21050478518009186) + | > loss_disc_real_2: 0.24446028470993042 (0.24446028470993042) + | > loss_disc_real_3: 0.24507740139961243 (0.24507740139961243) + | > loss_disc_real_4: 0.16547884047031403 (0.16547884047031403) + | > loss_disc_real_5: 0.24210499227046967 (0.24210499227046967) + | > loss_0: 2.49183988571167 (2.49183988571167) + | > loss_gen: 2.556461811065674 (2.556461811065674) + | > loss_kl: 2.4145748615264893 (2.4145748615264893) + | > loss_feat: 7.560917377471924 (7.560917377471924) + | > loss_mel: 19.54974937438965 (19.54974937438965) + | > loss_duration: 1.6599164009094238 (1.6599164009094238) + | > loss_1: 33.74161911010742 (33.74161911010742) + + --> STEP: 1 + | > loss_disc: 2.414961099624634 (2.414961099624634) + | > loss_disc_real_0: 0.17244403064250946 (0.17244403064250946) + | > loss_disc_real_1: 0.19007188081741333 (0.19007188081741333) + | > loss_disc_real_2: 0.25409257411956787 (0.25409257411956787) + | > loss_disc_real_3: 0.1958790421485901 (0.1958790421485901) + | > loss_disc_real_4: 0.1949940174818039 (0.1949940174818039) + | > loss_disc_real_5: 0.22035391628742218 (0.22035391628742218) + | > loss_0: 2.414961099624634 (2.414961099624634) + | > loss_gen: 2.352362632751465 (2.352362632751465) + | > loss_kl: 2.4739315509796143 (2.4739315509796143) + | > loss_feat: 7.439085960388184 (7.439085960388184) + | > loss_mel: 18.76494598388672 (18.76494598388672) + | > loss_duration: 1.694861888885498 (1.694861888885498) + | > loss_1: 32.725189208984375 (32.725189208984375) + + --> STEP: 2 + | > loss_disc: 2.4609763622283936 (2.4379687309265137) + | > loss_disc_real_0: 0.1713973581790924 (0.17192069441080093) + | > loss_disc_real_1: 0.21936699748039246 (0.2047194391489029) + | > loss_disc_real_2: 0.25959882140159607 (0.25684569776058197) + | > loss_disc_real_3: 0.2247917205095291 (0.2103353813290596) + | > loss_disc_real_4: 0.1878744661808014 (0.19143424183130264) + | > loss_disc_real_5: 0.2577674090862274 (0.2390606626868248) + | > loss_0: 2.4609763622283936 (2.4379687309265137) + | > loss_gen: 2.305361032485962 (2.3288618326187134) + | > loss_kl: 2.2858612537384033 (2.379896402359009) + | > loss_feat: 6.372036933898926 (6.905561447143555) + | > loss_mel: 17.052440643310547 (17.908693313598633) + | > loss_duration: 1.6140968799591064 (1.6544793844223022) + | > loss_1: 29.629796981811523 (31.17749309539795) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00353848934173584 (-0.00046765804290771484) + | > avg_loss_disc: 2.4379687309265137 (-0.10934591293334961) + | > avg_loss_disc_real_0: 0.17192069441080093 (-0.025454312562942505) + | > avg_loss_disc_real_1: 0.2047194391489029 (-0.05542232096195221) + | > avg_loss_disc_real_2: 0.25684569776058197 (-0.03453068435192108) + | > avg_loss_disc_real_3: 0.2103353813290596 (-0.042339205741882324) + | > avg_loss_disc_real_4: 0.19143424183130264 (-0.05545748025178909) + | > avg_loss_disc_real_5: 0.2390606626868248 (-0.04540460556745529) + | > avg_loss_0: 2.4379687309265137 (-0.10934591293334961) + | > avg_loss_gen: 2.3288618326187134 (-0.34219491481781006) + | > avg_loss_kl: 2.379896402359009 (+0.0007725954055786133) + | > avg_loss_feat: 6.905561447143555 (-0.9188356399536133) + | > avg_loss_mel: 17.908693313598633 (-0.43862247467041016) + | > avg_loss_duration: 1.6544793844223022 (-0.022014200687408447) + | > avg_loss_1: 31.17749309539795 (-1.7208938598632812) + + + > EPOCH: 155/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:32:59)  + + --> TIME: 2025-04-19 01:33:02 -- STEP: 8/142 -- GLOBAL_STEP: 401300 + | > loss_disc: 2.226761817932129 (2.3829453587532043) + | > loss_disc_real_0: 0.13725070655345917 (0.10519226267933846) + | > loss_disc_real_1: 0.1847456842660904 (0.20422352477908134) + | > loss_disc_real_2: 0.20338352024555206 (0.21923594176769257) + | > loss_disc_real_3: 0.18176090717315674 (0.21819592639803886) + | > loss_disc_real_4: 0.17804905772209167 (0.21234441176056862) + | > loss_disc_real_5: 0.20362932980060577 (0.24128489010035992) + | > loss_0: 2.226761817932129 (2.3829453587532043) + | > grad_norm_0: tensor(5.5878, device='cuda:0') (tensor(8.0175, device='cuda:0')) + | > loss_gen: 2.667205333709717 (2.4290947318077087) + | > loss_kl: 1.8880927562713623 (1.8952029645442963) + | > loss_feat: 7.907315254211426 (7.275486707687378) + | > loss_mel: 18.320999145507812 (17.863635301589966) + | > loss_duration: 1.4459477663040161 (1.4433226138353348) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.22956085205078 (30.906742334365845) + | > grad_norm_1: tensor(135.8430, device='cuda:0') (tensor(156.8484, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.3243 (0.33314448595046997) + | > loader_time: 0.0051 (0.004625290632247925) + + + --> TIME: 2025-04-19 01:33:11 -- STEP: 33/142 -- GLOBAL_STEP: 401325 + | > loss_disc: 2.591048002243042 (2.44963425578493) + | > loss_disc_real_0: 0.16877847909927368 (0.12150674877744733) + | > loss_disc_real_1: 0.2254028469324112 (0.20407366797779547) + | > loss_disc_real_2: 0.21924185752868652 (0.22478920131018668) + | > loss_disc_real_3: 0.24983763694763184 (0.22743667707298743) + | > loss_disc_real_4: 0.24010805785655975 (0.22676858576861295) + | > loss_disc_real_5: 0.22982098162174225 (0.23491003702987323) + | > loss_0: 2.591048002243042 (2.44963425578493) + | > grad_norm_0: tensor(9.3497, device='cuda:0') (tensor(9.0425, device='cuda:0')) + | > loss_gen: 2.2740461826324463 (2.365725445024895) + | > loss_kl: 1.8017529249191284 (1.892665642680544) + | > loss_feat: 6.2978010177612305 (7.122742233854352) + | > loss_mel: 16.361896514892578 (17.741080428614755) + | > loss_duration: 1.4498960971832275 (1.4510050397930723) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.185392379760742 (30.57321883692886) + | > grad_norm_1: tensor(62.0752, device='cuda:0') (tensor(183.4753, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.357 (0.34130685979669745) + | > loader_time: 0.0056 (0.005148902083888199) + + + --> TIME: 2025-04-19 01:33:20 -- STEP: 58/142 -- GLOBAL_STEP: 401350 + | > loss_disc: 2.587985038757324 (2.446725467155719) + | > loss_disc_real_0: 0.11701762676239014 (0.12204423302720332) + | > loss_disc_real_1: 0.1910533905029297 (0.20415691389092083) + | > loss_disc_real_2: 0.20790766179561615 (0.22280404408430232) + | > loss_disc_real_3: 0.2392774224281311 (0.22874246974443568) + | > loss_disc_real_4: 0.2326306700706482 (0.22751821089407492) + | > loss_disc_real_5: 0.24775907397270203 (0.23693695484564223) + | > loss_0: 2.587985038757324 (2.446725467155719) + | > grad_norm_0: tensor(6.6729, device='cuda:0') (tensor(11.3692, device='cuda:0')) + | > loss_gen: 2.1050665378570557 (2.3739812949608106) + | > loss_kl: 1.9257442951202393 (1.9068449509554897) + | > loss_feat: 6.458474159240723 (7.1536493136965) + | > loss_mel: 18.13787078857422 (17.814956813023) + | > loss_duration: 1.4818103313446045 (1.4616228650356164) + | > amp_scaler: 256.0 (436.9655172413793) + | > loss_1: 30.108964920043945 (30.71105519656477) + | > grad_norm_1: tensor(155.2755, device='cuda:0') (tensor(228.1647, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.3747 (0.35130775385889507) + | > loader_time: 0.0063 (0.005460023880004884) + + + --> TIME: 2025-04-19 01:33:30 -- STEP: 83/142 -- GLOBAL_STEP: 401375 + | > loss_disc: 2.5175466537475586 (2.453487152076629) + | > loss_disc_real_0: 0.23506584763526917 (0.12415956203118865) + | > loss_disc_real_1: 0.2121235430240631 (0.20529161016625094) + | > loss_disc_real_2: 0.21118465065956116 (0.22415568491062485) + | > loss_disc_real_3: 0.2160007804632187 (0.22879893718713737) + | > loss_disc_real_4: 0.24098652601242065 (0.22712771971541715) + | > loss_disc_real_5: 0.22390222549438477 (0.2360298296055162) + | > loss_0: 2.5175466537475586 (2.453487152076629) + | > grad_norm_0: tensor(10.4186, device='cuda:0') (tensor(11.1002, device='cuda:0')) + | > loss_gen: 2.2263402938842773 (2.3631831709160864) + | > loss_kl: 1.9505661725997925 (1.9201450146824481) + | > loss_feat: 6.118978023529053 (7.048202675509166) + | > loss_mel: 16.326704025268555 (17.66259320959987) + | > loss_duration: 1.574998140335083 (1.4730742968708643) + | > amp_scaler: 256.0 (382.45783132530124) + | > loss_1: 28.197587966918945 (30.467198360397155) + | > grad_norm_1: tensor(365.9059, device='cuda:0') (tensor(244.2081, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.3978 (0.3637400851192244) + | > loader_time: 0.0064 (0.005779407110558935) + + + --> TIME: 2025-04-19 01:33:41 -- STEP: 108/142 -- GLOBAL_STEP: 401400 + | > loss_disc: 2.580549478530884 (2.4689360636251942) + | > loss_disc_real_0: 0.1530393809080124 (0.1277839164077132) + | > loss_disc_real_1: 0.2527078092098236 (0.20682236372872634) + | > loss_disc_real_2: 0.2240649312734604 (0.22519149148353823) + | > loss_disc_real_3: 0.24648870527744293 (0.22941557911259156) + | > loss_disc_real_4: 0.24179723858833313 (0.22873033362406273) + | > loss_disc_real_5: 0.22994807362556458 (0.23523628504739869) + | > loss_0: 2.580549478530884 (2.4689360636251942) + | > grad_norm_0: tensor(18.6116, device='cuda:0') (tensor(11.4466, device='cuda:0')) + | > loss_gen: 2.1502344608306885 (2.3456936809751716) + | > loss_kl: 1.9996604919433594 (1.9240937751752358) + | > loss_feat: 5.815238952636719 (7.004007087813483) + | > loss_mel: 15.605578422546387 (17.710007588068635) + | > loss_duration: 1.5115433931350708 (1.4817657437589438) + | > amp_scaler: 256.0 (353.1851851851853) + | > loss_1: 27.082256317138672 (30.465567853715683) + | > grad_norm_1: tensor(254.4794, device='cuda:0') (tensor(249.8139, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.4308 (0.37491586914768926) + | > loader_time: 0.007 (0.006012552314334446) + + + --> TIME: 2025-04-19 01:33:52 -- STEP: 133/142 -- GLOBAL_STEP: 401425 + | > loss_disc: 2.5837321281433105 (2.4730348730445804) + | > loss_disc_real_0: 0.1450231373310089 (0.12989768615566705) + | > loss_disc_real_1: 0.18767832219600677 (0.2069249061265386) + | > loss_disc_real_2: 0.253819078207016 (0.2258460406298028) + | > loss_disc_real_3: 0.23266951739788055 (0.22920735034727513) + | > loss_disc_real_4: 0.19833233952522278 (0.22845337030134702) + | > loss_disc_real_5: 0.2239457219839096 (0.2354451386552108) + | > loss_0: 2.5837321281433105 (2.4730348730445804) + | > grad_norm_0: tensor(10.4125, device='cuda:0') (tensor(11.5844, device='cuda:0')) + | > loss_gen: 2.1123361587524414 (2.3403480554881844) + | > loss_kl: 2.0100972652435303 (1.9253498970117784) + | > loss_feat: 6.641541481018066 (6.956322257680104) + | > loss_mel: 17.598388671875 (17.702272988799812) + | > loss_duration: 1.5679250955581665 (1.4878621397161846) + | > amp_scaler: 256.0 (334.9172932330828) + | > loss_1: 29.930286407470703 (30.41215533779976) + | > grad_norm_1: tensor(282.1992, device='cuda:0') (tensor(248.6262, device='cuda:0')) + | > current_lr_0: 0.00019616206023256638 + | > current_lr_1: 0.00019616206023256638 + | > step_time: 0.4484 (0.3880093294875066) + | > loader_time: 0.0074 (0.006265561383469661) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.421964406967163 (2.421964406967163) + | > loss_disc_real_0: 0.12370182573795319 (0.12370182573795319) + | > loss_disc_real_1: 0.23686476051807404 (0.23686476051807404) + | > loss_disc_real_2: 0.23625288903713226 (0.23625288903713226) + | > loss_disc_real_3: 0.23210129141807556 (0.23210129141807556) + | > loss_disc_real_4: 0.21925902366638184 (0.21925902366638184) + | > loss_disc_real_5: 0.2266940474510193 (0.2266940474510193) + | > loss_0: 2.421964406967163 (2.421964406967163) + | > loss_gen: 2.3280158042907715 (2.3280158042907715) + | > loss_kl: 2.4045629501342773 (2.4045629501342773) + | > loss_feat: 6.459539413452148 (6.459539413452148) + | > loss_mel: 18.479103088378906 (18.479103088378906) + | > loss_duration: 1.6415119171142578 (1.6415119171142578) + | > loss_1: 31.312732696533203 (31.312732696533203) + + --> STEP: 1 + | > loss_disc: 2.6069552898406982 (2.6069552898406982) + | > loss_disc_real_0: 0.18466326594352722 (0.18466326594352722) + | > loss_disc_real_1: 0.2251378744840622 (0.2251378744840622) + | > loss_disc_real_2: 0.2545762062072754 (0.2545762062072754) + | > loss_disc_real_3: 0.25955960154533386 (0.25955960154533386) + | > loss_disc_real_4: 0.23937910795211792 (0.23937910795211792) + | > loss_disc_real_5: 0.26588377356529236 (0.26588377356529236) + | > loss_0: 2.6069552898406982 (2.6069552898406982) + | > loss_gen: 2.311103343963623 (2.311103343963623) + | > loss_kl: 2.727548837661743 (2.727548837661743) + | > loss_feat: 6.000794410705566 (6.000794410705566) + | > loss_mel: 18.257238388061523 (18.257238388061523) + | > loss_duration: 1.6911303997039795 (1.6911303997039795) + | > loss_1: 30.98781394958496 (30.98781394958496) + + --> STEP: 2 + | > loss_disc: 2.6784236431121826 (2.6426894664764404) + | > loss_disc_real_0: 0.191087543964386 (0.1878754049539566) + | > loss_disc_real_1: 0.2570697069168091 (0.24110379070043564) + | > loss_disc_real_2: 0.27938538789749146 (0.2669807970523834) + | > loss_disc_real_3: 0.2808878421783447 (0.2702237218618393) + | > loss_disc_real_4: 0.2812816798686981 (0.260330393910408) + | > loss_disc_real_5: 0.2979215681552887 (0.2819026708602905) + | > loss_0: 2.6784236431121826 (2.6426894664764404) + | > loss_gen: 2.291919708251953 (2.301511526107788) + | > loss_kl: 1.9658026695251465 (2.346675753593445) + | > loss_feat: 5.383387088775635 (5.692090749740601) + | > loss_mel: 17.759876251220703 (18.008557319641113) + | > loss_duration: 1.6581319570541382 (1.6746311783790588) + | > loss_1: 29.059118270874023 (30.023466110229492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033113956451416016 (-0.00022709369659423828) + | > avg_loss_disc: 2.6426894664764404 (+0.20472073554992676) + | > avg_loss_disc_real_0: 0.1878754049539566 (+0.01595471054315567) + | > avg_loss_disc_real_1: 0.24110379070043564 (+0.036384351551532745) + | > avg_loss_disc_real_2: 0.2669807970523834 (+0.010135099291801453) + | > avg_loss_disc_real_3: 0.2702237218618393 (+0.059888340532779694) + | > avg_loss_disc_real_4: 0.260330393910408 (+0.06889615207910538) + | > avg_loss_disc_real_5: 0.2819026708602905 (+0.04284200817346573) + | > avg_loss_0: 2.6426894664764404 (+0.20472073554992676) + | > avg_loss_gen: 2.301511526107788 (-0.027350306510925293) + | > avg_loss_kl: 2.346675753593445 (-0.033220648765563965) + | > avg_loss_feat: 5.692090749740601 (-1.213470697402954) + | > avg_loss_mel: 18.008557319641113 (+0.09986400604248047) + | > avg_loss_duration: 1.6746311783790588 (+0.020151793956756592) + | > avg_loss_1: 30.023466110229492 (-1.154026985168457) + + + > EPOCH: 156/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:33:58)  + + --> TIME: 2025-04-19 01:34:04 -- STEP: 16/142 -- GLOBAL_STEP: 401450 + | > loss_disc: 2.426527976989746 (2.460011526942253) + | > loss_disc_real_0: 0.11343824863433838 (0.12025889102369547) + | > loss_disc_real_1: 0.17811310291290283 (0.2096713799983263) + | > loss_disc_real_2: 0.21998924016952515 (0.2309597237035632) + | > loss_disc_real_3: 0.24463264644145966 (0.22814802918583155) + | > loss_disc_real_4: 0.26086553931236267 (0.23790621664375067) + | > loss_disc_real_5: 0.2864748537540436 (0.22954932134598494) + | > loss_0: 2.426527976989746 (2.460011526942253) + | > grad_norm_0: tensor(6.5603, device='cuda:0') (tensor(11.3143, device='cuda:0')) + | > loss_gen: 2.5666744709014893 (2.3606996089220047) + | > loss_kl: 1.8293516635894775 (1.8433690667152405) + | > loss_feat: 7.339040756225586 (7.072178214788437) + | > loss_mel: 19.595762252807617 (17.817344069480896) + | > loss_duration: 1.4332411289215088 (1.4371759593486786) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.76407241821289 (30.53076708316803) + | > grad_norm_1: tensor(169.1306, device='cuda:0') (tensor(267.5527, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.3384 (0.328045517206192) + | > loader_time: 0.005 (0.004850476980209351) + + + --> TIME: 2025-04-19 01:34:13 -- STEP: 41/142 -- GLOBAL_STEP: 401475 + | > loss_disc: 2.5111196041107178 (2.4609467518038866) + | > loss_disc_real_0: 0.15716934204101562 (0.12478101162648783) + | > loss_disc_real_1: 0.18845707178115845 (0.20797568414269424) + | > loss_disc_real_2: 0.22378912568092346 (0.22610365871976063) + | > loss_disc_real_3: 0.24657517671585083 (0.22874822558426275) + | > loss_disc_real_4: 0.24608691036701202 (0.2319781169658754) + | > loss_disc_real_5: 0.2788359224796295 (0.23304670612986494) + | > loss_0: 2.5111196041107178 (2.4609467518038866) + | > grad_norm_0: tensor(10.7192, device='cuda:0') (tensor(10.5966, device='cuda:0')) + | > loss_gen: 2.317861795425415 (2.3478987565854705) + | > loss_kl: 2.0115087032318115 (1.8982578603232778) + | > loss_feat: 6.590297698974609 (7.040362486025182) + | > loss_mel: 17.345983505249023 (17.739499557309035) + | > loss_duration: 1.4419761896133423 (1.4511820281424173) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70762825012207 (30.477200763981518) + | > grad_norm_1: tensor(276.2363, device='cuda:0') (tensor(247.1785, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.3621 (0.34128322252413124) + | > loader_time: 0.0059 (0.005292933161665753) + + + --> TIME: 2025-04-19 01:34:23 -- STEP: 66/142 -- GLOBAL_STEP: 401500 + | > loss_disc: 2.4595425128936768 (2.466335805979642) + | > loss_disc_real_0: 0.12340221554040909 (0.1262805207886479) + | > loss_disc_real_1: 0.20416492223739624 (0.2081500136039474) + | > loss_disc_real_2: 0.21985788643360138 (0.22530375330737143) + | > loss_disc_real_3: 0.235024094581604 (0.2272936821435437) + | > loss_disc_real_4: 0.22034300863742828 (0.2326736773053805) + | > loss_disc_real_5: 0.250474750995636 (0.23496026636073084) + | > loss_0: 2.4595425128936768 (2.466335805979642) + | > grad_norm_0: tensor(16.2189, device='cuda:0') (tensor(9.9093, device='cuda:0')) + | > loss_gen: 2.2607061862945557 (2.3308056159452963) + | > loss_kl: 2.016467571258545 (1.903284338387576) + | > loss_feat: 6.767299175262451 (6.990863619428692) + | > loss_mel: 18.283367156982422 (17.770474982984137) + | > loss_duration: 1.463450312614441 (1.4647296504540874) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.791290283203125 (30.46015837698272) + | > grad_norm_1: tensor(209.3682, device='cuda:0') (tensor(219.8381, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.3847 (0.35590058023279364) + | > loader_time: 0.0061 (0.005614909258755771) + + + --> TIME: 2025-04-19 01:34:33 -- STEP: 91/142 -- GLOBAL_STEP: 401525 + | > loss_disc: 2.452460765838623 (2.463503476027603) + | > loss_disc_real_0: 0.12989191710948944 (0.12838949230346045) + | > loss_disc_real_1: 0.2008051723241806 (0.2077098847418041) + | > loss_disc_real_2: 0.2222236841917038 (0.2239921896994769) + | > loss_disc_real_3: 0.19968469440937042 (0.2266039001745182) + | > loss_disc_real_4: 0.1905413568019867 (0.22985413817913977) + | > loss_disc_real_5: 0.2183038741350174 (0.2342281290790537) + | > loss_0: 2.452460765838623 (2.463503476027603) + | > grad_norm_0: tensor(28.0504, device='cuda:0') (tensor(10.2000, device='cuda:0')) + | > loss_gen: 2.263063669204712 (2.336016657588246) + | > loss_kl: 2.052992343902588 (1.911375838321644) + | > loss_feat: 6.801455974578857 (7.013344555110722) + | > loss_mel: 17.56229019165039 (17.856934998061632) + | > loss_duration: 1.5037243366241455 (1.4766919494985224) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.18352699279785 (30.594364145299892) + | > grad_norm_1: tensor(552.6478, device='cuda:0') (tensor(222.9261, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.4064 (0.36765300834571923) + | > loader_time: 0.0068 (0.0058494190593342205) + + + --> TIME: 2025-04-19 01:34:43 -- STEP: 116/142 -- GLOBAL_STEP: 401550 + | > loss_disc: 2.5146727561950684 (2.4664845816020287) + | > loss_disc_real_0: 0.13438396155834198 (0.13046998883886582) + | > loss_disc_real_1: 0.20524542033672333 (0.20780295518965558) + | > loss_disc_real_2: 0.26802387833595276 (0.2245372392494103) + | > loss_disc_real_3: 0.25998586416244507 (0.2267252792572153) + | > loss_disc_real_4: 0.23892799019813538 (0.22924697103685346) + | > loss_disc_real_5: 0.243193581700325 (0.2339198156420527) + | > loss_0: 2.5146727561950684 (2.4664845816020287) + | > grad_norm_0: tensor(28.4389, device='cuda:0') (tensor(12.2775, device='cuda:0')) + | > loss_gen: 2.3628389835357666 (2.3398573871316573) + | > loss_kl: 1.9844132661819458 (1.9192004912886127) + | > loss_feat: 6.7708353996276855 (7.007196200305018) + | > loss_mel: 16.937898635864258 (17.84786623921888) + | > loss_duration: 1.4730684757232666 (1.4830021847938668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.529054641723633 (30.597122504793365) + | > grad_norm_1: tensor(241.2318, device='cuda:0') (tensor(248.7211, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.4336 (0.37894378653888044) + | > loader_time: 0.0071 (0.006125824204806624) + + + --> TIME: 2025-04-19 01:34:55 -- STEP: 141/142 -- GLOBAL_STEP: 401575 + | > loss_disc: 2.2110166549682617 (2.4677957220280407) + | > loss_disc_real_0: 0.06616707891225815 (0.12981048363743095) + | > loss_disc_real_1: 0.14856043457984924 (0.2070372605577428) + | > loss_disc_real_2: 0.22692441940307617 (0.22464510904136278) + | > loss_disc_real_3: 0.19617857038974762 (0.226961372697607) + | > loss_disc_real_4: 0.16355596482753754 (0.22906784332813102) + | > loss_disc_real_5: 0.1666211187839508 (0.23417446355447702) + | > loss_0: 2.2110166549682617 (2.4677957220280407) + | > grad_norm_0: tensor(14.3583, device='cuda:0') (tensor(12.6430, device='cuda:0')) + | > loss_gen: 2.6736538410186768 (2.343830731743616) + | > loss_kl: 1.7153048515319824 (1.9225175862616681) + | > loss_feat: 9.534658432006836 (6.991994052914017) + | > loss_mel: 18.90799331665039 (17.80519394840755) + | > loss_duration: 1.3459758758544922 (1.4884242825474299) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.177589416503906 (30.55196064076525) + | > grad_norm_1: tensor(535.5235, device='cuda:0') (tensor(255.6385, device='cuda:0')) + | > current_lr_0: 0.0001961375399750373 + | > current_lr_1: 0.0001961375399750373 + | > step_time: 0.2041 (0.39025774238802863) + | > loader_time: 0.0024 (0.006315659123955044) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.504909038543701 (2.504909038543701) + | > loss_disc_real_0: 0.1683403104543686 (0.1683403104543686) + | > loss_disc_real_1: 0.24388004839420319 (0.24388004839420319) + | > loss_disc_real_2: 0.21494081616401672 (0.21494081616401672) + | > loss_disc_real_3: 0.2185937762260437 (0.2185937762260437) + | > loss_disc_real_4: 0.20182090997695923 (0.20182090997695923) + | > loss_disc_real_5: 0.2532758116722107 (0.2532758116722107) + | > loss_0: 2.504909038543701 (2.504909038543701) + | > loss_gen: 2.242903470993042 (2.242903470993042) + | > loss_kl: 2.2019424438476562 (2.2019424438476562) + | > loss_feat: 6.2804741859436035 (6.2804741859436035) + | > loss_mel: 17.216989517211914 (17.216989517211914) + | > loss_duration: 1.6548165082931519 (1.6548165082931519) + | > loss_1: 29.597124099731445 (29.597124099731445) + + --> STEP: 1 + | > loss_disc: 2.5465822219848633 (2.5465822219848633) + | > loss_disc_real_0: 0.20650744438171387 (0.20650744438171387) + | > loss_disc_real_1: 0.2284972071647644 (0.2284972071647644) + | > loss_disc_real_2: 0.24893267452716827 (0.24893267452716827) + | > loss_disc_real_3: 0.2634289562702179 (0.2634289562702179) + | > loss_disc_real_4: 0.24557296931743622 (0.24557296931743622) + | > loss_disc_real_5: 0.20339646935462952 (0.20339646935462952) + | > loss_0: 2.5465822219848633 (2.5465822219848633) + | > loss_gen: 2.4175398349761963 (2.4175398349761963) + | > loss_kl: 2.5677993297576904 (2.5677993297576904) + | > loss_feat: 7.295757293701172 (7.295757293701172) + | > loss_mel: 19.315828323364258 (19.315828323364258) + | > loss_duration: 1.697115182876587 (1.697115182876587) + | > loss_1: 33.29404067993164 (33.29404067993164) + + --> STEP: 2 + | > loss_disc: 2.3476686477661133 (2.4471254348754883) + | > loss_disc_real_0: 0.16833677887916565 (0.18742211163043976) + | > loss_disc_real_1: 0.23902501165866852 (0.23376110941171646) + | > loss_disc_real_2: 0.26656773686408997 (0.2577502056956291) + | > loss_disc_real_3: 0.24757422506809235 (0.2555015906691551) + | > loss_disc_real_4: 0.2473078817129135 (0.24644042551517487) + | > loss_disc_real_5: 0.21940669417381287 (0.2114015817642212) + | > loss_0: 2.3476686477661133 (2.4471254348754883) + | > loss_gen: 2.767336845397949 (2.5924383401870728) + | > loss_kl: 2.3553946018218994 (2.461596965789795) + | > loss_feat: 7.717300891876221 (7.506529092788696) + | > loss_mel: 18.507122039794922 (18.91147518157959) + | > loss_duration: 1.6514265537261963 (1.6742708683013916) + | > loss_1: 32.99858474731445 (33.14631271362305) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034782886505126953 (+0.00016689300537109375) + | > avg_loss_disc: 2.4471254348754883 (-0.19556403160095215) + | > avg_loss_disc_real_0: 0.18742211163043976 (-0.0004532933235168457) + | > avg_loss_disc_real_1: 0.23376110941171646 (-0.007342681288719177) + | > avg_loss_disc_real_2: 0.2577502056956291 (-0.009230591356754303) + | > avg_loss_disc_real_3: 0.2555015906691551 (-0.014722131192684174) + | > avg_loss_disc_real_4: 0.24644042551517487 (-0.013889968395233154) + | > avg_loss_disc_real_5: 0.2114015817642212 (-0.07050108909606934) + | > avg_loss_0: 2.4471254348754883 (-0.19556403160095215) + | > avg_loss_gen: 2.5924383401870728 (+0.29092681407928467) + | > avg_loss_kl: 2.461596965789795 (+0.1149212121963501) + | > avg_loss_feat: 7.506529092788696 (+1.8144383430480957) + | > avg_loss_mel: 18.91147518157959 (+0.9029178619384766) + | > avg_loss_duration: 1.6742708683013916 (-0.00036031007766723633) + | > avg_loss_1: 33.14631271362305 (+3.1228466033935547) + + + > EPOCH: 157/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:34:57)  + + --> TIME: 2025-04-19 01:35:06 -- STEP: 24/142 -- GLOBAL_STEP: 401600 + | > loss_disc: 2.5805060863494873 (2.4822601477305093) + | > loss_disc_real_0: 0.15026500821113586 (0.12435493307809035) + | > loss_disc_real_1: 0.2008499801158905 (0.2103351534654697) + | > loss_disc_real_2: 0.23503674566745758 (0.22970554046332836) + | > loss_disc_real_3: 0.2525808811187744 (0.23386904348929724) + | > loss_disc_real_4: 0.26025599241256714 (0.2381586047510306) + | > loss_disc_real_5: 0.2741163671016693 (0.2335939904054006) + | > loss_0: 2.5805060863494873 (2.4822601477305093) + | > grad_norm_0: tensor(8.4872, device='cuda:0') (tensor(8.6918, device='cuda:0')) + | > loss_gen: 2.1646227836608887 (2.3541251718997955) + | > loss_kl: 1.8688163757324219 (1.8477935244639714) + | > loss_feat: 6.04516077041626 (7.155757228533427) + | > loss_mel: 17.46367073059082 (18.139903624852504) + | > loss_duration: 1.4864567518234253 (1.4465541541576385) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.02872657775879 (30.944133281707764) + | > grad_norm_1: tensor(161.4299, device='cuda:0') (tensor(207.4830, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.3454 (0.33292717734972643) + | > loader_time: 0.0051 (0.00461996595064799) + + + --> TIME: 2025-04-19 01:35:15 -- STEP: 49/142 -- GLOBAL_STEP: 401625 + | > loss_disc: 2.5023865699768066 (2.466696554300736) + | > loss_disc_real_0: 0.12886954843997955 (0.1226773838303527) + | > loss_disc_real_1: 0.2225896269083023 (0.2107859634015025) + | > loss_disc_real_2: 0.24192185699939728 (0.2263709504385384) + | > loss_disc_real_3: 0.2513056993484497 (0.22871126264941935) + | > loss_disc_real_4: 0.2320372313261032 (0.23111616257502107) + | > loss_disc_real_5: 0.22792983055114746 (0.23285931227158527) + | > loss_0: 2.5023865699768066 (2.466696554300736) + | > grad_norm_0: tensor(10.8635, device='cuda:0') (tensor(9.7305, device='cuda:0')) + | > loss_gen: 2.315232515335083 (2.3485483782632017) + | > loss_kl: 1.880171298980713 (1.8599598115804243) + | > loss_feat: 6.248414516448975 (7.095854720290826) + | > loss_mel: 17.31931495666504 (18.131565093994148) + | > loss_duration: 1.4801907539367676 (1.4564377069473267) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.243324279785156 (30.89236559186663) + | > grad_norm_1: tensor(322.0106, device='cuda:0') (tensor(232.1110, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.3725 (0.3457503708041445) + | > loader_time: 0.0056 (0.005037565620578065) + + + --> TIME: 2025-04-19 01:35:25 -- STEP: 74/142 -- GLOBAL_STEP: 401650 + | > loss_disc: 2.597102642059326 (2.4780497196558358) + | > loss_disc_real_0: 0.22656694054603577 (0.12705784744104823) + | > loss_disc_real_1: 0.20529800653457642 (0.20800217825013237) + | > loss_disc_real_2: 0.21833108365535736 (0.22699142932086377) + | > loss_disc_real_3: 0.24598641693592072 (0.2306087053305394) + | > loss_disc_real_4: 0.2406063973903656 (0.23120269179344177) + | > loss_disc_real_5: 0.27324169874191284 (0.23430560066087827) + | > loss_0: 2.597102642059326 (2.4780497196558358) + | > grad_norm_0: tensor(23.4595, device='cuda:0') (tensor(10.4216, device='cuda:0')) + | > loss_gen: 2.3264212608337402 (2.339314132123381) + | > loss_kl: 1.9799201488494873 (1.8735029439668398) + | > loss_feat: 6.375331401824951 (7.042424917221071) + | > loss_mel: 17.477428436279297 (17.96923753377554) + | > loss_duration: 1.4927537441253662 (1.4682170072117366) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65185546875 (30.692696365150244) + | > grad_norm_1: tensor(350.8000, device='cuda:0') (tensor(242.3483, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.3875 (0.3576494036494075) + | > loader_time: 0.0063 (0.005336162206288931) + + + --> TIME: 2025-04-19 01:35:35 -- STEP: 99/142 -- GLOBAL_STEP: 401675 + | > loss_disc: 2.5370354652404785 (2.487323445503158) + | > loss_disc_real_0: 0.102483369410038 (0.12842487905061603) + | > loss_disc_real_1: 0.26001212000846863 (0.21014753902199293) + | > loss_disc_real_2: 0.24766243994235992 (0.22689196798536512) + | > loss_disc_real_3: 0.2550775706768036 (0.23004470419402073) + | > loss_disc_real_4: 0.26186850666999817 (0.23160691920555) + | > loss_disc_real_5: 0.2622949182987213 (0.235025621875368) + | > loss_0: 2.5370354652404785 (2.487323445503158) + | > grad_norm_0: tensor(13.1626, device='cuda:0') (tensor(11.4332, device='cuda:0')) + | > loss_gen: 2.439277410507202 (2.3286192537558197) + | > loss_kl: 1.9576979875564575 (1.8879947698477544) + | > loss_feat: 6.586407661437988 (7.001049099546491) + | > loss_mel: 18.156734466552734 (17.970873553343495) + | > loss_duration: 1.519794225692749 (1.4786805239590732) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.659912109375 (30.66721708124334) + | > grad_norm_1: tensor(420.1612, device='cuda:0') (tensor(258.3323, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.4039 (0.3682709125557332) + | > loader_time: 0.0069 (0.00562661826008498) + + + --> TIME: 2025-04-19 01:35:46 -- STEP: 124/142 -- GLOBAL_STEP: 401700 + | > loss_disc: 2.357617139816284 (2.4860611231096335) + | > loss_disc_real_0: 0.08776607364416122 (0.129033189627432) + | > loss_disc_real_1: 0.2020736038684845 (0.21038314543904796) + | > loss_disc_real_2: 0.19904150068759918 (0.2267358519377247) + | > loss_disc_real_3: 0.2020748108625412 (0.22969031177701488) + | > loss_disc_real_4: 0.22188235819339752 (0.23092561491554783) + | > loss_disc_real_5: 0.23075290024280548 (0.2343573234975338) + | > loss_0: 2.357617139816284 (2.4860611231096335) + | > grad_norm_0: tensor(11.9870, device='cuda:0') (tensor(12.1081, device='cuda:0')) + | > loss_gen: 2.428170919418335 (2.327177176552434) + | > loss_kl: 2.069728136062622 (1.9028686534973882) + | > loss_feat: 7.3651933670043945 (6.9523990615721685) + | > loss_mel: 17.628934860229492 (17.893886689216867) + | > loss_duration: 1.5257534980773926 (1.485329560695156) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.017780303955078 (30.561660966565533) + | > grad_norm_1: tensor(238.8133, device='cuda:0') (tensor(261.5937, device='cuda:0')) + | > current_lr_0: 0.0001961130227825404 + | > current_lr_1: 0.0001961130227825404 + | > step_time: 0.4403 (0.379775349170931) + | > loader_time: 0.0074 (0.005909141032926498) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.429978370666504 (2.429978370666504) + | > loss_disc_real_0: 0.13201744854450226 (0.13201744854450226) + | > loss_disc_real_1: 0.1985866278409958 (0.1985866278409958) + | > loss_disc_real_2: 0.25684812664985657 (0.25684812664985657) + | > loss_disc_real_3: 0.22357939183712006 (0.22357939183712006) + | > loss_disc_real_4: 0.2084248811006546 (0.2084248811006546) + | > loss_disc_real_5: 0.24394750595092773 (0.24394750595092773) + | > loss_0: 2.429978370666504 (2.429978370666504) + | > loss_gen: 2.327537775039673 (2.327537775039673) + | > loss_kl: 2.237395763397217 (2.237395763397217) + | > loss_feat: 6.787867069244385 (6.787867069244385) + | > loss_mel: 18.249412536621094 (18.249412536621094) + | > loss_duration: 1.6692808866500854 (1.6692808866500854) + | > loss_1: 31.271493911743164 (31.271493911743164) + + --> STEP: 1 + | > loss_disc: 2.3851373195648193 (2.3851373195648193) + | > loss_disc_real_0: 0.1137903481721878 (0.1137903481721878) + | > loss_disc_real_1: 0.19862015545368195 (0.19862015545368195) + | > loss_disc_real_2: 0.21261055767536163 (0.21261055767536163) + | > loss_disc_real_3: 0.20546714961528778 (0.20546714961528778) + | > loss_disc_real_4: 0.16609182953834534 (0.16609182953834534) + | > loss_disc_real_5: 0.18310348689556122 (0.18310348689556122) + | > loss_0: 2.3851373195648193 (2.3851373195648193) + | > loss_gen: 2.0760180950164795 (2.0760180950164795) + | > loss_kl: 2.626772165298462 (2.626772165298462) + | > loss_feat: 7.290275573730469 (7.290275573730469) + | > loss_mel: 20.106565475463867 (20.106565475463867) + | > loss_duration: 1.682361364364624 (1.682361364364624) + | > loss_1: 33.7819938659668 (33.7819938659668) + + --> STEP: 2 + | > loss_disc: 2.4157063961029053 (2.4004218578338623) + | > loss_disc_real_0: 0.22272515296936035 (0.16825775057077408) + | > loss_disc_real_1: 0.19526413083076477 (0.19694214314222336) + | > loss_disc_real_2: 0.22449445724487305 (0.21855250746011734) + | > loss_disc_real_3: 0.20230521261692047 (0.20388618111610413) + | > loss_disc_real_4: 0.19484083354473114 (0.18046633154153824) + | > loss_disc_real_5: 0.20445479452610016 (0.1937791407108307) + | > loss_0: 2.4157063961029053 (2.4004218578338623) + | > loss_gen: 2.3721120357513428 (2.224065065383911) + | > loss_kl: 2.191164970397949 (2.4089685678482056) + | > loss_feat: 8.151667594909668 (7.720971584320068) + | > loss_mel: 19.40292739868164 (19.754746437072754) + | > loss_duration: 1.6388015747070312 (1.6605814695358276) + | > loss_1: 33.75667190551758 (33.76933288574219) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033273696899414062 (-0.00015091896057128906) + | > avg_loss_disc: 2.4004218578338623 (-0.04670357704162598) + | > avg_loss_disc_real_0: 0.16825775057077408 (-0.01916436105966568) + | > avg_loss_disc_real_1: 0.19694214314222336 (-0.0368189662694931) + | > avg_loss_disc_real_2: 0.21855250746011734 (-0.03919769823551178) + | > avg_loss_disc_real_3: 0.20388618111610413 (-0.051615409553050995) + | > avg_loss_disc_real_4: 0.18046633154153824 (-0.06597409397363663) + | > avg_loss_disc_real_5: 0.1937791407108307 (-0.017622441053390503) + | > avg_loss_0: 2.4004218578338623 (-0.04670357704162598) + | > avg_loss_gen: 2.224065065383911 (-0.3683732748031616) + | > avg_loss_kl: 2.4089685678482056 (-0.052628397941589355) + | > avg_loss_feat: 7.720971584320068 (+0.21444249153137207) + | > avg_loss_mel: 19.754746437072754 (+0.8432712554931641) + | > avg_loss_duration: 1.6605814695358276 (-0.013689398765563965) + | > avg_loss_1: 33.76933288574219 (+0.6230201721191406) + + + > EPOCH: 158/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:35:55)  + + --> TIME: 2025-04-19 01:35:58 -- STEP: 7/142 -- GLOBAL_STEP: 401725 + | > loss_disc: 2.461322784423828 (2.4771035739353726) + | > loss_disc_real_0: 0.15667055547237396 (0.12356099379914147) + | > loss_disc_real_1: 0.21352817118167877 (0.2031618527003697) + | > loss_disc_real_2: 0.23855598270893097 (0.22750480898788997) + | > loss_disc_real_3: 0.211094930768013 (0.24460437042372568) + | > loss_disc_real_4: 0.20671601593494415 (0.2356161198445729) + | > loss_disc_real_5: 0.2536104917526245 (0.2588316010577338) + | > loss_0: 2.461322784423828 (2.4771035739353726) + | > grad_norm_0: tensor(7.6375, device='cuda:0') (tensor(8.1755, device='cuda:0')) + | > loss_gen: 2.3634328842163086 (2.3648231710706438) + | > loss_kl: 1.766487717628479 (1.8253151859555925) + | > loss_feat: 6.1383376121521 (6.994849477495466) + | > loss_mel: 16.559410095214844 (17.811454364231654) + | > loss_duration: 1.4100689888000488 (1.4343749625342233) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.237735748291016 (30.430817195347377) + | > grad_norm_1: tensor(89.1172, device='cuda:0') (tensor(157.5946, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.324 (0.3253781795501709) + | > loader_time: 0.005 (0.0046473571232387) + + + --> TIME: 2025-04-19 01:36:07 -- STEP: 32/142 -- GLOBAL_STEP: 401750 + | > loss_disc: 2.2877235412597656 (2.4451988339424133) + | > loss_disc_real_0: 0.09745524078607559 (0.11769064026884735) + | > loss_disc_real_1: 0.19235140085220337 (0.2059620013460517) + | > loss_disc_real_2: 0.16779732704162598 (0.2225830708630383) + | > loss_disc_real_3: 0.21117861568927765 (0.22951621282845736) + | > loss_disc_real_4: 0.23187455534934998 (0.23096670815721154) + | > loss_disc_real_5: 0.22473929822444916 (0.238058318849653) + | > loss_0: 2.2877235412597656 (2.4451988339424133) + | > grad_norm_0: tensor(6.1696, device='cuda:0') (tensor(7.7037, device='cuda:0')) + | > loss_gen: 2.3983139991760254 (2.357636108994484) + | > loss_kl: 1.931713581085205 (1.8504662215709686) + | > loss_feat: 7.219151496887207 (7.115712001919746) + | > loss_mel: 17.668190002441406 (18.087113052606583) + | > loss_duration: 1.4335534572601318 (1.4446328058838844) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.650922775268555 (30.85556036233902) + | > grad_norm_1: tensor(69.7552, device='cuda:0') (tensor(117.4998, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.3621 (0.34055253863334656) + | > loader_time: 0.0051 (0.005079291760921478) + + + --> TIME: 2025-04-19 01:36:17 -- STEP: 57/142 -- GLOBAL_STEP: 401775 + | > loss_disc: 2.5276548862457275 (2.4540163090354516) + | > loss_disc_real_0: 0.13685239851474762 (0.12248551231204417) + | > loss_disc_real_1: 0.19243086874485016 (0.20384392952709868) + | > loss_disc_real_2: 0.2381003201007843 (0.22566186989608564) + | > loss_disc_real_3: 0.24997195601463318 (0.22839776491909697) + | > loss_disc_real_4: 0.23022040724754333 (0.2294498718621438) + | > loss_disc_real_5: 0.3045358955860138 (0.23840008623767317) + | > loss_0: 2.5276548862457275 (2.4540163090354516) + | > grad_norm_0: tensor(4.7465, device='cuda:0') (tensor(7.4289, device='cuda:0')) + | > loss_gen: 2.540907144546509 (2.3596685016364387) + | > loss_kl: 1.7995524406433105 (1.8688266235485411) + | > loss_feat: 7.638474941253662 (7.1119697805036575) + | > loss_mel: 18.677762985229492 (18.04162452095433) + | > loss_duration: 1.527266263961792 (1.457950966400013) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.183963775634766 (30.84004040768272) + | > grad_norm_1: tensor(33.9113, device='cuda:0') (tensor(113.7674, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.3742 (0.3577867348988851) + | > loader_time: 0.0055 (0.00535809784604792) + + + --> TIME: 2025-04-19 01:36:27 -- STEP: 82/142 -- GLOBAL_STEP: 401800 + | > loss_disc: 2.5365967750549316 (2.462849425106514) + | > loss_disc_real_0: 0.19640925526618958 (0.12641662113913674) + | > loss_disc_real_1: 0.22068218886852264 (0.20484977815209374) + | > loss_disc_real_2: 0.1847887933254242 (0.22553535877931408) + | > loss_disc_real_3: 0.2314329445362091 (0.22782986847365774) + | > loss_disc_real_4: 0.2683661878108978 (0.22898283310052825) + | > loss_disc_real_5: 0.22303083539009094 (0.23514294969599422) + | > loss_0: 2.5365967750549316 (2.462849425106514) + | > grad_norm_0: tensor(6.1874, device='cuda:0') (tensor(7.5375, device='cuda:0')) + | > loss_gen: 2.2891621589660645 (2.339741657419902) + | > loss_kl: 2.038691997528076 (1.8841148076987848) + | > loss_feat: 6.7731828689575195 (6.999559175677415) + | > loss_mel: 18.240734100341797 (17.895411351831946) + | > loss_duration: 1.5766777992248535 (1.469643296265021) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.918447494506836 (30.58847031942228) + | > grad_norm_1: tensor(84.3224, device='cuda:0') (tensor(108.3257, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.4059 (0.3685564326076972) + | > loader_time: 0.0079 (0.005670056110475121) + + + --> TIME: 2025-04-19 01:36:37 -- STEP: 107/142 -- GLOBAL_STEP: 401825 + | > loss_disc: 2.5653607845306396 (2.462160495954139) + | > loss_disc_real_0: 0.10742979496717453 (0.12723766765165548) + | > loss_disc_real_1: 0.19596262276172638 (0.2049095232353033) + | > loss_disc_real_2: 0.2432158887386322 (0.22494438248817053) + | > loss_disc_real_3: 0.20379094779491425 (0.22760884477713397) + | > loss_disc_real_4: 0.22573617100715637 (0.22922572967048002) + | > loss_disc_real_5: 0.22054961323738098 (0.23412258839495828) + | > loss_0: 2.5653607845306396 (2.462160495954139) + | > grad_norm_0: tensor(7.3599, device='cuda:0') (tensor(7.9907, device='cuda:0')) + | > loss_gen: 2.2901175022125244 (2.3379634451643327) + | > loss_kl: 2.0005788803100586 (1.896305976627029) + | > loss_feat: 6.929917335510254 (6.986267713742835) + | > loss_mel: 17.98375701904297 (17.914777987471258) + | > loss_duration: 1.4920032024383545 (1.4785817571889577) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.696372985839844 (30.61389682448913) + | > grad_norm_1: tensor(231.5330, device='cuda:0') (tensor(135.2088, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.416 (0.37837252661446547) + | > loader_time: 0.0071 (0.005948093449958016) + + + --> TIME: 2025-04-19 01:36:48 -- STEP: 132/142 -- GLOBAL_STEP: 401850 + | > loss_disc: 2.461735725402832 (2.4688693393360484) + | > loss_disc_real_0: 0.15944208204746246 (0.1279797667750355) + | > loss_disc_real_1: 0.19146956503391266 (0.20560848125905706) + | > loss_disc_real_2: 0.2559705674648285 (0.22575728742010664) + | > loss_disc_real_3: 0.23998983204364777 (0.22848841277035803) + | > loss_disc_real_4: 0.23089924454689026 (0.22999680934078764) + | > loss_disc_real_5: 0.2685896158218384 (0.23558644058577943) + | > loss_0: 2.461735725402832 (2.4688693393360484) + | > grad_norm_0: tensor(8.0515, device='cuda:0') (tensor(8.2441, device='cuda:0')) + | > loss_gen: 2.3111624717712402 (2.3351150967858056) + | > loss_kl: 1.9176480770111084 (1.902388587142482) + | > loss_feat: 6.713297367095947 (6.942329124970869) + | > loss_mel: 17.66094398498535 (17.867550647620007) + | > loss_duration: 1.494433045387268 (1.4845838889931189) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.097484588623047 (30.531967220884383) + | > grad_norm_1: tensor(141.6833, device='cuda:0') (tensor(148.1560, device='cuda:0')) + | > current_lr_0: 0.00019608850865469258 + | > current_lr_1: 0.00019608850865469258 + | > step_time: 0.4455 (0.3891326893459667) + | > loader_time: 0.0076 (0.006236228075894442) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.552551746368408 (2.552551746368408) + | > loss_disc_real_0: 0.18363609910011292 (0.18363609910011292) + | > loss_disc_real_1: 0.23363368213176727 (0.23363368213176727) + | > loss_disc_real_2: 0.2823810875415802 (0.2823810875415802) + | > loss_disc_real_3: 0.21377244591712952 (0.21377244591712952) + | > loss_disc_real_4: 0.2590023875236511 (0.2590023875236511) + | > loss_disc_real_5: 0.25396785140037537 (0.25396785140037537) + | > loss_0: 2.552551746368408 (2.552551746368408) + | > loss_gen: 2.4643802642822266 (2.4643802642822266) + | > loss_kl: 2.2982335090637207 (2.2982335090637207) + | > loss_feat: 7.046683311462402 (7.046683311462402) + | > loss_mel: 17.864151000976562 (17.864151000976562) + | > loss_duration: 1.6429979801177979 (1.6429979801177979) + | > loss_1: 31.31644630432129 (31.31644630432129) + + --> STEP: 1 + | > loss_disc: 2.422426223754883 (2.422426223754883) + | > loss_disc_real_0: 0.17604108154773712 (0.17604108154773712) + | > loss_disc_real_1: 0.23529937863349915 (0.23529937863349915) + | > loss_disc_real_2: 0.21885225176811218 (0.21885225176811218) + | > loss_disc_real_3: 0.20055513083934784 (0.20055513083934784) + | > loss_disc_real_4: 0.20250903069972992 (0.20250903069972992) + | > loss_disc_real_5: 0.20900210738182068 (0.20900210738182068) + | > loss_0: 2.422426223754883 (2.422426223754883) + | > loss_gen: 2.4574055671691895 (2.4574055671691895) + | > loss_kl: 2.482921838760376 (2.482921838760376) + | > loss_feat: 7.675820827484131 (7.675820827484131) + | > loss_mel: 19.14237403869629 (19.14237403869629) + | > loss_duration: 1.7148641347885132 (1.7148641347885132) + | > loss_1: 33.473384857177734 (33.473384857177734) + + --> STEP: 2 + | > loss_disc: 2.5610129833221436 (2.491719603538513) + | > loss_disc_real_0: 0.23271824419498444 (0.20437966287136078) + | > loss_disc_real_1: 0.2298637181520462 (0.23258154839277267) + | > loss_disc_real_2: 0.2152046412229538 (0.217028446495533) + | > loss_disc_real_3: 0.2537381947040558 (0.2271466627717018) + | > loss_disc_real_4: 0.2555042505264282 (0.22900664061307907) + | > loss_disc_real_5: 0.2616659998893738 (0.23533405363559723) + | > loss_0: 2.5610129833221436 (2.491719603538513) + | > loss_gen: 2.3510968685150146 (2.404251217842102) + | > loss_kl: 2.076842784881592 (2.279882311820984) + | > loss_feat: 5.518597602844238 (6.597209215164185) + | > loss_mel: 16.95540428161621 (18.04888916015625) + | > loss_duration: 1.6516034603118896 (1.6832337975502014) + | > loss_1: 28.553544998168945 (31.01346492767334) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003724813461303711 (+0.0003974437713623047) + | > avg_loss_disc: 2.491719603538513 (+0.09129774570465088) + | > avg_loss_disc_real_0: 0.20437966287136078 (+0.0361219123005867) + | > avg_loss_disc_real_1: 0.23258154839277267 (+0.035639405250549316) + | > avg_loss_disc_real_2: 0.217028446495533 (-0.0015240609645843506) + | > avg_loss_disc_real_3: 0.2271466627717018 (+0.023260481655597687) + | > avg_loss_disc_real_4: 0.22900664061307907 (+0.04854030907154083) + | > avg_loss_disc_real_5: 0.23533405363559723 (+0.04155491292476654) + | > avg_loss_0: 2.491719603538513 (+0.09129774570465088) + | > avg_loss_gen: 2.404251217842102 (+0.18018615245819092) + | > avg_loss_kl: 2.279882311820984 (-0.12908625602722168) + | > avg_loss_feat: 6.597209215164185 (-1.1237623691558838) + | > avg_loss_mel: 18.04888916015625 (-1.705857276916504) + | > avg_loss_duration: 1.6832337975502014 (+0.02265232801437378) + | > avg_loss_1: 31.01346492767334 (-2.7558679580688477) + + + > EPOCH: 159/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:36:54)  + + --> TIME: 2025-04-19 01:37:00 -- STEP: 15/142 -- GLOBAL_STEP: 401875 + | > loss_disc: 2.5077273845672607 (2.513049666086833) + | > loss_disc_real_0: 0.13381114602088928 (0.12553719729185103) + | > loss_disc_real_1: 0.2230423390865326 (0.21609881321589153) + | > loss_disc_real_2: 0.21924322843551636 (0.22886896630128226) + | > loss_disc_real_3: 0.23802216351032257 (0.23157902856667836) + | > loss_disc_real_4: 0.26252663135528564 (0.22951887945334118) + | > loss_disc_real_5: 0.2043292075395584 (0.2331598420937856) + | > loss_0: 2.5077273845672607 (2.513049666086833) + | > grad_norm_0: tensor(5.9323, device='cuda:0') (tensor(7.0064, device='cuda:0')) + | > loss_gen: 2.297985315322876 (2.3160448710123687) + | > loss_kl: 1.9026377201080322 (1.8572677850723267) + | > loss_feat: 7.080135822296143 (7.248094813028971) + | > loss_mel: 17.792709350585938 (18.615532811482748) + | > loss_duration: 1.4465864896774292 (1.4411162773768107) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.52005386352539 (31.478055699666342) + | > grad_norm_1: tensor(194.6962, device='cuda:0') (tensor(139.6451, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.3352 (0.3272844791412354) + | > loader_time: 0.0049 (0.004682858784993489) + + + --> TIME: 2025-04-19 01:37:09 -- STEP: 40/142 -- GLOBAL_STEP: 401900 + | > loss_disc: 2.372391700744629 (2.4737078726291655) + | > loss_disc_real_0: 0.06908994168043137 (0.12333866879343983) + | > loss_disc_real_1: 0.19250932335853577 (0.21191229969263076) + | > loss_disc_real_2: 0.24657759070396423 (0.22383929155766963) + | > loss_disc_real_3: 0.2272196114063263 (0.22778447903692722) + | > loss_disc_real_4: 0.21938283741474152 (0.22875210978090763) + | > loss_disc_real_5: 0.2581494152545929 (0.23419118411839007) + | > loss_0: 2.372391700744629 (2.4737078726291655) + | > grad_norm_0: tensor(17.7001, device='cuda:0') (tensor(8.3143, device='cuda:0')) + | > loss_gen: 2.3034770488739014 (2.33459050655365) + | > loss_kl: 1.8437886238098145 (1.8559128642082214) + | > loss_feat: 6.78499698638916 (7.204856884479523) + | > loss_mel: 18.46587371826172 (18.393614006042476) + | > loss_duration: 1.437774419784546 (1.4531061351299284) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.83591079711914 (31.24207992553711) + | > grad_norm_1: tensor(348.1919, device='cuda:0') (tensor(187.2595, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.3611 (0.34105793237686155) + | > loader_time: 0.0058 (0.005057573318481445) + + + --> TIME: 2025-04-19 01:37:18 -- STEP: 65/142 -- GLOBAL_STEP: 401925 + | > loss_disc: 2.6012885570526123 (2.481793986834013) + | > loss_disc_real_0: 0.14593306183815002 (0.12461019399074408) + | > loss_disc_real_1: 0.22457866370677948 (0.20998116112672366) + | > loss_disc_real_2: 0.23036108911037445 (0.225591651063699) + | > loss_disc_real_3: 0.27415916323661804 (0.2298558867894686) + | > loss_disc_real_4: 0.25881025195121765 (0.2315455796626898) + | > loss_disc_real_5: 0.2680938243865967 (0.2361750997029818) + | > loss_0: 2.6012885570526123 (2.481793986834013) + | > grad_norm_0: tensor(16.8101, device='cuda:0') (tensor(8.8324, device='cuda:0')) + | > loss_gen: 2.3084473609924316 (2.3393401329333967) + | > loss_kl: 1.8415471315383911 (1.8750498679968026) + | > loss_feat: 6.581161975860596 (7.054583982320932) + | > loss_mel: 17.13460350036621 (18.066985849233774) + | > loss_duration: 1.47672438621521 (1.4654557723265427) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.342483520507812 (30.80141528202937) + | > grad_norm_1: tensor(237.7822, device='cuda:0') (tensor(201.6085, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.3887 (0.3528307988093449) + | > loader_time: 0.0065 (0.005457071157602163) + + + --> TIME: 2025-04-19 01:37:29 -- STEP: 90/142 -- GLOBAL_STEP: 401950 + | > loss_disc: 2.4921202659606934 (2.480825363265144) + | > loss_disc_real_0: 0.09210740029811859 (0.12471632829142941) + | > loss_disc_real_1: 0.18653231859207153 (0.209520294268926) + | > loss_disc_real_2: 0.22387786209583282 (0.22577205333444808) + | > loss_disc_real_3: 0.22600504755973816 (0.22790433681673475) + | > loss_disc_real_4: 0.24134670197963715 (0.23103997326559492) + | > loss_disc_real_5: 0.24461756646633148 (0.23564728647470473) + | > loss_0: 2.4921202659606934 (2.480825363265144) + | > grad_norm_0: tensor(13.2094, device='cuda:0') (tensor(9.6605, device='cuda:0')) + | > loss_gen: 2.281613349914551 (2.3407592932383223) + | > loss_kl: 1.778948187828064 (1.8904963374137878) + | > loss_feat: 6.724641799926758 (7.009130069944594) + | > loss_mel: 17.826698303222656 (18.013503540886777) + | > loss_duration: 1.5158241987228394 (1.4760184605916342) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.127723693847656 (30.72990737491184) + | > grad_norm_1: tensor(45.1303, device='cuda:0') (tensor(222.7668, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.4046 (0.3641474564870197) + | > loader_time: 0.0068 (0.0056857559416029204) + + + --> TIME: 2025-04-19 01:37:39 -- STEP: 115/142 -- GLOBAL_STEP: 401975 + | > loss_disc: 2.5142364501953125 (2.483139556387198) + | > loss_disc_real_0: 0.238901287317276 (0.12916273098924888) + | > loss_disc_real_1: 0.20128123462200165 (0.20912860631942748) + | > loss_disc_real_2: 0.22526249289512634 (0.22607503040977148) + | > loss_disc_real_3: 0.2089577168226242 (0.22896705697412076) + | > loss_disc_real_4: 0.22168627381324768 (0.2311959023061006) + | > loss_disc_real_5: 0.24625347554683685 (0.23527193406353827) + | > loss_0: 2.5142364501953125 (2.483139556387198) + | > grad_norm_0: tensor(36.9106, device='cuda:0') (tensor(12.6674, device='cuda:0')) + | > loss_gen: 2.4482808113098145 (2.3554546107416563) + | > loss_kl: 1.91423761844635 (1.898129651857459) + | > loss_feat: 7.783559322357178 (7.006126681618069) + | > loss_mel: 18.486141204833984 (17.905584426548177) + | > loss_duration: 1.5180983543395996 (1.482635672196098) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.15031433105469 (30.64793070917544) + | > grad_norm_1: tensor(389.7714, device='cuda:0') (tensor(262.4113, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.4239 (0.37501630368440037) + | > loader_time: 0.0071 (0.005938415941984758) + + + --> TIME: 2025-04-19 01:37:50 -- STEP: 140/142 -- GLOBAL_STEP: 402000 + | > loss_disc: 2.412611961364746 (2.4837250794683197) + | > loss_disc_real_0: 0.13895907998085022 (0.12838088742324283) + | > loss_disc_real_1: 0.18417879939079285 (0.2091601184436253) + | > loss_disc_real_2: 0.16970521211624146 (0.22639260398490088) + | > loss_disc_real_3: 0.21037720143795013 (0.22878345283014434) + | > loss_disc_real_4: 0.21824851632118225 (0.2305905734854085) + | > loss_disc_real_5: 0.20825029909610748 (0.23491206275565282) + | > loss_0: 2.412611961364746 (2.4837250794683197) + | > grad_norm_0: tensor(11.8535, device='cuda:0') (tensor(12.7042, device='cuda:0')) + | > loss_gen: 2.2657628059387207 (2.3494533283369883) + | > loss_kl: 2.006006956100464 (1.905541991336005) + | > loss_feat: 6.646921634674072 (6.958992048672267) + | > loss_mel: 15.99579906463623 (17.83745089939663) + | > loss_duration: 1.557529330253601 (1.4888099772589547) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.47201919555664 (30.54024795804705) + | > grad_norm_1: tensor(318.6595, device='cuda:0') (tensor(270.1399, device='cuda:0')) + | > current_lr_0: 0.00019606399759111075 + | > current_lr_1: 0.00019606399759111075 + | > step_time: 0.457 (0.3871047326496668) + | > loader_time: 0.0076 (0.006200458322252547) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4254987239837646 (2.4254987239837646) + | > loss_disc_real_0: 0.19238224625587463 (0.19238224625587463) + | > loss_disc_real_1: 0.2104157656431198 (0.2104157656431198) + | > loss_disc_real_2: 0.2607959508895874 (0.2607959508895874) + | > loss_disc_real_3: 0.25549694895744324 (0.25549694895744324) + | > loss_disc_real_4: 0.24778035283088684 (0.24778035283088684) + | > loss_disc_real_5: 0.2630392611026764 (0.2630392611026764) + | > loss_0: 2.4254987239837646 (2.4254987239837646) + | > loss_gen: 2.532083511352539 (2.532083511352539) + | > loss_kl: 2.348984479904175 (2.348984479904175) + | > loss_feat: 5.821692943572998 (5.821692943572998) + | > loss_mel: 17.76882553100586 (17.76882553100586) + | > loss_duration: 1.6350886821746826 (1.6350886821746826) + | > loss_1: 30.106674194335938 (30.106674194335938) + + --> STEP: 1 + | > loss_disc: 2.4757776260375977 (2.4757776260375977) + | > loss_disc_real_0: 0.1287052482366562 (0.1287052482366562) + | > loss_disc_real_1: 0.20977258682250977 (0.20977258682250977) + | > loss_disc_real_2: 0.22099819779396057 (0.22099819779396057) + | > loss_disc_real_3: 0.20700518786907196 (0.20700518786907196) + | > loss_disc_real_4: 0.25273042917251587 (0.25273042917251587) + | > loss_disc_real_5: 0.21721452474594116 (0.21721452474594116) + | > loss_0: 2.4757776260375977 (2.4757776260375977) + | > loss_gen: 2.229701042175293 (2.229701042175293) + | > loss_kl: 2.532386302947998 (2.532386302947998) + | > loss_feat: 6.856813907623291 (6.856813907623291) + | > loss_mel: 18.375259399414062 (18.375259399414062) + | > loss_duration: 1.6590721607208252 (1.6590721607208252) + | > loss_1: 31.65323257446289 (31.65323257446289) + + --> STEP: 2 + | > loss_disc: 2.496403217315674 (2.4860904216766357) + | > loss_disc_real_0: 0.11621962487697601 (0.1224624365568161) + | > loss_disc_real_1: 0.2509787976741791 (0.23037569224834442) + | > loss_disc_real_2: 0.28321772813796997 (0.25210796296596527) + | > loss_disc_real_3: 0.25362443923950195 (0.23031481355428696) + | > loss_disc_real_4: 0.2419854700565338 (0.24735794961452484) + | > loss_disc_real_5: 0.2749548852443695 (0.24608470499515533) + | > loss_0: 2.496403217315674 (2.4860904216766357) + | > loss_gen: 2.4581644535064697 (2.3439327478408813) + | > loss_kl: 1.9568930864334106 (2.2446396946907043) + | > loss_feat: 7.6014933586120605 (7.229153633117676) + | > loss_mel: 19.412519454956055 (18.89388942718506) + | > loss_duration: 1.6362512111663818 (1.6476616859436035) + | > loss_1: 33.0653190612793 (32.359275817871094) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003529071807861328 (-0.0001957416534423828) + | > avg_loss_disc: 2.4860904216766357 (-0.005629181861877441) + | > avg_loss_disc_real_0: 0.1224624365568161 (-0.08191722631454468) + | > avg_loss_disc_real_1: 0.23037569224834442 (-0.002205856144428253) + | > avg_loss_disc_real_2: 0.25210796296596527 (+0.03507951647043228) + | > avg_loss_disc_real_3: 0.23031481355428696 (+0.003168150782585144) + | > avg_loss_disc_real_4: 0.24735794961452484 (+0.01835130900144577) + | > avg_loss_disc_real_5: 0.24608470499515533 (+0.010750651359558105) + | > avg_loss_0: 2.4860904216766357 (-0.005629181861877441) + | > avg_loss_gen: 2.3439327478408813 (-0.0603184700012207) + | > avg_loss_kl: 2.2446396946907043 (-0.03524261713027954) + | > avg_loss_feat: 7.229153633117676 (+0.6319444179534912) + | > avg_loss_mel: 18.89388942718506 (+0.8450002670288086) + | > avg_loss_duration: 1.6476616859436035 (-0.0355721116065979) + | > avg_loss_1: 32.359275817871094 (+1.345810890197754) + + + > EPOCH: 160/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:37:53)  + + --> TIME: 2025-04-19 01:38:01 -- STEP: 23/142 -- GLOBAL_STEP: 402025 + | > loss_disc: 2.5653531551361084 (2.4289324905561362) + | > loss_disc_real_0: 0.13015058636665344 (0.1225466488496117) + | > loss_disc_real_1: 0.1981777399778366 (0.20870505402917447) + | > loss_disc_real_2: 0.21121565997600555 (0.22280180583829465) + | > loss_disc_real_3: 0.21226175129413605 (0.22471997400988702) + | > loss_disc_real_4: 0.2171987146139145 (0.2271228937999062) + | > loss_disc_real_5: 0.2574880123138428 (0.22585271558035974) + | > loss_0: 2.5653531551361084 (2.4289324905561362) + | > grad_norm_0: tensor(5.1899, device='cuda:0') (tensor(10.5682, device='cuda:0')) + | > loss_gen: 2.1980273723602295 (2.362058276715486) + | > loss_kl: 1.9308704137802124 (1.851524829864502) + | > loss_feat: 6.998047828674316 (7.145199464715046) + | > loss_mel: 17.787797927856445 (17.749313064243484) + | > loss_duration: 1.4588016271591187 (1.4462538439294566) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.373544692993164 (30.554349070009977) + | > grad_norm_1: tensor(162.3684, device='cuda:0') (tensor(250.2293, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.344 (0.3339411901391071) + | > loader_time: 0.0051 (0.004925541255785071) + + + --> TIME: 2025-04-19 01:38:11 -- STEP: 48/142 -- GLOBAL_STEP: 402050 + | > loss_disc: 2.459826707839966 (2.4501737952232356) + | > loss_disc_real_0: 0.14915695786476135 (0.12460415375729404) + | > loss_disc_real_1: 0.20615330338478088 (0.20628771123786768) + | > loss_disc_real_2: 0.24720154702663422 (0.22520933703829846) + | > loss_disc_real_3: 0.23127683997154236 (0.22830360538015762) + | > loss_disc_real_4: 0.26824915409088135 (0.22996775464465222) + | > loss_disc_real_5: 0.24766063690185547 (0.23156946462889513) + | > loss_0: 2.459826707839966 (2.4501737952232356) + | > grad_norm_0: tensor(7.3999, device='cuda:0') (tensor(10.1102, device='cuda:0')) + | > loss_gen: 2.447695255279541 (2.3605160365502047) + | > loss_kl: 2.0644898414611816 (1.8865663781762123) + | > loss_feat: 6.723483085632324 (7.127845525741577) + | > loss_mel: 17.91629981994629 (17.73694183429083) + | > loss_duration: 1.442509412765503 (1.4521651739875476) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.5944766998291 (30.564034740130108) + | > grad_norm_1: tensor(193.8077, device='cuda:0') (tensor(248.9454, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.364 (0.3455283691485722) + | > loader_time: 0.0058 (0.005235756436983745) + + + --> TIME: 2025-04-19 01:38:20 -- STEP: 73/142 -- GLOBAL_STEP: 402075 + | > loss_disc: 2.6249520778656006 (2.4540362815334364) + | > loss_disc_real_0: 0.16641603410243988 (0.12421992359912559) + | > loss_disc_real_1: 0.19850021600723267 (0.20608441694958568) + | > loss_disc_real_2: 0.24951942265033722 (0.22568120919678308) + | > loss_disc_real_3: 0.1946931630373001 (0.227936228660688) + | > loss_disc_real_4: 0.21573252975940704 (0.2290705040301362) + | > loss_disc_real_5: 0.22872750461101532 (0.2306101385452976) + | > loss_0: 2.6249520778656006 (2.4540362815334364) + | > grad_norm_0: tensor(5.5625, device='cuda:0') (tensor(8.8994, device='cuda:0')) + | > loss_gen: 2.2554352283477783 (2.35013720434006) + | > loss_kl: 1.9136604070663452 (1.8981711521540603) + | > loss_feat: 6.541844844818115 (7.062760307364266) + | > loss_mel: 17.410505294799805 (17.664295941183017) + | > loss_duration: 1.516839861869812 (1.4640201444495213) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.638286590576172 (30.43938446044922) + | > grad_norm_1: tensor(112.8140, device='cuda:0') (tensor(206.5919, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.3952 (0.3576451915584198) + | > loader_time: 0.006 (0.005507384261039839) + + + --> TIME: 2025-04-19 01:38:30 -- STEP: 98/142 -- GLOBAL_STEP: 402100 + | > loss_disc: 2.4585790634155273 (2.467956971149054) + | > loss_disc_real_0: 0.11516808718442917 (0.12743414079352297) + | > loss_disc_real_1: 0.1965472400188446 (0.20674222479669416) + | > loss_disc_real_2: 0.2115052342414856 (0.2261902308281587) + | > loss_disc_real_3: 0.21096643805503845 (0.22867338648255991) + | > loss_disc_real_4: 0.20202776789665222 (0.22843428898830803) + | > loss_disc_real_5: 0.19048503041267395 (0.23147375349487578) + | > loss_0: 2.4585790634155273 (2.467956971149054) + | > grad_norm_0: tensor(5.0103, device='cuda:0') (tensor(8.3154, device='cuda:0')) + | > loss_gen: 2.3796491622924805 (2.3300639026019048) + | > loss_kl: 2.1153180599212646 (1.916294412953513) + | > loss_feat: 7.78314733505249 (6.967154585585301) + | > loss_mel: 17.61298179626465 (17.588619193252256) + | > loss_duration: 1.4604086875915527 (1.4752204041091763) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.35150718688965 (30.277352255217885) + | > grad_norm_1: tensor(87.5426, device='cuda:0') (tensor(182.3367, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.4204 (0.36847882368126694) + | > loader_time: 0.0071 (0.005789360221551389) + + + --> TIME: 2025-04-19 01:38:41 -- STEP: 123/142 -- GLOBAL_STEP: 402125 + | > loss_disc: 2.391207218170166 (2.4699174000964894) + | > loss_disc_real_0: 0.11114075034856796 (0.12910271060418313) + | > loss_disc_real_1: 0.2045000195503235 (0.20652811132310853) + | > loss_disc_real_2: 0.2536260187625885 (0.22587109871996128) + | > loss_disc_real_3: 0.20845791697502136 (0.22908141167183232) + | > loss_disc_real_4: 0.22642406821250916 (0.2295551451483393) + | > loss_disc_real_5: 0.20535947382450104 (0.2325345650921023) + | > loss_0: 2.391207218170166 (2.4699174000964894) + | > grad_norm_0: tensor(6.8692, device='cuda:0') (tensor(8.2095, device='cuda:0')) + | > loss_gen: 2.3481552600860596 (2.3258427081069337) + | > loss_kl: 1.871914267539978 (1.9169704904401206) + | > loss_feat: 7.272669315338135 (6.928943095168446) + | > loss_mel: 17.394821166992188 (17.567811299145713) + | > loss_duration: 1.4908051490783691 (1.4828482158784944) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.378366470336914 (30.22241559842738) + | > grad_norm_1: tensor(75.6741, device='cuda:0') (tensor(177.2144, device='cuda:0')) + | > current_lr_0: 0.00019603948959141186 + | > current_lr_1: 0.00019603948959141186 + | > step_time: 0.438 (0.38018578823989) + | > loader_time: 0.0075 (0.006097037617753192) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4085073471069336 (2.4085073471069336) + | > loss_disc_real_0: 0.14759156107902527 (0.14759156107902527) + | > loss_disc_real_1: 0.2471914440393448 (0.2471914440393448) + | > loss_disc_real_2: 0.2013227939605713 (0.2013227939605713) + | > loss_disc_real_3: 0.22827649116516113 (0.22827649116516113) + | > loss_disc_real_4: 0.32560113072395325 (0.32560113072395325) + | > loss_disc_real_5: 0.23083797097206116 (0.23083797097206116) + | > loss_0: 2.4085073471069336 (2.4085073471069336) + | > loss_gen: 2.5951924324035645 (2.5951924324035645) + | > loss_kl: 2.232102394104004 (2.232102394104004) + | > loss_feat: 7.2290873527526855 (7.2290873527526855) + | > loss_mel: 19.008390426635742 (19.008390426635742) + | > loss_duration: 1.6811573505401611 (1.6811573505401611) + | > loss_1: 32.74592971801758 (32.74592971801758) + + --> STEP: 1 + | > loss_disc: 2.4965012073516846 (2.4965012073516846) + | > loss_disc_real_0: 0.13168255984783173 (0.13168255984783173) + | > loss_disc_real_1: 0.26347920298576355 (0.26347920298576355) + | > loss_disc_real_2: 0.24167169630527496 (0.24167169630527496) + | > loss_disc_real_3: 0.21454821527004242 (0.21454821527004242) + | > loss_disc_real_4: 0.27703797817230225 (0.27703797817230225) + | > loss_disc_real_5: 0.21948614716529846 (0.21948614716529846) + | > loss_0: 2.4965012073516846 (2.4965012073516846) + | > loss_gen: 2.4640183448791504 (2.4640183448791504) + | > loss_kl: 2.4012012481689453 (2.4012012481689453) + | > loss_feat: 6.693819999694824 (6.693819999694824) + | > loss_mel: 19.04244041442871 (19.04244041442871) + | > loss_duration: 1.6822367906570435 (1.6822367906570435) + | > loss_1: 32.28371810913086 (32.28371810913086) + + --> STEP: 2 + | > loss_disc: 2.5417380332946777 (2.519119620323181) + | > loss_disc_real_0: 0.1768534779548645 (0.15426801890134811) + | > loss_disc_real_1: 0.2511266767978668 (0.2573029398918152) + | > loss_disc_real_2: 0.18647149205207825 (0.2140715941786766) + | > loss_disc_real_3: 0.25140392780303955 (0.23297607153654099) + | > loss_disc_real_4: 0.2876141369342804 (0.2823260575532913) + | > loss_disc_real_5: 0.23959600925445557 (0.22954107820987701) + | > loss_0: 2.5417380332946777 (2.519119620323181) + | > loss_gen: 2.430996894836426 (2.447507619857788) + | > loss_kl: 1.9893544912338257 (2.1952778697013855) + | > loss_feat: 7.175469398498535 (6.93464469909668) + | > loss_mel: 18.951683044433594 (18.997061729431152) + | > loss_duration: 1.637787103652954 (1.6600119471549988) + | > loss_1: 32.1852912902832 (32.23450469970703) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003551483154296875 (+2.2411346435546875e-05) + | > avg_loss_disc: 2.519119620323181 (+0.03302919864654541) + | > avg_loss_disc_real_0: 0.15426801890134811 (+0.03180558234453201) + | > avg_loss_disc_real_1: 0.2573029398918152 (+0.026927247643470764) + | > avg_loss_disc_real_2: 0.2140715941786766 (-0.038036368787288666) + | > avg_loss_disc_real_3: 0.23297607153654099 (+0.0026612579822540283) + | > avg_loss_disc_real_4: 0.2823260575532913 (+0.03496810793876648) + | > avg_loss_disc_real_5: 0.22954107820987701 (-0.01654362678527832) + | > avg_loss_0: 2.519119620323181 (+0.03302919864654541) + | > avg_loss_gen: 2.447507619857788 (+0.10357487201690674) + | > avg_loss_kl: 2.1952778697013855 (-0.04936182498931885) + | > avg_loss_feat: 6.93464469909668 (-0.2945089340209961) + | > avg_loss_mel: 18.997061729431152 (+0.10317230224609375) + | > avg_loss_duration: 1.6600119471549988 (+0.012350261211395264) + | > avg_loss_1: 32.23450469970703 (-0.1247711181640625) + + + > EPOCH: 161/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:38:51)  + + --> TIME: 2025-04-19 01:38:54 -- STEP: 6/142 -- GLOBAL_STEP: 402150 + | > loss_disc: 2.4544482231140137 (2.4149680534998574) + | > loss_disc_real_0: 0.12487395107746124 (0.11231967185934384) + | > loss_disc_real_1: 0.21004053950309753 (0.20396259427070618) + | > loss_disc_real_2: 0.25011539459228516 (0.21077135453621546) + | > loss_disc_real_3: 0.23464852571487427 (0.23384677867094675) + | > loss_disc_real_4: 0.20467138290405273 (0.2179319808880488) + | > loss_disc_real_5: 0.22388014197349548 (0.24044409145911536) + | > loss_0: 2.4544482231140137 (2.4149680534998574) + | > grad_norm_0: tensor(7.3732, device='cuda:0') (tensor(9.6241, device='cuda:0')) + | > loss_gen: 2.3148956298828125 (2.384155035018921) + | > loss_kl: 1.7742010354995728 (1.8254679044087727) + | > loss_feat: 6.720626354217529 (7.052790562311809) + | > loss_mel: 16.920747756958008 (17.639517466227215) + | > loss_duration: 1.4437015056610107 (1.4375682473182678) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.174171447753906 (30.33949851989746) + | > grad_norm_1: tensor(285.4843, device='cuda:0') (tensor(303.2898, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.3293 (0.3255076805750529) + | > loader_time: 0.004 (0.004113475481669108) + + + --> TIME: 2025-04-19 01:39:03 -- STEP: 31/142 -- GLOBAL_STEP: 402175 + | > loss_disc: 2.5173211097717285 (2.446439596914476) + | > loss_disc_real_0: 0.10383640974760056 (0.11853232787501428) + | > loss_disc_real_1: 0.19843463599681854 (0.20857544724018343) + | > loss_disc_real_2: 0.22264991700649261 (0.2259132597715624) + | > loss_disc_real_3: 0.21368472278118134 (0.2269061126055256) + | > loss_disc_real_4: 0.2722679376602173 (0.22826144820259464) + | > loss_disc_real_5: 0.21027185022830963 (0.22740114215881593) + | > loss_0: 2.5173211097717285 (2.446439596914476) + | > grad_norm_0: tensor(11.9534, device='cuda:0') (tensor(11.4424, device='cuda:0')) + | > loss_gen: 2.326594352722168 (2.3746660217162097) + | > loss_kl: 1.7860933542251587 (1.8476288126360985) + | > loss_feat: 7.386500358581543 (7.176404445402084) + | > loss_mel: 16.353734970092773 (17.95941875826928) + | > loss_duration: 1.4189388751983643 (1.442749342610759) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.27186393737793 (30.800867265270604) + | > grad_norm_1: tensor(463.0808, device='cuda:0') (tensor(298.6427, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.3504 (0.33783212784797917) + | > loader_time: 0.0055 (0.004790121509182838) + + + --> TIME: 2025-04-19 01:39:12 -- STEP: 56/142 -- GLOBAL_STEP: 402200 + | > loss_disc: 2.501190423965454 (2.4609108822686325) + | > loss_disc_real_0: 0.1008671298623085 (0.12078236236370035) + | > loss_disc_real_1: 0.20293015241622925 (0.20731680866863048) + | > loss_disc_real_2: 0.22592495381832123 (0.22637026757001877) + | > loss_disc_real_3: 0.25157126784324646 (0.22879831812211446) + | > loss_disc_real_4: 0.2514284551143646 (0.2309821154922247) + | > loss_disc_real_5: 0.23702381551265717 (0.23187619181615965) + | > loss_0: 2.501190423965454 (2.4609108822686325) + | > grad_norm_0: tensor(12.2010, device='cuda:0') (tensor(12.4047, device='cuda:0')) + | > loss_gen: 2.2690725326538086 (2.3706027013914928) + | > loss_kl: 1.8673653602600098 (1.8567906383957182) + | > loss_feat: 6.794688701629639 (7.10548551593508) + | > loss_mel: 18.903345108032227 (17.933379854474754) + | > loss_duration: 1.4996020793914795 (1.4540299262319292) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.334074020385742 (30.72028878756932) + | > grad_norm_1: tensor(278.8917, device='cuda:0') (tensor(304.9601, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.3737 (0.348929830959865) + | > loader_time: 0.0061 (0.005146230970110212) + + + --> TIME: 2025-04-19 01:39:22 -- STEP: 81/142 -- GLOBAL_STEP: 402225 + | > loss_disc: 2.4862942695617676 (2.465705285837621) + | > loss_disc_real_0: 0.12114720791578293 (0.12367556727042904) + | > loss_disc_real_1: 0.23249246180057526 (0.20675514712009901) + | > loss_disc_real_2: 0.217972069978714 (0.22565891142980551) + | > loss_disc_real_3: 0.20384657382965088 (0.22690364847212663) + | > loss_disc_real_4: 0.254723459482193 (0.23078461506484468) + | > loss_disc_real_5: 0.24874825775623322 (0.23326372180455995) + | > loss_0: 2.4862942695617676 (2.465705285837621) + | > grad_norm_0: tensor(13.1358, device='cuda:0') (tensor(11.3684, device='cuda:0')) + | > loss_gen: 2.4031901359558105 (2.3488237092524393) + | > loss_kl: 1.8472343683242798 (1.8771525385939045) + | > loss_feat: 6.6479291915893555 (7.024013001241801) + | > loss_mel: 19.200105667114258 (17.87224983874663) + | > loss_duration: 1.4762234687805176 (1.466148903340469) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.574684143066406 (30.588388136875484) + | > grad_norm_1: tensor(358.8467, device='cuda:0') (tensor(268.9981, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.3926 (0.36126905900460704) + | > loader_time: 0.0063 (0.00548583784221131) + + + --> TIME: 2025-04-19 01:39:32 -- STEP: 106/142 -- GLOBAL_STEP: 402250 + | > loss_disc: 2.546201705932617 (2.467806442728583) + | > loss_disc_real_0: 0.16682672500610352 (0.12634153604366866) + | > loss_disc_real_1: 0.1923566609621048 (0.2066681756163543) + | > loss_disc_real_2: 0.24108274281024933 (0.22552840355432258) + | > loss_disc_real_3: 0.2555091977119446 (0.22772553661521874) + | > loss_disc_real_4: 0.23842079937458038 (0.23019104985133657) + | > loss_disc_real_5: 0.2341199815273285 (0.23364287305553005) + | > loss_0: 2.546201705932617 (2.467806442728583) + | > grad_norm_0: tensor(4.2400, device='cuda:0') (tensor(11.6637, device='cuda:0')) + | > loss_gen: 2.288209915161133 (2.345417013708151) + | > loss_kl: 1.8439435958862305 (1.889645831764869) + | > loss_feat: 6.441903591156006 (6.976717705996531) + | > loss_mel: 16.79888916015625 (17.80298913199946) + | > loss_duration: 1.5076123476028442 (1.4776512676814817) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.880558013916016 (30.4924210782321) + | > grad_norm_1: tensor(112.5566, device='cuda:0') (tensor(267.1865, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.4278 (0.3719683255789415) + | > loader_time: 0.0071 (0.005728921800289513) + + + --> TIME: 2025-04-19 01:39:44 -- STEP: 131/142 -- GLOBAL_STEP: 402275 + | > loss_disc: 2.554744243621826 (2.470518436140687) + | > loss_disc_real_0: 0.18817733228206635 (0.12709165492471844) + | > loss_disc_real_1: 0.22788748145103455 (0.20728573316836174) + | > loss_disc_real_2: 0.24396246671676636 (0.2265081671813062) + | > loss_disc_real_3: 0.2625649869441986 (0.2286557862321839) + | > loss_disc_real_4: 0.2661619782447815 (0.22960133224953222) + | > loss_disc_real_5: 0.23924106359481812 (0.23318448765132263) + | > loss_0: 2.554744243621826 (2.470518436140687) + | > grad_norm_0: tensor(12.3680, device='cuda:0') (tensor(11.2630, device='cuda:0')) + | > loss_gen: 2.360013723373413 (2.3435292917353507) + | > loss_kl: 1.9561870098114014 (1.8926763434446494) + | > loss_feat: 6.762694358825684 (6.983231850252807) + | > loss_mel: 17.64851951599121 (17.801314586901483) + | > loss_duration: 1.506413459777832 (1.4838621807462387) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.233829498291016 (30.50461440049965) + | > grad_norm_1: tensor(114.9900, device='cuda:0') (tensor(249.0785, device='cuda:0')) + | > current_lr_0: 0.0001960149846552129 + | > current_lr_1: 0.0001960149846552129 + | > step_time: 0.4492 (0.3838500157567381) + | > loader_time: 0.0071 (0.0060058914068091) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5061211585998535 (2.5061211585998535) + | > loss_disc_real_0: 0.1851886510848999 (0.1851886510848999) + | > loss_disc_real_1: 0.20998960733413696 (0.20998960733413696) + | > loss_disc_real_2: 0.22808776795864105 (0.22808776795864105) + | > loss_disc_real_3: 0.2416936755180359 (0.2416936755180359) + | > loss_disc_real_4: 0.2233744114637375 (0.2233744114637375) + | > loss_disc_real_5: 0.23650653660297394 (0.23650653660297394) + | > loss_0: 2.5061211585998535 (2.5061211585998535) + | > loss_gen: 2.3675169944763184 (2.3675169944763184) + | > loss_kl: 2.339829683303833 (2.339829683303833) + | > loss_feat: 7.662934303283691 (7.662934303283691) + | > loss_mel: 20.57867431640625 (20.57867431640625) + | > loss_duration: 1.6544861793518066 (1.6544861793518066) + | > loss_1: 34.60344314575195 (34.60344314575195) + + --> STEP: 1 + | > loss_disc: 2.450162887573242 (2.450162887573242) + | > loss_disc_real_0: 0.20088759064674377 (0.20088759064674377) + | > loss_disc_real_1: 0.2239193618297577 (0.2239193618297577) + | > loss_disc_real_2: 0.23466047644615173 (0.23466047644615173) + | > loss_disc_real_3: 0.21948012709617615 (0.21948012709617615) + | > loss_disc_real_4: 0.27527791261672974 (0.27527791261672974) + | > loss_disc_real_5: 0.1782912164926529 (0.1782912164926529) + | > loss_0: 2.450162887573242 (2.450162887573242) + | > loss_gen: 2.461454391479492 (2.461454391479492) + | > loss_kl: 2.587801218032837 (2.587801218032837) + | > loss_feat: 7.370443344116211 (7.370443344116211) + | > loss_mel: 19.108095169067383 (19.108095169067383) + | > loss_duration: 1.7018104791641235 (1.7018104791641235) + | > loss_1: 33.2296028137207 (33.2296028137207) + + --> STEP: 2 + | > loss_disc: 2.5911524295806885 (2.5206576585769653) + | > loss_disc_real_0: 0.19203031063079834 (0.19645895063877106) + | > loss_disc_real_1: 0.2091985046863556 (0.21655893325805664) + | > loss_disc_real_2: 0.2268679440021515 (0.2307642102241516) + | > loss_disc_real_3: 0.23891602456569672 (0.22919807583093643) + | > loss_disc_real_4: 0.24230968952178955 (0.25879380106925964) + | > loss_disc_real_5: 0.23356789350509644 (0.20592955499887466) + | > loss_0: 2.5911524295806885 (2.5206576585769653) + | > loss_gen: 2.34089994430542 (2.401177167892456) + | > loss_kl: 1.8630043268203735 (2.2254027724266052) + | > loss_feat: 7.239311695098877 (7.304877519607544) + | > loss_mel: 19.72073745727539 (19.414416313171387) + | > loss_duration: 1.6560966968536377 (1.6789535880088806) + | > loss_1: 32.82004928588867 (33.02482604980469) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037418603897094727 (+0.00019037723541259766) + | > avg_loss_disc: 2.5206576585769653 (+0.0015380382537841797) + | > avg_loss_disc_real_0: 0.19645895063877106 (+0.04219093173742294) + | > avg_loss_disc_real_1: 0.21655893325805664 (-0.040744006633758545) + | > avg_loss_disc_real_2: 0.2307642102241516 (+0.016692616045475006) + | > avg_loss_disc_real_3: 0.22919807583093643 (-0.0037779957056045532) + | > avg_loss_disc_real_4: 0.25879380106925964 (-0.023532256484031677) + | > avg_loss_disc_real_5: 0.20592955499887466 (-0.02361152321100235) + | > avg_loss_0: 2.5206576585769653 (+0.0015380382537841797) + | > avg_loss_gen: 2.401177167892456 (-0.04633045196533203) + | > avg_loss_kl: 2.2254027724266052 (+0.030124902725219727) + | > avg_loss_feat: 7.304877519607544 (+0.37023282051086426) + | > avg_loss_mel: 19.414416313171387 (+0.4173545837402344) + | > avg_loss_duration: 1.6789535880088806 (+0.018941640853881836) + | > avg_loss_1: 33.02482604980469 (+0.7903213500976562) + + + > EPOCH: 162/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:39:50)  + + --> TIME: 2025-04-19 01:39:56 -- STEP: 14/142 -- GLOBAL_STEP: 402300 + | > loss_disc: 2.464616298675537 (2.4701137713023593) + | > loss_disc_real_0: 0.09782671183347702 (0.1208734240915094) + | > loss_disc_real_1: 0.20020140707492828 (0.21200023697955267) + | > loss_disc_real_2: 0.22518691420555115 (0.22892243840864726) + | > loss_disc_real_3: 0.2472352534532547 (0.2294308011020933) + | > loss_disc_real_4: 0.2357271909713745 (0.2271442243031093) + | > loss_disc_real_5: 0.1983589380979538 (0.23210697833980834) + | > loss_0: 2.464616298675537 (2.4701137713023593) + | > grad_norm_0: tensor(5.3212, device='cuda:0') (tensor(7.9301, device='cuda:0')) + | > loss_gen: 2.0887832641601562 (2.309003676686968) + | > loss_kl: 1.8582260608673096 (1.8550443649291992) + | > loss_feat: 7.412558555603027 (7.2154655115945) + | > loss_mel: 17.772197723388672 (18.031004905700684) + | > loss_duration: 1.4489107131958008 (1.4406778727258955) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.580677032470703 (30.851196425301687) + | > grad_norm_1: tensor(90.0346, device='cuda:0') (tensor(155.6296, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.3349 (0.3275430202484131) + | > loader_time: 0.0048 (0.004395621163504465) + + + --> TIME: 2025-04-19 01:40:04 -- STEP: 39/142 -- GLOBAL_STEP: 402325 + | > loss_disc: 2.5362491607666016 (2.475336123735477) + | > loss_disc_real_0: 0.12955938279628754 (0.12395862929331951) + | > loss_disc_real_1: 0.25727182626724243 (0.20703756045072508) + | > loss_disc_real_2: 0.25054892897605896 (0.22633898984163237) + | > loss_disc_real_3: 0.2345922887325287 (0.22980321905551812) + | > loss_disc_real_4: 0.24792851507663727 (0.22980547256958792) + | > loss_disc_real_5: 0.26560482382774353 (0.23556842444798884) + | > loss_0: 2.5362491607666016 (2.475336123735477) + | > grad_norm_0: tensor(11.5570, device='cuda:0') (tensor(8.8875, device='cuda:0')) + | > loss_gen: 2.207576036453247 (2.3128470029586405) + | > loss_kl: 1.9483954906463623 (1.8572130386645977) + | > loss_feat: 5.809024333953857 (6.993929166060227) + | > loss_mel: 15.156989097595215 (17.639522870381672) + | > loss_duration: 1.493625283241272 (1.452196011176476) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.615612030029297 (30.255708303207005) + | > grad_norm_1: tensor(224.4307, device='cuda:0') (tensor(200.0731, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.3556 (0.3411872937129094) + | > loader_time: 0.0052 (0.0049208127535306494) + + + --> TIME: 2025-04-19 01:40:14 -- STEP: 64/142 -- GLOBAL_STEP: 402350 + | > loss_disc: 2.5372426509857178 (2.4597993791103363) + | > loss_disc_real_0: 0.1650371253490448 (0.12115792371332645) + | > loss_disc_real_1: 0.24124401807785034 (0.20617100526578724) + | > loss_disc_real_2: 0.2515054941177368 (0.22592705790884793) + | > loss_disc_real_3: 0.22945356369018555 (0.2296016262844205) + | > loss_disc_real_4: 0.2625370919704437 (0.22821758477948606) + | > loss_disc_real_5: 0.2519073784351349 (0.23574168886989355) + | > loss_0: 2.5372426509857178 (2.4597993791103363) + | > grad_norm_0: tensor(9.6107, device='cuda:0') (tensor(8.8409, device='cuda:0')) + | > loss_gen: 2.2907822132110596 (2.334290839731694) + | > loss_kl: 1.9109218120574951 (1.8703353703022003) + | > loss_feat: 6.023702621459961 (7.030571594834328) + | > loss_mel: 17.253171920776367 (17.786138221621506) + | > loss_duration: 1.5113868713378906 (1.4631875846534965) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.989967346191406 (30.48452377319336) + | > grad_norm_1: tensor(205.5589, device='cuda:0') (tensor(228.3651, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.3841 (0.3526846505701543) + | > loader_time: 0.0059 (0.005224697291851045) + + + --> TIME: 2025-04-19 01:40:24 -- STEP: 89/142 -- GLOBAL_STEP: 402375 + | > loss_disc: 2.541367530822754 (2.4660996988918016) + | > loss_disc_real_0: 0.150013267993927 (0.12397046981567747) + | > loss_disc_real_1: 0.20979686081409454 (0.2065521456887213) + | > loss_disc_real_2: 0.23600873351097107 (0.2256807175914893) + | > loss_disc_real_3: 0.2355143129825592 (0.2295300116699733) + | > loss_disc_real_4: 0.2570216655731201 (0.22903359338138882) + | > loss_disc_real_5: 0.2725234031677246 (0.2354044057010265) + | > loss_0: 2.541367530822754 (2.4660996988918016) + | > grad_norm_0: tensor(11.2967, device='cuda:0') (tensor(10.1921, device='cuda:0')) + | > loss_gen: 2.327791213989258 (2.3370589942074895) + | > loss_kl: 2.0890514850616455 (1.8884932097424283) + | > loss_feat: 6.533156394958496 (6.984004063552685) + | > loss_mel: 16.17564582824707 (17.752376674266337) + | > loss_duration: 1.496597409248352 (1.4744085890523504) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.622241973876953 (30.436341703607795) + | > grad_norm_1: tensor(403.5967, device='cuda:0') (tensor(241.0311, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.4015 (0.36462901147563803) + | > loader_time: 0.0065 (0.005500268400385139) + + + --> TIME: 2025-04-19 01:40:35 -- STEP: 114/142 -- GLOBAL_STEP: 402400 + | > loss_disc: 2.4601550102233887 (2.465968928839032) + | > loss_disc_real_0: 0.17568247020244598 (0.12520037944379608) + | > loss_disc_real_1: 0.15505941212177277 (0.20591269917132562) + | > loss_disc_real_2: 0.19756288826465607 (0.22536684611910268) + | > loss_disc_real_3: 0.218339204788208 (0.22922142647337496) + | > loss_disc_real_4: 0.18891507387161255 (0.22857214314372917) + | > loss_disc_real_5: 0.1973714679479599 (0.234364920148724) + | > loss_0: 2.4601550102233887 (2.465968928839032) + | > grad_norm_0: tensor(12.8319, device='cuda:0') (tensor(11.0658, device='cuda:0')) + | > loss_gen: 2.5070228576660156 (2.3393152471174283) + | > loss_kl: 1.8855074644088745 (1.9020085041983086) + | > loss_feat: 6.97959566116333 (6.95783805847168) + | > loss_mel: 17.821340560913086 (17.723453580287455) + | > loss_duration: 1.507151484489441 (1.4813246643334104) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.70061683654785 (30.403940217536793) + | > grad_norm_1: tensor(190.7715, device='cuda:0') (tensor(262.4335, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.4237 (0.3754649371431585) + | > loader_time: 0.0067 (0.00573017722681949) + + + --> TIME: 2025-04-19 01:40:46 -- STEP: 139/142 -- GLOBAL_STEP: 402425 + | > loss_disc: 2.6279659271240234 (2.469940773874736) + | > loss_disc_real_0: 0.1360715627670288 (0.12641412496781182) + | > loss_disc_real_1: 0.20147427916526794 (0.20636732061561064) + | > loss_disc_real_2: 0.24650579690933228 (0.2259462939963924) + | > loss_disc_real_3: 0.2086382657289505 (0.22871748674258907) + | > loss_disc_real_4: 0.24671229720115662 (0.22946428791653337) + | > loss_disc_real_5: 0.2686164975166321 (0.23463816222527045) + | > loss_0: 2.6279659271240234 (2.469940773874736) + | > grad_norm_0: tensor(8.6797, device='cuda:0') (tensor(12.0657, device='cuda:0')) + | > loss_gen: 2.177076816558838 (2.3508019841832226) + | > loss_kl: 1.927765130996704 (1.9139835182711376) + | > loss_feat: 6.125138282775879 (6.981409052293078) + | > loss_mel: 15.116991996765137 (17.76753773449136) + | > loss_duration: 1.504058599472046 (1.4873539972648346) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.851030349731445 (30.50108644773634) + | > grad_norm_1: tensor(401.8649, device='cuda:0') (tensor(272.0574, device='cuda:0')) + | > current_lr_0: 0.000195990482782131 + | > current_lr_1: 0.000195990482782131 + | > step_time: 0.4552 (0.3885781662069636) + | > loader_time: 0.0076 (0.006052006920464605) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5501999855041504 (2.5501999855041504) + | > loss_disc_real_0: 0.14074678719043732 (0.14074678719043732) + | > loss_disc_real_1: 0.21678484976291656 (0.21678484976291656) + | > loss_disc_real_2: 0.26989731192588806 (0.26989731192588806) + | > loss_disc_real_3: 0.26691800355911255 (0.26691800355911255) + | > loss_disc_real_4: 0.25940242409706116 (0.25940242409706116) + | > loss_disc_real_5: 0.2674500644207001 (0.2674500644207001) + | > loss_0: 2.5501999855041504 (2.5501999855041504) + | > loss_gen: 2.383521318435669 (2.383521318435669) + | > loss_kl: 2.4909017086029053 (2.4909017086029053) + | > loss_feat: 7.09968900680542 (7.09968900680542) + | > loss_mel: 18.03978729248047 (18.03978729248047) + | > loss_duration: 1.6591730117797852 (1.6591730117797852) + | > loss_1: 31.673072814941406 (31.673072814941406) + + --> STEP: 1 + | > loss_disc: 2.654578685760498 (2.654578685760498) + | > loss_disc_real_0: 0.17380987107753754 (0.17380987107753754) + | > loss_disc_real_1: 0.24708881974220276 (0.24708881974220276) + | > loss_disc_real_2: 0.2553781569004059 (0.2553781569004059) + | > loss_disc_real_3: 0.2669338881969452 (0.2669338881969452) + | > loss_disc_real_4: 0.24428367614746094 (0.24428367614746094) + | > loss_disc_real_5: 0.27970799803733826 (0.27970799803733826) + | > loss_0: 2.654578685760498 (2.654578685760498) + | > loss_gen: 2.210766315460205 (2.210766315460205) + | > loss_kl: 2.5794196128845215 (2.5794196128845215) + | > loss_feat: 5.936257839202881 (5.936257839202881) + | > loss_mel: 18.244670867919922 (18.244670867919922) + | > loss_duration: 1.7162091732025146 (1.7162091732025146) + | > loss_1: 30.68732452392578 (30.68732452392578) + + --> STEP: 2 + | > loss_disc: 2.687020778656006 (2.670799732208252) + | > loss_disc_real_0: 0.17523235082626343 (0.17452111095190048) + | > loss_disc_real_1: 0.22045595943927765 (0.2337723895907402) + | > loss_disc_real_2: 0.2596403956413269 (0.2575092762708664) + | > loss_disc_real_3: 0.25153225660324097 (0.2592330724000931) + | > loss_disc_real_4: 0.2674524784088135 (0.2558680772781372) + | > loss_disc_real_5: 0.25138401985168457 (0.2655460089445114) + | > loss_0: 2.687020778656006 (2.670799732208252) + | > loss_gen: 2.073293685913086 (2.1420300006866455) + | > loss_kl: 2.144991874694824 (2.362205743789673) + | > loss_feat: 6.028146266937256 (5.982202053070068) + | > loss_mel: 16.140165328979492 (17.192418098449707) + | > loss_duration: 1.6579171419143677 (1.6870631575584412) + | > loss_1: 28.044513702392578 (29.36591911315918) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003654956817626953 (-8.690357208251953e-05) + | > avg_loss_disc: 2.670799732208252 (+0.15014207363128662) + | > avg_loss_disc_real_0: 0.17452111095190048 (-0.021937839686870575) + | > avg_loss_disc_real_1: 0.2337723895907402 (+0.017213456332683563) + | > avg_loss_disc_real_2: 0.2575092762708664 (+0.026745066046714783) + | > avg_loss_disc_real_3: 0.2592330724000931 (+0.030034996569156647) + | > avg_loss_disc_real_4: 0.2558680772781372 (-0.0029257237911224365) + | > avg_loss_disc_real_5: 0.2655460089445114 (+0.05961645394563675) + | > avg_loss_0: 2.670799732208252 (+0.15014207363128662) + | > avg_loss_gen: 2.1420300006866455 (-0.25914716720581055) + | > avg_loss_kl: 2.362205743789673 (+0.13680297136306763) + | > avg_loss_feat: 5.982202053070068 (-1.3226754665374756) + | > avg_loss_mel: 17.192418098449707 (-2.2219982147216797) + | > avg_loss_duration: 1.6870631575584412 (+0.008109569549560547) + | > avg_loss_1: 29.36591911315918 (-3.658906936645508) + + + > EPOCH: 163/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:40:49)  + + --> TIME: 2025-04-19 01:40:57 -- STEP: 22/142 -- GLOBAL_STEP: 402450 + | > loss_disc: 2.5090808868408203 (2.4019550518556074) + | > loss_disc_real_0: 0.19410383701324463 (0.12324568900195035) + | > loss_disc_real_1: 0.24181856215000153 (0.20516334745016965) + | > loss_disc_real_2: 0.24567195773124695 (0.22361371530727905) + | > loss_disc_real_3: 0.2745608687400818 (0.22750471464612268) + | > loss_disc_real_4: 0.2683318257331848 (0.22081641040065073) + | > loss_disc_real_5: 0.2851758301258087 (0.22442418336868286) + | > loss_0: 2.5090808868408203 (2.4019550518556074) + | > grad_norm_0: tensor(10.5924, device='cuda:0') (tensor(10.9864, device='cuda:0')) + | > loss_gen: 2.548074722290039 (2.402835791761225) + | > loss_kl: 2.011951208114624 (1.8636001511053606) + | > loss_feat: 6.849320888519287 (7.190953168002042) + | > loss_mel: 16.948274612426758 (17.468651684847746) + | > loss_duration: 1.4083211421966553 (1.4368124712597241) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.76594352722168 (30.36285365711559) + | > grad_norm_1: tensor(87.2113, device='cuda:0') (tensor(234.3536, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.3466 (0.3328259858218106) + | > loader_time: 0.0049 (0.004615241830999201) + + + --> TIME: 2025-04-19 01:41:06 -- STEP: 47/142 -- GLOBAL_STEP: 402475 + | > loss_disc: 2.4270846843719482 (2.439756332559788) + | > loss_disc_real_0: 0.11619832366704941 (0.12368274543513642) + | > loss_disc_real_1: 0.21104592084884644 (0.20719984427411506) + | > loss_disc_real_2: 0.19146417081356049 (0.22280376895945123) + | > loss_disc_real_3: 0.21507379412651062 (0.22588182637032042) + | > loss_disc_real_4: 0.20486819744110107 (0.2272221287514301) + | > loss_disc_real_5: 0.22987547516822815 (0.22893311178430598) + | > loss_0: 2.4270846843719482 (2.439756332559788) + | > grad_norm_0: tensor(5.4046, device='cuda:0') (tensor(8.9987, device='cuda:0')) + | > loss_gen: 2.3063948154449463 (2.347671113115675) + | > loss_kl: 1.9211277961730957 (1.8649016086091386) + | > loss_feat: 6.813615322113037 (7.1035798052524) + | > loss_mel: 17.158523559570312 (17.638802426926635) + | > loss_duration: 1.43471097946167 (1.4512716227389393) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63437271118164 (30.40622658425189) + | > grad_norm_1: tensor(98.0334, device='cuda:0') (tensor(169.1866, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.3644 (0.3453896603685745) + | > loader_time: 0.0056 (0.005089399662423642) + + + --> TIME: 2025-04-19 01:41:16 -- STEP: 72/142 -- GLOBAL_STEP: 402500 + | > loss_disc: 2.415255308151245 (2.4380401803387537) + | > loss_disc_real_0: 0.08947135508060455 (0.12339488944659631) + | > loss_disc_real_1: 0.1922423541545868 (0.20473080314695832) + | > loss_disc_real_2: 0.215828076004982 (0.22427208866510126) + | > loss_disc_real_3: 0.2260647416114807 (0.22637504215041795) + | > loss_disc_real_4: 0.2122846394777298 (0.22665765343440902) + | > loss_disc_real_5: 0.2095182090997696 (0.230292154269086) + | > loss_0: 2.415255308151245 (2.4380401803387537) + | > grad_norm_0: tensor(12.2296, device='cuda:0') (tensor(8.6522, device='cuda:0')) + | > loss_gen: 2.3538947105407715 (2.3523769213093653) + | > loss_kl: 1.9856083393096924 (1.8761046247349844) + | > loss_feat: 7.434194564819336 (7.102436727947659) + | > loss_mel: 17.822566986083984 (17.770950330628292) + | > loss_duration: 1.460493564605713 (1.4643827544318304) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.056758880615234 (30.56625133090549) + | > grad_norm_1: tensor(282.6779, device='cuda:0') (tensor(180.4577, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.398 (0.3574767344527775) + | > loader_time: 0.0059 (0.005456798606448705) + + + --> TIME: 2025-04-19 01:41:26 -- STEP: 97/142 -- GLOBAL_STEP: 402525 + | > loss_disc: 2.4674203395843506 (2.4602552812124032) + | > loss_disc_real_0: 0.14268124103546143 (0.1276227028406772) + | > loss_disc_real_1: 0.25144630670547485 (0.20666810501482066) + | > loss_disc_real_2: 0.21789810061454773 (0.2251483246530454) + | > loss_disc_real_3: 0.21165570616722107 (0.2285323143005371) + | > loss_disc_real_4: 0.21686139702796936 (0.2286963106430683) + | > loss_disc_real_5: 0.2630944848060608 (0.23122009189472986) + | > loss_0: 2.4674203395843506 (2.4602552812124032) + | > grad_norm_0: tensor(5.0696, device='cuda:0') (tensor(9.1960, device='cuda:0')) + | > loss_gen: 2.318230390548706 (2.3439046574622093) + | > loss_kl: 1.9761919975280762 (1.888136025556584) + | > loss_feat: 7.041347503662109 (7.0175532999726915) + | > loss_mel: 18.5223331451416 (17.75475230659407) + | > loss_duration: 1.5062837600708008 (1.4759164458697605) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.36438751220703 (30.480262677694103) + | > grad_norm_1: tensor(210.9755, device='cuda:0') (tensor(193.3445, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.4122 (0.3687930303750582) + | > loader_time: 0.007 (0.005732632174934311) + + + --> TIME: 2025-04-19 01:41:37 -- STEP: 122/142 -- GLOBAL_STEP: 402550 + | > loss_disc: 2.436708927154541 (2.4678963989507956) + | > loss_disc_real_0: 0.13959747552871704 (0.1299516273448701) + | > loss_disc_real_1: 0.20243588089942932 (0.20725355431681772) + | > loss_disc_real_2: 0.2098509967327118 (0.22549184057556215) + | > loss_disc_real_3: 0.2009316384792328 (0.22773427565078266) + | > loss_disc_real_4: 0.20176143944263458 (0.22948329272817392) + | > loss_disc_real_5: 0.2018507421016693 (0.23023973625214372) + | > loss_0: 2.436708927154541 (2.4678963989507956) + | > grad_norm_0: tensor(11.5258, device='cuda:0') (tensor(8.9368, device='cuda:0')) + | > loss_gen: 2.324195146560669 (2.332541041686888) + | > loss_kl: 1.9432172775268555 (1.8971069191322951) + | > loss_feat: 6.918652057647705 (6.947163050291969) + | > loss_mel: 18.218172073364258 (17.717012014545386) + | > loss_duration: 1.5028107166290283 (1.4829400513992934) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.90704917907715 (30.376763015496927) + | > grad_norm_1: tensor(256.7032, device='cuda:0') (tensor(175.9081, device='cuda:0')) + | > current_lr_0: 0.00019596598397178324 + | > current_lr_1: 0.00019596598397178324 + | > step_time: 0.4514 (0.3807681447169822) + | > loader_time: 0.007 (0.005989676616230951) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.548971652984619 (2.548971652984619) + | > loss_disc_real_0: 0.1805674284696579 (0.1805674284696579) + | > loss_disc_real_1: 0.22663220763206482 (0.22663220763206482) + | > loss_disc_real_2: 0.2758239209651947 (0.2758239209651947) + | > loss_disc_real_3: 0.30808913707733154 (0.30808913707733154) + | > loss_disc_real_4: 0.21588605642318726 (0.21588605642318726) + | > loss_disc_real_5: 0.2861378490924835 (0.2861378490924835) + | > loss_0: 2.548971652984619 (2.548971652984619) + | > loss_gen: 2.527508497238159 (2.527508497238159) + | > loss_kl: 2.454371929168701 (2.454371929168701) + | > loss_feat: 6.929494380950928 (6.929494380950928) + | > loss_mel: 19.085468292236328 (19.085468292236328) + | > loss_duration: 1.6598074436187744 (1.6598074436187744) + | > loss_1: 32.65665054321289 (32.65665054321289) + + --> STEP: 1 + | > loss_disc: 2.4348716735839844 (2.4348716735839844) + | > loss_disc_real_0: 0.1666896641254425 (0.1666896641254425) + | > loss_disc_real_1: 0.2407582402229309 (0.2407582402229309) + | > loss_disc_real_2: 0.2121901512145996 (0.2121901512145996) + | > loss_disc_real_3: 0.2437596470117569 (0.2437596470117569) + | > loss_disc_real_4: 0.2120768427848816 (0.2120768427848816) + | > loss_disc_real_5: 0.21692007780075073 (0.21692007780075073) + | > loss_0: 2.4348716735839844 (2.4348716735839844) + | > loss_gen: 2.4090216159820557 (2.4090216159820557) + | > loss_kl: 2.5604946613311768 (2.5604946613311768) + | > loss_feat: 7.504300117492676 (7.504300117492676) + | > loss_mel: 18.275474548339844 (18.275474548339844) + | > loss_duration: 1.7161023616790771 (1.7161023616790771) + | > loss_1: 32.46539306640625 (32.46539306640625) + + --> STEP: 2 + | > loss_disc: 2.640348196029663 (2.5376099348068237) + | > loss_disc_real_0: 0.20070840418338776 (0.18369903415441513) + | > loss_disc_real_1: 0.2349441796541214 (0.23785120993852615) + | > loss_disc_real_2: 0.2864566445350647 (0.24932339787483215) + | > loss_disc_real_3: 0.2867121398448944 (0.26523589342832565) + | > loss_disc_real_4: 0.249095156788826 (0.2305859997868538) + | > loss_disc_real_5: 0.2598288655281067 (0.2383744716644287) + | > loss_0: 2.640348196029663 (2.5376099348068237) + | > loss_gen: 2.4778847694396973 (2.4434531927108765) + | > loss_kl: 2.1913270950317383 (2.3759108781814575) + | > loss_feat: 6.873817443847656 (7.189058780670166) + | > loss_mel: 17.69185447692871 (17.983664512634277) + | > loss_duration: 1.6506165266036987 (1.683359444141388) + | > loss_1: 30.885499954223633 (31.67544651031494) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003937244415283203 (+0.00028228759765625) + | > avg_loss_disc: 2.5376099348068237 (-0.13318979740142822) + | > avg_loss_disc_real_0: 0.18369903415441513 (+0.009177923202514648) + | > avg_loss_disc_real_1: 0.23785120993852615 (+0.00407882034778595) + | > avg_loss_disc_real_2: 0.24932339787483215 (-0.00818587839603424) + | > avg_loss_disc_real_3: 0.26523589342832565 (+0.0060028210282325745) + | > avg_loss_disc_real_4: 0.2305859997868538 (-0.025282077491283417) + | > avg_loss_disc_real_5: 0.2383744716644287 (-0.027171537280082703) + | > avg_loss_0: 2.5376099348068237 (-0.13318979740142822) + | > avg_loss_gen: 2.4434531927108765 (+0.30142319202423096) + | > avg_loss_kl: 2.3759108781814575 (+0.013705134391784668) + | > avg_loss_feat: 7.189058780670166 (+1.2068567276000977) + | > avg_loss_mel: 17.983664512634277 (+0.7912464141845703) + | > avg_loss_duration: 1.683359444141388 (-0.0037037134170532227) + | > avg_loss_1: 31.67544651031494 (+2.3095273971557617) + + + > EPOCH: 164/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:41:48)  + + --> TIME: 2025-04-19 01:41:50 -- STEP: 5/142 -- GLOBAL_STEP: 402575 + | > loss_disc: 2.4632296562194824 (2.457337760925293) + | > loss_disc_real_0: 0.12776407599449158 (0.1171925351023674) + | > loss_disc_real_1: 0.19775116443634033 (0.20257979929447173) + | > loss_disc_real_2: 0.1800069808959961 (0.2189990222454071) + | > loss_disc_real_3: 0.21336045861244202 (0.23096518218517303) + | > loss_disc_real_4: 0.2126338630914688 (0.23053566813468934) + | > loss_disc_real_5: 0.237043097615242 (0.26017839312553404) + | > loss_0: 2.4632296562194824 (2.457337760925293) + | > grad_norm_0: tensor(22.2608, device='cuda:0') (tensor(14.7733, device='cuda:0')) + | > loss_gen: 2.462998151779175 (2.379579257965088) + | > loss_kl: 1.7314701080322266 (1.7843781471252442) + | > loss_feat: 7.741144180297852 (7.285140514373779) + | > loss_mel: 18.937833786010742 (18.069183349609375) + | > loss_duration: 1.4254450798034668 (1.4338681936264037) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.29888916015625 (30.952149200439454) + | > grad_norm_1: tensor(459.4452, device='cuda:0') (tensor(318.1328, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.3244 (0.32316269874572756) + | > loader_time: 0.0043 (0.004111957550048828) + + + --> TIME: 2025-04-19 01:41:59 -- STEP: 30/142 -- GLOBAL_STEP: 402600 + | > loss_disc: 2.344014883041382 (2.4452013333638507) + | > loss_disc_real_0: 0.10884162038564682 (0.12003421485424041) + | > loss_disc_real_1: 0.22251737117767334 (0.2070469468832016) + | > loss_disc_real_2: 0.21458971500396729 (0.2257276768485705) + | > loss_disc_real_3: 0.2271258383989334 (0.23040779531002045) + | > loss_disc_real_4: 0.23891179263591766 (0.22910401125748953) + | > loss_disc_real_5: 0.2532278597354889 (0.23317663768927258) + | > loss_0: 2.344014883041382 (2.4452013333638507) + | > grad_norm_0: tensor(9.5704, device='cuda:0') (tensor(10.9184, device='cuda:0')) + | > loss_gen: 2.419191837310791 (2.373898394902547) + | > loss_kl: 1.793954610824585 (1.836460276444753) + | > loss_feat: 7.086946964263916 (7.174680391947429) + | > loss_mel: 18.557435989379883 (17.827494049072264) + | > loss_duration: 1.437185287475586 (1.4449460665384928) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.294713973999023 (30.657479286193848) + | > grad_norm_1: tensor(209.2842, device='cuda:0') (tensor(259.5236, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.3494 (0.33646650314331056) + | > loader_time: 0.0053 (0.004711381594340007) + + + --> TIME: 2025-04-19 01:42:08 -- STEP: 55/142 -- GLOBAL_STEP: 402625 + | > loss_disc: 2.5139992237091064 (2.4538936831734386) + | > loss_disc_real_0: 0.19735407829284668 (0.12408406924117696) + | > loss_disc_real_1: 0.2288004755973816 (0.20976950959725812) + | > loss_disc_real_2: 0.2532781958580017 (0.22561339302496478) + | > loss_disc_real_3: 0.24668502807617188 (0.2280752176588232) + | > loss_disc_real_4: 0.2264821082353592 (0.22760391587560827) + | > loss_disc_real_5: 0.278697669506073 (0.23397628394040196) + | > loss_0: 2.5139992237091064 (2.4538936831734386) + | > grad_norm_0: tensor(15.5131, device='cuda:0') (tensor(12.1723, device='cuda:0')) + | > loss_gen: 2.3338661193847656 (2.3847291339527485) + | > loss_kl: 1.904539942741394 (1.863056445121765) + | > loss_feat: 6.650945663452148 (7.191677891124378) + | > loss_mel: 17.54676055908203 (18.014855818314985) + | > loss_duration: 1.5167157649993896 (1.4541240258650345) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.95282745361328 (30.908443173495208) + | > grad_norm_1: tensor(329.2911, device='cuda:0') (tensor(277.6772, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.3762 (0.34844396764581853) + | > loader_time: 0.0057 (0.005081662264737216) + + + --> TIME: 2025-04-19 01:42:18 -- STEP: 80/142 -- GLOBAL_STEP: 402650 + | > loss_disc: 2.4663445949554443 (2.4663361489772786) + | > loss_disc_real_0: 0.09231399744749069 (0.12588630681857466) + | > loss_disc_real_1: 0.19185875356197357 (0.21045361887663602) + | > loss_disc_real_2: 0.20581522583961487 (0.22507219668477774) + | > loss_disc_real_3: 0.2338605523109436 (0.22823317665606738) + | > loss_disc_real_4: 0.21659302711486816 (0.2271213959902525) + | > loss_disc_real_5: 0.1953757405281067 (0.2331200672313571) + | > loss_0: 2.4663445949554443 (2.4663361489772786) + | > grad_norm_0: tensor(16.5404, device='cuda:0') (tensor(14.1349, device='cuda:0')) + | > loss_gen: 2.431694746017456 (2.3691372573375706) + | > loss_kl: 1.855628252029419 (1.871124117076397) + | > loss_feat: 7.42488431930542 (7.073584574460983) + | > loss_mel: 18.276147842407227 (17.8838342666626) + | > loss_duration: 1.4974526166915894 (1.46659240424633) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.48580551147461 (30.664272475242615) + | > grad_norm_1: tensor(529.4739, device='cuda:0') (tensor(289.1072, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.409 (0.36081239879131316) + | > loader_time: 0.0061 (0.005433958768844604) + + + --> TIME: 2025-04-19 01:42:28 -- STEP: 105/142 -- GLOBAL_STEP: 402675 + | > loss_disc: 2.6142332553863525 (2.469091574350992) + | > loss_disc_real_0: 0.08940473943948746 (0.12519426565794722) + | > loss_disc_real_1: 0.19672656059265137 (0.2098815749088923) + | > loss_disc_real_2: 0.22110159695148468 (0.2269655099936894) + | > loss_disc_real_3: 0.20407432317733765 (0.22840831066880907) + | > loss_disc_real_4: 0.21136526763439178 (0.22754770233517602) + | > loss_disc_real_5: 0.22869277000427246 (0.23255666494369506) + | > loss_0: 2.6142332553863525 (2.469091574350992) + | > grad_norm_0: tensor(8.4210, device='cuda:0') (tensor(14.7979, device='cuda:0')) + | > loss_gen: 2.2636001110076904 (2.370933439618066) + | > loss_kl: 2.0128214359283447 (1.883954964365278) + | > loss_feat: 6.510944843292236 (7.060473124186198) + | > loss_mel: 18.044538497924805 (17.847773906162818) + | > loss_duration: 1.5171947479248047 (1.4771338735307968) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.34910011291504 (30.640269306727816) + | > grad_norm_1: tensor(296.3985, device='cuda:0') (tensor(303.2960, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.4175 (0.3732054097311837) + | > loader_time: 0.0069 (0.0056786196572440025) + + + --> TIME: 2025-04-19 01:42:39 -- STEP: 130/142 -- GLOBAL_STEP: 402700 + | > loss_disc: 2.482865571975708 (2.4718124848145697) + | > loss_disc_real_0: 0.09988187998533249 (0.12738702784364045) + | > loss_disc_real_1: 0.2246886044740677 (0.21013388599340732) + | > loss_disc_real_2: 0.18643273413181305 (0.22609571115328717) + | > loss_disc_real_3: 0.2200118750333786 (0.2288195224908682) + | > loss_disc_real_4: 0.22250156104564667 (0.22822810124892454) + | > loss_disc_real_5: 0.22682277858257294 (0.23166580291894767) + | > loss_0: 2.482865571975708 (2.4718124848145697) + | > grad_norm_0: tensor(9.7791, device='cuda:0') (tensor(14.9770, device='cuda:0')) + | > loss_gen: 2.1801347732543945 (2.360579716242277) + | > loss_kl: 1.9368425607681274 (1.8833268073888925) + | > loss_feat: 6.5053391456604 (7.045079858486469) + | > loss_mel: 16.89170265197754 (17.864447505657502) + | > loss_duration: 1.5018473863601685 (1.48355068060068) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.015867233276367 (30.636984590383676) + | > grad_norm_1: tensor(213.7871, device='cuda:0') (tensor(300.7583, device='cuda:0')) + | > current_lr_0: 0.00019594148822378677 + | > current_lr_1: 0.00019594148822378677 + | > step_time: 0.4509 (0.3845960121888381) + | > loader_time: 0.0077 (0.005931980793292707) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.55232834815979 (2.55232834815979) + | > loss_disc_real_0: 0.09154355525970459 (0.09154355525970459) + | > loss_disc_real_1: 0.2464417815208435 (0.2464417815208435) + | > loss_disc_real_2: 0.22827094793319702 (0.22827094793319702) + | > loss_disc_real_3: 0.23545223474502563 (0.23545223474502563) + | > loss_disc_real_4: 0.26031413674354553 (0.26031413674354553) + | > loss_disc_real_5: 0.24474363029003143 (0.24474363029003143) + | > loss_0: 2.55232834815979 (2.55232834815979) + | > loss_gen: 2.2686269283294678 (2.2686269283294678) + | > loss_kl: 2.3164985179901123 (2.3164985179901123) + | > loss_feat: 7.1269145011901855 (7.1269145011901855) + | > loss_mel: 18.57216453552246 (18.57216453552246) + | > loss_duration: 1.6619627475738525 (1.6619627475738525) + | > loss_1: 31.9461669921875 (31.9461669921875) + + --> STEP: 1 + | > loss_disc: 2.594208240509033 (2.594208240509033) + | > loss_disc_real_0: 0.11103945970535278 (0.11103945970535278) + | > loss_disc_real_1: 0.23829752206802368 (0.23829752206802368) + | > loss_disc_real_2: 0.23605336248874664 (0.23605336248874664) + | > loss_disc_real_3: 0.21435093879699707 (0.21435093879699707) + | > loss_disc_real_4: 0.2505353093147278 (0.2505353093147278) + | > loss_disc_real_5: 0.28341513872146606 (0.28341513872146606) + | > loss_0: 2.594208240509033 (2.594208240509033) + | > loss_gen: 2.052816867828369 (2.052816867828369) + | > loss_kl: 2.6126937866210938 (2.6126937866210938) + | > loss_feat: 5.356156826019287 (5.356156826019287) + | > loss_mel: 16.860279083251953 (16.860279083251953) + | > loss_duration: 1.7087067365646362 (1.7087067365646362) + | > loss_1: 28.590652465820312 (28.590652465820312) + + --> STEP: 2 + | > loss_disc: 2.6336112022399902 (2.6139097213745117) + | > loss_disc_real_0: 0.1379621922969818 (0.1245008260011673) + | > loss_disc_real_1: 0.2677960991859436 (0.25304681062698364) + | > loss_disc_real_2: 0.23600679636001587 (0.23603007942438126) + | > loss_disc_real_3: 0.2391400784254074 (0.22674550861120224) + | > loss_disc_real_4: 0.2585850656032562 (0.254560187458992) + | > loss_disc_real_5: 0.291767418384552 (0.28759127855300903) + | > loss_0: 2.6336112022399902 (2.6139097213745117) + | > loss_gen: 2.1356611251831055 (2.0942389965057373) + | > loss_kl: 2.2695229053497314 (2.4411083459854126) + | > loss_feat: 6.3783345222473145 (5.867245674133301) + | > loss_mel: 16.566757202148438 (16.713518142700195) + | > loss_duration: 1.6443827152252197 (1.676544725894928) + | > loss_1: 28.994659423828125 (28.79265594482422) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004398345947265625 (+0.0004611015319824219) + | > avg_loss_disc: 2.6139097213745117 (+0.07629978656768799) + | > avg_loss_disc_real_0: 0.1245008260011673 (-0.05919820815324783) + | > avg_loss_disc_real_1: 0.25304681062698364 (+0.015195600688457489) + | > avg_loss_disc_real_2: 0.23603007942438126 (-0.013293318450450897) + | > avg_loss_disc_real_3: 0.22674550861120224 (-0.03849038481712341) + | > avg_loss_disc_real_4: 0.254560187458992 (+0.023974187672138214) + | > avg_loss_disc_real_5: 0.28759127855300903 (+0.04921680688858032) + | > avg_loss_0: 2.6139097213745117 (+0.07629978656768799) + | > avg_loss_gen: 2.0942389965057373 (-0.34921419620513916) + | > avg_loss_kl: 2.4411083459854126 (+0.06519746780395508) + | > avg_loss_feat: 5.867245674133301 (-1.3218131065368652) + | > avg_loss_mel: 16.713518142700195 (-1.270146369934082) + | > avg_loss_duration: 1.676544725894928 (-0.006814718246459961) + | > avg_loss_1: 28.79265594482422 (-2.8827905654907227) + + + > EPOCH: 165/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:42:46)  + + --> TIME: 2025-04-19 01:42:51 -- STEP: 13/142 -- GLOBAL_STEP: 402725 + | > loss_disc: 2.4594788551330566 (2.4323008977449856) + | > loss_disc_real_0: 0.09664920717477798 (0.12018082806697258) + | > loss_disc_real_1: 0.20970502495765686 (0.21027478461082166) + | > loss_disc_real_2: 0.21914757788181305 (0.22615097119258) + | > loss_disc_real_3: 0.21077467501163483 (0.22538337340721717) + | > loss_disc_real_4: 0.23772870004177094 (0.2235954782137504) + | > loss_disc_real_5: 0.2019060254096985 (0.2246802208515314) + | > loss_0: 2.4594788551330566 (2.4323008977449856) + | > grad_norm_0: tensor(7.9576, device='cuda:0') (tensor(9.7044, device='cuda:0')) + | > loss_gen: 2.3575499057769775 (2.3631427471454325) + | > loss_kl: 1.7771611213684082 (1.8288254279356737) + | > loss_feat: 7.0981621742248535 (7.034847296201265) + | > loss_mel: 17.9990291595459 (17.65705548799955) + | > loss_duration: 1.4493604898452759 (1.4369573409740741) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.68126106262207 (30.320827924288235) + | > grad_norm_1: tensor(201.2404, device='cuda:0') (tensor(217.9106, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.3296 (0.3266037610860971) + | > loader_time: 0.005 (0.0046171775230994605) + + + --> TIME: 2025-04-19 01:43:00 -- STEP: 38/142 -- GLOBAL_STEP: 402750 + | > loss_disc: 2.466660499572754 (2.44136413147575) + | > loss_disc_real_0: 0.10483918339014053 (0.11869037465045326) + | > loss_disc_real_1: 0.2049458920955658 (0.20920598546141073) + | > loss_disc_real_2: 0.21034839749336243 (0.22517618223240501) + | > loss_disc_real_3: 0.24603183567523956 (0.2280405093180506) + | > loss_disc_real_4: 0.2375813126564026 (0.22535053405322528) + | > loss_disc_real_5: 0.27668702602386475 (0.22929296603328303) + | > loss_0: 2.466660499572754 (2.44136413147575) + | > grad_norm_0: tensor(13.0593, device='cuda:0') (tensor(9.0539, device='cuda:0')) + | > loss_gen: 2.2993080615997314 (2.3466517486070337) + | > loss_kl: 1.8726115226745605 (1.85552443014948) + | > loss_feat: 6.970688819885254 (7.0427493045204566) + | > loss_mel: 18.311708450317383 (17.677229228772614) + | > loss_duration: 1.470445156097412 (1.4452705665638572) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.924762725830078 (30.367425065291556) + | > grad_norm_1: tensor(265.9301, device='cuda:0') (tensor(214.0619, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.353 (0.33987116186242355) + | > loader_time: 0.0056 (0.005171236238981549) + + + --> TIME: 2025-04-19 01:43:10 -- STEP: 63/142 -- GLOBAL_STEP: 402775 + | > loss_disc: 2.516028881072998 (2.4448692495860755) + | > loss_disc_real_0: 0.15090535581111908 (0.12111293347108931) + | > loss_disc_real_1: 0.20265072584152222 (0.20685500190371559) + | > loss_disc_real_2: 0.24182070791721344 (0.22361517047125196) + | > loss_disc_real_3: 0.22213943302631378 (0.2287936903654583) + | > loss_disc_real_4: 0.24049127101898193 (0.22660936486153377) + | > loss_disc_real_5: 0.24424457550048828 (0.23042154950754984) + | > loss_0: 2.516028881072998 (2.4448692495860755) + | > grad_norm_0: tensor(8.4287, device='cuda:0') (tensor(8.5497, device='cuda:0')) + | > loss_gen: 2.2382636070251465 (2.335980381284442) + | > loss_kl: 2.034343719482422 (1.8650271211351668) + | > loss_feat: 6.197825908660889 (7.011839571453276) + | > loss_mel: 17.165264129638672 (17.651216007414323) + | > loss_duration: 1.5350584983825684 (1.459306868295821) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.170757293701172 (30.32336982848152) + | > grad_norm_1: tensor(218.8284, device='cuda:0') (tensor(198.4705, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.3873 (0.3519174333602663) + | > loader_time: 0.0059 (0.005508309318905785) + + + --> TIME: 2025-04-19 01:43:20 -- STEP: 88/142 -- GLOBAL_STEP: 402800 + | > loss_disc: 2.442018508911133 (2.4512467330152337) + | > loss_disc_real_0: 0.11325263231992722 (0.12183371211656115) + | > loss_disc_real_1: 0.22936603426933289 (0.20579696920785037) + | > loss_disc_real_2: 0.2029087245464325 (0.22448028861121697) + | > loss_disc_real_3: 0.21252189576625824 (0.22863082926381717) + | > loss_disc_real_4: 0.18508346378803253 (0.22768082639033144) + | > loss_disc_real_5: 0.23548609018325806 (0.23080555942248215) + | > loss_0: 2.442018508911133 (2.4512467330152337) + | > grad_norm_0: tensor(11.9054, device='cuda:0') (tensor(8.5641, device='cuda:0')) + | > loss_gen: 2.365872859954834 (2.3341710323637197) + | > loss_kl: 1.9809799194335938 (1.8795926597985355) + | > loss_feat: 7.300227642059326 (6.970232866027138) + | > loss_mel: 19.37896728515625 (17.671983816406946) + | > loss_duration: 1.5486030578613281 (1.4719682864167467) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.57465362548828 (30.32794865694913) + | > grad_norm_1: tensor(286.0490, device='cuda:0') (tensor(199.5839, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.3964 (0.3639003580266778) + | > loader_time: 0.0067 (0.00580007650635459) + + + --> TIME: 2025-04-19 01:43:30 -- STEP: 113/142 -- GLOBAL_STEP: 402825 + | > loss_disc: 2.565351963043213 (2.4633654978422985) + | > loss_disc_real_0: 0.15820646286010742 (0.12392621045618986) + | > loss_disc_real_1: 0.2140325903892517 (0.20620800989919005) + | > loss_disc_real_2: 0.22917084395885468 (0.22584632236873153) + | > loss_disc_real_3: 0.22739341855049133 (0.22951401704180557) + | > loss_disc_real_4: 0.2557182312011719 (0.2289952915326684) + | > loss_disc_real_5: 0.22523991763591766 (0.23206699276919915) + | > loss_0: 2.565351963043213 (2.4633654978422985) + | > grad_norm_0: tensor(15.4013, device='cuda:0') (tensor(9.0999, device='cuda:0')) + | > loss_gen: 2.1907799243927 (2.3271942307463798) + | > loss_kl: 1.7977195978164673 (1.8896631065723115) + | > loss_feat: 6.055419921875 (6.931316367292826) + | > loss_mel: 16.66958236694336 (17.674727456759555) + | > loss_duration: 1.5125364065170288 (1.4794137709963633) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.22603988647461 (30.30231497773027) + | > grad_norm_1: tensor(254.4680, device='cuda:0') (tensor(209.8914, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.4275 (0.3745479625938212) + | > loader_time: 0.0074 (0.00604952542127761) + + + --> TIME: 2025-04-19 01:43:42 -- STEP: 138/142 -- GLOBAL_STEP: 402850 + | > loss_disc: 2.5265488624572754 (2.463041618250418) + | > loss_disc_real_0: 0.13382750749588013 (0.12498470563171567) + | > loss_disc_real_1: 0.23090563714504242 (0.20638160429139069) + | > loss_disc_real_2: 0.2312261313199997 (0.22569095073402792) + | > loss_disc_real_3: 0.2369944155216217 (0.22930459872536038) + | > loss_disc_real_4: 0.20687170326709747 (0.22808186439932257) + | > loss_disc_real_5: 0.26404669880867004 (0.23244848944570706) + | > loss_0: 2.5265488624572754 (2.463041618250418) + | > grad_norm_0: tensor(10.3309, device='cuda:0') (tensor(9.7612, device='cuda:0')) + | > loss_gen: 2.248638868331909 (2.3291556748791025) + | > loss_kl: 1.9478232860565186 (1.8938528843547986) + | > loss_feat: 6.917473316192627 (6.926949293717094) + | > loss_mel: 18.580970764160156 (17.68761227096336) + | > loss_duration: 1.5499017238616943 (1.48547122184781) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.24480628967285 (30.323041390681613) + | > grad_norm_1: tensor(231.1136, device='cuda:0') (tensor(219.1260, device='cuda:0')) + | > current_lr_0: 0.00019591699553775878 + | > current_lr_1: 0.00019591699553775878 + | > step_time: 0.4525 (0.3864859929983166) + | > loader_time: 0.0078 (0.006303864976634149) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.547538995742798 (2.547538995742798) + | > loss_disc_real_0: 0.13323722779750824 (0.13323722779750824) + | > loss_disc_real_1: 0.25630924105644226 (0.25630924105644226) + | > loss_disc_real_2: 0.24609415233135223 (0.24609415233135223) + | > loss_disc_real_3: 0.23038405179977417 (0.23038405179977417) + | > loss_disc_real_4: 0.22566787898540497 (0.22566787898540497) + | > loss_disc_real_5: 0.3112138509750366 (0.3112138509750366) + | > loss_0: 2.547538995742798 (2.547538995742798) + | > loss_gen: 2.409294366836548 (2.409294366836548) + | > loss_kl: 2.2789955139160156 (2.2789955139160156) + | > loss_feat: 6.794196605682373 (6.794196605682373) + | > loss_mel: 18.681413650512695 (18.681413650512695) + | > loss_duration: 1.67910635471344 (1.67910635471344) + | > loss_1: 31.843006134033203 (31.843006134033203) + + --> STEP: 1 + | > loss_disc: 2.6122617721557617 (2.6122617721557617) + | > loss_disc_real_0: 0.14703018963336945 (0.14703018963336945) + | > loss_disc_real_1: 0.29367396235466003 (0.29367396235466003) + | > loss_disc_real_2: 0.24686385691165924 (0.24686385691165924) + | > loss_disc_real_3: 0.25337713956832886 (0.25337713956832886) + | > loss_disc_real_4: 0.24527424573898315 (0.24527424573898315) + | > loss_disc_real_5: 0.2751713991165161 (0.2751713991165161) + | > loss_0: 2.6122617721557617 (2.6122617721557617) + | > loss_gen: 2.451052665710449 (2.451052665710449) + | > loss_kl: 2.4248945713043213 (2.4248945713043213) + | > loss_feat: 6.539217948913574 (6.539217948913574) + | > loss_mel: 19.159648895263672 (19.159648895263672) + | > loss_duration: 1.7020037174224854 (1.7020037174224854) + | > loss_1: 32.276817321777344 (32.276817321777344) + + --> STEP: 2 + | > loss_disc: 2.607285261154175 (2.6097735166549683) + | > loss_disc_real_0: 0.1748599112033844 (0.16094505041837692) + | > loss_disc_real_1: 0.3646102845668793 (0.32914212346076965) + | > loss_disc_real_2: 0.26224806904792786 (0.25455596297979355) + | > loss_disc_real_3: 0.2580183148384094 (0.25569772720336914) + | > loss_disc_real_4: 0.27695032954216003 (0.2611122876405716) + | > loss_disc_real_5: 0.28916266560554504 (0.2821670323610306) + | > loss_0: 2.607285261154175 (2.6097735166549683) + | > loss_gen: 2.6724438667297363 (2.5617482662200928) + | > loss_kl: 2.1409718990325928 (2.282933235168457) + | > loss_feat: 7.019137859344482 (6.779177904129028) + | > loss_mel: 17.774951934814453 (18.467300415039062) + | > loss_duration: 1.6319992542266846 (1.667001485824585) + | > loss_1: 31.239505767822266 (31.758161544799805) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034465789794921875 (-0.0009517669677734375) + | > avg_loss_disc: 2.6097735166549683 (-0.004136204719543457) + | > avg_loss_disc_real_0: 0.16094505041837692 (+0.036444224417209625) + | > avg_loss_disc_real_1: 0.32914212346076965 (+0.07609531283378601) + | > avg_loss_disc_real_2: 0.25455596297979355 (+0.018525883555412292) + | > avg_loss_disc_real_3: 0.25569772720336914 (+0.0289522185921669) + | > avg_loss_disc_real_4: 0.2611122876405716 (+0.00655210018157959) + | > avg_loss_disc_real_5: 0.2821670323610306 (-0.005424246191978455) + | > avg_loss_0: 2.6097735166549683 (-0.004136204719543457) + | > avg_loss_gen: 2.5617482662200928 (+0.46750926971435547) + | > avg_loss_kl: 2.282933235168457 (-0.15817511081695557) + | > avg_loss_feat: 6.779177904129028 (+0.9119322299957275) + | > avg_loss_mel: 18.467300415039062 (+1.7537822723388672) + | > avg_loss_duration: 1.667001485824585 (-0.009543240070343018) + | > avg_loss_1: 31.758161544799805 (+2.965505599975586) + + + > EPOCH: 166/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:43:45)  + + --> TIME: 2025-04-19 01:43:53 -- STEP: 21/142 -- GLOBAL_STEP: 402875 + | > loss_disc: 2.3850998878479004 (2.488468272345407) + | > loss_disc_real_0: 0.11261317878961563 (0.12454944210393089) + | > loss_disc_real_1: 0.21056827902793884 (0.2112986920844941) + | > loss_disc_real_2: 0.23460270464420319 (0.2254857456400281) + | > loss_disc_real_3: 0.2338368147611618 (0.23149047579084123) + | > loss_disc_real_4: 0.242366224527359 (0.2322666503134228) + | > loss_disc_real_5: 0.20747072994709015 (0.23450409230731784) + | > loss_0: 2.3850998878479004 (2.488468272345407) + | > grad_norm_0: tensor(15.3976, device='cuda:0') (tensor(10.0664, device='cuda:0')) + | > loss_gen: 2.561359405517578 (2.3629538218180337) + | > loss_kl: 1.8644518852233887 (1.8637169315701438) + | > loss_feat: 7.9083709716796875 (7.250439030783517) + | > loss_mel: 19.010269165039062 (18.24048342023577) + | > loss_duration: 1.4594924449920654 (1.442536717369443) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.8039436340332 (31.16012991042364) + | > grad_norm_1: tensor(387.6789, device='cuda:0') (tensor(222.9158, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.3445 (0.3418138481321789) + | > loader_time: 0.0048 (0.0047416800544375455) + + + --> TIME: 2025-04-19 01:44:02 -- STEP: 46/142 -- GLOBAL_STEP: 402900 + | > loss_disc: 2.4104015827178955 (2.472324993299402) + | > loss_disc_real_0: 0.1557028740644455 (0.12386190956053526) + | > loss_disc_real_1: 0.2194683700799942 (0.21169710483240045) + | > loss_disc_real_2: 0.22301159799098969 (0.22898420691490173) + | > loss_disc_real_3: 0.21723873913288116 (0.2300828530088715) + | > loss_disc_real_4: 0.2525210976600647 (0.22877324890831244) + | > loss_disc_real_5: 0.1807917207479477 (0.23212747794130575) + | > loss_0: 2.4104015827178955 (2.472324993299402) + | > grad_norm_0: tensor(15.3910, device='cuda:0') (tensor(10.9421, device='cuda:0')) + | > loss_gen: 2.4272148609161377 (2.3612367433050396) + | > loss_kl: 1.8908880949020386 (1.843896570413009) + | > loss_feat: 7.3383283615112305 (7.125739201255467) + | > loss_mel: 17.504863739013672 (18.0108583699102) + | > loss_duration: 1.432405710220337 (1.4518242763436362) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.59370231628418 (30.79355538409689) + | > grad_norm_1: tensor(544.0573, device='cuda:0') (tensor(282.6785, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.3675 (0.34902224851691205) + | > loader_time: 0.0058 (0.0051218374915744935) + + + --> TIME: 2025-04-19 01:44:12 -- STEP: 71/142 -- GLOBAL_STEP: 402925 + | > loss_disc: 2.586022138595581 (2.4691037661592725) + | > loss_disc_real_0: 0.09080363065004349 (0.12355365383792931) + | > loss_disc_real_1: 0.2341555804014206 (0.21004764672736048) + | > loss_disc_real_2: 0.24595488607883453 (0.22870805376852063) + | > loss_disc_real_3: 0.2377522736787796 (0.229731442433008) + | > loss_disc_real_4: 0.24277092516422272 (0.2277894179585954) + | > loss_disc_real_5: 0.25506895780563354 (0.23273491040921548) + | > loss_0: 2.586022138595581 (2.4691037661592725) + | > grad_norm_0: tensor(7.3931, device='cuda:0') (tensor(11.2482, device='cuda:0')) + | > loss_gen: 2.237539529800415 (2.3584825556043167) + | > loss_kl: 1.9501656293869019 (1.8575255299957705) + | > loss_feat: 6.101593971252441 (7.0847639836056135) + | > loss_mel: 17.062490463256836 (17.93940374884806) + | > loss_duration: 1.4950227737426758 (1.4633942254832095) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.846813201904297 (30.703570191289337) + | > grad_norm_1: tensor(131.7780, device='cuda:0') (tensor(286.4700, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.3851 (0.35936313615718357) + | > loader_time: 0.006 (0.0054452184220434895) + + + --> TIME: 2025-04-19 01:44:22 -- STEP: 96/142 -- GLOBAL_STEP: 402950 + | > loss_disc: 2.583599328994751 (2.476173192262649) + | > loss_disc_real_0: 0.12300663441419601 (0.12558387763177353) + | > loss_disc_real_1: 0.21615754067897797 (0.21132281329482794) + | > loss_disc_real_2: 0.21467752754688263 (0.2290928956742088) + | > loss_disc_real_3: 0.23074749112129211 (0.22910343514134487) + | > loss_disc_real_4: 0.2253050059080124 (0.2280163623703023) + | > loss_disc_real_5: 0.20931822061538696 (0.23162204182396331) + | > loss_0: 2.583599328994751 (2.476173192262649) + | > grad_norm_0: tensor(8.0797, device='cuda:0') (tensor(10.4675, device='cuda:0')) + | > loss_gen: 2.186609983444214 (2.347439376016457) + | > loss_kl: 1.8435821533203125 (1.8751998605827491) + | > loss_feat: 6.715229511260986 (7.01948810617129) + | > loss_mel: 16.93117332458496 (17.911927521228794) + | > loss_duration: 1.5020732879638672 (1.474596766134103) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.178667068481445 (30.62865173816681) + | > grad_norm_1: tensor(279.5984, device='cuda:0') (tensor(259.7440, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.4041 (0.3695220972100893) + | > loader_time: 0.0068 (0.005711687107880909) + + + --> TIME: 2025-04-19 01:44:33 -- STEP: 121/142 -- GLOBAL_STEP: 402975 + | > loss_disc: 2.392186164855957 (2.4775049430279683) + | > loss_disc_real_0: 0.10729892551898956 (0.12719289724491842) + | > loss_disc_real_1: 0.1856512427330017 (0.21047038160079767) + | > loss_disc_real_2: 0.21744924783706665 (0.228604773598269) + | > loss_disc_real_3: 0.22285409271717072 (0.22937667468362602) + | > loss_disc_real_4: 0.21484030783176422 (0.22899335339542262) + | > loss_disc_real_5: 0.2069367915391922 (0.23195071296751008) + | > loss_0: 2.392186164855957 (2.4775049430279683) + | > grad_norm_0: tensor(10.2512, device='cuda:0') (tensor(10.5896, device='cuda:0')) + | > loss_gen: 2.3172414302825928 (2.338478021385256) + | > loss_kl: 2.0944550037384033 (1.8848041355117293) + | > loss_feat: 7.251056671142578 (6.94445505221028) + | > loss_mel: 18.099796295166016 (17.816558246770178) + | > loss_duration: 1.5517652034759521 (1.481429930560845) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.314315795898438 (30.465725496780774) + | > grad_norm_1: tensor(246.0501, device='cuda:0') (tensor(250.5022, device='cuda:0')) + | > current_lr_0: 0.00019589250591331656 + | > current_lr_1: 0.00019589250591331656 + | > step_time: 0.452 (0.3804540220370962) + | > loader_time: 0.0079 (0.005944502255148138) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.530876398086548 (2.530876398086548) + | > loss_disc_real_0: 0.1032547876238823 (0.1032547876238823) + | > loss_disc_real_1: 0.22232364118099213 (0.22232364118099213) + | > loss_disc_real_2: 0.22542032599449158 (0.22542032599449158) + | > loss_disc_real_3: 0.2753256857395172 (0.2753256857395172) + | > loss_disc_real_4: 0.2212730348110199 (0.2212730348110199) + | > loss_disc_real_5: 0.25221365690231323 (0.25221365690231323) + | > loss_0: 2.530876398086548 (2.530876398086548) + | > loss_gen: 2.236165761947632 (2.236165761947632) + | > loss_kl: 2.658600091934204 (2.658600091934204) + | > loss_feat: 7.870993614196777 (7.870993614196777) + | > loss_mel: 19.128807067871094 (19.128807067871094) + | > loss_duration: 1.6566253900527954 (1.6566253900527954) + | > loss_1: 33.55119323730469 (33.55119323730469) + + --> STEP: 1 + | > loss_disc: 2.6827406883239746 (2.6827406883239746) + | > loss_disc_real_0: 0.1458338350057602 (0.1458338350057602) + | > loss_disc_real_1: 0.2714654803276062 (0.2714654803276062) + | > loss_disc_real_2: 0.2173870950937271 (0.2173870950937271) + | > loss_disc_real_3: 0.25476011633872986 (0.25476011633872986) + | > loss_disc_real_4: 0.2301589846611023 (0.2301589846611023) + | > loss_disc_real_5: 0.2620100975036621 (0.2620100975036621) + | > loss_0: 2.6827406883239746 (2.6827406883239746) + | > loss_gen: 2.012071132659912 (2.012071132659912) + | > loss_kl: 2.397245168685913 (2.397245168685913) + | > loss_feat: 6.212799549102783 (6.212799549102783) + | > loss_mel: 15.731013298034668 (15.731013298034668) + | > loss_duration: 1.698824167251587 (1.698824167251587) + | > loss_1: 28.051952362060547 (28.051952362060547) + + --> STEP: 2 + | > loss_disc: 2.4886510372161865 (2.5856958627700806) + | > loss_disc_real_0: 0.1580236405134201 (0.15192873775959015) + | > loss_disc_real_1: 0.2011864185333252 (0.2363259494304657) + | > loss_disc_real_2: 0.21601711213588715 (0.21670210361480713) + | > loss_disc_real_3: 0.24099700152873993 (0.2478785589337349) + | > loss_disc_real_4: 0.2089443802833557 (0.219551682472229) + | > loss_disc_real_5: 0.23850208520889282 (0.25025609135627747) + | > loss_0: 2.4886510372161865 (2.5856958627700806) + | > loss_gen: 2.1882495880126953 (2.1001603603363037) + | > loss_kl: 2.0856821537017822 (2.2414636611938477) + | > loss_feat: 6.556183815002441 (6.384491682052612) + | > loss_mel: 17.21847152709961 (16.47474241256714) + | > loss_duration: 1.6415505409240723 (1.6701873540878296) + | > loss_1: 29.690135955810547 (28.871044158935547) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037779808044433594 (+0.0003314018249511719) + | > avg_loss_disc: 2.5856958627700806 (-0.024077653884887695) + | > avg_loss_disc_real_0: 0.15192873775959015 (-0.009016312658786774) + | > avg_loss_disc_real_1: 0.2363259494304657 (-0.09281617403030396) + | > avg_loss_disc_real_2: 0.21670210361480713 (-0.03785385936498642) + | > avg_loss_disc_real_3: 0.2478785589337349 (-0.007819168269634247) + | > avg_loss_disc_real_4: 0.219551682472229 (-0.04156060516834259) + | > avg_loss_disc_real_5: 0.25025609135627747 (-0.03191094100475311) + | > avg_loss_0: 2.5856958627700806 (-0.024077653884887695) + | > avg_loss_gen: 2.1001603603363037 (-0.46158790588378906) + | > avg_loss_kl: 2.2414636611938477 (-0.041469573974609375) + | > avg_loss_feat: 6.384491682052612 (-0.394686222076416) + | > avg_loss_mel: 16.47474241256714 (-1.9925580024719238) + | > avg_loss_duration: 1.6701873540878296 (+0.003185868263244629) + | > avg_loss_1: 28.871044158935547 (-2.887117385864258) + + + > EPOCH: 167/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:44:44)  + + --> TIME: 2025-04-19 01:44:46 -- STEP: 4/142 -- GLOBAL_STEP: 403000 + | > loss_disc: 2.463758707046509 (2.491587996482849) + | > loss_disc_real_0: 0.06828349828720093 (0.13352929800748825) + | > loss_disc_real_1: 0.22540393471717834 (0.229835607111454) + | > loss_disc_real_2: 0.23054388165473938 (0.24614587426185608) + | > loss_disc_real_3: 0.20624348521232605 (0.2461625337600708) + | > loss_disc_real_4: 0.25157177448272705 (0.24617193639278412) + | > loss_disc_real_5: 0.20941703021526337 (0.24478325620293617) + | > loss_0: 2.463758707046509 (2.491587996482849) + | > grad_norm_0: tensor(7.2837, device='cuda:0') (tensor(8.3529, device='cuda:0')) + | > loss_gen: 2.285165309906006 (2.4239226579666138) + | > loss_kl: 1.9220943450927734 (1.9005194306373596) + | > loss_feat: 8.280633926391602 (7.417357683181763) + | > loss_mel: 19.427082061767578 (18.221299171447754) + | > loss_duration: 1.450778841972351 (1.4431798160076141) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.365753173828125 (31.40627908706665) + | > grad_norm_1: tensor(242.8556, device='cuda:0') (tensor(241.9736, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.3214 (0.32404667139053345) + | > loader_time: 0.0051 (0.004113495349884033) + + + --> TIME: 2025-04-19 01:44:54 -- STEP: 29/142 -- GLOBAL_STEP: 403025 + | > loss_disc: 2.424492597579956 (2.458285331726074) + | > loss_disc_real_0: 0.14338067173957825 (0.12426158246295206) + | > loss_disc_real_1: 0.21224413812160492 (0.21020220528388844) + | > loss_disc_real_2: 0.22241643071174622 (0.22722541874852673) + | > loss_disc_real_3: 0.2444320172071457 (0.22460187509142118) + | > loss_disc_real_4: 0.24157488346099854 (0.23054035367636844) + | > loss_disc_real_5: 0.18574878573417664 (0.23366108279803705) + | > loss_0: 2.424492597579956 (2.458285331726074) + | > grad_norm_0: tensor(18.1996, device='cuda:0') (tensor(10.4016, device='cuda:0')) + | > loss_gen: 2.430572509765625 (2.39735379712335) + | > loss_kl: 1.7938635349273682 (1.8750004316198414) + | > loss_feat: 7.441938400268555 (7.390406608581543) + | > loss_mel: 17.433406829833984 (18.227286240150185) + | > loss_duration: 1.449636459350586 (1.4437259600080292) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.54941749572754 (31.333773119696254) + | > grad_norm_1: tensor(426.7239, device='cuda:0') (tensor(259.9240, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.3464 (0.34182839558042327) + | > loader_time: 0.005 (0.004855920528543407) + + + --> TIME: 2025-04-19 01:45:04 -- STEP: 54/142 -- GLOBAL_STEP: 403050 + | > loss_disc: 2.3863885402679443 (2.449053472942777) + | > loss_disc_real_0: 0.07159002870321274 (0.12595771032351036) + | > loss_disc_real_1: 0.20369641482830048 (0.20741544001632267) + | > loss_disc_real_2: 0.22183221578598022 (0.22571650947685595) + | > loss_disc_real_3: 0.24053548276424408 (0.22475095727929362) + | > loss_disc_real_4: 0.20144540071487427 (0.22659310781293446) + | > loss_disc_real_5: 0.19508181512355804 (0.23063558947156976) + | > loss_0: 2.3863885402679443 (2.449053472942777) + | > grad_norm_0: tensor(12.8137, device='cuda:0') (tensor(13.8469, device='cuda:0')) + | > loss_gen: 2.292349338531494 (2.3940381915481) + | > loss_kl: 1.9520697593688965 (1.884534275090253) + | > loss_feat: 7.649092197418213 (7.329255978266398) + | > loss_mel: 17.70876121520996 (18.04901144239637) + | > loss_duration: 1.4721472263336182 (1.4525755643844604) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.074419021606445 (31.109415548819083) + | > grad_norm_1: tensor(391.8113, device='cuda:0') (tensor(307.1145, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.3696 (0.3507124053107369) + | > loader_time: 0.0059 (0.005246471475671839) + + + --> TIME: 2025-04-19 01:45:14 -- STEP: 79/142 -- GLOBAL_STEP: 403075 + | > loss_disc: 2.423896312713623 (2.4510655765292024) + | > loss_disc_real_0: 0.1116405501961708 (0.12413613479348677) + | > loss_disc_real_1: 0.17934457957744598 (0.20546660864654975) + | > loss_disc_real_2: 0.20360518991947174 (0.2271926206878469) + | > loss_disc_real_3: 0.22185547649860382 (0.22680185131634337) + | > loss_disc_real_4: 0.24712982773780823 (0.22825095585629906) + | > loss_disc_real_5: 0.2613632082939148 (0.23400624813158302) + | > loss_0: 2.423896312713623 (2.4510655765292024) + | > grad_norm_0: tensor(9.2099, device='cuda:0') (tensor(13.1661, device='cuda:0')) + | > loss_gen: 2.383481740951538 (2.3803208930582946) + | > loss_kl: 1.9823254346847534 (1.8944999900045274) + | > loss_feat: 6.978154182434082 (7.179878240899194) + | > loss_mel: 17.081806182861328 (17.79180151299585) + | > loss_duration: 1.5212485790252686 (1.4651338254349144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9470157623291 (30.711634551422506) + | > grad_norm_1: tensor(302.8858, device='cuda:0') (tensor(301.6740, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.3852 (0.3615325372430343) + | > loader_time: 0.0066 (0.005593631840959381) + + + --> TIME: 2025-04-19 01:45:24 -- STEP: 104/142 -- GLOBAL_STEP: 403100 + | > loss_disc: 2.5440611839294434 (2.4557310572037343) + | > loss_disc_real_0: 0.10781145095825195 (0.1256723792936939) + | > loss_disc_real_1: 0.20485664904117584 (0.20708318069004095) + | > loss_disc_real_2: 0.23255373537540436 (0.22667813057509753) + | > loss_disc_real_3: 0.24243074655532837 (0.22718156788211602) + | > loss_disc_real_4: 0.24281300604343414 (0.2269636588887526) + | > loss_disc_real_5: 0.22179023921489716 (0.2336514242566549) + | > loss_0: 2.5440611839294434 (2.4557310572037343) + | > grad_norm_0: tensor(12.6263, device='cuda:0') (tensor(12.3333, device='cuda:0')) + | > loss_gen: 2.210317373275757 (2.3699591847566452) + | > loss_kl: 1.9946284294128418 (1.9067540019750595) + | > loss_feat: 7.332474231719971 (7.135682564515334) + | > loss_mel: 17.748241424560547 (17.82118408496564) + | > loss_duration: 1.526857614517212 (1.4764662350599589) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.812519073486328 (30.710046144632194) + | > grad_norm_1: tensor(370.6961, device='cuda:0') (tensor(287.1013, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.4081 (0.3726288378238678) + | > loader_time: 0.0069 (0.0058712432017693165) + + + --> TIME: 2025-04-19 01:45:35 -- STEP: 129/142 -- GLOBAL_STEP: 403125 + | > loss_disc: 2.501849889755249 (2.4621287408725245) + | > loss_disc_real_0: 0.09158717095851898 (0.12568986283023223) + | > loss_disc_real_1: 0.18033500015735626 (0.20717031907203587) + | > loss_disc_real_2: 0.21503444015979767 (0.2267169937375904) + | > loss_disc_real_3: 0.23069539666175842 (0.22702331829440686) + | > loss_disc_real_4: 0.20642277598381042 (0.22844758953234942) + | > loss_disc_real_5: 0.20749667286872864 (0.23428585247475972) + | > loss_0: 2.501849889755249 (2.4621287408725245) + | > grad_norm_0: tensor(11.1190, device='cuda:0') (tensor(11.6864, device='cuda:0')) + | > loss_gen: 2.19667649269104 (2.3536412734393926) + | > loss_kl: 1.966383695602417 (1.9133108484652614) + | > loss_feat: 7.011504173278809 (7.070288277411645) + | > loss_mel: 17.44231605529785 (17.765190715937653) + | > loss_duration: 1.501222848892212 (1.483109972273657) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.11810302734375 (30.585541111554285) + | > grad_norm_1: tensor(351.5311, device='cuda:0') (tensor(270.8328, device='cuda:0')) + | > current_lr_0: 0.0001958680193500774 + | > current_lr_1: 0.0001958680193500774 + | > step_time: 0.4434 (0.383771632068841) + | > loader_time: 0.0075 (0.006139675776163738) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5141170024871826 (2.5141170024871826) + | > loss_disc_real_0: 0.14684133231639862 (0.14684133231639862) + | > loss_disc_real_1: 0.25411006808280945 (0.25411006808280945) + | > loss_disc_real_2: 0.21972239017486572 (0.21972239017486572) + | > loss_disc_real_3: 0.24834434688091278 (0.24834434688091278) + | > loss_disc_real_4: 0.22480922937393188 (0.22480922937393188) + | > loss_disc_real_5: 0.23403006792068481 (0.23403006792068481) + | > loss_0: 2.5141170024871826 (2.5141170024871826) + | > loss_gen: 2.318248748779297 (2.318248748779297) + | > loss_kl: 2.1324870586395264 (2.1324870586395264) + | > loss_feat: 6.426450252532959 (6.426450252532959) + | > loss_mel: 18.186283111572266 (18.186283111572266) + | > loss_duration: 1.6511623859405518 (1.6511623859405518) + | > loss_1: 30.714632034301758 (30.714632034301758) + + --> STEP: 1 + | > loss_disc: 2.4239866733551025 (2.4239866733551025) + | > loss_disc_real_0: 0.14391791820526123 (0.14391791820526123) + | > loss_disc_real_1: 0.20478974282741547 (0.20478974282741547) + | > loss_disc_real_2: 0.20815874636173248 (0.20815874636173248) + | > loss_disc_real_3: 0.24222606420516968 (0.24222606420516968) + | > loss_disc_real_4: 0.20909138023853302 (0.20909138023853302) + | > loss_disc_real_5: 0.2002423107624054 (0.2002423107624054) + | > loss_0: 2.4239866733551025 (2.4239866733551025) + | > loss_gen: 2.168898344039917 (2.168898344039917) + | > loss_kl: 2.6381168365478516 (2.6381168365478516) + | > loss_feat: 6.48962926864624 (6.48962926864624) + | > loss_mel: 17.88149070739746 (17.88149070739746) + | > loss_duration: 1.6912175416946411 (1.6912175416946411) + | > loss_1: 30.869352340698242 (30.869352340698242) + + --> STEP: 2 + | > loss_disc: 2.489917755126953 (2.456952214241028) + | > loss_disc_real_0: 0.15516838431358337 (0.1495431512594223) + | > loss_disc_real_1: 0.22929510474205017 (0.21704242378473282) + | > loss_disc_real_2: 0.2184261679649353 (0.2132924571633339) + | > loss_disc_real_3: 0.2323572188615799 (0.2372916415333748) + | > loss_disc_real_4: 0.20448791980743408 (0.20678965002298355) + | > loss_disc_real_5: 0.21036586165428162 (0.2053040862083435) + | > loss_0: 2.489917755126953 (2.456952214241028) + | > loss_gen: 2.1967997550964355 (2.1828490495681763) + | > loss_kl: 2.3293919563293457 (2.4837543964385986) + | > loss_feat: 7.573921203613281 (7.031775236129761) + | > loss_mel: 19.138710021972656 (18.51010036468506) + | > loss_duration: 1.669010877609253 (1.680114209651947) + | > loss_1: 32.907833099365234 (31.88859272003174) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032252073287963867 (-0.0005527734756469727) + | > avg_loss_disc: 2.456952214241028 (-0.12874364852905273) + | > avg_loss_disc_real_0: 0.1495431512594223 (-0.0023855865001678467) + | > avg_loss_disc_real_1: 0.21704242378473282 (-0.01928352564573288) + | > avg_loss_disc_real_2: 0.2132924571633339 (-0.003409646451473236) + | > avg_loss_disc_real_3: 0.2372916415333748 (-0.010586917400360107) + | > avg_loss_disc_real_4: 0.20678965002298355 (-0.012762032449245453) + | > avg_loss_disc_real_5: 0.2053040862083435 (-0.04495200514793396) + | > avg_loss_0: 2.456952214241028 (-0.12874364852905273) + | > avg_loss_gen: 2.1828490495681763 (+0.08268868923187256) + | > avg_loss_kl: 2.4837543964385986 (+0.24229073524475098) + | > avg_loss_feat: 7.031775236129761 (+0.6472835540771484) + | > avg_loss_mel: 18.51010036468506 (+2.03535795211792) + | > avg_loss_duration: 1.680114209651947 (+0.009926855564117432) + | > avg_loss_1: 31.88859272003174 (+3.0175485610961914) + + + > EPOCH: 168/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:45:43)  + + --> TIME: 2025-04-19 01:45:47 -- STEP: 12/142 -- GLOBAL_STEP: 403150 + | > loss_disc: 2.462991952896118 (2.432013432184855) + | > loss_disc_real_0: 0.08133500814437866 (0.11137018352746964) + | > loss_disc_real_1: 0.200474351644516 (0.20144804318745932) + | > loss_disc_real_2: 0.2111133486032486 (0.22841075186928114) + | > loss_disc_real_3: 0.21602457761764526 (0.22843619684378305) + | > loss_disc_real_4: 0.21346548199653625 (0.22944493343432745) + | > loss_disc_real_5: 0.22344687581062317 (0.2292564163605372) + | > loss_0: 2.462991952896118 (2.432013432184855) + | > grad_norm_0: tensor(8.5066, device='cuda:0') (tensor(8.2275, device='cuda:0')) + | > loss_gen: 2.3354580402374268 (2.3720774054527283) + | > loss_kl: 1.8506271839141846 (1.8076939086119335) + | > loss_feat: 7.435379981994629 (7.353730122248332) + | > loss_mel: 19.2958927154541 (18.063485940297443) + | > loss_duration: 1.4377036094665527 (1.4361942907174428) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.35506057739258 (31.03318166732788) + | > grad_norm_1: tensor(60.6555, device='cuda:0') (tensor(223.3990, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.3303 (0.32995231946309406) + | > loader_time: 0.0045 (0.004454354445139567) + + + --> TIME: 2025-04-19 01:45:56 -- STEP: 37/142 -- GLOBAL_STEP: 403175 + | > loss_disc: 2.452925682067871 (2.443565465308524) + | > loss_disc_real_0: 0.15481074154376984 (0.11712226815320351) + | > loss_disc_real_1: 0.18396078050136566 (0.2046228284771378) + | > loss_disc_real_2: 0.22436872124671936 (0.22752878875345797) + | > loss_disc_real_3: 0.22311817109584808 (0.22576008616267024) + | > loss_disc_real_4: 0.25580868124961853 (0.22755624192792015) + | > loss_disc_real_5: 0.2304648458957672 (0.23585561844142708) + | > loss_0: 2.452925682067871 (2.443565465308524) + | > grad_norm_0: tensor(10.8568, device='cuda:0') (tensor(9.5547, device='cuda:0')) + | > loss_gen: 2.343331813812256 (2.3732461220509298) + | > loss_kl: 1.7435368299484253 (1.8331026515445195) + | > loss_feat: 6.8409247398376465 (7.245153040499301) + | > loss_mel: 16.921791076660156 (17.973365190866833) + | > loss_duration: 1.4829145669937134 (1.4487254684035842) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.33249855041504 (30.87359242825895) + | > grad_norm_1: tensor(230.4235, device='cuda:0') (tensor(239.3402, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.3547 (0.3413720195357864) + | > loader_time: 0.0054 (0.0049303995596395945) + + + --> TIME: 2025-04-19 01:46:06 -- STEP: 62/142 -- GLOBAL_STEP: 403200 + | > loss_disc: 2.3954389095306396 (2.44843574493162) + | > loss_disc_real_0: 0.05796172842383385 (0.1209384625357005) + | > loss_disc_real_1: 0.219826802611351 (0.20357909750553868) + | > loss_disc_real_2: 0.23294265568256378 (0.22502214629803935) + | > loss_disc_real_3: 0.20541754364967346 (0.22563026052328847) + | > loss_disc_real_4: 0.207615464925766 (0.22852332433385233) + | > loss_disc_real_5: 0.19840195775032043 (0.23472418756254257) + | > loss_0: 2.3954389095306396 (2.44843574493162) + | > grad_norm_0: tensor(12.2689, device='cuda:0') (tensor(12.3004, device='cuda:0')) + | > loss_gen: 2.5653457641601562 (2.3847459670036066) + | > loss_kl: 2.0168581008911133 (1.8445703771806532) + | > loss_feat: 8.445253372192383 (7.157171210935039) + | > loss_mel: 19.53026580810547 (17.839453343422182) + | > loss_duration: 1.4977188110351562 (1.4596487456752405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.055442810058594 (30.685589544234738) + | > grad_norm_1: tensor(470.2911, device='cuda:0') (tensor(277.8105, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.3803 (0.35255833595029773) + | > loader_time: 0.006 (0.0052292654591221946) + + + --> TIME: 2025-04-19 01:46:16 -- STEP: 87/142 -- GLOBAL_STEP: 403225 + | > loss_disc: 2.508535861968994 (2.4708076455127226) + | > loss_disc_real_0: 0.07532133162021637 (0.12650533782682197) + | > loss_disc_real_1: 0.20366887748241425 (0.20460202382213769) + | > loss_disc_real_2: 0.25039437413215637 (0.2262951608361869) + | > loss_disc_real_3: 0.21046409010887146 (0.22854110341647577) + | > loss_disc_real_4: 0.2178100049495697 (0.22954985824809676) + | > loss_disc_real_5: 0.21499396860599518 (0.23426157493015815) + | > loss_0: 2.508535861968994 (2.4708076455127226) + | > grad_norm_0: tensor(13.7365, device='cuda:0') (tensor(15.9630, device='cuda:0')) + | > loss_gen: 2.39471697807312 (2.38389210591371) + | > loss_kl: 2.0631792545318604 (1.8697853910511937) + | > loss_feat: 6.530604839324951 (7.0889641224652875) + | > loss_mel: 18.402729034423828 (17.810501723453918) + | > loss_duration: 1.5288779735565186 (1.4719891904414386) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.920106887817383 (30.625132462074017) + | > grad_norm_1: tensor(265.8445, device='cuda:0') (tensor(317.8803, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.4036 (0.3639563829049298) + | > loader_time: 0.0065 (0.0055372577974165975) + + + --> TIME: 2025-04-19 01:46:26 -- STEP: 112/142 -- GLOBAL_STEP: 403250 + | > loss_disc: 2.5404016971588135 (2.4776924167360583) + | > loss_disc_real_0: 0.11112918704748154 (0.12902985440034956) + | > loss_disc_real_1: 0.21501289308071136 (0.2055788648181728) + | > loss_disc_real_2: 0.21966882050037384 (0.22685736643948726) + | > loss_disc_real_3: 0.21832086145877838 (0.22881395702383348) + | > loss_disc_real_4: 0.2416856437921524 (0.22995654360524245) + | > loss_disc_real_5: 0.20713849365711212 (0.23540103808045387) + | > loss_0: 2.5404016971588135 (2.4776924167360583) + | > grad_norm_0: tensor(7.8662, device='cuda:0') (tensor(15.5129, device='cuda:0')) + | > loss_gen: 2.0972418785095215 (2.368056188736642) + | > loss_kl: 1.8382915258407593 (1.8807862933192934) + | > loss_feat: 6.0114288330078125 (7.036098923001971) + | > loss_mel: 16.38451385498047 (17.787014714309144) + | > loss_duration: 1.461101770401001 (1.4804729904447282) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.792577743530273 (30.552429062979563) + | > grad_norm_1: tensor(101.3697, device='cuda:0') (tensor(302.0338, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.4264 (0.37463134952953897) + | > loader_time: 0.0071 (0.005810873849051337) + + + --> TIME: 2025-04-19 01:46:37 -- STEP: 137/142 -- GLOBAL_STEP: 403275 + | > loss_disc: 2.5849509239196777 (2.4751630734353176) + | > loss_disc_real_0: 0.14202561974525452 (0.12943009270803774) + | > loss_disc_real_1: 0.21303024888038635 (0.20547664905116506) + | > loss_disc_real_2: 0.2686161696910858 (0.2266370633440296) + | > loss_disc_real_3: 0.270744264125824 (0.22938407446346143) + | > loss_disc_real_4: 0.29915255308151245 (0.22948896906671734) + | > loss_disc_real_5: 0.27928951382637024 (0.23453183189360766) + | > loss_0: 2.5849509239196777 (2.4751630734353176) + | > grad_norm_0: tensor(4.0403, device='cuda:0') (tensor(14.0212, device='cuda:0')) + | > loss_gen: 2.3614206314086914 (2.359058919614248) + | > loss_kl: 1.905964970588684 (1.88535645495366) + | > loss_feat: 7.051553726196289 (7.027806341213031) + | > loss_mel: 17.64288902282715 (17.781726718819048) + | > loss_duration: 1.513363242149353 (1.486852491859102) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.475189208984375 (30.540800860328396) + | > grad_norm_1: tensor(77.5233, device='cuda:0') (tensor(269.8116, device='cuda:0')) + | > current_lr_0: 0.00019584353584765863 + | > current_lr_1: 0.00019584353584765863 + | > step_time: 0.4572 (0.38659310688937676) + | > loader_time: 0.0072 (0.0060543665920730905) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.531964063644409 (2.531964063644409) + | > loss_disc_real_0: 0.21308378875255585 (0.21308378875255585) + | > loss_disc_real_1: 0.24015207588672638 (0.24015207588672638) + | > loss_disc_real_2: 0.29433751106262207 (0.29433751106262207) + | > loss_disc_real_3: 0.2946788966655731 (0.2946788966655731) + | > loss_disc_real_4: 0.24399501085281372 (0.24399501085281372) + | > loss_disc_real_5: 0.24903744459152222 (0.24903744459152222) + | > loss_0: 2.531964063644409 (2.531964063644409) + | > loss_gen: 2.62746262550354 (2.62746262550354) + | > loss_kl: 2.4421913623809814 (2.4421913623809814) + | > loss_feat: 6.897793292999268 (6.897793292999268) + | > loss_mel: 19.70296859741211 (19.70296859741211) + | > loss_duration: 1.6751763820648193 (1.6751763820648193) + | > loss_1: 33.3455924987793 (33.3455924987793) + + --> STEP: 1 + | > loss_disc: 2.5054028034210205 (2.5054028034210205) + | > loss_disc_real_0: 0.20357385277748108 (0.20357385277748108) + | > loss_disc_real_1: 0.18369418382644653 (0.18369418382644653) + | > loss_disc_real_2: 0.26434531807899475 (0.26434531807899475) + | > loss_disc_real_3: 0.22325733304023743 (0.22325733304023743) + | > loss_disc_real_4: 0.2962258458137512 (0.2962258458137512) + | > loss_disc_real_5: 0.23921628296375275 (0.23921628296375275) + | > loss_0: 2.5054028034210205 (2.5054028034210205) + | > loss_gen: 2.511500120162964 (2.511500120162964) + | > loss_kl: 2.594431161880493 (2.594431161880493) + | > loss_feat: 7.2756500244140625 (7.2756500244140625) + | > loss_mel: 19.5446720123291 (19.5446720123291) + | > loss_duration: 1.6854865550994873 (1.6854865550994873) + | > loss_1: 33.61174011230469 (33.61174011230469) + + --> STEP: 2 + | > loss_disc: 2.5207419395446777 (2.513072371482849) + | > loss_disc_real_0: 0.1810150444507599 (0.19229444861412048) + | > loss_disc_real_1: 0.21225500106811523 (0.19797459244728088) + | > loss_disc_real_2: 0.3066639304161072 (0.28550462424755096) + | > loss_disc_real_3: 0.2680490016937256 (0.2456531673669815) + | > loss_disc_real_4: 0.19999314844608307 (0.24810949712991714) + | > loss_disc_real_5: 0.246408149600029 (0.24281221628189087) + | > loss_0: 2.5207419395446777 (2.513072371482849) + | > loss_gen: 2.4266903400421143 (2.469095230102539) + | > loss_kl: 2.1061325073242188 (2.350281834602356) + | > loss_feat: 7.433060169219971 (7.354355096817017) + | > loss_mel: 19.37575912475586 (19.46021556854248) + | > loss_duration: 1.6449041366577148 (1.665195345878601) + | > loss_1: 32.98654556274414 (33.299142837524414) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036405324935913086 (+0.0004153251647949219) + | > avg_loss_disc: 2.513072371482849 (+0.05612015724182129) + | > avg_loss_disc_real_0: 0.19229444861412048 (+0.04275129735469818) + | > avg_loss_disc_real_1: 0.19797459244728088 (-0.019067831337451935) + | > avg_loss_disc_real_2: 0.28550462424755096 (+0.07221216708421707) + | > avg_loss_disc_real_3: 0.2456531673669815 (+0.00836152583360672) + | > avg_loss_disc_real_4: 0.24810949712991714 (+0.041319847106933594) + | > avg_loss_disc_real_5: 0.24281221628189087 (+0.03750813007354736) + | > avg_loss_0: 2.513072371482849 (+0.05612015724182129) + | > avg_loss_gen: 2.469095230102539 (+0.2862461805343628) + | > avg_loss_kl: 2.350281834602356 (-0.13347256183624268) + | > avg_loss_feat: 7.354355096817017 (+0.32257986068725586) + | > avg_loss_mel: 19.46021556854248 (+0.9501152038574219) + | > avg_loss_duration: 1.665195345878601 (-0.014918863773345947) + | > avg_loss_1: 33.299142837524414 (+1.4105501174926758) + + + > EPOCH: 169/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:46:41)  + + --> TIME: 2025-04-19 01:46:48 -- STEP: 20/142 -- GLOBAL_STEP: 403300 + | > loss_disc: 2.4676032066345215 (2.4499300599098204) + | > loss_disc_real_0: 0.10831707715988159 (0.1226508654654026) + | > loss_disc_real_1: 0.211282879114151 (0.20649363175034524) + | > loss_disc_real_2: 0.2666274905204773 (0.23193623274564742) + | > loss_disc_real_3: 0.2501375377178192 (0.22492440342903136) + | > loss_disc_real_4: 0.23437689244747162 (0.22646498084068298) + | > loss_disc_real_5: 0.2506483197212219 (0.22964921966195107) + | > loss_0: 2.4676032066345215 (2.4499300599098204) + | > grad_norm_0: tensor(4.3565, device='cuda:0') (tensor(6.5087, device='cuda:0')) + | > loss_gen: 2.3666832447052 (2.375515401363373) + | > loss_kl: 1.846720576286316 (1.8617339372634887) + | > loss_feat: 7.091471195220947 (7.2523945569992065) + | > loss_mel: 18.23790168762207 (17.971825695037843) + | > loss_duration: 1.5009698867797852 (1.4378821074962616) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.043746948242188 (30.89935131072998) + | > grad_norm_1: tensor(89.8455, device='cuda:0') (tensor(131.6798, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.3451 (0.33031914234161375) + | > loader_time: 0.0057 (0.004797780513763427) + + + --> TIME: 2025-04-19 01:46:58 -- STEP: 45/142 -- GLOBAL_STEP: 403325 + | > loss_disc: 2.476850748062134 (2.471675258212619) + | > loss_disc_real_0: 0.09809579700231552 (0.12408474700318442) + | > loss_disc_real_1: 0.15628798305988312 (0.20889896882904901) + | > loss_disc_real_2: 0.22079376876354218 (0.2296906034151713) + | > loss_disc_real_3: 0.22320164740085602 (0.22697201205624473) + | > loss_disc_real_4: 0.21470746397972107 (0.2284246348672443) + | > loss_disc_real_5: 0.20191431045532227 (0.23210838834444683) + | > loss_0: 2.476850748062134 (2.471675258212619) + | > grad_norm_0: tensor(7.4360, device='cuda:0') (tensor(7.2301, device='cuda:0')) + | > loss_gen: 2.204435348510742 (2.3244090080261235) + | > loss_kl: 1.80373215675354 (1.8534544812308418) + | > loss_feat: 6.979915618896484 (7.057288032107883) + | > loss_mel: 17.63509178161621 (17.862564807467987) + | > loss_duration: 1.4793272018432617 (1.4505267964469062) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.102500915527344 (30.548242823282877) + | > grad_norm_1: tensor(308.6660, device='cuda:0') (tensor(158.7190, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.3752 (0.3511290974087186) + | > loader_time: 0.006 (0.005295583936903212) + + + --> TIME: 2025-04-19 01:47:08 -- STEP: 70/142 -- GLOBAL_STEP: 403350 + | > loss_disc: 2.349911689758301 (2.464359467370169) + | > loss_disc_real_0: 0.13487665355205536 (0.12354371994733811) + | > loss_disc_real_1: 0.1605483442544937 (0.20709196094955717) + | > loss_disc_real_2: 0.21182802319526672 (0.22746497286217554) + | > loss_disc_real_3: 0.24135549366474152 (0.2282132293496813) + | > loss_disc_real_4: 0.1781688630580902 (0.22857750015599387) + | > loss_disc_real_5: 0.21498163044452667 (0.23339090432439533) + | > loss_0: 2.349911689758301 (2.464359467370169) + | > grad_norm_0: tensor(6.3785, device='cuda:0') (tensor(7.0880, device='cuda:0')) + | > loss_gen: 2.3351736068725586 (2.331568745204381) + | > loss_kl: 1.928603172302246 (1.8745956914765494) + | > loss_feat: 7.4640116691589355 (7.0542922224317275) + | > loss_mel: 18.254140853881836 (17.86372955867222) + | > loss_duration: 1.5091454982757568 (1.461632558277675) + | > amp_scaler: 512.0 (318.1714285714286) + | > loss_1: 31.49107551574707 (30.585818590436663) + | > grad_norm_1: tensor(85.3549, device='cuda:0') (tensor(153.0345, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.4016 (0.36418925012860986) + | > loader_time: 0.0064 (0.005604563440595354) + + + --> TIME: 2025-04-19 01:47:18 -- STEP: 95/142 -- GLOBAL_STEP: 403375 + | > loss_disc: 2.5552239418029785 (2.473321229533145) + | > loss_disc_real_0: 0.1709297001361847 (0.1265015729948094) + | > loss_disc_real_1: 0.20363746583461761 (0.20834061079903654) + | > loss_disc_real_2: 0.21629118919372559 (0.22799856945088035) + | > loss_disc_real_3: 0.2483922690153122 (0.2284952251534713) + | > loss_disc_real_4: 0.24782156944274902 (0.2294858306646347) + | > loss_disc_real_5: 0.24975602328777313 (0.23330554021032232) + | > loss_0: 2.5552239418029785 (2.473321229533145) + | > grad_norm_0: tensor(13.9822, device='cuda:0') (tensor(7.3141, device='cuda:0')) + | > loss_gen: 2.2431867122650146 (2.3231015481446926) + | > loss_kl: 1.8045604228973389 (1.8829994803980776) + | > loss_feat: 6.133207321166992 (6.962629669591) + | > loss_mel: 17.355562210083008 (17.8570101085462) + | > loss_duration: 1.5150525569915771 (1.4737846198834872) + | > amp_scaler: 512.0 (369.17894736842106) + | > loss_1: 29.051570892333984 (30.499525311118678) + | > grad_norm_1: tensor(133.4846, device='cuda:0') (tensor(153.2120, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.4228 (0.3762935889394661) + | > loader_time: 0.0068 (0.005902157331767835) + + + --> TIME: 2025-04-19 01:47:29 -- STEP: 120/142 -- GLOBAL_STEP: 403400 + | > loss_disc: 2.5212464332580566 (2.470355151096979) + | > loss_disc_real_0: 0.1334172785282135 (0.12713239658623937) + | > loss_disc_real_1: 0.2280491292476654 (0.2074653449157874) + | > loss_disc_real_2: 0.23150429129600525 (0.22722927468518417) + | > loss_disc_real_3: 0.27118349075317383 (0.22881449088454248) + | > loss_disc_real_4: 0.27670934796333313 (0.23011592030525208) + | > loss_disc_real_5: 0.24561110138893127 (0.2333969636509816) + | > loss_0: 2.5212464332580566 (2.470355151096979) + | > grad_norm_0: tensor(8.4473, device='cuda:0') (tensor(7.7294, device='cuda:0')) + | > loss_gen: 2.344013214111328 (2.32324014902115) + | > loss_kl: 1.9163103103637695 (1.8930959105491638) + | > loss_feat: 6.968196392059326 (6.960446612040202) + | > loss_mel: 17.208093643188477 (17.785044256846103) + | > loss_duration: 1.5210126638412476 (1.4808352251847585) + | > amp_scaler: 512.0 (398.93333333333334) + | > loss_1: 29.957626342773438 (30.442662080128986) + | > grad_norm_1: tensor(269.6828, device='cuda:0') (tensor(159.2537, device='cuda:0')) + | > current_lr_0: 0.00019581905540567768 + | > current_lr_1: 0.00019581905540567768 + | > step_time: 0.4509 (0.38809539278348293) + | > loader_time: 0.0073 (0.006149520476659139) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5073082447052 (2.5073082447052) + | > loss_disc_real_0: 0.14555656909942627 (0.14555656909942627) + | > loss_disc_real_1: 0.16686099767684937 (0.16686099767684937) + | > loss_disc_real_2: 0.22550415992736816 (0.22550415992736816) + | > loss_disc_real_3: 0.18802738189697266 (0.18802738189697266) + | > loss_disc_real_4: 0.25210949778556824 (0.25210949778556824) + | > loss_disc_real_5: 0.21394787728786469 (0.21394787728786469) + | > loss_0: 2.5073082447052 (2.5073082447052) + | > loss_gen: 2.1853318214416504 (2.1853318214416504) + | > loss_kl: 2.335892915725708 (2.335892915725708) + | > loss_feat: 6.991516590118408 (6.991516590118408) + | > loss_mel: 18.228206634521484 (18.228206634521484) + | > loss_duration: 1.680128574371338 (1.680128574371338) + | > loss_1: 31.42107582092285 (31.42107582092285) + + --> STEP: 1 + | > loss_disc: 2.5595109462738037 (2.5595109462738037) + | > loss_disc_real_0: 0.16769786179065704 (0.16769786179065704) + | > loss_disc_real_1: 0.1754167228937149 (0.1754167228937149) + | > loss_disc_real_2: 0.19605688750743866 (0.19605688750743866) + | > loss_disc_real_3: 0.18990424275398254 (0.18990424275398254) + | > loss_disc_real_4: 0.19426830112934113 (0.19426830112934113) + | > loss_disc_real_5: 0.23215864598751068 (0.23215864598751068) + | > loss_0: 2.5595109462738037 (2.5595109462738037) + | > loss_gen: 2.0865118503570557 (2.0865118503570557) + | > loss_kl: 2.5452353954315186 (2.5452353954315186) + | > loss_feat: 7.6024489402771 (7.6024489402771) + | > loss_mel: 19.088951110839844 (19.088951110839844) + | > loss_duration: 1.7188748121261597 (1.7188748121261597) + | > loss_1: 33.042022705078125 (33.042022705078125) + + --> STEP: 2 + | > loss_disc: 2.5807809829711914 (2.5701459646224976) + | > loss_disc_real_0: 0.20617789030075073 (0.1869378760457039) + | > loss_disc_real_1: 0.20695234835147858 (0.19118453562259674) + | > loss_disc_real_2: 0.22513020038604736 (0.210593543946743) + | > loss_disc_real_3: 0.22386333346366882 (0.20688378810882568) + | > loss_disc_real_4: 0.2846637964248657 (0.23946604877710342) + | > loss_disc_real_5: 0.2717084586620331 (0.2519335523247719) + | > loss_0: 2.5807809829711914 (2.5701459646224976) + | > loss_gen: 2.3641357421875 (2.225323796272278) + | > loss_kl: 2.099165439605713 (2.3222004175186157) + | > loss_feat: 7.164768695831299 (7.383608818054199) + | > loss_mel: 18.9473934173584 (19.01817226409912) + | > loss_duration: 1.6400835514068604 (1.67947918176651) + | > loss_1: 32.215545654296875 (32.6287841796875) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033295154571533203 (-0.0003110170364379883) + | > avg_loss_disc: 2.5701459646224976 (+0.05707359313964844) + | > avg_loss_disc_real_0: 0.1869378760457039 (-0.0053565725684165955) + | > avg_loss_disc_real_1: 0.19118453562259674 (-0.006790056824684143) + | > avg_loss_disc_real_2: 0.210593543946743 (-0.07491108030080795) + | > avg_loss_disc_real_3: 0.20688378810882568 (-0.03876937925815582) + | > avg_loss_disc_real_4: 0.23946604877710342 (-0.00864344835281372) + | > avg_loss_disc_real_5: 0.2519335523247719 (+0.009121336042881012) + | > avg_loss_0: 2.5701459646224976 (+0.05707359313964844) + | > avg_loss_gen: 2.225323796272278 (-0.24377143383026123) + | > avg_loss_kl: 2.3222004175186157 (-0.028081417083740234) + | > avg_loss_feat: 7.383608818054199 (+0.029253721237182617) + | > avg_loss_mel: 19.01817226409912 (-0.4420433044433594) + | > avg_loss_duration: 1.67947918176651 (+0.014283835887908936) + | > avg_loss_1: 32.6287841796875 (-0.6703586578369141) + + + > EPOCH: 170/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:47:41)  + + --> TIME: 2025-04-19 01:47:43 -- STEP: 3/142 -- GLOBAL_STEP: 403425 + | > loss_disc: 2.418769121170044 (2.4215997060139975) + | > loss_disc_real_0: 0.07099085301160812 (0.1034863864382108) + | > loss_disc_real_1: 0.23878484964370728 (0.21112209558486938) + | > loss_disc_real_2: 0.2757958173751831 (0.2565063287814458) + | > loss_disc_real_3: 0.24033021926879883 (0.22111176451047262) + | > loss_disc_real_4: 0.23066484928131104 (0.23985202113787332) + | > loss_disc_real_5: 0.2510758936405182 (0.23824910819530487) + | > loss_0: 2.418769121170044 (2.4215997060139975) + | > grad_norm_0: tensor(10.6886, device='cuda:0') (tensor(8.8249, device='cuda:0')) + | > loss_gen: 2.46795654296875 (2.4491795698801675) + | > loss_kl: 1.8527073860168457 (1.8779863119125366) + | > loss_feat: 6.985546112060547 (7.396193504333496) + | > loss_mel: 18.78255271911621 (18.96883265177409) + | > loss_duration: 1.4482709169387817 (1.4436346292495728) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.537033081054688 (32.135826110839844) + | > grad_norm_1: tensor(218.0510, device='cuda:0') (tensor(274.1935, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.3242 (0.32502301534016925) + | > loader_time: 0.0043 (0.0045197804768880205) + + + --> TIME: 2025-04-19 01:47:51 -- STEP: 28/142 -- GLOBAL_STEP: 403450 + | > loss_disc: 2.497199296951294 (2.426863133907318) + | > loss_disc_real_0: 0.1202847957611084 (0.11761904800576824) + | > loss_disc_real_1: 0.21808241307735443 (0.20455328534756387) + | > loss_disc_real_2: 0.23586493730545044 (0.2270285151898861) + | > loss_disc_real_3: 0.21878041326999664 (0.22574118045823915) + | > loss_disc_real_4: 0.2400476634502411 (0.22685390497956956) + | > loss_disc_real_5: 0.22957272827625275 (0.23285594316465513) + | > loss_0: 2.497199296951294 (2.426863133907318) + | > grad_norm_0: tensor(5.3035, device='cuda:0') (tensor(7.8275, device='cuda:0')) + | > loss_gen: 2.410250663757324 (2.3894608531679427) + | > loss_kl: 1.9759973287582397 (1.8459595058645522) + | > loss_feat: 6.777576446533203 (7.206950221742902) + | > loss_mel: 17.931583404541016 (17.958790847233367) + | > loss_duration: 1.4936599731445312 (1.4458945308412825) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.589069366455078 (30.847056116376603) + | > grad_norm_1: tensor(100.6659, device='cuda:0') (tensor(203.1929, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.3435 (0.33570684705461773) + | > loader_time: 0.0052 (0.005033654826028007) + + + --> TIME: 2025-04-19 01:48:01 -- STEP: 53/142 -- GLOBAL_STEP: 403475 + | > loss_disc: 2.40700101852417 (2.451768357798738) + | > loss_disc_real_0: 0.10755015909671783 (0.12169132550370018) + | > loss_disc_real_1: 0.18457701802253723 (0.2089191717359255) + | > loss_disc_real_2: 0.23807604610919952 (0.22684402342112558) + | > loss_disc_real_3: 0.2372143268585205 (0.2266138108833781) + | > loss_disc_real_4: 0.25112709403038025 (0.23056590191598209) + | > loss_disc_real_5: 0.19879142940044403 (0.22961535853035045) + | > loss_0: 2.40700101852417 (2.451768357798738) + | > grad_norm_0: tensor(12.0929, device='cuda:0') (tensor(7.9754, device='cuda:0')) + | > loss_gen: 2.487074851989746 (2.361721605624793) + | > loss_kl: 1.8730539083480835 (1.849842166000942) + | > loss_feat: 8.386963844299316 (7.147531761313385) + | > loss_mel: 17.953176498413086 (17.810639597334944) + | > loss_duration: 1.4532073736190796 (1.453432431760824) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.15347671508789 (30.62316761376723) + | > grad_norm_1: tensor(384.9656, device='cuda:0') (tensor(192.6079, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.3682 (0.34709671308409495) + | > loader_time: 0.0062 (0.00533169170595565) + + + --> TIME: 2025-04-19 01:48:10 -- STEP: 78/142 -- GLOBAL_STEP: 403500 + | > loss_disc: 2.422916889190674 (2.4538682729769974) + | > loss_disc_real_0: 0.1051228791475296 (0.12325303953809616) + | > loss_disc_real_1: 0.16832828521728516 (0.2065291828834093) + | > loss_disc_real_2: 0.22602519392967224 (0.22589516505981103) + | > loss_disc_real_3: 0.22607526183128357 (0.2269965954698049) + | > loss_disc_real_4: 0.21388165652751923 (0.22948582871602133) + | > loss_disc_real_5: 0.21019421517848969 (0.22961021864261383) + | > loss_0: 2.422916889190674 (2.4538682729769974) + | > grad_norm_0: tensor(10.7080, device='cuda:0') (tensor(8.8809, device='cuda:0')) + | > loss_gen: 2.448927402496338 (2.3492261660404696) + | > loss_kl: 1.9423332214355469 (1.8599631648797255) + | > loss_feat: 7.411279678344727 (7.043727520184639) + | > loss_mel: 17.942890167236328 (17.719614591353974) + | > loss_duration: 1.50596022605896 (1.4663573121413207) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.25139045715332 (30.438888794336563) + | > grad_norm_1: tensor(451.6566, device='cuda:0') (tensor(209.3620, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.3923 (0.35915639461615145) + | > loader_time: 0.0061 (0.005617887545854618) + + + --> TIME: 2025-04-19 01:48:21 -- STEP: 103/142 -- GLOBAL_STEP: 403525 + | > loss_disc: 2.5119245052337646 (2.461230449306154) + | > loss_disc_real_0: 0.10171464830636978 (0.1259549778786678) + | > loss_disc_real_1: 0.17709100246429443 (0.20664225388499136) + | > loss_disc_real_2: 0.24522686004638672 (0.2276375350153562) + | > loss_disc_real_3: 0.26913294196128845 (0.22874864270386186) + | > loss_disc_real_4: 0.27661824226379395 (0.23036891569211645) + | > loss_disc_real_5: 0.2623109221458435 (0.22908569062219084) + | > loss_0: 2.5119245052337646 (2.461230449306154) + | > grad_norm_0: tensor(8.5061, device='cuda:0') (tensor(10.2557, device='cuda:0')) + | > loss_gen: 2.4530413150787354 (2.3496476515982923) + | > loss_kl: 1.8295210599899292 (1.8752996389148304) + | > loss_feat: 7.220762252807617 (7.013842068829582) + | > loss_mel: 19.050678253173828 (17.728278521195204) + | > loss_duration: 1.555802345275879 (1.4778302051488637) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.109806060791016 (30.444898105362086) + | > grad_norm_1: tensor(130.9348, device='cuda:0') (tensor(231.3654, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.4119 (0.37008915595637953) + | > loader_time: 0.0064 (0.005859050935911903) + + + --> TIME: 2025-04-19 01:48:32 -- STEP: 128/142 -- GLOBAL_STEP: 403550 + | > loss_disc: 2.5899786949157715 (2.463608672842383) + | > loss_disc_real_0: 0.12687401473522186 (0.12731613405048853) + | > loss_disc_real_1: 0.19328264892101288 (0.20700755703728643) + | > loss_disc_real_2: 0.22385409474372864 (0.22674764576368034) + | > loss_disc_real_3: 0.24808424711227417 (0.22823300235904753) + | > loss_disc_real_4: 0.23810173571109772 (0.22976237861439586) + | > loss_disc_real_5: 0.2673466205596924 (0.22949946078006178) + | > loss_0: 2.5899786949157715 (2.463608672842383) + | > grad_norm_0: tensor(6.9550, device='cuda:0') (tensor(10.7547, device='cuda:0')) + | > loss_gen: 2.2499523162841797 (2.34779173322022) + | > loss_kl: 1.8579801321029663 (1.884242537431419) + | > loss_feat: 6.048604965209961 (7.010219112038612) + | > loss_mel: 17.15924644470215 (17.712953016161915) + | > loss_duration: 1.5119202136993408 (1.4843689696863294) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.82770538330078 (30.43957543373108) + | > grad_norm_1: tensor(219.4282, device='cuda:0') (tensor(234.8812, device='cuda:0')) + | > current_lr_0: 0.00019579457802375197 + | > current_lr_1: 0.00019579457802375197 + | > step_time: 0.4474 (0.38226552307605755) + | > loader_time: 0.0071 (0.0060996990650892275) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.519657850265503 (2.519657850265503) + | > loss_disc_real_0: 0.10791479796171188 (0.10791479796171188) + | > loss_disc_real_1: 0.19784341752529144 (0.19784341752529144) + | > loss_disc_real_2: 0.22961409389972687 (0.22961409389972687) + | > loss_disc_real_3: 0.2946736812591553 (0.2946736812591553) + | > loss_disc_real_4: 0.2907106280326843 (0.2907106280326843) + | > loss_disc_real_5: 0.28660961985588074 (0.28660961985588074) + | > loss_0: 2.519657850265503 (2.519657850265503) + | > loss_gen: 2.2932045459747314 (2.2932045459747314) + | > loss_kl: 2.2862913608551025 (2.2862913608551025) + | > loss_feat: 6.752913951873779 (6.752913951873779) + | > loss_mel: 18.859643936157227 (18.859643936157227) + | > loss_duration: 1.6550670862197876 (1.6550670862197876) + | > loss_1: 31.847122192382812 (31.847122192382812) + + --> STEP: 1 + | > loss_disc: 2.586562156677246 (2.586562156677246) + | > loss_disc_real_0: 0.15310750901699066 (0.15310750901699066) + | > loss_disc_real_1: 0.21499472856521606 (0.21499472856521606) + | > loss_disc_real_2: 0.2049873322248459 (0.2049873322248459) + | > loss_disc_real_3: 0.23956984281539917 (0.23956984281539917) + | > loss_disc_real_4: 0.26208192110061646 (0.26208192110061646) + | > loss_disc_real_5: 0.2993418574333191 (0.2993418574333191) + | > loss_0: 2.586562156677246 (2.586562156677246) + | > loss_gen: 2.2143969535827637 (2.2143969535827637) + | > loss_kl: 2.6506214141845703 (2.6506214141845703) + | > loss_feat: 6.963572978973389 (6.963572978973389) + | > loss_mel: 18.72974967956543 (18.72974967956543) + | > loss_duration: 1.7023625373840332 (1.7023625373840332) + | > loss_1: 32.260704040527344 (32.260704040527344) + + --> STEP: 2 + | > loss_disc: 2.63618803024292 (2.611375093460083) + | > loss_disc_real_0: 0.17947985231876373 (0.1662936806678772) + | > loss_disc_real_1: 0.20806317031383514 (0.2115289494395256) + | > loss_disc_real_2: 0.20527103543281555 (0.20512918382883072) + | > loss_disc_real_3: 0.24383442103862762 (0.2417021319270134) + | > loss_disc_real_4: 0.26674214005470276 (0.2644120305776596) + | > loss_disc_real_5: 0.22620585560798645 (0.26277385652065277) + | > loss_0: 2.63618803024292 (2.611375093460083) + | > loss_gen: 2.1432156562805176 (2.1788063049316406) + | > loss_kl: 2.115187406539917 (2.3829044103622437) + | > loss_feat: 7.392096519470215 (7.177834749221802) + | > loss_mel: 18.88130760192871 (18.80552864074707) + | > loss_duration: 1.6462883949279785 (1.6743254661560059) + | > loss_1: 32.178096771240234 (32.21940040588379) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003931879997253418 (+0.0006023645401000977) + | > avg_loss_disc: 2.611375093460083 (+0.04122912883758545) + | > avg_loss_disc_real_0: 0.1662936806678772 (-0.02064419537782669) + | > avg_loss_disc_real_1: 0.2115289494395256 (+0.020344413816928864) + | > avg_loss_disc_real_2: 0.20512918382883072 (-0.0054643601179122925) + | > avg_loss_disc_real_3: 0.2417021319270134 (+0.034818343818187714) + | > avg_loss_disc_real_4: 0.2644120305776596 (+0.024945981800556183) + | > avg_loss_disc_real_5: 0.26277385652065277 (+0.01084030419588089) + | > avg_loss_0: 2.611375093460083 (+0.04122912883758545) + | > avg_loss_gen: 2.1788063049316406 (-0.04651749134063721) + | > avg_loss_kl: 2.3829044103622437 (+0.06070399284362793) + | > avg_loss_feat: 7.177834749221802 (-0.20577406883239746) + | > avg_loss_mel: 18.80552864074707 (-0.21264362335205078) + | > avg_loss_duration: 1.6743254661560059 (-0.00515371561050415) + | > avg_loss_1: 32.21940040588379 (-0.40938377380371094) + + + > EPOCH: 171/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:48:40)  + + --> TIME: 2025-04-19 01:48:44 -- STEP: 11/142 -- GLOBAL_STEP: 403575 + | > loss_disc: 2.4381606578826904 (2.431645631790161) + | > loss_disc_real_0: 0.08112510293722153 (0.11825668337670239) + | > loss_disc_real_1: 0.21335181593894958 (0.20731697434728796) + | > loss_disc_real_2: 0.2160683274269104 (0.227137953042984) + | > loss_disc_real_3: 0.25717073678970337 (0.2323850759051063) + | > loss_disc_real_4: 0.24441859126091003 (0.23003711890090595) + | > loss_disc_real_5: 0.25857284665107727 (0.22929922423579477) + | > loss_0: 2.4381606578826904 (2.431645631790161) + | > grad_norm_0: tensor(8.0510, device='cuda:0') (tensor(10.7630, device='cuda:0')) + | > loss_gen: 2.2496981620788574 (2.3334090926430444) + | > loss_kl: 1.7519978284835815 (1.8530278964476152) + | > loss_feat: 6.808446884155273 (7.1420863324945625) + | > loss_mel: 18.327775955200195 (17.970921082930133) + | > loss_duration: 1.4198360443115234 (1.4326705174012617) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.557756423950195 (30.73211531205611) + | > grad_norm_1: tensor(219.1378, device='cuda:0') (tensor(218.5033, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.3318 (0.32907245375893335) + | > loader_time: 0.0051 (0.004368565299294211) + + + --> TIME: 2025-04-19 01:48:53 -- STEP: 36/142 -- GLOBAL_STEP: 403600 + | > loss_disc: 2.3795032501220703 (2.429844319820403) + | > loss_disc_real_0: 0.10081759840250015 (0.11589199987550576) + | > loss_disc_real_1: 0.20974676311016083 (0.20181208683384788) + | > loss_disc_real_2: 0.21835704147815704 (0.227657543702258) + | > loss_disc_real_3: 0.2303503006696701 (0.2266868609521124) + | > loss_disc_real_4: 0.20661422610282898 (0.2255160477426317) + | > loss_disc_real_5: 0.24427995085716248 (0.23034380541907418) + | > loss_0: 2.3795032501220703 (2.429844319820403) + | > grad_norm_0: tensor(6.3960, device='cuda:0') (tensor(9.8822, device='cuda:0')) + | > loss_gen: 2.6295952796936035 (2.379186948140462) + | > loss_kl: 1.9666087627410889 (1.8587788376543257) + | > loss_feat: 7.184572219848633 (7.2529155015945435) + | > loss_mel: 17.89699363708496 (17.920818275875515) + | > loss_duration: 1.4285825490951538 (1.4474288920561473) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.106353759765625 (30.85912863413493) + | > grad_norm_1: tensor(192.7654, device='cuda:0') (tensor(240.8357, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.3569 (0.345216797457801) + | > loader_time: 0.0056 (0.004920151498582628) + + + --> TIME: 2025-04-19 01:49:03 -- STEP: 61/142 -- GLOBAL_STEP: 403625 + | > loss_disc: 2.558915138244629 (2.454154393712028) + | > loss_disc_real_0: 0.10716255754232407 (0.12172924287495065) + | > loss_disc_real_1: 0.2432124763727188 (0.20508760189423797) + | > loss_disc_real_2: 0.2448529750108719 (0.2290343805903294) + | > loss_disc_real_3: 0.23103471100330353 (0.2282087412036833) + | > loss_disc_real_4: 0.22876521944999695 (0.2290273543264045) + | > loss_disc_real_5: 0.2904444634914398 (0.2322811860530103) + | > loss_0: 2.558915138244629 (2.454154393712028) + | > grad_norm_0: tensor(10.1975, device='cuda:0') (tensor(8.6150, device='cuda:0')) + | > loss_gen: 2.2604472637176514 (2.3531478233024714) + | > loss_kl: 1.7608659267425537 (1.8650305290691187) + | > loss_feat: 6.798356056213379 (7.154261604684298) + | > loss_mel: 16.76483154296875 (17.86336762787866) + | > loss_duration: 1.475504994392395 (1.459726022892311) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.060007095336914 (30.695533752441406) + | > grad_norm_1: tensor(171.6666, device='cuda:0') (tensor(195.5010, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.3871 (0.35594620470140803) + | > loader_time: 0.0067 (0.005343644345392948) + + + --> TIME: 2025-04-19 01:49:13 -- STEP: 86/142 -- GLOBAL_STEP: 403650 + | > loss_disc: 2.527892589569092 (2.463908203812531) + | > loss_disc_real_0: 0.12806661427021027 (0.1254895114621451) + | > loss_disc_real_1: 0.2031238079071045 (0.20651523023843765) + | > loss_disc_real_2: 0.18853047490119934 (0.22751082722530808) + | > loss_disc_real_3: 0.23264539241790771 (0.22725849477357643) + | > loss_disc_real_4: 0.19715361297130585 (0.22827624183061512) + | > loss_disc_real_5: 0.18629178404808044 (0.23252943435380624) + | > loss_0: 2.527892589569092 (2.463908203812531) + | > grad_norm_0: tensor(7.2848, device='cuda:0') (tensor(8.3597, device='cuda:0')) + | > loss_gen: 2.31095552444458 (2.34811455704445) + | > loss_kl: 2.025261163711548 (1.8754245569539625) + | > loss_feat: 7.099689960479736 (7.033310823662337) + | > loss_mel: 17.619140625 (17.680118904557343) + | > loss_duration: 1.524312973022461 (1.4721800615621172) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.57935905456543 (30.409149147743403) + | > grad_norm_1: tensor(120.5690, device='cuda:0') (tensor(181.4496, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.3982 (0.36627625864605573) + | > loader_time: 0.0065 (0.005639397820761036) + + + --> TIME: 2025-04-19 01:49:23 -- STEP: 111/142 -- GLOBAL_STEP: 403675 + | > loss_disc: 2.483849287033081 (2.478858458029257) + | > loss_disc_real_0: 0.13194318115711212 (0.1288830277484817) + | > loss_disc_real_1: 0.20158229768276215 (0.20647062938492577) + | > loss_disc_real_2: 0.2672469913959503 (0.22896707863421054) + | > loss_disc_real_3: 0.2562306821346283 (0.2298818923331596) + | > loss_disc_real_4: 0.25058919191360474 (0.23088476276612496) + | > loss_disc_real_5: 0.24449002742767334 (0.2334336063883326) + | > loss_0: 2.483849287033081 (2.478858458029257) + | > grad_norm_0: tensor(11.4128, device='cuda:0') (tensor(9.2567, device='cuda:0')) + | > loss_gen: 2.3755674362182617 (2.338119590604626) + | > loss_kl: 1.893304467201233 (1.8888749807804555) + | > loss_feat: 5.807230472564697 (6.970024933686128) + | > loss_mel: 17.100908279418945 (17.711243595088927) + | > loss_duration: 1.5407825708389282 (1.4805815778337088) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.717792510986328 (30.388844928225957) + | > grad_norm_1: tensor(283.3200, device='cuda:0') (tensor(196.2143, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.4232 (0.3761713418874655) + | > loader_time: 0.0074 (0.005927403767903645) + + + --> TIME: 2025-04-19 01:49:34 -- STEP: 136/142 -- GLOBAL_STEP: 403700 + | > loss_disc: 2.5390820503234863 (2.473682559588376) + | > loss_disc_real_0: 0.14588940143585205 (0.12977707232622548) + | > loss_disc_real_1: 0.19454361498355865 (0.20593653947991483) + | > loss_disc_real_2: 0.21968860924243927 (0.22717089972951832) + | > loss_disc_real_3: 0.21010610461235046 (0.22854547114933238) + | > loss_disc_real_4: 0.2409670203924179 (0.2296459967380061) + | > loss_disc_real_5: 0.22167380154132843 (0.23300978289369276) + | > loss_0: 2.5390820503234863 (2.473682559588376) + | > grad_norm_0: tensor(10.9120, device='cuda:0') (tensor(10.6791, device='cuda:0')) + | > loss_gen: 2.1942646503448486 (2.341725012835333) + | > loss_kl: 1.8442127704620361 (1.8965531710316152) + | > loss_feat: 6.751536846160889 (6.967821520917556) + | > loss_mel: 16.336355209350586 (17.704042322495408) + | > loss_duration: 1.5171568393707275 (1.48613413291819) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.64352798461914 (30.39627638985129) + | > grad_norm_1: tensor(475.9984, device='cuda:0') (tensor(220.3675, device='cuda:0')) + | > current_lr_0: 0.000195770103701499 + | > current_lr_1: 0.000195770103701499 + | > step_time: 0.4483 (0.3878231381668764) + | > loader_time: 0.0075 (0.006234169006347655) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.371274948120117 (2.371274948120117) + | > loss_disc_real_0: 0.17461562156677246 (0.17461562156677246) + | > loss_disc_real_1: 0.16660916805267334 (0.16660916805267334) + | > loss_disc_real_2: 0.2228788286447525 (0.2228788286447525) + | > loss_disc_real_3: 0.2442505657672882 (0.2442505657672882) + | > loss_disc_real_4: 0.22121639549732208 (0.22121639549732208) + | > loss_disc_real_5: 0.24023662507534027 (0.24023662507534027) + | > loss_0: 2.371274948120117 (2.371274948120117) + | > loss_gen: 2.6826465129852295 (2.6826465129852295) + | > loss_kl: 2.2335054874420166 (2.2335054874420166) + | > loss_feat: 7.552648544311523 (7.552648544311523) + | > loss_mel: 19.43347930908203 (19.43347930908203) + | > loss_duration: 1.6656222343444824 (1.6656222343444824) + | > loss_1: 33.567901611328125 (33.567901611328125) + + --> STEP: 1 + | > loss_disc: 2.396639347076416 (2.396639347076416) + | > loss_disc_real_0: 0.16764996945858002 (0.16764996945858002) + | > loss_disc_real_1: 0.2480422407388687 (0.2480422407388687) + | > loss_disc_real_2: 0.19993314146995544 (0.19993314146995544) + | > loss_disc_real_3: 0.206099271774292 (0.206099271774292) + | > loss_disc_real_4: 0.18922358751296997 (0.18922358751296997) + | > loss_disc_real_5: 0.1982961744070053 (0.1982961744070053) + | > loss_0: 2.396639347076416 (2.396639347076416) + | > loss_gen: 2.4123620986938477 (2.4123620986938477) + | > loss_kl: 2.634392261505127 (2.634392261505127) + | > loss_feat: 7.830205917358398 (7.830205917358398) + | > loss_mel: 20.45993423461914 (20.45993423461914) + | > loss_duration: 1.71147620677948 (1.71147620677948) + | > loss_1: 35.048370361328125 (35.048370361328125) + + --> STEP: 2 + | > loss_disc: 2.4357311725616455 (2.4161852598190308) + | > loss_disc_real_0: 0.2353750616312027 (0.20151251554489136) + | > loss_disc_real_1: 0.20785903930664062 (0.22795064002275467) + | > loss_disc_real_2: 0.23880252242088318 (0.2193678319454193) + | > loss_disc_real_3: 0.2277970016002655 (0.21694813668727875) + | > loss_disc_real_4: 0.21976915001869202 (0.204496368765831) + | > loss_disc_real_5: 0.21974140405654907 (0.2090187892317772) + | > loss_0: 2.4357311725616455 (2.4161852598190308) + | > loss_gen: 2.5309956073760986 (2.471678853034973) + | > loss_kl: 2.1492903232574463 (2.3918412923812866) + | > loss_feat: 6.205684185028076 (7.017945051193237) + | > loss_mel: 18.198293685913086 (19.329113960266113) + | > loss_duration: 1.6594314575195312 (1.6854538321495056) + | > loss_1: 30.743696212768555 (32.89603328704834) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037914514541625977 (-0.0001404285430908203) + | > avg_loss_disc: 2.4161852598190308 (-0.19518983364105225) + | > avg_loss_disc_real_0: 0.20151251554489136 (+0.03521883487701416) + | > avg_loss_disc_real_1: 0.22795064002275467 (+0.016421690583229065) + | > avg_loss_disc_real_2: 0.2193678319454193 (+0.014238648116588593) + | > avg_loss_disc_real_3: 0.21694813668727875 (-0.02475399523973465) + | > avg_loss_disc_real_4: 0.204496368765831 (-0.05991566181182861) + | > avg_loss_disc_real_5: 0.2090187892317772 (-0.05375506728887558) + | > avg_loss_0: 2.4161852598190308 (-0.19518983364105225) + | > avg_loss_gen: 2.471678853034973 (+0.2928725481033325) + | > avg_loss_kl: 2.3918412923812866 (+0.008936882019042969) + | > avg_loss_feat: 7.017945051193237 (-0.15988969802856445) + | > avg_loss_mel: 19.329113960266113 (+0.523585319519043) + | > avg_loss_duration: 1.6854538321495056 (+0.011128365993499756) + | > avg_loss_1: 32.89603328704834 (+0.6766328811645508) + + + > EPOCH: 172/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:49:38)  + + --> TIME: 2025-04-19 01:49:46 -- STEP: 19/142 -- GLOBAL_STEP: 403725 + | > loss_disc: 2.4452109336853027 (2.486745646125392) + | > loss_disc_real_0: 0.09815820306539536 (0.12804318572345533) + | > loss_disc_real_1: 0.20152845978736877 (0.2109720440287339) + | > loss_disc_real_2: 0.23652654886245728 (0.23412707840141497) + | > loss_disc_real_3: 0.2189335972070694 (0.22899067715594643) + | > loss_disc_real_4: 0.22728103399276733 (0.23018719804914375) + | > loss_disc_real_5: 0.2427424192428589 (0.23530368585335582) + | > loss_0: 2.4452109336853027 (2.486745646125392) + | > grad_norm_0: tensor(18.9189, device='cuda:0') (tensor(12.1432, device='cuda:0')) + | > loss_gen: 2.446136951446533 (2.3279472526751066) + | > loss_kl: 1.7269494533538818 (1.8178074673602456) + | > loss_feat: 7.591935634613037 (7.125026125656931) + | > loss_mel: 19.03064727783203 (18.202481420416582) + | > loss_duration: 1.4328569173812866 (1.4340838821310746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.2285270690918 (30.907346022756474) + | > grad_norm_1: tensor(413.7679, device='cuda:0') (tensor(247.6905, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.3397 (0.330407832798205) + | > loader_time: 0.0051 (0.004731040251882453) + + + --> TIME: 2025-04-19 01:49:55 -- STEP: 44/142 -- GLOBAL_STEP: 403750 + | > loss_disc: 2.495851755142212 (2.4693136486140164) + | > loss_disc_real_0: 0.10280578583478928 (0.12406517429785296) + | > loss_disc_real_1: 0.2224641740322113 (0.20950946042483504) + | > loss_disc_real_2: 0.24097740650177002 (0.22851901230486957) + | > loss_disc_real_3: 0.23790837824344635 (0.22882464155554771) + | > loss_disc_real_4: 0.22102169692516327 (0.22749335488135164) + | > loss_disc_real_5: 0.21408818662166595 (0.23405754735524004) + | > loss_0: 2.495851755142212 (2.4693136486140164) + | > grad_norm_0: tensor(17.0972, device='cuda:0') (tensor(12.2027, device='cuda:0')) + | > loss_gen: 2.322425365447998 (2.3308737277984615) + | > loss_kl: 1.9596974849700928 (1.8238840401172638) + | > loss_feat: 6.921041011810303 (7.099210652438077) + | > loss_mel: 18.3649959564209 (17.992706147107214) + | > loss_duration: 1.4783999919891357 (1.4511822055686603) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.046560287475586 (30.697856816378508) + | > grad_norm_1: tensor(339.1233, device='cuda:0') (tensor(267.7152, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.3576 (0.3433523665774953) + | > loader_time: 0.0055 (0.00509975173256614) + + + --> TIME: 2025-04-19 01:50:04 -- STEP: 69/142 -- GLOBAL_STEP: 403775 + | > loss_disc: 2.530059337615967 (2.4653882151064663) + | > loss_disc_real_0: 0.10781707614660263 (0.12563995768626535) + | > loss_disc_real_1: 0.21362347900867462 (0.2089288508978443) + | > loss_disc_real_2: 0.2384548932313919 (0.22611944312634674) + | > loss_disc_real_3: 0.26822754740715027 (0.22857932983965112) + | > loss_disc_real_4: 0.2587244212627411 (0.22789876918861832) + | > loss_disc_real_5: 0.29728344082832336 (0.23422323793604755) + | > loss_0: 2.530059337615967 (2.4653882151064663) + | > grad_norm_0: tensor(21.1294, device='cuda:0') (tensor(12.1229, device='cuda:0')) + | > loss_gen: 2.4264092445373535 (2.3386347121086666) + | > loss_kl: 1.9390910863876343 (1.847203963044761) + | > loss_feat: 7.209568977355957 (7.092576794002367) + | > loss_mel: 17.914669036865234 (17.919335392938144) + | > loss_duration: 1.5344278812408447 (1.4629269769226296) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.0241641998291 (30.660677896029707) + | > grad_norm_1: tensor(444.6430, device='cuda:0') (tensor(271.1503, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.3942 (0.355316200118134) + | > loader_time: 0.006 (0.005390481672425201) + + + --> TIME: 2025-04-19 01:50:14 -- STEP: 94/142 -- GLOBAL_STEP: 403800 + | > loss_disc: 2.5789105892181396 (2.4678463834397326) + | > loss_disc_real_0: 0.11829239130020142 (0.12769846523061718) + | > loss_disc_real_1: 0.20256367325782776 (0.2087045261200438) + | > loss_disc_real_2: 0.23725545406341553 (0.22574435554920358) + | > loss_disc_real_3: 0.26518502831459045 (0.22767201430620032) + | > loss_disc_real_4: 0.23608992993831635 (0.22631120158636825) + | > loss_disc_real_5: 0.2705298960208893 (0.2346236767603996) + | > loss_0: 2.5789105892181396 (2.4678463834397326) + | > grad_norm_0: tensor(6.3153, device='cuda:0') (tensor(11.3155, device='cuda:0')) + | > loss_gen: 2.2126574516296387 (2.329443000732584) + | > loss_kl: 2.022998332977295 (1.8572883504502318) + | > loss_feat: 6.565722942352295 (7.015504831963397) + | > loss_mel: 18.09723663330078 (17.812315048055446) + | > loss_duration: 1.5163556337356567 (1.4745586501791117) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.41497230529785 (30.489109911817184) + | > grad_norm_1: tensor(116.9694, device='cuda:0') (tensor(248.2813, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.3941 (0.36611829920017974) + | > loader_time: 0.0067 (0.005708904976540423) + + + --> TIME: 2025-04-19 01:50:25 -- STEP: 119/142 -- GLOBAL_STEP: 403825 + | > loss_disc: 2.4717178344726562 (2.4673909519900796) + | > loss_disc_real_0: 0.12182122468948364 (0.1283252106494263) + | > loss_disc_real_1: 0.25001242756843567 (0.20871899834200114) + | > loss_disc_real_2: 0.22316934168338776 (0.2257426685395361) + | > loss_disc_real_3: 0.20599365234375 (0.22763572382826766) + | > loss_disc_real_4: 0.24084696173667908 (0.22599402846408492) + | > loss_disc_real_5: 0.21997514367103577 (0.23305254323141916) + | > loss_0: 2.4717178344726562 (2.4673909519900796) + | > grad_norm_0: tensor(6.4574, device='cuda:0') (tensor(10.7344, device='cuda:0')) + | > loss_gen: 2.3707516193389893 (2.329842477285562) + | > loss_kl: 1.9600237607955933 (1.8758142986217468) + | > loss_feat: 7.221868515014648 (7.0199372027100635) + | > loss_mel: 17.9826717376709 (17.83551147204488) + | > loss_duration: 1.5676192045211792 (1.4809115724403312) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.102935791015625 (30.542017031116647) + | > grad_norm_1: tensor(173.5707, device='cuda:0') (tensor(228.4697, device='cuda:0')) + | > current_lr_0: 0.0001957456324385363 + | > current_lr_1: 0.0001957456324385363 + | > step_time: 0.4268 (0.37718221520175454) + | > loader_time: 0.0074 (0.005959715162004743) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4560494422912598 (2.4560494422912598) + | > loss_disc_real_0: 0.14581246674060822 (0.14581246674060822) + | > loss_disc_real_1: 0.2564680576324463 (0.2564680576324463) + | > loss_disc_real_2: 0.2650311291217804 (0.2650311291217804) + | > loss_disc_real_3: 0.25536927580833435 (0.25536927580833435) + | > loss_disc_real_4: 0.24931061267852783 (0.24931061267852783) + | > loss_disc_real_5: 0.25615036487579346 (0.25615036487579346) + | > loss_0: 2.4560494422912598 (2.4560494422912598) + | > loss_gen: 2.6225147247314453 (2.6225147247314453) + | > loss_kl: 2.3106088638305664 (2.3106088638305664) + | > loss_feat: 7.2191386222839355 (7.2191386222839355) + | > loss_mel: 20.22597885131836 (20.22597885131836) + | > loss_duration: 1.6602627038955688 (1.6602627038955688) + | > loss_1: 34.03850555419922 (34.03850555419922) + + --> STEP: 1 + | > loss_disc: 2.548057794570923 (2.548057794570923) + | > loss_disc_real_0: 0.16011446714401245 (0.16011446714401245) + | > loss_disc_real_1: 0.2701490521430969 (0.2701490521430969) + | > loss_disc_real_2: 0.26232030987739563 (0.26232030987739563) + | > loss_disc_real_3: 0.2686665654182434 (0.2686665654182434) + | > loss_disc_real_4: 0.2678532004356384 (0.2678532004356384) + | > loss_disc_real_5: 0.2557910084724426 (0.2557910084724426) + | > loss_0: 2.548057794570923 (2.548057794570923) + | > loss_gen: 2.526975631713867 (2.526975631713867) + | > loss_kl: 2.640599489212036 (2.640599489212036) + | > loss_feat: 6.210231781005859 (6.210231781005859) + | > loss_mel: 17.664974212646484 (17.664974212646484) + | > loss_duration: 1.733750581741333 (1.733750581741333) + | > loss_1: 30.776531219482422 (30.776531219482422) + + --> STEP: 2 + | > loss_disc: 2.611896514892578 (2.5799771547317505) + | > loss_disc_real_0: 0.19227658212184906 (0.17619552463293076) + | > loss_disc_real_1: 0.25594770908355713 (0.263048380613327) + | > loss_disc_real_2: 0.2765060365200043 (0.26941317319869995) + | > loss_disc_real_3: 0.2607118785381317 (0.26468922197818756) + | > loss_disc_real_4: 0.29620110988616943 (0.28202715516090393) + | > loss_disc_real_5: 0.2544412910938263 (0.25511614978313446) + | > loss_0: 2.611896514892578 (2.5799771547317505) + | > loss_gen: 2.499678134918213 (2.51332688331604) + | > loss_kl: 2.2150356769561768 (2.4278175830841064) + | > loss_feat: 7.094619274139404 (6.652425527572632) + | > loss_mel: 19.672582626342773 (18.66877841949463) + | > loss_duration: 1.6330158710479736 (1.6833832263946533) + | > loss_1: 33.114933013916016 (31.94573211669922) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003863096237182617 (+7.164478302001953e-05) + | > avg_loss_disc: 2.5799771547317505 (+0.16379189491271973) + | > avg_loss_disc_real_0: 0.17619552463293076 (-0.025316990911960602) + | > avg_loss_disc_real_1: 0.263048380613327 (+0.03509774059057236) + | > avg_loss_disc_real_2: 0.26941317319869995 (+0.05004534125328064) + | > avg_loss_disc_real_3: 0.26468922197818756 (+0.047741085290908813) + | > avg_loss_disc_real_4: 0.28202715516090393 (+0.07753078639507294) + | > avg_loss_disc_real_5: 0.25511614978313446 (+0.04609736055135727) + | > avg_loss_0: 2.5799771547317505 (+0.16379189491271973) + | > avg_loss_gen: 2.51332688331604 (+0.041648030281066895) + | > avg_loss_kl: 2.4278175830841064 (+0.035976290702819824) + | > avg_loss_feat: 6.652425527572632 (-0.36551952362060547) + | > avg_loss_mel: 18.66877841949463 (-0.6603355407714844) + | > avg_loss_duration: 1.6833832263946533 (-0.002070605754852295) + | > avg_loss_1: 31.94573211669922 (-0.9503011703491211) + + + > EPOCH: 173/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:50:37)  + + --> TIME: 2025-04-19 01:50:39 -- STEP: 2/142 -- GLOBAL_STEP: 403850 + | > loss_disc: 2.5894172191619873 (2.5074750185012817) + | > loss_disc_real_0: 0.15370038151741028 (0.13570332899689674) + | > loss_disc_real_1: 0.23683306574821472 (0.24803070724010468) + | > loss_disc_real_2: 0.28565481305122375 (0.2811344116926193) + | > loss_disc_real_3: 0.2927127480506897 (0.2796356827020645) + | > loss_disc_real_4: 0.2766822576522827 (0.27728013694286346) + | > loss_disc_real_5: 0.24398232996463776 (0.23423991352319717) + | > loss_0: 2.5894172191619873 (2.5074750185012817) + | > grad_norm_0: tensor(7.1927, device='cuda:0') (tensor(9.2063, device='cuda:0')) + | > loss_gen: 2.2372090816497803 (2.425127387046814) + | > loss_kl: 1.8743650913238525 (1.8999086618423462) + | > loss_feat: 6.976071357727051 (7.1696507930755615) + | > loss_mel: 17.92827606201172 (17.756479263305664) + | > loss_duration: 1.4476450681686401 (1.422131896018982) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.46356773376465 (30.673298835754395) + | > grad_norm_1: tensor(124.2045, device='cuda:0') (tensor(259.5006, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.3284 (0.42624032497406006) + | > loader_time: 0.0039 (0.003931522369384766) + + + --> TIME: 2025-04-19 01:50:47 -- STEP: 27/142 -- GLOBAL_STEP: 403875 + | > loss_disc: 2.443922996520996 (2.455556834185564) + | > loss_disc_real_0: 0.08760973066091537 (0.11904025657309426) + | > loss_disc_real_1: 0.20490410923957825 (0.21057352092530993) + | > loss_disc_real_2: 0.22192683815956116 (0.2283712018419195) + | > loss_disc_real_3: 0.22409285604953766 (0.22943347250973736) + | > loss_disc_real_4: 0.1922905594110489 (0.22958450792012391) + | > loss_disc_real_5: 0.22159041464328766 (0.23402516985381092) + | > loss_0: 2.443922996520996 (2.455556834185564) + | > grad_norm_0: tensor(6.6240, device='cuda:0') (tensor(12.0385, device='cuda:0')) + | > loss_gen: 2.3496408462524414 (2.387866876743458) + | > loss_kl: 1.98758864402771 (1.8408701640588265) + | > loss_feat: 7.267060279846191 (7.12374155609696) + | > loss_mel: 17.9610538482666 (17.97400725329364) + | > loss_duration: 1.4589273929595947 (1.4434183306164212) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.024269104003906 (30.76990424262153) + | > grad_norm_1: tensor(150.6183, device='cuda:0') (tensor(316.0724, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.3519 (0.34444840749104816) + | > loader_time: 0.0052 (0.004799851664790401) + + + --> TIME: 2025-04-19 01:50:56 -- STEP: 52/142 -- GLOBAL_STEP: 403900 + | > loss_disc: 2.4194176197052 (2.465164143305558) + | > loss_disc_real_0: 0.1350834220647812 (0.12251609348906921) + | > loss_disc_real_1: 0.2418947070837021 (0.20953090202349883) + | > loss_disc_real_2: 0.21154220402240753 (0.22665522648737982) + | > loss_disc_real_3: 0.24766626954078674 (0.2308889598800586) + | > loss_disc_real_4: 0.23342636227607727 (0.23083289426106673) + | > loss_disc_real_5: 0.2614138126373291 (0.23447764693544462) + | > loss_0: 2.4194176197052 (2.465164143305558) + | > grad_norm_0: tensor(12.1469, device='cuda:0') (tensor(11.6517, device='cuda:0')) + | > loss_gen: 2.264608144760132 (2.3586898950430064) + | > loss_kl: 1.837269902229309 (1.8710944652557373) + | > loss_feat: 6.809537410736084 (7.055723685484666) + | > loss_mel: 18.34009552001953 (17.969149461159336) + | > loss_duration: 1.5170549154281616 (1.4519158487136552) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.76856803894043 (30.70657363304725) + | > grad_norm_1: tensor(286.5566, device='cuda:0') (tensor(284.6753, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.3673 (0.351873150238624) + | > loader_time: 0.0056 (0.005175778499016395) + + + --> TIME: 2025-04-19 01:51:06 -- STEP: 77/142 -- GLOBAL_STEP: 403925 + | > loss_disc: 2.4000208377838135 (2.465195123251383) + | > loss_disc_real_0: 0.09842324256896973 (0.12416802075776187) + | > loss_disc_real_1: 0.15484148263931274 (0.2071956736700875) + | > loss_disc_real_2: 0.25233832001686096 (0.22634534537792206) + | > loss_disc_real_3: 0.22404302656650543 (0.22952538225557897) + | > loss_disc_real_4: 0.2073637992143631 (0.22847394103353674) + | > loss_disc_real_5: 0.2041073739528656 (0.23315246306456527) + | > loss_0: 2.4000208377838135 (2.465195123251383) + | > grad_norm_0: tensor(11.4404, device='cuda:0') (tensor(11.8464, device='cuda:0')) + | > loss_gen: 2.433279037475586 (2.3499315497163056) + | > loss_kl: 1.8806520700454712 (1.882021097393779) + | > loss_feat: 7.809281349182129 (7.012195413762873) + | > loss_mel: 18.451387405395508 (17.81074038418856) + | > loss_duration: 1.4832959175109863 (1.4641838569145702) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.05789566040039 (30.519072445956144) + | > grad_norm_1: tensor(425.9224, device='cuda:0') (tensor(286.2672, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.3969 (0.36274079843000934) + | > loader_time: 0.0065 (0.0055146031565480435) + + + --> TIME: 2025-04-19 01:51:17 -- STEP: 102/142 -- GLOBAL_STEP: 403950 + | > loss_disc: 2.6068012714385986 (2.479118852054371) + | > loss_disc_real_0: 0.09990036487579346 (0.12485703172198696) + | > loss_disc_real_1: 0.24126657843589783 (0.20841573865390292) + | > loss_disc_real_2: 0.2609752118587494 (0.227761518721487) + | > loss_disc_real_3: 0.2604807913303375 (0.2301205893357595) + | > loss_disc_real_4: 0.27946335077285767 (0.23072058751302607) + | > loss_disc_real_5: 0.2574906647205353 (0.2345597987958029) + | > loss_0: 2.6068012714385986 (2.479118852054371) + | > grad_norm_0: tensor(13.9047, device='cuda:0') (tensor(12.4619, device='cuda:0')) + | > loss_gen: 2.3081722259521484 (2.347693048271477) + | > loss_kl: 1.90680730342865 (1.8831014913671158) + | > loss_feat: 5.700585842132568 (6.963715642106299) + | > loss_mel: 17.009450912475586 (17.84571911306941) + | > loss_duration: 1.4896351099014282 (1.4750772621117392) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.41465187072754 (30.515306753270767) + | > grad_norm_1: tensor(151.3936, device='cuda:0') (tensor(285.5187, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.4205 (0.3731309409235038) + | > loader_time: 0.007 (0.005793520048552871) + + + --> TIME: 2025-04-19 01:51:28 -- STEP: 127/142 -- GLOBAL_STEP: 403975 + | > loss_disc: 2.4648356437683105 (2.474194909643939) + | > loss_disc_real_0: 0.1759916990995407 (0.12548989306871344) + | > loss_disc_real_1: 0.20997555553913116 (0.2074135593541964) + | > loss_disc_real_2: 0.22330285608768463 (0.22748511419521542) + | > loss_disc_real_3: 0.2227412909269333 (0.22905894128356394) + | > loss_disc_real_4: 0.24729683995246887 (0.23031096101745846) + | > loss_disc_real_5: 0.21165238320827484 (0.234180222698084) + | > loss_0: 2.4648356437683105 (2.474194909643939) + | > grad_norm_0: tensor(14.3737, device='cuda:0') (tensor(12.7716, device='cuda:0')) + | > loss_gen: 2.3997209072113037 (2.343368025276604) + | > loss_kl: 2.007951021194458 (1.8919367020524394) + | > loss_feat: 6.4896721839904785 (6.941002695579228) + | > loss_mel: 16.382835388183594 (17.842757758193123) + | > loss_duration: 1.5255558490753174 (1.4829863816734379) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.805736541748047 (30.502051676352192) + | > grad_norm_1: tensor(265.2359, device='cuda:0') (tensor(280.8814, device='cuda:0')) + | > current_lr_0: 0.00019572116423448148 + | > current_lr_1: 0.00019572116423448148 + | > step_time: 0.4397 (0.3839780326903336) + | > loader_time: 0.0075 (0.006082401500912165) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4947807788848877 (2.4947807788848877) + | > loss_disc_real_0: 0.07971347868442535 (0.07971347868442535) + | > loss_disc_real_1: 0.22233939170837402 (0.22233939170837402) + | > loss_disc_real_2: 0.26151034235954285 (0.26151034235954285) + | > loss_disc_real_3: 0.25384119153022766 (0.25384119153022766) + | > loss_disc_real_4: 0.20659205317497253 (0.20659205317497253) + | > loss_disc_real_5: 0.26766839623451233 (0.26766839623451233) + | > loss_0: 2.4947807788848877 (2.4947807788848877) + | > loss_gen: 2.270704984664917 (2.270704984664917) + | > loss_kl: 2.1906557083129883 (2.1906557083129883) + | > loss_feat: 6.620312690734863 (6.620312690734863) + | > loss_mel: 17.23819923400879 (17.23819923400879) + | > loss_duration: 1.670220136642456 (1.670220136642456) + | > loss_1: 29.99009132385254 (29.99009132385254) + + --> STEP: 1 + | > loss_disc: 2.4223251342773438 (2.4223251342773438) + | > loss_disc_real_0: 0.09589561074972153 (0.09589561074972153) + | > loss_disc_real_1: 0.23735952377319336 (0.23735952377319336) + | > loss_disc_real_2: 0.26138338446617126 (0.26138338446617126) + | > loss_disc_real_3: 0.23662234842777252 (0.23662234842777252) + | > loss_disc_real_4: 0.2220495343208313 (0.2220495343208313) + | > loss_disc_real_5: 0.2311389297246933 (0.2311389297246933) + | > loss_0: 2.4223251342773438 (2.4223251342773438) + | > loss_gen: 2.385932445526123 (2.385932445526123) + | > loss_kl: 2.534407377243042 (2.534407377243042) + | > loss_feat: 6.941102504730225 (6.941102504730225) + | > loss_mel: 19.05274772644043 (19.05274772644043) + | > loss_duration: 1.7053974866867065 (1.7053974866867065) + | > loss_1: 32.61958694458008 (32.61958694458008) + + --> STEP: 2 + | > loss_disc: 2.304861545562744 (2.363593339920044) + | > loss_disc_real_0: 0.07707487046718597 (0.08648524060845375) + | > loss_disc_real_1: 0.3202125132083893 (0.2787860184907913) + | > loss_disc_real_2: 0.22230055928230286 (0.24184197187423706) + | > loss_disc_real_3: 0.23729518055915833 (0.23695876449346542) + | > loss_disc_real_4: 0.21341654658317566 (0.21773304045200348) + | > loss_disc_real_5: 0.26305821537971497 (0.24709857255220413) + | > loss_0: 2.304861545562744 (2.363593339920044) + | > loss_gen: 2.6463825702667236 (2.5161575078964233) + | > loss_kl: 2.081921339035034 (2.308164358139038) + | > loss_feat: 8.306965827941895 (7.62403416633606) + | > loss_mel: 18.86834716796875 (18.96054744720459) + | > loss_duration: 1.6398133039474487 (1.6726053953170776) + | > loss_1: 33.54343032836914 (33.08150863647461) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003803730010986328 (-5.936622619628906e-05) + | > avg_loss_disc: 2.363593339920044 (-0.21638381481170654) + | > avg_loss_disc_real_0: 0.08648524060845375 (-0.089710284024477) + | > avg_loss_disc_real_1: 0.2787860184907913 (+0.015737637877464294) + | > avg_loss_disc_real_2: 0.24184197187423706 (-0.02757120132446289) + | > avg_loss_disc_real_3: 0.23695876449346542 (-0.027730457484722137) + | > avg_loss_disc_real_4: 0.21773304045200348 (-0.06429411470890045) + | > avg_loss_disc_real_5: 0.24709857255220413 (-0.008017577230930328) + | > avg_loss_0: 2.363593339920044 (-0.21638381481170654) + | > avg_loss_gen: 2.5161575078964233 (+0.0028306245803833008) + | > avg_loss_kl: 2.308164358139038 (-0.11965322494506836) + | > avg_loss_feat: 7.62403416633606 (+0.9716086387634277) + | > avg_loss_mel: 18.96054744720459 (+0.29176902770996094) + | > avg_loss_duration: 1.6726053953170776 (-0.010777831077575684) + | > avg_loss_1: 33.08150863647461 (+1.1357765197753906) + + + > EPOCH: 174/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:51:36)  + + --> TIME: 2025-04-19 01:51:40 -- STEP: 10/142 -- GLOBAL_STEP: 404000 + | > loss_disc: 2.403071403503418 (2.444047379493713) + | > loss_disc_real_0: 0.11778217554092407 (0.13100456744432448) + | > loss_disc_real_1: 0.21483150124549866 (0.20097219794988633) + | > loss_disc_real_2: 0.23381195962429047 (0.23540587574243546) + | > loss_disc_real_3: 0.23929990828037262 (0.23412164598703383) + | > loss_disc_real_4: 0.23710997402668 (0.22873718589544295) + | > loss_disc_real_5: 0.20518319308757782 (0.23634272068738937) + | > loss_0: 2.403071403503418 (2.444047379493713) + | > grad_norm_0: tensor(5.1688, device='cuda:0') (tensor(10.0645, device='cuda:0')) + | > loss_gen: 2.277911424636841 (2.3556148529052736) + | > loss_kl: 2.041965961456299 (1.8499886989593506) + | > loss_feat: 6.857169151306152 (7.103074836730957) + | > loss_mel: 16.934738159179688 (17.475973320007324) + | > loss_duration: 1.3929569721221924 (1.4321137428283692) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.504743576049805 (30.21676540374756) + | > grad_norm_1: tensor(150.0191, device='cuda:0') (tensor(205.7734, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.3407 (0.32729592323303225) + | > loader_time: 0.005 (0.004461145401000977) + + + --> TIME: 2025-04-19 01:51:49 -- STEP: 35/142 -- GLOBAL_STEP: 404025 + | > loss_disc: 2.3377585411071777 (2.4412811824253637) + | > loss_disc_real_0: 0.1150292158126831 (0.12022454653467451) + | > loss_disc_real_1: 0.1648034006357193 (0.2043690025806427) + | > loss_disc_real_2: 0.21020545065402985 (0.22877479110445295) + | > loss_disc_real_3: 0.19082798063755035 (0.22961057211671557) + | > loss_disc_real_4: 0.20408838987350464 (0.22716617030756814) + | > loss_disc_real_5: 0.22363854944705963 (0.22977840985570636) + | > loss_0: 2.3377585411071777 (2.4412811824253637) + | > grad_norm_0: tensor(6.8613, device='cuda:0') (tensor(8.1624, device='cuda:0')) + | > loss_gen: 2.466538429260254 (2.3498157501220702) + | > loss_kl: 2.0152742862701416 (1.8453576939446585) + | > loss_feat: 7.301279544830322 (7.108227743421282) + | > loss_mel: 17.981155395507812 (17.855504008701867) + | > loss_duration: 1.463322639465332 (1.4458295175007412) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.227569580078125 (30.604734802246092) + | > grad_norm_1: tensor(227.0389, device='cuda:0') (tensor(194.1398, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.3567 (0.3394287518092564) + | > loader_time: 0.0056 (0.0050360815865652915) + + + --> TIME: 2025-04-19 01:51:58 -- STEP: 60/142 -- GLOBAL_STEP: 404050 + | > loss_disc: 2.4924628734588623 (2.4449355800946555) + | > loss_disc_real_0: 0.1003279909491539 (0.11965456157922745) + | > loss_disc_real_1: 0.20693425834178925 (0.20658169438441595) + | > loss_disc_real_2: 0.2315697968006134 (0.22578705176711084) + | > loss_disc_real_3: 0.2173582911491394 (0.22946787054340045) + | > loss_disc_real_4: 0.20617161691188812 (0.2272999182343483) + | > loss_disc_real_5: 0.22148804366588593 (0.23133047570784887) + | > loss_0: 2.4924628734588623 (2.4449355800946555) + | > grad_norm_0: tensor(14.6501, device='cuda:0') (tensor(9.3481, device='cuda:0')) + | > loss_gen: 2.363131046295166 (2.3648862004280096) + | > loss_kl: 1.8929363489151 (1.8600687066713968) + | > loss_feat: 7.538407325744629 (7.11504700978597) + | > loss_mel: 18.36284637451172 (17.846534935633336) + | > loss_duration: 1.4977715015411377 (1.4568376262982687) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.655092239379883 (30.6433744430542) + | > grad_norm_1: tensor(372.0067, device='cuda:0') (tensor(227.8625, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.3766 (0.3511953592300415) + | > loader_time: 0.0062 (0.00542437235514323) + + + --> TIME: 2025-04-19 01:52:08 -- STEP: 85/142 -- GLOBAL_STEP: 404075 + | > loss_disc: 2.35305118560791 (2.4584659043480364) + | > loss_disc_real_0: 0.15144413709640503 (0.12128869715858909) + | > loss_disc_real_1: 0.18366281688213348 (0.2066372019403121) + | > loss_disc_real_2: 0.2289692461490631 (0.22693018650307376) + | > loss_disc_real_3: 0.22115471959114075 (0.2295282400706235) + | > loss_disc_real_4: 0.23598657548427582 (0.2309649900478475) + | > loss_disc_real_5: 0.23396021127700806 (0.23263977713444653) + | > loss_0: 2.35305118560791 (2.4584659043480364) + | > grad_norm_0: tensor(11.9251, device='cuda:0') (tensor(10.1887, device='cuda:0')) + | > loss_gen: 2.4435536861419678 (2.3538091126610263) + | > loss_kl: 1.969269037246704 (1.8736298785490149) + | > loss_feat: 7.433713912963867 (7.06371015660903) + | > loss_mel: 18.39512825012207 (17.867299831614776) + | > loss_duration: 1.5058317184448242 (1.469443514767815) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.74749755859375 (30.62789255030015) + | > grad_norm_1: tensor(440.5618, device='cuda:0') (tensor(257.9050, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.3957 (0.36331883318283986) + | > loader_time: 0.0064 (0.005729349921731388) + + + --> TIME: 2025-04-19 01:52:19 -- STEP: 110/142 -- GLOBAL_STEP: 404100 + | > loss_disc: 2.5173215866088867 (2.464967474070462) + | > loss_disc_real_0: 0.11899112910032272 (0.12182610062035647) + | > loss_disc_real_1: 0.20426738262176514 (0.20729436711831525) + | > loss_disc_real_2: 0.2025444656610489 (0.22689866626804525) + | > loss_disc_real_3: 0.20928621292114258 (0.229920877922665) + | > loss_disc_real_4: 0.2077913135290146 (0.23095215586098758) + | > loss_disc_real_5: 0.21753095090389252 (0.23376749699765986) + | > loss_0: 2.5173215866088867 (2.464967474070462) + | > grad_norm_0: tensor(4.9324, device='cuda:0') (tensor(11.3318, device='cuda:0')) + | > loss_gen: 2.2326722145080566 (2.3485715215856384) + | > loss_kl: 2.0540692806243896 (1.887437448718331) + | > loss_feat: 6.661574363708496 (7.015048421512951) + | > loss_mel: 17.535202026367188 (17.737698095495052) + | > loss_duration: 1.5025479793548584 (1.4785117517818105) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.986066818237305 (30.46726727919145) + | > grad_norm_1: tensor(99.5933, device='cuda:0') (tensor(275.5014, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.429 (0.3734731392426925) + | > loader_time: 0.0067 (0.005978382717479359) + + + --> TIME: 2025-04-19 01:52:30 -- STEP: 135/142 -- GLOBAL_STEP: 404125 + | > loss_disc: 2.607541084289551 (2.4698348416222466) + | > loss_disc_real_0: 0.22307096421718597 (0.12498773949013817) + | > loss_disc_real_1: 0.26959356665611267 (0.20724374972007892) + | > loss_disc_real_2: 0.22835808992385864 (0.22763229244285157) + | > loss_disc_real_3: 0.24832798540592194 (0.22975201573636797) + | > loss_disc_real_4: 0.24940679967403412 (0.230563865436448) + | > loss_disc_real_5: 0.260304719209671 (0.23223899448359453) + | > loss_0: 2.607541084289551 (2.4698348416222466) + | > grad_norm_0: tensor(10.2635, device='cuda:0') (tensor(12.7345, device='cuda:0')) + | > loss_gen: 2.3912248611450195 (2.3511137450182886) + | > loss_kl: 2.0172579288482666 (1.8957769994382505) + | > loss_feat: 5.7713422775268555 (6.98405748296667) + | > loss_mel: 15.029306411743164 (17.726443389610008) + | > loss_duration: 1.4836277961730957 (1.4840372659541943) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.69275665283203 (30.44142891212746) + | > grad_norm_1: tensor(88.1543, device='cuda:0') (tensor(285.0902, device='cuda:0')) + | > current_lr_0: 0.00019569669908895215 + | > current_lr_1: 0.00019569669908895215 + | > step_time: 0.4566 (0.386540338728163) + | > loader_time: 0.0075 (0.006219719074390553) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5491347312927246 (2.5491347312927246) + | > loss_disc_real_0: 0.13798034191131592 (0.13798034191131592) + | > loss_disc_real_1: 0.18213534355163574 (0.18213534355163574) + | > loss_disc_real_2: 0.2507954239845276 (0.2507954239845276) + | > loss_disc_real_3: 0.2503119111061096 (0.2503119111061096) + | > loss_disc_real_4: 0.2350044846534729 (0.2350044846534729) + | > loss_disc_real_5: 0.23492006957530975 (0.23492006957530975) + | > loss_0: 2.5491347312927246 (2.5491347312927246) + | > loss_gen: 2.085397720336914 (2.085397720336914) + | > loss_kl: 2.347966194152832 (2.347966194152832) + | > loss_feat: 5.2229413986206055 (5.2229413986206055) + | > loss_mel: 14.570627212524414 (14.570627212524414) + | > loss_duration: 1.6663998365402222 (1.6663998365402222) + | > loss_1: 25.89333152770996 (25.89333152770996) + + --> STEP: 1 + | > loss_disc: 2.481158494949341 (2.481158494949341) + | > loss_disc_real_0: 0.18273189663887024 (0.18273189663887024) + | > loss_disc_real_1: 0.21413622796535492 (0.21413622796535492) + | > loss_disc_real_2: 0.21618664264678955 (0.21618664264678955) + | > loss_disc_real_3: 0.22967012226581573 (0.22967012226581573) + | > loss_disc_real_4: 0.23278188705444336 (0.23278188705444336) + | > loss_disc_real_5: 0.20731909573078156 (0.20731909573078156) + | > loss_0: 2.481158494949341 (2.481158494949341) + | > loss_gen: 2.3280222415924072 (2.3280222415924072) + | > loss_kl: 2.4980344772338867 (2.4980344772338867) + | > loss_feat: 7.1918253898620605 (7.1918253898620605) + | > loss_mel: 18.953834533691406 (18.953834533691406) + | > loss_duration: 1.6936147212982178 (1.6936147212982178) + | > loss_1: 32.66532897949219 (32.66532897949219) + + --> STEP: 2 + | > loss_disc: 2.4805338382720947 (2.4808461666107178) + | > loss_disc_real_0: 0.16826260089874268 (0.17549724876880646) + | > loss_disc_real_1: 0.2091723084449768 (0.21165426820516586) + | > loss_disc_real_2: 0.25354450941085815 (0.23486557602882385) + | > loss_disc_real_3: 0.22770190238952637 (0.22868601232767105) + | > loss_disc_real_4: 0.2115750014781952 (0.22217844426631927) + | > loss_disc_real_5: 0.2509562373161316 (0.22913766652345657) + | > loss_0: 2.4805338382720947 (2.4808461666107178) + | > loss_gen: 2.3240346908569336 (2.3260284662246704) + | > loss_kl: 2.2275145053863525 (2.3627744913101196) + | > loss_feat: 6.7413482666015625 (6.9665868282318115) + | > loss_mel: 18.381582260131836 (18.66770839691162) + | > loss_duration: 1.6326286792755127 (1.6631217002868652) + | > loss_1: 31.307109832763672 (31.98621940612793) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037211179733276367 (-8.26120376586914e-05) + | > avg_loss_disc: 2.4808461666107178 (+0.11725282669067383) + | > avg_loss_disc_real_0: 0.17549724876880646 (+0.0890120081603527) + | > avg_loss_disc_real_1: 0.21165426820516586 (-0.06713175028562546) + | > avg_loss_disc_real_2: 0.23486557602882385 (-0.006976395845413208) + | > avg_loss_disc_real_3: 0.22868601232767105 (-0.008272752165794373) + | > avg_loss_disc_real_4: 0.22217844426631927 (+0.004445403814315796) + | > avg_loss_disc_real_5: 0.22913766652345657 (-0.01796090602874756) + | > avg_loss_0: 2.4808461666107178 (+0.11725282669067383) + | > avg_loss_gen: 2.3260284662246704 (-0.19012904167175293) + | > avg_loss_kl: 2.3627744913101196 (+0.05461013317108154) + | > avg_loss_feat: 6.9665868282318115 (-0.657447338104248) + | > avg_loss_mel: 18.66770839691162 (-0.29283905029296875) + | > avg_loss_duration: 1.6631217002868652 (-0.009483695030212402) + | > avg_loss_1: 31.98621940612793 (-1.0952892303466797) + + + > EPOCH: 175/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:52:35)  + + --> TIME: 2025-04-19 01:52:41 -- STEP: 18/142 -- GLOBAL_STEP: 404150 + | > loss_disc: 2.4778337478637695 (2.417562034395006) + | > loss_disc_real_0: 0.22050762176513672 (0.11640158378415638) + | > loss_disc_real_1: 0.1764068454504013 (0.2028484990199407) + | > loss_disc_real_2: 0.20947231352329254 (0.22090168545643488) + | > loss_disc_real_3: 0.22355979681015015 (0.2271027316649755) + | > loss_disc_real_4: 0.21388272941112518 (0.22209153489934075) + | > loss_disc_real_5: 0.24380071461200714 (0.22491353998581567) + | > loss_0: 2.4778337478637695 (2.417562034395006) + | > grad_norm_0: tensor(29.6933, device='cuda:0') (tensor(17.8387, device='cuda:0')) + | > loss_gen: 2.276595115661621 (2.3856646484798856) + | > loss_kl: 1.858627200126648 (1.841431392563714) + | > loss_feat: 6.773829936981201 (7.126256889767117) + | > loss_mel: 17.258037567138672 (17.56549506717258) + | > loss_duration: 1.4425957202911377 (1.436096681488885) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.60968780517578 (30.35494539472792) + | > grad_norm_1: tensor(390.6612, device='cuda:0') (tensor(341.9996, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.3392 (0.3303961753845215) + | > loader_time: 0.005 (0.004729363653394911) + + + --> TIME: 2025-04-19 01:52:50 -- STEP: 43/142 -- GLOBAL_STEP: 404175 + | > loss_disc: 2.48275089263916 (2.4356047885362497) + | > loss_disc_real_0: 0.14259925484657288 (0.11933273874049963) + | > loss_disc_real_1: 0.20201504230499268 (0.2054044534993726) + | > loss_disc_real_2: 0.21202105283737183 (0.2232579699782438) + | > loss_disc_real_3: 0.21332263946533203 (0.22747834893160088) + | > loss_disc_real_4: 0.2539118826389313 (0.22637662534103836) + | > loss_disc_real_5: 0.275279700756073 (0.23159181274646937) + | > loss_0: 2.48275089263916 (2.4356047885362497) + | > grad_norm_0: tensor(9.2313, device='cuda:0') (tensor(15.7489, device='cuda:0')) + | > loss_gen: 2.300030469894409 (2.3683516202970987) + | > loss_kl: 1.7654136419296265 (1.8506919533707376) + | > loss_feat: 6.9025068283081055 (7.137293161347855) + | > loss_mel: 17.789505004882812 (17.697400824968202) + | > loss_duration: 1.4690208435058594 (1.4479968520097954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.22647476196289 (30.501734578332236) + | > grad_norm_1: tensor(342.4967, device='cuda:0') (tensor(345.6887, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.3631 (0.3428008001904155) + | > loader_time: 0.0057 (0.0051793553108392745) + + + --> TIME: 2025-04-19 01:53:00 -- STEP: 68/142 -- GLOBAL_STEP: 404200 + | > loss_disc: 2.4904589653015137 (2.438946201520807) + | > loss_disc_real_0: 0.12279784679412842 (0.12130911231917493) + | > loss_disc_real_1: 0.22394108772277832 (0.20456655652207487) + | > loss_disc_real_2: 0.2612290680408478 (0.22306491105872042) + | > loss_disc_real_3: 0.22414572536945343 (0.2277924271629137) + | > loss_disc_real_4: 0.20345446467399597 (0.2246287519879201) + | > loss_disc_real_5: 0.24900569021701813 (0.23021205120226917) + | > loss_0: 2.4904589653015137 (2.438946201520807) + | > grad_norm_0: tensor(6.3546, device='cuda:0') (tensor(12.8461, device='cuda:0')) + | > loss_gen: 2.5020923614501953 (2.3643645819495696) + | > loss_kl: 1.8896229267120361 (1.8591800647623398) + | > loss_feat: 6.8435378074646 (7.139049480943119) + | > loss_mel: 16.261512756347656 (17.675024537479175) + | > loss_duration: 1.4679886102676392 (1.4592184810077442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.96475601196289 (30.496837195228128) + | > grad_norm_1: tensor(41.3858, device='cuda:0') (tensor(270.4754, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.3886 (0.3551742820178761) + | > loader_time: 0.0057 (0.005456815747653735) + + + --> TIME: 2025-04-19 01:53:10 -- STEP: 93/142 -- GLOBAL_STEP: 404225 + | > loss_disc: 2.5848827362060547 (2.4474163363056802) + | > loss_disc_real_0: 0.11916308850049973 (0.12487567792977056) + | > loss_disc_real_1: 0.22108194231987 (0.20521002735501975) + | > loss_disc_real_2: 0.24183689057826996 (0.2242028865442481) + | > loss_disc_real_3: 0.23936523497104645 (0.22840781301580448) + | > loss_disc_real_4: 0.24732919037342072 (0.2253327997781897) + | > loss_disc_real_5: 0.2352694720029831 (0.23225796238709523) + | > loss_0: 2.5848827362060547 (2.4474163363056802) + | > grad_norm_0: tensor(5.0435, device='cuda:0') (tensor(11.2916, device='cuda:0')) + | > loss_gen: 2.2702040672302246 (2.3613212057339252) + | > loss_kl: 1.9002612829208374 (1.8853385153637137) + | > loss_feat: 5.857518672943115 (7.084042149205362) + | > loss_mel: 17.390512466430664 (17.725634298016942) + | > loss_duration: 1.5116009712219238 (1.4716742230999849) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.930097579956055 (30.528010481147355) + | > grad_norm_1: tensor(107.1539, device='cuda:0') (tensor(222.9923, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.4008 (0.36642341203587037) + | > loader_time: 0.0075 (0.005722384299001384) + + + --> TIME: 2025-04-19 01:53:21 -- STEP: 118/142 -- GLOBAL_STEP: 404250 + | > loss_disc: 2.5403826236724854 (2.4669050119690974) + | > loss_disc_real_0: 0.11147167533636093 (0.12890447051090706) + | > loss_disc_real_1: 0.2258070856332779 (0.20746525187613601) + | > loss_disc_real_2: 0.22369906306266785 (0.22534747295460458) + | > loss_disc_real_3: 0.2349720299243927 (0.22905118371975625) + | > loss_disc_real_4: 0.2389858514070511 (0.22807636137230922) + | > loss_disc_real_5: 0.2587199807167053 (0.23200500769130253) + | > loss_0: 2.5403826236724854 (2.4669050119690974) + | > grad_norm_0: tensor(6.5151, device='cuda:0') (tensor(10.2056, device='cuda:0')) + | > loss_gen: 2.188974618911743 (2.3382538456027793) + | > loss_kl: 1.9679867029190063 (1.8946147793430392) + | > loss_feat: 6.431788444519043 (6.987074302414716) + | > loss_mel: 15.929571151733398 (17.673562809572374) + | > loss_duration: 1.4919862747192383 (1.4789163136886334) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.01030731201172 (30.372422137502898) + | > grad_norm_1: tensor(49.2442, device='cuda:0') (tensor(192.4408, device='cuda:0')) + | > current_lr_0: 0.00019567223700156603 + | > current_lr_1: 0.00019567223700156603 + | > step_time: 0.4393 (0.3783119411791786) + | > loader_time: 0.0072 (0.006052726406162066) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5269644260406494 (2.5269644260406494) + | > loss_disc_real_0: 0.15868216753005981 (0.15868216753005981) + | > loss_disc_real_1: 0.19326481223106384 (0.19326481223106384) + | > loss_disc_real_2: 0.21535784006118774 (0.21535784006118774) + | > loss_disc_real_3: 0.24856708943843842 (0.24856708943843842) + | > loss_disc_real_4: 0.2101253867149353 (0.2101253867149353) + | > loss_disc_real_5: 0.21658872067928314 (0.21658872067928314) + | > loss_0: 2.5269644260406494 (2.5269644260406494) + | > loss_gen: 2.20253586769104 (2.20253586769104) + | > loss_kl: 2.2779393196105957 (2.2779393196105957) + | > loss_feat: 7.180487155914307 (7.180487155914307) + | > loss_mel: 18.348159790039062 (18.348159790039062) + | > loss_duration: 1.67743980884552 (1.67743980884552) + | > loss_1: 31.68656349182129 (31.68656349182129) + + --> STEP: 1 + | > loss_disc: 2.6658525466918945 (2.6658525466918945) + | > loss_disc_real_0: 0.15872584283351898 (0.15872584283351898) + | > loss_disc_real_1: 0.19268757104873657 (0.19268757104873657) + | > loss_disc_real_2: 0.25038081407546997 (0.25038081407546997) + | > loss_disc_real_3: 0.24490611255168915 (0.24490611255168915) + | > loss_disc_real_4: 0.19643594324588776 (0.19643594324588776) + | > loss_disc_real_5: 0.2185986042022705 (0.2185986042022705) + | > loss_0: 2.6658525466918945 (2.6658525466918945) + | > loss_gen: 2.0739974975585938 (2.0739974975585938) + | > loss_kl: 2.7117013931274414 (2.7117013931274414) + | > loss_feat: 6.618649005889893 (6.618649005889893) + | > loss_mel: 18.067909240722656 (18.067909240722656) + | > loss_duration: 1.7217981815338135 (1.7217981815338135) + | > loss_1: 31.194055557250977 (31.194055557250977) + + --> STEP: 2 + | > loss_disc: 2.5172154903411865 (2.5915340185165405) + | > loss_disc_real_0: 0.12420140206813812 (0.14146362245082855) + | > loss_disc_real_1: 0.2110079675912857 (0.20184776932001114) + | > loss_disc_real_2: 0.21538017690181732 (0.23288049548864365) + | > loss_disc_real_3: 0.23900367319583893 (0.24195489287376404) + | > loss_disc_real_4: 0.23059242963790894 (0.21351418644189835) + | > loss_disc_real_5: 0.175851508975029 (0.19722505658864975) + | > loss_0: 2.5172154903411865 (2.5915340185165405) + | > loss_gen: 2.0659360885620117 (2.0699667930603027) + | > loss_kl: 2.048874616622925 (2.380288004875183) + | > loss_feat: 6.68185567855835 (6.650252342224121) + | > loss_mel: 16.93315887451172 (17.500534057617188) + | > loss_duration: 1.662218689918518 (1.6920084357261658) + | > loss_1: 29.39204216003418 (30.293048858642578) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003370046615600586 (-0.0003510713577270508) + | > avg_loss_disc: 2.5915340185165405 (+0.11068785190582275) + | > avg_loss_disc_real_0: 0.14146362245082855 (-0.034033626317977905) + | > avg_loss_disc_real_1: 0.20184776932001114 (-0.009806498885154724) + | > avg_loss_disc_real_2: 0.23288049548864365 (-0.0019850805401802063) + | > avg_loss_disc_real_3: 0.24195489287376404 (+0.013268880546092987) + | > avg_loss_disc_real_4: 0.21351418644189835 (-0.008664257824420929) + | > avg_loss_disc_real_5: 0.19722505658864975 (-0.031912609934806824) + | > avg_loss_0: 2.5915340185165405 (+0.11068785190582275) + | > avg_loss_gen: 2.0699667930603027 (-0.2560616731643677) + | > avg_loss_kl: 2.380288004875183 (+0.017513513565063477) + | > avg_loss_feat: 6.650252342224121 (-0.31633448600769043) + | > avg_loss_mel: 17.500534057617188 (-1.1671743392944336) + | > avg_loss_duration: 1.6920084357261658 (+0.028886735439300537) + | > avg_loss_1: 30.293048858642578 (-1.6931705474853516) + + + > EPOCH: 176/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:53:33)  + + --> TIME: 2025-04-19 01:53:34 -- STEP: 1/142 -- GLOBAL_STEP: 404275 + | > loss_disc: 2.381770133972168 (2.381770133972168) + | > loss_disc_real_0: 0.14291881024837494 (0.14291881024837494) + | > loss_disc_real_1: 0.18686164915561676 (0.18686164915561676) + | > loss_disc_real_2: 0.21785010397434235 (0.21785010397434235) + | > loss_disc_real_3: 0.2549208402633667 (0.2549208402633667) + | > loss_disc_real_4: 0.24130463600158691 (0.24130463600158691) + | > loss_disc_real_5: 0.23274965584278107 (0.23274965584278107) + | > loss_0: 2.381770133972168 (2.381770133972168) + | > grad_norm_0: tensor(9.1830, device='cuda:0') (tensor(9.1830, device='cuda:0')) + | > loss_gen: 2.5890660285949707 (2.5890660285949707) + | > loss_kl: 1.8595436811447144 (1.8595436811447144) + | > loss_feat: 7.969950199127197 (7.969950199127197) + | > loss_mel: 19.030502319335938 (19.030502319335938) + | > loss_duration: 1.409470796585083 (1.409470796585083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.8585319519043 (32.8585319519043) + | > grad_norm_1: tensor(160.0610, device='cuda:0') (tensor(160.0610, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.3296 (0.3295929431915283) + | > loader_time: 0.0039 (0.0038700103759765625) + + + --> TIME: 2025-04-19 01:53:43 -- STEP: 26/142 -- GLOBAL_STEP: 404300 + | > loss_disc: 2.313828229904175 (2.456577200155992) + | > loss_disc_real_0: 0.0980452373623848 (0.12549927085638046) + | > loss_disc_real_1: 0.17086787521839142 (0.2051205915900377) + | > loss_disc_real_2: 0.2254018634557724 (0.2260634990838858) + | > loss_disc_real_3: 0.25393247604370117 (0.22985273656936792) + | > loss_disc_real_4: 0.2701754868030548 (0.22970280051231384) + | > loss_disc_real_5: 0.266944944858551 (0.23496298835827753) + | > loss_0: 2.313828229904175 (2.456577200155992) + | > grad_norm_0: tensor(6.5735, device='cuda:0') (tensor(6.5757, device='cuda:0')) + | > loss_gen: 2.517404556274414 (2.3689596469585705) + | > loss_kl: 2.016465187072754 (1.8726628101789033) + | > loss_feat: 7.639379501342773 (7.217727514413687) + | > loss_mel: 17.940845489501953 (17.9410652747521) + | > loss_duration: 1.4639168977737427 (1.4422423151823192) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.578012466430664 (30.842657235952522) + | > grad_norm_1: tensor(167.6002, device='cuda:0') (tensor(108.9826, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.3592 (0.33588334230276257) + | > loader_time: 0.0049 (0.004740238189697266) + + + --> TIME: 2025-04-19 01:53:52 -- STEP: 51/142 -- GLOBAL_STEP: 404325 + | > loss_disc: 2.4402947425842285 (2.4533177029852773) + | > loss_disc_real_0: 0.15886667370796204 (0.12567968944124144) + | > loss_disc_real_1: 0.2113698571920395 (0.2063448493387185) + | > loss_disc_real_2: 0.23483790457248688 (0.22590180062780194) + | > loss_disc_real_3: 0.2189403623342514 (0.22840630862058378) + | > loss_disc_real_4: 0.2320890575647354 (0.22883491276525983) + | > loss_disc_real_5: 0.24679416418075562 (0.23321653318171406) + | > loss_0: 2.4402947425842285 (2.4533177029852773) + | > grad_norm_0: tensor(6.1428, device='cuda:0') (tensor(7.5130, device='cuda:0')) + | > loss_gen: 2.3752434253692627 (2.3776131003510703) + | > loss_kl: 1.9006972312927246 (1.8625253111708397) + | > loss_feat: 7.373226642608643 (7.212820183997061) + | > loss_mel: 18.37197494506836 (18.012692208383594) + | > loss_duration: 1.470753788948059 (1.4509627234702016) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.491897583007812 (30.916613447899913) + | > grad_norm_1: tensor(74.8641, device='cuda:0') (tensor(157.2689, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.3794 (0.3518403324426389) + | > loader_time: 0.0056 (0.005196108537561753) + + + --> TIME: 2025-04-19 01:54:03 -- STEP: 76/142 -- GLOBAL_STEP: 404350 + | > loss_disc: 2.5567102432250977 (2.459227201185729) + | > loss_disc_real_0: 0.1360377073287964 (0.12748768210018932) + | > loss_disc_real_1: 0.19827358424663544 (0.20671872755414558) + | > loss_disc_real_2: 0.2552747428417206 (0.22465416436132632) + | > loss_disc_real_3: 0.25002017617225647 (0.22947095452170624) + | > loss_disc_real_4: 0.24571047723293304 (0.2288343588772573) + | > loss_disc_real_5: 0.26508212089538574 (0.23278607114365227) + | > loss_0: 2.5567102432250977 (2.459227201185729) + | > grad_norm_0: tensor(9.8603, device='cuda:0') (tensor(9.2763, device='cuda:0')) + | > loss_gen: 2.3703927993774414 (2.378275789712604) + | > loss_kl: 1.8359607458114624 (1.8646202479538165) + | > loss_feat: 6.252408027648926 (7.1599937740125155) + | > loss_mel: 18.05683135986328 (17.969991232219495) + | > loss_duration: 1.5075366497039795 (1.4642314816776074) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.02312660217285 (30.83711250204789) + | > grad_norm_1: tensor(232.5956, device='cuda:0') (tensor(197.6354, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.3958 (0.36761692636891413) + | > loader_time: 0.0061 (0.005537284047980057) + + + --> TIME: 2025-04-19 01:54:13 -- STEP: 101/142 -- GLOBAL_STEP: 404375 + | > loss_disc: 2.544785976409912 (2.471026073588004) + | > loss_disc_real_0: 0.1373545527458191 (0.12956502613159682) + | > loss_disc_real_1: 0.17636148631572723 (0.20779215846911514) + | > loss_disc_real_2: 0.29243341088294983 (0.2267587298508918) + | > loss_disc_real_3: 0.2722727060317993 (0.22935380572729772) + | > loss_disc_real_4: 0.22492513060569763 (0.22900058476641627) + | > loss_disc_real_5: 0.28490540385246277 (0.2338253056058789) + | > loss_0: 2.544785976409912 (2.471026073588004) + | > grad_norm_0: tensor(23.7524, device='cuda:0') (tensor(11.4905, device='cuda:0')) + | > loss_gen: 2.3543765544891357 (2.3721036061201928) + | > loss_kl: 1.918958067893982 (1.882053791886509) + | > loss_feat: 6.85333776473999 (7.090728419842106) + | > loss_mel: 18.19978141784668 (17.940035980526762) + | > loss_duration: 1.5291740894317627 (1.4744887989346345) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.855627059936523 (30.75941065042326) + | > grad_norm_1: tensor(380.5008, device='cuda:0') (tensor(237.9547, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.4144 (0.37615916752579187) + | > loader_time: 0.0065 (0.005767510669066176) + + + --> TIME: 2025-04-19 01:54:24 -- STEP: 126/142 -- GLOBAL_STEP: 404400 + | > loss_disc: 2.4388039112091064 (2.4707494879525815) + | > loss_disc_real_0: 0.12612302601337433 (0.13040043457987763) + | > loss_disc_real_1: 0.1582745462656021 (0.20755388457623739) + | > loss_disc_real_2: 0.20115339756011963 (0.22661039800870986) + | > loss_disc_real_3: 0.22700145840644836 (0.22911169841175988) + | > loss_disc_real_4: 0.2122117131948471 (0.22877598672159136) + | > loss_disc_real_5: 0.23191805183887482 (0.23263761495787) + | > loss_0: 2.4388039112091064 (2.4707494879525815) + | > grad_norm_0: tensor(15.1899, device='cuda:0') (tensor(13.0123, device='cuda:0')) + | > loss_gen: 2.314544677734375 (2.363523965790157) + | > loss_kl: 1.8606271743774414 (1.8892148411463177) + | > loss_feat: 7.823580741882324 (7.031578779220581) + | > loss_mel: 18.090499877929688 (17.857113247825986) + | > loss_duration: 1.513107180595398 (1.48126886288325) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.602359771728516 (30.622699752686515) + | > grad_norm_1: tensor(531.8383, device='cuda:0') (tensor(258.9845, device='cuda:0')) + | > current_lr_0: 0.00019564777797194082 + | > current_lr_1: 0.00019564777797194082 + | > step_time: 0.4402 (0.38633948091476683) + | > loader_time: 0.0068 (0.006037439618791853) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4900717735290527 (2.4900717735290527) + | > loss_disc_real_0: 0.14365993440151215 (0.14365993440151215) + | > loss_disc_real_1: 0.27617618441581726 (0.27617618441581726) + | > loss_disc_real_2: 0.2702852487564087 (0.2702852487564087) + | > loss_disc_real_3: 0.2627222239971161 (0.2627222239971161) + | > loss_disc_real_4: 0.2752254009246826 (0.2752254009246826) + | > loss_disc_real_5: 0.24758434295654297 (0.24758434295654297) + | > loss_0: 2.4900717735290527 (2.4900717735290527) + | > loss_gen: 2.5402908325195312 (2.5402908325195312) + | > loss_kl: 2.488844394683838 (2.488844394683838) + | > loss_feat: 6.287694931030273 (6.287694931030273) + | > loss_mel: 17.0000057220459 (17.0000057220459) + | > loss_duration: 1.6468455791473389 (1.6468455791473389) + | > loss_1: 29.963680267333984 (29.963680267333984) + + --> STEP: 1 + | > loss_disc: 2.4466753005981445 (2.4466753005981445) + | > loss_disc_real_0: 0.14113448560237885 (0.14113448560237885) + | > loss_disc_real_1: 0.2543385922908783 (0.2543385922908783) + | > loss_disc_real_2: 0.254230797290802 (0.254230797290802) + | > loss_disc_real_3: 0.24692891538143158 (0.24692891538143158) + | > loss_disc_real_4: 0.23108968138694763 (0.23108968138694763) + | > loss_disc_real_5: 0.24524083733558655 (0.24524083733558655) + | > loss_0: 2.4466753005981445 (2.4466753005981445) + | > loss_gen: 2.4131462574005127 (2.4131462574005127) + | > loss_kl: 2.652376890182495 (2.652376890182495) + | > loss_feat: 7.167685508728027 (7.167685508728027) + | > loss_mel: 17.607763290405273 (17.607763290405273) + | > loss_duration: 1.688075065612793 (1.688075065612793) + | > loss_1: 31.529048919677734 (31.529048919677734) + + --> STEP: 2 + | > loss_disc: 2.479661464691162 (2.4631683826446533) + | > loss_disc_real_0: 0.16340136528015137 (0.1522679254412651) + | > loss_disc_real_1: 0.21712273359298706 (0.23573066294193268) + | > loss_disc_real_2: 0.23702514171600342 (0.2456279695034027) + | > loss_disc_real_3: 0.28091010451316833 (0.26391950994729996) + | > loss_disc_real_4: 0.22036737203598022 (0.22572852671146393) + | > loss_disc_real_5: 0.2614009380340576 (0.2533208876848221) + | > loss_0: 2.479661464691162 (2.4631683826446533) + | > loss_gen: 2.5377304553985596 (2.475438356399536) + | > loss_kl: 2.020695924758911 (2.336536407470703) + | > loss_feat: 7.441305637359619 (7.304495573043823) + | > loss_mel: 19.757108688354492 (18.682435989379883) + | > loss_duration: 1.656406044960022 (1.6722405552864075) + | > loss_1: 33.413246154785156 (32.471147537231445) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032033920288085938 (-0.0001666545867919922) + | > avg_loss_disc: 2.4631683826446533 (-0.1283656358718872) + | > avg_loss_disc_real_0: 0.1522679254412651 (+0.010804302990436554) + | > avg_loss_disc_real_1: 0.23573066294193268 (+0.03388289362192154) + | > avg_loss_disc_real_2: 0.2456279695034027 (+0.012747474014759064) + | > avg_loss_disc_real_3: 0.26391950994729996 (+0.02196461707353592) + | > avg_loss_disc_real_4: 0.22572852671146393 (+0.012214340269565582) + | > avg_loss_disc_real_5: 0.2533208876848221 (+0.05609583109617233) + | > avg_loss_0: 2.4631683826446533 (-0.1283656358718872) + | > avg_loss_gen: 2.475438356399536 (+0.4054715633392334) + | > avg_loss_kl: 2.336536407470703 (-0.04375159740447998) + | > avg_loss_feat: 7.304495573043823 (+0.6542432308197021) + | > avg_loss_mel: 18.682435989379883 (+1.1819019317626953) + | > avg_loss_duration: 1.6722405552864075 (-0.0197678804397583) + | > avg_loss_1: 32.471147537231445 (+2.178098678588867) + + + > EPOCH: 177/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:54:32)  + + --> TIME: 2025-04-19 01:54:36 -- STEP: 9/142 -- GLOBAL_STEP: 404425 + | > loss_disc: 2.396169424057007 (2.4050600263807507) + | > loss_disc_real_0: 0.09631537646055222 (0.11189435919125874) + | > loss_disc_real_1: 0.21677151322364807 (0.20303184456295437) + | > loss_disc_real_2: 0.2073187232017517 (0.21938243673907387) + | > loss_disc_real_3: 0.22136136889457703 (0.2230664243300756) + | > loss_disc_real_4: 0.21857105195522308 (0.22407925460073683) + | > loss_disc_real_5: 0.23998408019542694 (0.22944054669804043) + | > loss_0: 2.396169424057007 (2.4050600263807507) + | > grad_norm_0: tensor(11.5590, device='cuda:0') (tensor(8.0236, device='cuda:0')) + | > loss_gen: 2.4114832878112793 (2.381204605102539) + | > loss_kl: 1.9358930587768555 (1.8089318010542128) + | > loss_feat: 7.491033554077148 (7.265462769402398) + | > loss_mel: 17.6118221282959 (17.86016994052463) + | > loss_duration: 1.419619083404541 (1.4387450085745916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.86985206604004 (30.75451448228624) + | > grad_norm_1: tensor(118.3803, device='cuda:0') (tensor(196.1172, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.324 (0.326019287109375) + | > loader_time: 0.0049 (0.004503938886854384) + + + --> TIME: 2025-04-19 01:54:45 -- STEP: 34/142 -- GLOBAL_STEP: 404450 + | > loss_disc: 2.506185293197632 (2.4136115943684295) + | > loss_disc_real_0: 0.13508738577365875 (0.11654684342005674) + | > loss_disc_real_1: 0.20449118316173553 (0.2036365213639596) + | > loss_disc_real_2: 0.22585241496562958 (0.2208251111647662) + | > loss_disc_real_3: 0.17095264792442322 (0.22468644687358072) + | > loss_disc_real_4: 0.21334998309612274 (0.221668062841191) + | > loss_disc_real_5: 0.23740151524543762 (0.2318611723535201) + | > loss_0: 2.506185293197632 (2.4136115943684295) + | > grad_norm_0: tensor(10.2744, device='cuda:0') (tensor(6.8053, device='cuda:0')) + | > loss_gen: 2.3264899253845215 (2.3786251194336834) + | > loss_kl: 1.923332929611206 (1.8511604701771456) + | > loss_feat: 7.2825493812561035 (7.196590493707096) + | > loss_mel: 17.775108337402344 (17.648439743939562) + | > loss_duration: 1.4898922443389893 (1.4443399064681108) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.797374725341797 (30.519156119402716) + | > grad_norm_1: tensor(93.3096, device='cuda:0') (tensor(135.2453, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.3549 (0.3382942325928632) + | > loader_time: 0.0055 (0.004965705030104693) + + + --> TIME: 2025-04-19 01:54:54 -- STEP: 59/142 -- GLOBAL_STEP: 404475 + | > loss_disc: 2.4530317783355713 (2.437713865506447) + | > loss_disc_real_0: 0.13017497956752777 (0.11962121636685678) + | > loss_disc_real_1: 0.23494268953800201 (0.20549179967177117) + | > loss_disc_real_2: 0.25446343421936035 (0.22289426624774933) + | > loss_disc_real_3: 0.2539093494415283 (0.22620194923069517) + | > loss_disc_real_4: 0.2532958388328552 (0.2265624411530414) + | > loss_disc_real_5: 0.26727333664894104 (0.2331237969762188) + | > loss_0: 2.4530317783355713 (2.437713865506447) + | > grad_norm_0: tensor(9.7749, device='cuda:0') (tensor(7.6280, device='cuda:0')) + | > loss_gen: 2.517418384552002 (2.3655565310332736) + | > loss_kl: 1.9837759733200073 (1.8710528410087197) + | > loss_feat: 7.477380752563477 (7.164402258598198) + | > loss_mel: 18.704641342163086 (17.688190460205075) + | > loss_duration: 1.460875153541565 (1.4548119185334545) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.14409255981445 (30.544014138690496) + | > grad_norm_1: tensor(313.3636, device='cuda:0') (tensor(170.8942, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.3785 (0.3509469598026599) + | > loader_time: 0.0059 (0.005313845004065563) + + + --> TIME: 2025-04-19 01:55:04 -- STEP: 84/142 -- GLOBAL_STEP: 404500 + | > loss_disc: 2.5946922302246094 (2.4552418759890964) + | > loss_disc_real_0: 0.14724038541316986 (0.12439643698079246) + | > loss_disc_real_1: 0.2708493769168854 (0.20742584374688922) + | > loss_disc_real_2: 0.2391170859336853 (0.22410210080090023) + | > loss_disc_real_3: 0.25327858328819275 (0.22741611532512165) + | > loss_disc_real_4: 0.25090277194976807 (0.22829986345909892) + | > loss_disc_real_5: 0.23819568753242493 (0.23244485081661315) + | > loss_0: 2.5946922302246094 (2.4552418759890964) + | > grad_norm_0: tensor(8.5425, device='cuda:0') (tensor(8.0238, device='cuda:0')) + | > loss_gen: 2.436030149459839 (2.3557522098223354) + | > loss_kl: 1.9490888118743896 (1.8930976305689131) + | > loss_feat: 6.286342144012451 (7.077142397562663) + | > loss_mel: 16.286975860595703 (17.779054028647284) + | > loss_duration: 1.5278723239898682 (1.4678856489204224) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.486309051513672 (30.57293197086879) + | > grad_norm_1: tensor(291.1779, device='cuda:0') (tensor(183.0568, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.3932 (0.36234149478730704) + | > loader_time: 0.0063 (0.005561357452755882) + + + --> TIME: 2025-04-19 01:55:15 -- STEP: 109/142 -- GLOBAL_STEP: 404525 + | > loss_disc: 2.5075767040252686 (2.4718539977292395) + | > loss_disc_real_0: 0.13719923794269562 (0.1284571016600373) + | > loss_disc_real_1: 0.21290579438209534 (0.20900086812469937) + | > loss_disc_real_2: 0.2186489999294281 (0.22530685214821353) + | > loss_disc_real_3: 0.2247985452413559 (0.22806454128628478) + | > loss_disc_real_4: 0.2543369233608246 (0.2297559868578517) + | > loss_disc_real_5: 0.2647137939929962 (0.23282118565445648) + | > loss_0: 2.5075767040252686 (2.4718539977292395) + | > grad_norm_0: tensor(3.7893, device='cuda:0') (tensor(8.1620, device='cuda:0')) + | > loss_gen: 2.292104482650757 (2.3363684251767762) + | > loss_kl: 1.9499800205230713 (1.9022956638161195) + | > loss_feat: 6.615359306335449 (6.97521298084784) + | > loss_mel: 18.332456588745117 (17.709433398115525) + | > loss_duration: 1.5053715705871582 (1.4772775479412952) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.69527244567871 (30.400588044332803) + | > grad_norm_1: tensor(133.2384, device='cuda:0') (tensor(181.6607, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.41 (0.3727119734527868) + | > loader_time: 0.0066 (0.005794183923563825) + + + --> TIME: 2025-04-19 01:55:26 -- STEP: 134/142 -- GLOBAL_STEP: 404550 + | > loss_disc: 2.42911696434021 (2.473141958464437) + | > loss_disc_real_0: 0.15389028191566467 (0.13074484817794896) + | > loss_disc_real_1: 0.20366305112838745 (0.20787165736529364) + | > loss_disc_real_2: 0.19900016486644745 (0.22486903351634296) + | > loss_disc_real_3: 0.2136947512626648 (0.22718628662735668) + | > loss_disc_real_4: 0.22894079983234406 (0.22872399116185174) + | > loss_disc_real_5: 0.23767589032649994 (0.23306701599217172) + | > loss_0: 2.42911696434021 (2.473141958464437) + | > grad_norm_0: tensor(7.9702, device='cuda:0') (tensor(8.0731, device='cuda:0')) + | > loss_gen: 2.4299492835998535 (2.328475576728136) + | > loss_kl: 1.9399057626724243 (1.9063994635396928) + | > loss_feat: 7.266740798950195 (6.913862637619474) + | > loss_mel: 16.94373893737793 (17.605167922688945) + | > loss_duration: 1.5389857292175293 (1.4833817855635683) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.119321823120117 (30.237287492894414) + | > grad_norm_1: tensor(145.1820, device='cuda:0') (tensor(174.2806, device='cuda:0')) + | > current_lr_0: 0.00019562332199969432 + | > current_lr_1: 0.00019562332199969432 + | > step_time: 0.4386 (0.38547815671607627) + | > loader_time: 0.0072 (0.006056011612735577) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3384454250335693 (2.3384454250335693) + | > loss_disc_real_0: 0.18252165615558624 (0.18252165615558624) + | > loss_disc_real_1: 0.22352536022663116 (0.22352536022663116) + | > loss_disc_real_2: 0.1599780172109604 (0.1599780172109604) + | > loss_disc_real_3: 0.21741323173046112 (0.21741323173046112) + | > loss_disc_real_4: 0.17398427426815033 (0.17398427426815033) + | > loss_disc_real_5: 0.24160067737102509 (0.24160067737102509) + | > loss_0: 2.3384454250335693 (2.3384454250335693) + | > loss_gen: 2.6379222869873047 (2.6379222869873047) + | > loss_kl: 2.3527705669403076 (2.3527705669403076) + | > loss_feat: 7.922006130218506 (7.922006130218506) + | > loss_mel: 20.01361846923828 (20.01361846923828) + | > loss_duration: 1.6565897464752197 (1.6565897464752197) + | > loss_1: 34.58290481567383 (34.58290481567383) + + --> STEP: 1 + | > loss_disc: 2.584257125854492 (2.584257125854492) + | > loss_disc_real_0: 0.2083725929260254 (0.2083725929260254) + | > loss_disc_real_1: 0.2694619297981262 (0.2694619297981262) + | > loss_disc_real_2: 0.264937162399292 (0.264937162399292) + | > loss_disc_real_3: 0.2555241584777832 (0.2555241584777832) + | > loss_disc_real_4: 0.22947652637958527 (0.22947652637958527) + | > loss_disc_real_5: 0.27672865986824036 (0.27672865986824036) + | > loss_0: 2.584257125854492 (2.584257125854492) + | > loss_gen: 2.552971839904785 (2.552971839904785) + | > loss_kl: 2.5769360065460205 (2.5769360065460205) + | > loss_feat: 7.013905048370361 (7.013905048370361) + | > loss_mel: 18.240793228149414 (18.240793228149414) + | > loss_duration: 1.724663496017456 (1.724663496017456) + | > loss_1: 32.10926818847656 (32.10926818847656) + + --> STEP: 2 + | > loss_disc: 2.4787237644195557 (2.531490445137024) + | > loss_disc_real_0: 0.2747791111469269 (0.24157585203647614) + | > loss_disc_real_1: 0.2626863420009613 (0.26607413589954376) + | > loss_disc_real_2: 0.21537257730960846 (0.24015486985445023) + | > loss_disc_real_3: 0.21951547265052795 (0.23751981556415558) + | > loss_disc_real_4: 0.21683305501937866 (0.22315479069948196) + | > loss_disc_real_5: 0.20836125314235687 (0.24254495650529861) + | > loss_0: 2.4787237644195557 (2.531490445137024) + | > loss_gen: 2.6915411949157715 (2.6222565174102783) + | > loss_kl: 2.0852279663085938 (2.331081986427307) + | > loss_feat: 7.5784687995910645 (7.296186923980713) + | > loss_mel: 18.918323516845703 (18.57955837249756) + | > loss_duration: 1.6470096111297607 (1.6858365535736084) + | > loss_1: 32.920570373535156 (32.51491928100586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036052465438842773 (+0.0004018545150756836) + | > avg_loss_disc: 2.531490445137024 (+0.0683220624923706) + | > avg_loss_disc_real_0: 0.24157585203647614 (+0.08930792659521103) + | > avg_loss_disc_real_1: 0.26607413589954376 (+0.030343472957611084) + | > avg_loss_disc_real_2: 0.24015486985445023 (-0.005473099648952484) + | > avg_loss_disc_real_3: 0.23751981556415558 (-0.02639969438314438) + | > avg_loss_disc_real_4: 0.22315479069948196 (-0.002573736011981964) + | > avg_loss_disc_real_5: 0.24254495650529861 (-0.010775931179523468) + | > avg_loss_0: 2.531490445137024 (+0.0683220624923706) + | > avg_loss_gen: 2.6222565174102783 (+0.1468181610107422) + | > avg_loss_kl: 2.331081986427307 (-0.005454421043395996) + | > avg_loss_feat: 7.296186923980713 (-0.008308649063110352) + | > avg_loss_mel: 18.57955837249756 (-0.10287761688232422) + | > avg_loss_duration: 1.6858365535736084 (+0.013595998287200928) + | > avg_loss_1: 32.51491928100586 (+0.04377174377441406) + + + > EPOCH: 178/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:55:31)  + + --> TIME: 2025-04-19 01:55:37 -- STEP: 17/142 -- GLOBAL_STEP: 404575 + | > loss_disc: 2.484023094177246 (2.434919231078204) + | > loss_disc_real_0: 0.11532817035913467 (0.12063635491272982) + | > loss_disc_real_1: 0.22915324568748474 (0.209479281130959) + | > loss_disc_real_2: 0.234646737575531 (0.22218698350822225) + | > loss_disc_real_3: 0.2647590637207031 (0.23090725874199586) + | > loss_disc_real_4: 0.24867719411849976 (0.23344774544239044) + | > loss_disc_real_5: 0.22643765807151794 (0.23427046835422516) + | > loss_0: 2.484023094177246 (2.434919231078204) + | > grad_norm_0: tensor(12.3795, device='cuda:0') (tensor(8.0993, device='cuda:0')) + | > loss_gen: 2.413621187210083 (2.3849818846758675) + | > loss_kl: 1.8833036422729492 (1.7982689843458288) + | > loss_feat: 6.769067287445068 (7.336719933678122) + | > loss_mel: 15.869722366333008 (17.986970228307385) + | > loss_duration: 1.4353978633880615 (1.4327116643681246) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.371112823486328 (30.939652835621555) + | > grad_norm_1: tensor(229.3724, device='cuda:0') (tensor(173.8687, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.3401 (0.3292993377236759) + | > loader_time: 0.0048 (0.004633637035594267) + + + --> TIME: 2025-04-19 01:55:47 -- STEP: 42/142 -- GLOBAL_STEP: 404600 + | > loss_disc: 2.5299668312072754 (2.425329316230047) + | > loss_disc_real_0: 0.10795028507709503 (0.12121272353189332) + | > loss_disc_real_1: 0.2172784060239792 (0.2064355987878073) + | > loss_disc_real_2: 0.2248651534318924 (0.22109925817875636) + | > loss_disc_real_3: 0.18688005208969116 (0.22783721699601128) + | > loss_disc_real_4: 0.2419995665550232 (0.22765738055819557) + | > loss_disc_real_5: 0.20440511405467987 (0.23231896722600573) + | > loss_0: 2.5299668312072754 (2.425329316230047) + | > grad_norm_0: tensor(10.6471, device='cuda:0') (tensor(7.8784, device='cuda:0')) + | > loss_gen: 2.22310209274292 (2.375356370494479) + | > loss_kl: 1.7974413633346558 (1.8300992136909848) + | > loss_feat: 6.642246246337891 (7.217670190901983) + | > loss_mel: 16.682846069335938 (17.80991563342866) + | > loss_duration: 1.4900357723236084 (1.4460936529295785) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.83567237854004 (30.67913514091855) + | > grad_norm_1: tensor(137.1588, device='cuda:0') (tensor(190.3782, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.3628 (0.348460662932623) + | > loader_time: 0.0056 (0.005115264937991187) + + + --> TIME: 2025-04-19 01:55:56 -- STEP: 67/142 -- GLOBAL_STEP: 404625 + | > loss_disc: 2.592897415161133 (2.4513342558447992) + | > loss_disc_real_0: 0.14767955243587494 (0.12413415946622393) + | > loss_disc_real_1: 0.18026922643184662 (0.20529084241212303) + | > loss_disc_real_2: 0.19787774980068207 (0.22463109577769663) + | > loss_disc_real_3: 0.23194943368434906 (0.23178009733335295) + | > loss_disc_real_4: 0.1994560807943344 (0.229560425254836) + | > loss_disc_real_5: 0.23698531091213226 (0.2349162186259654) + | > loss_0: 2.592897415161133 (2.4513342558447992) + | > grad_norm_0: tensor(8.3934, device='cuda:0') (tensor(8.0904, device='cuda:0')) + | > loss_gen: 2.1434805393218994 (2.367169631061269) + | > loss_kl: 1.8500174283981323 (1.861777216640871) + | > loss_feat: 6.142934322357178 (7.108702794829411) + | > loss_mel: 16.214244842529297 (17.82007245875116) + | > loss_duration: 1.4240624904632568 (1.4579661550806529) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.77474021911621 (30.61568826703883) + | > grad_norm_1: tensor(215.4584, device='cuda:0') (tensor(194.3287, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.3832 (0.35818942625131184) + | > loader_time: 0.0067 (0.0054257983592019155) + + + --> TIME: 2025-04-19 01:56:06 -- STEP: 92/142 -- GLOBAL_STEP: 404650 + | > loss_disc: 2.435624122619629 (2.4699535084807356) + | > loss_disc_real_0: 0.1655903309583664 (0.12713842968577924) + | > loss_disc_real_1: 0.19657190144062042 (0.20800849519993947) + | > loss_disc_real_2: 0.24296478927135468 (0.22694171236261076) + | > loss_disc_real_3: 0.22050994634628296 (0.23083081857665724) + | > loss_disc_real_4: 0.22266331315040588 (0.2304472607438979) + | > loss_disc_real_5: 0.23509976267814636 (0.23463067185619604) + | > loss_0: 2.435624122619629 (2.4699535084807356) + | > grad_norm_0: tensor(16.5612, device='cuda:0') (tensor(8.5680, device='cuda:0')) + | > loss_gen: 2.450598955154419 (2.354184250468793) + | > loss_kl: 1.8250387907028198 (1.8772727121477542) + | > loss_feat: 7.0408430099487305 (7.045245170593262) + | > loss_mel: 18.664812088012695 (17.881935275119275) + | > loss_duration: 1.4716548919677734 (1.4707356250804404) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.45294761657715 (30.62937307357788) + | > grad_norm_1: tensor(378.7692, device='cuda:0') (tensor(191.9373, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.4106 (0.3683949397957843) + | > loader_time: 0.0063 (0.00573801735173101) + + + --> TIME: 2025-04-19 01:56:17 -- STEP: 117/142 -- GLOBAL_STEP: 404675 + | > loss_disc: 2.487039566040039 (2.4721423010540824) + | > loss_disc_real_0: 0.10491068661212921 (0.1294703400160512) + | > loss_disc_real_1: 0.23458918929100037 (0.20836468168303499) + | > loss_disc_real_2: 0.2679491341114044 (0.2259614974515051) + | > loss_disc_real_3: 0.23118259012699127 (0.2298996307146855) + | > loss_disc_real_4: 0.265456885099411 (0.23044862477188435) + | > loss_disc_real_5: 0.24883785843849182 (0.2341749247832176) + | > loss_0: 2.487039566040039 (2.4721423010540824) + | > grad_norm_0: tensor(14.5654, device='cuda:0') (tensor(9.5917, device='cuda:0')) + | > loss_gen: 2.323002576828003 (2.3467957514982944) + | > loss_kl: 1.7825430631637573 (1.8914222982194688) + | > loss_feat: 6.668006896972656 (6.9451877683655825) + | > loss_mel: 16.989002227783203 (17.724684829385865) + | > loss_duration: 1.492537021636963 (1.4780240568340335) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.255090713500977 (30.386114723662025) + | > grad_norm_1: tensor(321.7157, device='cuda:0') (tensor(214.3290, device='cuda:0')) + | > current_lr_0: 0.00019559886908444434 + | > current_lr_1: 0.00019559886908444434 + | > step_time: 0.4374 (0.3787002115168123) + | > loader_time: 0.007 (0.005993631150987413) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3933005332946777 (2.3933005332946777) + | > loss_disc_real_0: 0.13575616478919983 (0.13575616478919983) + | > loss_disc_real_1: 0.21394236385822296 (0.21394236385822296) + | > loss_disc_real_2: 0.23012006282806396 (0.23012006282806396) + | > loss_disc_real_3: 0.25086361169815063 (0.25086361169815063) + | > loss_disc_real_4: 0.21731485426425934 (0.21731485426425934) + | > loss_disc_real_5: 0.2457129806280136 (0.2457129806280136) + | > loss_0: 2.3933005332946777 (2.3933005332946777) + | > loss_gen: 2.412421941757202 (2.412421941757202) + | > loss_kl: 2.4729361534118652 (2.4729361534118652) + | > loss_feat: 7.112707614898682 (7.112707614898682) + | > loss_mel: 18.867387771606445 (18.867387771606445) + | > loss_duration: 1.6757410764694214 (1.6757410764694214) + | > loss_1: 32.541194915771484 (32.541194915771484) + + --> STEP: 1 + | > loss_disc: 2.471801519393921 (2.471801519393921) + | > loss_disc_real_0: 0.14658783376216888 (0.14658783376216888) + | > loss_disc_real_1: 0.2687201201915741 (0.2687201201915741) + | > loss_disc_real_2: 0.23874835669994354 (0.23874835669994354) + | > loss_disc_real_3: 0.23470725119113922 (0.23470725119113922) + | > loss_disc_real_4: 0.2527312636375427 (0.2527312636375427) + | > loss_disc_real_5: 0.19746628403663635 (0.19746628403663635) + | > loss_0: 2.471801519393921 (2.471801519393921) + | > loss_gen: 2.306870937347412 (2.306870937347412) + | > loss_kl: 2.6626455783843994 (2.6626455783843994) + | > loss_feat: 5.890104293823242 (5.890104293823242) + | > loss_mel: 16.850383758544922 (16.850383758544922) + | > loss_duration: 1.732375144958496 (1.732375144958496) + | > loss_1: 29.442378997802734 (29.442378997802734) + + --> STEP: 2 + | > loss_disc: 2.5010650157928467 (2.486433267593384) + | > loss_disc_real_0: 0.13587291538715363 (0.14123037457466125) + | > loss_disc_real_1: 0.24962575733661652 (0.2591729387640953) + | > loss_disc_real_2: 0.2278742790222168 (0.23331131786108017) + | > loss_disc_real_3: 0.24940761923789978 (0.2420574352145195) + | > loss_disc_real_4: 0.19717006385326385 (0.2249506637454033) + | > loss_disc_real_5: 0.18378794193267822 (0.1906271129846573) + | > loss_0: 2.5010650157928467 (2.486433267593384) + | > loss_gen: 2.2832443714141846 (2.2950576543807983) + | > loss_kl: 2.198683500289917 (2.430664539337158) + | > loss_feat: 8.252683639526367 (7.071393966674805) + | > loss_mel: 19.582839965820312 (18.216611862182617) + | > loss_duration: 1.6715373992919922 (1.7019562721252441) + | > loss_1: 33.988990783691406 (31.71568489074707) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035001039505004883 (-0.00010514259338378906) + | > avg_loss_disc: 2.486433267593384 (-0.04505717754364014) + | > avg_loss_disc_real_0: 0.14123037457466125 (-0.10034547746181488) + | > avg_loss_disc_real_1: 0.2591729387640953 (-0.006901197135448456) + | > avg_loss_disc_real_2: 0.23331131786108017 (-0.006843551993370056) + | > avg_loss_disc_real_3: 0.2420574352145195 (+0.004537619650363922) + | > avg_loss_disc_real_4: 0.2249506637454033 (+0.0017958730459213257) + | > avg_loss_disc_real_5: 0.1906271129846573 (-0.05191784352064133) + | > avg_loss_0: 2.486433267593384 (-0.04505717754364014) + | > avg_loss_gen: 2.2950576543807983 (-0.32719886302948) + | > avg_loss_kl: 2.430664539337158 (+0.09958255290985107) + | > avg_loss_feat: 7.071393966674805 (-0.2247929573059082) + | > avg_loss_mel: 18.216611862182617 (-0.3629465103149414) + | > avg_loss_duration: 1.7019562721252441 (+0.016119718551635742) + | > avg_loss_1: 31.71568489074707 (-0.7992343902587891) + + + > EPOCH: 179/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:56:30)  + + --> TIME: 2025-04-19 01:56:31 -- STEP: 0/142 -- GLOBAL_STEP: 404700 + | > loss_disc: 2.448091506958008 (2.448091506958008) + | > loss_disc_real_0: 0.13954468071460724 (0.13954468071460724) + | > loss_disc_real_1: 0.23483619093894958 (0.23483619093894958) + | > loss_disc_real_2: 0.25265926122665405 (0.25265926122665405) + | > loss_disc_real_3: 0.2768899202346802 (0.2768899202346802) + | > loss_disc_real_4: 0.2690171003341675 (0.2690171003341675) + | > loss_disc_real_5: 0.2182510644197464 (0.2182510644197464) + | > loss_0: 2.448091506958008 (2.448091506958008) + | > grad_norm_0: tensor(18.4979, device='cuda:0') (tensor(18.4979, device='cuda:0')) + | > loss_gen: 2.582890748977661 (2.582890748977661) + | > loss_kl: 1.7492748498916626 (1.7492748498916626) + | > loss_feat: 6.525805950164795 (6.525805950164795) + | > loss_mel: 17.096296310424805 (17.096296310424805) + | > loss_duration: 1.4789113998413086 (1.4789113998413086) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.433177947998047 (29.433177947998047) + | > grad_norm_1: tensor(454.8466, device='cuda:0') (tensor(454.8466, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.3666 (0.36661219596862793) + | > loader_time: 0.3146 (0.3146097660064697) + + + --> TIME: 2025-04-19 01:56:39 -- STEP: 25/142 -- GLOBAL_STEP: 404725 + | > loss_disc: 2.4993484020233154 (2.4423282527923584) + | > loss_disc_real_0: 0.0914379358291626 (0.1130454695224762) + | > loss_disc_real_1: 0.18810828030109406 (0.20579156398773193) + | > loss_disc_real_2: 0.22666649520397186 (0.22666841447353364) + | > loss_disc_real_3: 0.22084154188632965 (0.22897449970245362) + | > loss_disc_real_4: 0.22099466621875763 (0.2296536707878113) + | > loss_disc_real_5: 0.23895655572414398 (0.23414088129997254) + | > loss_0: 2.4993484020233154 (2.4423282527923584) + | > grad_norm_0: tensor(13.7500, device='cuda:0') (tensor(9.5370, device='cuda:0')) + | > loss_gen: 2.395289897918701 (2.354251527786255) + | > loss_kl: 1.7171640396118164 (1.8469084787368775) + | > loss_feat: 7.316242694854736 (7.213465194702149) + | > loss_mel: 18.10926628112793 (17.83681594848632) + | > loss_duration: 1.4635112285614014 (1.4438257789611817) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.001474380493164 (30.69526725769043) + | > grad_norm_1: tensor(315.0100, device='cuda:0') (tensor(221.6884, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.3476 (0.3360606575012207) + | > loader_time: 0.0057 (0.00477825164794922) + + + --> TIME: 2025-04-19 01:56:48 -- STEP: 50/142 -- GLOBAL_STEP: 404750 + | > loss_disc: 2.4141745567321777 (2.443300333023071) + | > loss_disc_real_0: 0.08780594170093536 (0.1161403089761734) + | > loss_disc_real_1: 0.18546447157859802 (0.20463056802749635) + | > loss_disc_real_2: 0.2403537929058075 (0.22542832881212235) + | > loss_disc_real_3: 0.25516363978385925 (0.22797109097242355) + | > loss_disc_real_4: 0.2738035321235657 (0.2300485372543335) + | > loss_disc_real_5: 0.24051108956336975 (0.23519468307495117) + | > loss_0: 2.4141745567321777 (2.443300333023071) + | > grad_norm_0: tensor(5.0419, device='cuda:0') (tensor(11.0300, device='cuda:0')) + | > loss_gen: 2.4619181156158447 (2.36416974067688) + | > loss_kl: 1.8548643589019775 (1.838500669002533) + | > loss_feat: 7.51687479019165 (7.186255207061768) + | > loss_mel: 18.553720474243164 (17.892035369873046) + | > loss_duration: 1.487876296043396 (1.4511527943611144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.875255584716797 (30.732114028930663) + | > grad_norm_1: tensor(50.1359, device='cuda:0') (tensor(287.6958, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.3691 (0.3478089427947998) + | > loader_time: 0.0059 (0.005166811943054201) + + + --> TIME: 2025-04-19 01:56:58 -- STEP: 75/142 -- GLOBAL_STEP: 404775 + | > loss_disc: 2.5324783325195312 (2.45001062075297) + | > loss_disc_real_0: 0.09411700814962387 (0.1171578832467397) + | > loss_disc_real_1: 0.2283850759267807 (0.20591041068236032) + | > loss_disc_real_2: 0.2403913140296936 (0.22692168533802032) + | > loss_disc_real_3: 0.22920462489128113 (0.2282598493496577) + | > loss_disc_real_4: 0.2533131539821625 (0.23112707714239755) + | > loss_disc_real_5: 0.25767698884010315 (0.23643140852451325) + | > loss_0: 2.5324783325195312 (2.45001062075297) + | > grad_norm_0: tensor(21.3392, device='cuda:0') (tensor(11.3316, device='cuda:0')) + | > loss_gen: 2.212594985961914 (2.3670358498891195) + | > loss_kl: 1.8351514339447021 (1.8660225534439088) + | > loss_feat: 6.702700614929199 (7.1285869471232095) + | > loss_mel: 17.691715240478516 (17.90713251749674) + | > loss_duration: 1.4551986455917358 (1.4626800219217937) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.897361755371094 (30.731458053588867) + | > grad_norm_1: tensor(276.6227, device='cuda:0') (tensor(285.1257, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.3948 (0.35922454833984374) + | > loader_time: 0.0069 (0.005464932123819989) + + + --> TIME: 2025-04-19 01:57:08 -- STEP: 100/142 -- GLOBAL_STEP: 404800 + | > loss_disc: 2.4729394912719727 (2.45922656059265) + | > loss_disc_real_0: 0.13383938372135162 (0.12111661113798618) + | > loss_disc_real_1: 0.18423433601856232 (0.20591048181056976) + | > loss_disc_real_2: 0.2061128169298172 (0.22742767468094827) + | > loss_disc_real_3: 0.23509521782398224 (0.22819894015789033) + | > loss_disc_real_4: 0.20723755657672882 (0.22936217695474626) + | > loss_disc_real_5: 0.2252771109342575 (0.23482544407248496) + | > loss_0: 2.4729394912719727 (2.45922656059265) + | > grad_norm_0: tensor(14.9295, device='cuda:0') (tensor(11.7720, device='cuda:0')) + | > loss_gen: 2.2212812900543213 (2.351339619159698) + | > loss_kl: 1.9263339042663574 (1.8835922992229461) + | > loss_feat: 6.64475154876709 (7.013336958885193) + | > loss_mel: 17.09160614013672 (17.878874073028566) + | > loss_duration: 1.5058784484863281 (1.4744119715690613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.389850616455078 (30.601555080413817) + | > grad_norm_1: tensor(194.4194, device='cuda:0') (tensor(277.5369, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.413 (0.3693330669403076) + | > loader_time: 0.0064 (0.005720403194427491) + + + --> TIME: 2025-04-19 01:57:19 -- STEP: 125/142 -- GLOBAL_STEP: 404825 + | > loss_disc: 2.6531169414520264 (2.4637089786529534) + | > loss_disc_real_0: 0.20206384360790253 (0.12434836256504059) + | > loss_disc_real_1: 0.22857564687728882 (0.20680390667915344) + | > loss_disc_real_2: 0.2631658911705017 (0.2277775342464447) + | > loss_disc_real_3: 0.25844186544418335 (0.22840438878536223) + | > loss_disc_real_4: 0.2490624338388443 (0.2293152940273285) + | > loss_disc_real_5: 0.253452867269516 (0.23558728992938996) + | > loss_0: 2.6531169414520264 (2.4637089786529534) + | > grad_norm_0: tensor(25.1997, device='cuda:0') (tensor(12.6055, device='cuda:0')) + | > loss_gen: 2.2032065391540527 (2.353303174972534) + | > loss_kl: 1.8692057132720947 (1.8943895483016968) + | > loss_feat: 5.435566425323486 (6.968508232116699) + | > loss_mel: 16.677452087402344 (17.841236282348632) + | > loss_duration: 1.5384647846221924 (1.4820272035598754) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.723896026611328 (30.539464584350586) + | > grad_norm_1: tensor(232.0329, device='cuda:0') (tensor(278.3942, device='cuda:0')) + | > current_lr_0: 0.00019557441922580878 + | > current_lr_1: 0.00019557441922580878 + | > step_time: 0.4373 (0.3809135112762451) + | > loader_time: 0.0073 (0.005994173049926759) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.318359375 (2.318359375) + | > loss_disc_real_0: 0.16792809963226318 (0.16792809963226318) + | > loss_disc_real_1: 0.16631200909614563 (0.16631200909614563) + | > loss_disc_real_2: 0.2198488563299179 (0.2198488563299179) + | > loss_disc_real_3: 0.21716421842575073 (0.21716421842575073) + | > loss_disc_real_4: 0.16501112282276154 (0.16501112282276154) + | > loss_disc_real_5: 0.23389823734760284 (0.23389823734760284) + | > loss_0: 2.318359375 (2.318359375) + | > loss_gen: 2.2960379123687744 (2.2960379123687744) + | > loss_kl: 2.3760085105895996 (2.3760085105895996) + | > loss_feat: 6.622990131378174 (6.622990131378174) + | > loss_mel: 16.304418563842773 (16.304418563842773) + | > loss_duration: 1.6619675159454346 (1.6619675159454346) + | > loss_1: 29.261423110961914 (29.261423110961914) + + --> STEP: 1 + | > loss_disc: 2.4240171909332275 (2.4240171909332275) + | > loss_disc_real_0: 0.179199680685997 (0.179199680685997) + | > loss_disc_real_1: 0.18779385089874268 (0.18779385089874268) + | > loss_disc_real_2: 0.20460203289985657 (0.20460203289985657) + | > loss_disc_real_3: 0.2515093982219696 (0.2515093982219696) + | > loss_disc_real_4: 0.18835821747779846 (0.18835821747779846) + | > loss_disc_real_5: 0.21159473061561584 (0.21159473061561584) + | > loss_0: 2.4240171909332275 (2.4240171909332275) + | > loss_gen: 2.308506488800049 (2.308506488800049) + | > loss_kl: 2.5466716289520264 (2.5466716289520264) + | > loss_feat: 7.020550727844238 (7.020550727844238) + | > loss_mel: 18.98906135559082 (18.98906135559082) + | > loss_duration: 1.706121802330017 (1.706121802330017) + | > loss_1: 32.5709114074707 (32.5709114074707) + + --> STEP: 2 + | > loss_disc: 2.5053913593292236 (2.4647042751312256) + | > loss_disc_real_0: 0.1444971114397049 (0.16184839606285095) + | > loss_disc_real_1: 0.1867194026708603 (0.18725662678480148) + | > loss_disc_real_2: 0.2259558141231537 (0.21527892351150513) + | > loss_disc_real_3: 0.2396814376115799 (0.24559541791677475) + | > loss_disc_real_4: 0.23888979852199554 (0.213624007999897) + | > loss_disc_real_5: 0.23310869932174683 (0.22235171496868134) + | > loss_0: 2.5053913593292236 (2.4647042751312256) + | > loss_gen: 2.2904608249664307 (2.2994836568832397) + | > loss_kl: 2.3734524250030518 (2.460062026977539) + | > loss_feat: 6.997622013092041 (7.00908637046814) + | > loss_mel: 18.3947696685791 (18.69191551208496) + | > loss_duration: 1.659084439277649 (1.682603120803833) + | > loss_1: 31.715389251708984 (32.143150329589844) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003350377082824707 (-0.00014972686767578125) + | > avg_loss_disc: 2.4647042751312256 (-0.021728992462158203) + | > avg_loss_disc_real_0: 0.16184839606285095 (+0.020618021488189697) + | > avg_loss_disc_real_1: 0.18725662678480148 (-0.07191631197929382) + | > avg_loss_disc_real_2: 0.21527892351150513 (-0.018032394349575043) + | > avg_loss_disc_real_3: 0.24559541791677475 (+0.003537982702255249) + | > avg_loss_disc_real_4: 0.213624007999897 (-0.011326655745506287) + | > avg_loss_disc_real_5: 0.22235171496868134 (+0.03172460198402405) + | > avg_loss_0: 2.4647042751312256 (-0.021728992462158203) + | > avg_loss_gen: 2.2994836568832397 (+0.004426002502441406) + | > avg_loss_kl: 2.460062026977539 (+0.02939748764038086) + | > avg_loss_feat: 7.00908637046814 (-0.06230759620666504) + | > avg_loss_mel: 18.69191551208496 (+0.47530364990234375) + | > avg_loss_duration: 1.682603120803833 (-0.019353151321411133) + | > avg_loss_1: 32.143150329589844 (+0.42746543884277344) + + + > EPOCH: 180/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:57:29)  + + --> TIME: 2025-04-19 01:57:32 -- STEP: 8/142 -- GLOBAL_STEP: 404850 + | > loss_disc: 2.446439504623413 (2.395717114210129) + | > loss_disc_real_0: 0.15794728696346283 (0.13080790545791388) + | > loss_disc_real_1: 0.19999872148036957 (0.20572992600500584) + | > loss_disc_real_2: 0.21114817261695862 (0.21661632880568504) + | > loss_disc_real_3: 0.21057765185832977 (0.22680765017867088) + | > loss_disc_real_4: 0.19698762893676758 (0.21866548247635365) + | > loss_disc_real_5: 0.208805650472641 (0.23142492584884167) + | > loss_0: 2.446439504623413 (2.395717114210129) + | > grad_norm_0: tensor(23.0514, device='cuda:0') (tensor(22.2439, device='cuda:0')) + | > loss_gen: 2.309213876724243 (2.4331354200839996) + | > loss_kl: 1.8415662050247192 (1.8389830142259598) + | > loss_feat: 7.103691101074219 (7.226982593536377) + | > loss_mel: 17.642221450805664 (17.43914544582367) + | > loss_duration: 1.4368631839752197 (1.440351203083992) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.333555221557617 (30.378597259521484) + | > grad_norm_1: tensor(286.9472, device='cuda:0') (tensor(290.2053, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.326 (0.3247164785861969) + | > loader_time: 0.0041 (0.004203289747238159) + + + --> TIME: 2025-04-19 01:57:41 -- STEP: 33/142 -- GLOBAL_STEP: 404875 + | > loss_disc: 2.5736498832702637 (2.4185785741517036) + | > loss_disc_real_0: 0.1684383600950241 (0.12280320872863133) + | > loss_disc_real_1: 0.1981249451637268 (0.20371127715616516) + | > loss_disc_real_2: 0.26891225576400757 (0.22596468979662115) + | > loss_disc_real_3: 0.2494208663702011 (0.22837577895684677) + | > loss_disc_real_4: 0.2841864824295044 (0.22625557175188354) + | > loss_disc_real_5: 0.22757573425769806 (0.22873907107295413) + | > loss_0: 2.5736498832702637 (2.4185785741517036) + | > grad_norm_0: tensor(6.8427, device='cuda:0') (tensor(14.7999, device='cuda:0')) + | > loss_gen: 2.20752215385437 (2.40880739327633) + | > loss_kl: 1.8886862993240356 (1.8490347284259219) + | > loss_feat: 6.461773872375488 (7.195514924598463) + | > loss_mel: 17.739286422729492 (17.63396422068278) + | > loss_duration: 1.4543914794921875 (1.4439481424562859) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.751659393310547 (30.531269073486328) + | > grad_norm_1: tensor(182.7687, device='cuda:0') (tensor(256.6506, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.3568 (0.33916778275460907) + | > loader_time: 0.0053 (0.004912484775890005) + + + --> TIME: 2025-04-19 01:57:50 -- STEP: 58/142 -- GLOBAL_STEP: 404900 + | > loss_disc: 2.4237852096557617 (2.434229069742663) + | > loss_disc_real_0: 0.11090352386236191 (0.12200114156665473) + | > loss_disc_real_1: 0.15795192122459412 (0.20522549285970887) + | > loss_disc_real_2: 0.2281821072101593 (0.22563400196618047) + | > loss_disc_real_3: 0.21478819847106934 (0.22701238738051777) + | > loss_disc_real_4: 0.2529278099536896 (0.22721992458762794) + | > loss_disc_real_5: 0.27388298511505127 (0.23073177579148063) + | > loss_0: 2.4237852096557617 (2.434229069742663) + | > grad_norm_0: tensor(3.6406, device='cuda:0') (tensor(12.5639, device='cuda:0')) + | > loss_gen: 2.349684238433838 (2.3819444549494775) + | > loss_kl: 1.8529590368270874 (1.8687349270130027) + | > loss_feat: 7.478304862976074 (7.130556698503165) + | > loss_mel: 19.377695083618164 (17.707259852310706) + | > loss_duration: 1.4855231046676636 (1.4551696037424022) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.54416275024414 (30.54366532687483) + | > grad_norm_1: tensor(82.7760, device='cuda:0') (tensor(230.7249, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.3737 (0.35020072706814476) + | > loader_time: 0.0059 (0.005308936382162161) + + + --> TIME: 2025-04-19 01:58:00 -- STEP: 83/142 -- GLOBAL_STEP: 404925 + | > loss_disc: 2.547907590866089 (2.446832754525794) + | > loss_disc_real_0: 0.14258205890655518 (0.12476947957492736) + | > loss_disc_real_1: 0.23444552719593048 (0.20577497654650584) + | > loss_disc_real_2: 0.20628519356250763 (0.22502503337630306) + | > loss_disc_real_3: 0.20960961282253265 (0.2259532834391996) + | > loss_disc_real_4: 0.20997057855129242 (0.22743879707462816) + | > loss_disc_real_5: 0.22952492535114288 (0.23183269834662057) + | > loss_0: 2.547907590866089 (2.446832754525794) + | > grad_norm_0: tensor(5.5029, device='cuda:0') (tensor(10.4996, device='cuda:0')) + | > loss_gen: 2.2409870624542236 (2.3591820619192463) + | > loss_kl: 1.9887279272079468 (1.8835665332265648) + | > loss_feat: 6.287556171417236 (7.0856447392199415) + | > loss_mel: 18.081199645996094 (17.701768093798535) + | > loss_duration: 1.5545287132263184 (1.4684127468660653) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.152999877929688 (30.49857401560588) + | > grad_norm_1: tensor(34.0629, device='cuda:0') (tensor(188.5901, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.396 (0.362140098250056) + | > loader_time: 0.0063 (0.0056693352848650475) + + + --> TIME: 2025-04-19 01:58:11 -- STEP: 108/142 -- GLOBAL_STEP: 404950 + | > loss_disc: 2.4087841510772705 (2.457342916064793) + | > loss_disc_real_0: 0.11621647328138351 (0.12557426819370854) + | > loss_disc_real_1: 0.2228325754404068 (0.20645318552851677) + | > loss_disc_real_2: 0.23010680079460144 (0.22603943750814157) + | > loss_disc_real_3: 0.25044313073158264 (0.22745064590816144) + | > loss_disc_real_4: 0.17900365591049194 (0.22847621060079998) + | > loss_disc_real_5: 0.22240614891052246 (0.233268389409339) + | > loss_0: 2.4087841510772705 (2.457342916064793) + | > grad_norm_0: tensor(6.9411, device='cuda:0') (tensor(10.6194, device='cuda:0')) + | > loss_gen: 2.353822708129883 (2.354369854485547) + | > loss_kl: 1.9168161153793335 (1.8986490766207378) + | > loss_feat: 7.105640888214111 (7.052787418718691) + | > loss_mel: 17.487146377563477 (17.686585655918822) + | > loss_duration: 1.5059127807617188 (1.4773930112520854) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.369338989257812 (30.46978489557902) + | > grad_norm_1: tensor(70.2538, device='cuda:0') (tensor(204.6147, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.4191 (0.37298358148998695) + | > loader_time: 0.0067 (0.005910570974703189) + + + --> TIME: 2025-04-19 01:58:22 -- STEP: 133/142 -- GLOBAL_STEP: 404975 + | > loss_disc: 2.4733786582946777 (2.461459998797654) + | > loss_disc_real_0: 0.16179165244102478 (0.12658660046588216) + | > loss_disc_real_1: 0.22709257900714874 (0.2069960378838661) + | > loss_disc_real_2: 0.24788159132003784 (0.2263886261927454) + | > loss_disc_real_3: 0.206167533993721 (0.22694167001803117) + | > loss_disc_real_4: 0.23760995268821716 (0.22992731451540066) + | > loss_disc_real_5: 0.230741485953331 (0.23273559362816632) + | > loss_0: 2.4733786582946777 (2.461459998797654) + | > grad_norm_0: tensor(10.8687, device='cuda:0') (tensor(10.7145, device='cuda:0')) + | > loss_gen: 2.2949092388153076 (2.353428580707177) + | > loss_kl: 2.010146379470825 (1.8996685304139789) + | > loss_feat: 6.37831449508667 (7.020753709893477) + | > loss_mel: 17.096160888671875 (17.66389852538144) + | > loss_duration: 1.5610216856002808 (1.483916781898728) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.340553283691406 (30.421666023426486) + | > grad_norm_1: tensor(145.6895, device='cuda:0') (tensor(218.2190, device='cuda:0')) + | > current_lr_0: 0.00019554997242340555 + | > current_lr_1: 0.00019554997242340555 + | > step_time: 0.4375 (0.38471406743042463) + | > loader_time: 0.0071 (0.006181623702658747) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3798370361328125 (2.3798370361328125) + | > loss_disc_real_0: 0.140814408659935 (0.140814408659935) + | > loss_disc_real_1: 0.18133802711963654 (0.18133802711963654) + | > loss_disc_real_2: 0.2641623020172119 (0.2641623020172119) + | > loss_disc_real_3: 0.20858043432235718 (0.20858043432235718) + | > loss_disc_real_4: 0.23556506633758545 (0.23556506633758545) + | > loss_disc_real_5: 0.21319758892059326 (0.21319758892059326) + | > loss_0: 2.3798370361328125 (2.3798370361328125) + | > loss_gen: 2.3993353843688965 (2.3993353843688965) + | > loss_kl: 2.278555393218994 (2.278555393218994) + | > loss_feat: 7.372931957244873 (7.372931957244873) + | > loss_mel: 18.242597579956055 (18.242597579956055) + | > loss_duration: 1.6822600364685059 (1.6822600364685059) + | > loss_1: 31.97568130493164 (31.97568130493164) + + --> STEP: 1 + | > loss_disc: 2.5224781036376953 (2.5224781036376953) + | > loss_disc_real_0: 0.1544385403394699 (0.1544385403394699) + | > loss_disc_real_1: 0.21067702770233154 (0.21067702770233154) + | > loss_disc_real_2: 0.31010809540748596 (0.31010809540748596) + | > loss_disc_real_3: 0.24378249049186707 (0.24378249049186707) + | > loss_disc_real_4: 0.248984694480896 (0.248984694480896) + | > loss_disc_real_5: 0.23519769310951233 (0.23519769310951233) + | > loss_0: 2.5224781036376953 (2.5224781036376953) + | > loss_gen: 2.4865572452545166 (2.4865572452545166) + | > loss_kl: 2.414799928665161 (2.414799928665161) + | > loss_feat: 7.092501640319824 (7.092501640319824) + | > loss_mel: 17.939682006835938 (17.939682006835938) + | > loss_duration: 1.7004399299621582 (1.7004399299621582) + | > loss_1: 31.63397979736328 (31.63397979736328) + + --> STEP: 2 + | > loss_disc: 2.4342522621154785 (2.478365182876587) + | > loss_disc_real_0: 0.144739031791687 (0.14958878606557846) + | > loss_disc_real_1: 0.20188011229038239 (0.20627856999635696) + | > loss_disc_real_2: 0.3050384521484375 (0.30757327377796173) + | > loss_disc_real_3: 0.2457604855298996 (0.24477148801088333) + | > loss_disc_real_4: 0.197844997048378 (0.223414845764637) + | > loss_disc_real_5: 0.24053247272968292 (0.23786508291959763) + | > loss_0: 2.4342522621154785 (2.478365182876587) + | > loss_gen: 2.447317600250244 (2.4669374227523804) + | > loss_kl: 2.0094733238220215 (2.2121366262435913) + | > loss_feat: 7.902583122253418 (7.497542381286621) + | > loss_mel: 19.375308990478516 (18.657495498657227) + | > loss_duration: 1.6634708642959595 (1.6819553971290588) + | > loss_1: 33.39815139770508 (32.51606559753418) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033262968063354492 (-2.4080276489257812e-05) + | > avg_loss_disc: 2.478365182876587 (+0.013660907745361328) + | > avg_loss_disc_real_0: 0.14958878606557846 (-0.012259609997272491) + | > avg_loss_disc_real_1: 0.20627856999635696 (+0.01902194321155548) + | > avg_loss_disc_real_2: 0.30757327377796173 (+0.0922943502664566) + | > avg_loss_disc_real_3: 0.24477148801088333 (-0.0008239299058914185) + | > avg_loss_disc_real_4: 0.223414845764637 (+0.00979083776473999) + | > avg_loss_disc_real_5: 0.23786508291959763 (+0.01551336795091629) + | > avg_loss_0: 2.478365182876587 (+0.013660907745361328) + | > avg_loss_gen: 2.4669374227523804 (+0.16745376586914062) + | > avg_loss_kl: 2.2121366262435913 (-0.24792540073394775) + | > avg_loss_feat: 7.497542381286621 (+0.48845601081848145) + | > avg_loss_mel: 18.657495498657227 (-0.034420013427734375) + | > avg_loss_duration: 1.6819553971290588 (-0.0006477236747741699) + | > avg_loss_1: 32.51606559753418 (+0.37291526794433594) + + + > EPOCH: 181/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:58:27)  + + --> TIME: 2025-04-19 01:58:33 -- STEP: 16/142 -- GLOBAL_STEP: 405000 + | > loss_disc: 2.4434473514556885 (2.431931182742119) + | > loss_disc_real_0: 0.09698772430419922 (0.11980120791122317) + | > loss_disc_real_1: 0.20381386578083038 (0.20379319228231907) + | > loss_disc_real_2: 0.21746480464935303 (0.2231916394084692) + | > loss_disc_real_3: 0.19941604137420654 (0.23339474853128195) + | > loss_disc_real_4: 0.22829444706439972 (0.22555057611316442) + | > loss_disc_real_5: 0.20856767892837524 (0.2338289236649871) + | > loss_0: 2.4434473514556885 (2.431931182742119) + | > grad_norm_0: tensor(8.4478, device='cuda:0') (tensor(10.6735, device='cuda:0')) + | > loss_gen: 2.3122661113739014 (2.348897635936737) + | > loss_kl: 1.9177782535552979 (1.7819346711039543) + | > loss_feat: 7.168076515197754 (6.9883169531822205) + | > loss_mel: 16.987083435058594 (17.30726319551468) + | > loss_duration: 1.4308191537857056 (1.4352703243494034) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.816020965576172 (29.861683011054993) + | > grad_norm_1: tensor(277.5068, device='cuda:0') (tensor(249.7445, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.3399 (0.3277174085378647) + | > loader_time: 0.0052 (0.004513099789619446) + + + --> TIME: 2025-04-19 01:58:42 -- STEP: 41/142 -- GLOBAL_STEP: 405025 + | > loss_disc: 2.5180823802948 (2.445477532177437) + | > loss_disc_real_0: 0.12410340458154678 (0.1233333730843009) + | > loss_disc_real_1: 0.21329140663146973 (0.20480154382019508) + | > loss_disc_real_2: 0.23499517142772675 (0.22538504440610002) + | > loss_disc_real_3: 0.25686949491500854 (0.23023357296862254) + | > loss_disc_real_4: 0.24628899991512299 (0.22834730257348315) + | > loss_disc_real_5: 0.23722945153713226 (0.233464340247759) + | > loss_0: 2.5180823802948 (2.445477532177437) + | > grad_norm_0: tensor(6.8894, device='cuda:0') (tensor(10.7007, device='cuda:0')) + | > loss_gen: 2.2824764251708984 (2.362937723718038) + | > loss_kl: 1.863364577293396 (1.8177918980761272) + | > loss_feat: 6.198275089263916 (7.074569260201803) + | > loss_mel: 15.323362350463867 (17.57751067091779) + | > loss_duration: 1.427298665046692 (1.4488860485030384) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.094778060913086 (30.281695575248904) + | > grad_norm_1: tensor(230.5784, device='cuda:0') (tensor(261.4839, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.3711 (0.3448380667988847) + | > loader_time: 0.0056 (0.005060951884199933) + + + --> TIME: 2025-04-19 01:58:52 -- STEP: 66/142 -- GLOBAL_STEP: 405050 + | > loss_disc: 2.5062026977539062 (2.447621417768074) + | > loss_disc_real_0: 0.10671781748533249 (0.12260797010226683) + | > loss_disc_real_1: 0.18315625190734863 (0.20551014149730856) + | > loss_disc_real_2: 0.20788386464118958 (0.22548488866199146) + | > loss_disc_real_3: 0.23054981231689453 (0.22748086804693396) + | > loss_disc_real_4: 0.23025910556316376 (0.22819291326132687) + | > loss_disc_real_5: 0.23285305500030518 (0.23556366788618494) + | > loss_0: 2.5062026977539062 (2.447621417768074) + | > grad_norm_0: tensor(24.0807, device='cuda:0') (tensor(11.3990, device='cuda:0')) + | > loss_gen: 2.3151614665985107 (2.3579566803845493) + | > loss_kl: 1.975832223892212 (1.8475238969831755) + | > loss_feat: 7.123034477233887 (7.057626500274196) + | > loss_mel: 18.692232131958008 (17.679430542570174) + | > loss_duration: 1.4600534439086914 (1.4612326567823237) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.566314697265625 (30.403770273382012) + | > grad_norm_1: tensor(366.4446, device='cuda:0') (tensor(274.7032, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.3959 (0.3595080700787631) + | > loader_time: 0.007 (0.005432161417874423) + + + --> TIME: 2025-04-19 01:59:03 -- STEP: 91/142 -- GLOBAL_STEP: 405075 + | > loss_disc: 2.364299774169922 (2.4488847570104917) + | > loss_disc_real_0: 0.08370274305343628 (0.12257639245017544) + | > loss_disc_real_1: 0.188377246260643 (0.20578320913917417) + | > loss_disc_real_2: 0.20597867667675018 (0.22489369472304543) + | > loss_disc_real_3: 0.24933992326259613 (0.22864648765260046) + | > loss_disc_real_4: 0.22068065404891968 (0.22703261624325763) + | > loss_disc_real_5: 0.23100946843624115 (0.23573494612515627) + | > loss_0: 2.364299774169922 (2.4488847570104917) + | > grad_norm_0: tensor(22.6676, device='cuda:0') (tensor(12.2930, device='cuda:0')) + | > loss_gen: 2.450455904006958 (2.3628946189041975) + | > loss_kl: 1.9686806201934814 (1.8564285445999313) + | > loss_feat: 7.565825939178467 (7.046806314489344) + | > loss_mel: 18.225303649902344 (17.76278067159129) + | > loss_duration: 1.494266390800476 (1.4729646693219198) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.70453453063965 (30.50187494466593) + | > grad_norm_1: tensor(569.8977, device='cuda:0') (tensor(292.7240, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.4186 (0.3726157565693279) + | > loader_time: 0.0067 (0.005763163933387167) + + + --> TIME: 2025-04-19 01:59:14 -- STEP: 116/142 -- GLOBAL_STEP: 405100 + | > loss_disc: 2.510890007019043 (2.46155363937904) + | > loss_disc_real_0: 0.06910263746976852 (0.1250817678098021) + | > loss_disc_real_1: 0.2269974946975708 (0.20662906008033916) + | > loss_disc_real_2: 0.27037420868873596 (0.2260256209250154) + | > loss_disc_real_3: 0.24418585002422333 (0.2283716494667119) + | > loss_disc_real_4: 0.22749799489974976 (0.22772379078227897) + | > loss_disc_real_5: 0.2586079239845276 (0.23455237263235554) + | > loss_0: 2.510890007019043 (2.46155363937904) + | > grad_norm_0: tensor(22.9139, device='cuda:0') (tensor(13.8479, device='cuda:0')) + | > loss_gen: 2.5363903045654297 (2.3613034445663974) + | > loss_kl: 1.802609920501709 (1.8655312174353107) + | > loss_feat: 7.378044128417969 (7.01814787141208) + | > loss_mel: 18.40168571472168 (17.762049165265314) + | > loss_duration: 1.4715423583984375 (1.4790284993319678) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.590272903442383 (30.486060274058374) + | > grad_norm_1: tensor(492.0939, device='cuda:0') (tensor(300.6840, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.444 (0.384429302708856) + | > loader_time: 0.0075 (0.006064891815185545) + + + --> TIME: 2025-04-19 01:59:25 -- STEP: 141/142 -- GLOBAL_STEP: 405125 + | > loss_disc: 2.493539810180664 (2.4653861404310717) + | > loss_disc_real_0: 0.1337052285671234 (0.12697187631476856) + | > loss_disc_real_1: 0.24442961812019348 (0.20721727683611796) + | > loss_disc_real_2: 0.232821524143219 (0.22659767160178923) + | > loss_disc_real_3: 0.22802680730819702 (0.2289314312292329) + | > loss_disc_real_4: 0.24040377140045166 (0.22922537459972056) + | > loss_disc_real_5: 0.1907530575990677 (0.23504032405978398) + | > loss_0: 2.493539810180664 (2.4653861404310717) + | > grad_norm_0: tensor(19.0214, device='cuda:0') (tensor(15.2359, device='cuda:0')) + | > loss_gen: 2.343592643737793 (2.36294849375461) + | > loss_kl: 1.6841392517089844 (1.869744178251172) + | > loss_feat: 6.585351467132568 (6.998408419020633) + | > loss_mel: 15.008646965026855 (17.772482601463373) + | > loss_duration: 1.3470921516418457 (1.4846252583442852) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.968822479248047 (30.488209081879745) + | > grad_norm_1: tensor(326.2256, device='cuda:0') (tensor(310.3765, device='cuda:0')) + | > current_lr_0: 0.00019552552867685262 + | > current_lr_1: 0.00019552552867685262 + | > step_time: 0.2057 (0.3953141168499669) + | > loader_time: 0.0026 (0.006316494434437852) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5618667602539062 (2.5618667602539062) + | > loss_disc_real_0: 0.16174407303333282 (0.16174407303333282) + | > loss_disc_real_1: 0.18418870866298676 (0.18418870866298676) + | > loss_disc_real_2: 0.21794192492961884 (0.21794192492961884) + | > loss_disc_real_3: 0.2714410424232483 (0.2714410424232483) + | > loss_disc_real_4: 0.2680242657661438 (0.2680242657661438) + | > loss_disc_real_5: 0.21817460656166077 (0.21817460656166077) + | > loss_0: 2.5618667602539062 (2.5618667602539062) + | > loss_gen: 2.2307982444763184 (2.2307982444763184) + | > loss_kl: 2.2335948944091797 (2.2335948944091797) + | > loss_feat: 5.8894829750061035 (5.8894829750061035) + | > loss_mel: 16.685144424438477 (16.685144424438477) + | > loss_duration: 1.6472586393356323 (1.6472586393356323) + | > loss_1: 28.686279296875 (28.686279296875) + + --> STEP: 1 + | > loss_disc: 2.5880885124206543 (2.5880885124206543) + | > loss_disc_real_0: 0.15912450850009918 (0.15912450850009918) + | > loss_disc_real_1: 0.2146826982498169 (0.2146826982498169) + | > loss_disc_real_2: 0.2074001431465149 (0.2074001431465149) + | > loss_disc_real_3: 0.21721671521663666 (0.21721671521663666) + | > loss_disc_real_4: 0.2507723271846771 (0.2507723271846771) + | > loss_disc_real_5: 0.23383423686027527 (0.23383423686027527) + | > loss_0: 2.5880885124206543 (2.5880885124206543) + | > loss_gen: 2.10302734375 (2.10302734375) + | > loss_kl: 2.7074835300445557 (2.7074835300445557) + | > loss_feat: 5.8155598640441895 (5.8155598640441895) + | > loss_mel: 17.948591232299805 (17.948591232299805) + | > loss_duration: 1.7316765785217285 (1.7316765785217285) + | > loss_1: 30.306337356567383 (30.306337356567383) + + --> STEP: 2 + | > loss_disc: 2.533961772918701 (2.5610251426696777) + | > loss_disc_real_0: 0.1422739326953888 (0.150699220597744) + | > loss_disc_real_1: 0.18806618452072144 (0.20137444138526917) + | > loss_disc_real_2: 0.21872054040431976 (0.21306034177541733) + | > loss_disc_real_3: 0.22814106941223145 (0.22267889231443405) + | > loss_disc_real_4: 0.20796392858028412 (0.22936812788248062) + | > loss_disc_real_5: 0.2488863617181778 (0.24136029928922653) + | > loss_0: 2.533961772918701 (2.5610251426696777) + | > loss_gen: 2.2280232906341553 (2.1655253171920776) + | > loss_kl: 2.3018927574157715 (2.5046881437301636) + | > loss_feat: 6.839670658111572 (6.327615261077881) + | > loss_mel: 18.381263732910156 (18.16492748260498) + | > loss_duration: 1.662792682647705 (1.6972346305847168) + | > loss_1: 31.41364288330078 (30.859990119934082) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036094188690185547 (+0.00028312206268310547) + | > avg_loss_disc: 2.5610251426696777 (+0.08265995979309082) + | > avg_loss_disc_real_0: 0.150699220597744 (+0.0011104345321655273) + | > avg_loss_disc_real_1: 0.20137444138526917 (-0.004904128611087799) + | > avg_loss_disc_real_2: 0.21306034177541733 (-0.0945129320025444) + | > avg_loss_disc_real_3: 0.22267889231443405 (-0.02209259569644928) + | > avg_loss_disc_real_4: 0.22936812788248062 (+0.005953282117843628) + | > avg_loss_disc_real_5: 0.24136029928922653 (+0.0034952163696289062) + | > avg_loss_0: 2.5610251426696777 (+0.08265995979309082) + | > avg_loss_gen: 2.1655253171920776 (-0.30141210556030273) + | > avg_loss_kl: 2.5046881437301636 (+0.29255151748657227) + | > avg_loss_feat: 6.327615261077881 (-1.1699271202087402) + | > avg_loss_mel: 18.16492748260498 (-0.4925680160522461) + | > avg_loss_duration: 1.6972346305847168 (+0.015279233455657959) + | > avg_loss_1: 30.859990119934082 (-1.6560754776000977) + + + > EPOCH: 182/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 01:59:27)  + + --> TIME: 2025-04-19 01:59:36 -- STEP: 24/142 -- GLOBAL_STEP: 405150 + | > loss_disc: 2.571073055267334 (2.4636552631855015) + | > loss_disc_real_0: 0.13852477073669434 (0.1191121768206358) + | > loss_disc_real_1: 0.19906234741210938 (0.2076399369786183) + | > loss_disc_real_2: 0.2570345401763916 (0.22963228759666285) + | > loss_disc_real_3: 0.270644873380661 (0.23158524185419083) + | > loss_disc_real_4: 0.23557895421981812 (0.22903478145599365) + | > loss_disc_real_5: 0.28416913747787476 (0.22735934083660445) + | > loss_0: 2.571073055267334 (2.4636552631855015) + | > grad_norm_0: tensor(10.4077, device='cuda:0') (tensor(9.5821, device='cuda:0')) + | > loss_gen: 2.248612642288208 (2.3506767650445304) + | > loss_kl: 1.95517098903656 (1.8476126044988632) + | > loss_feat: 6.653801918029785 (7.010105113188426) + | > loss_mel: 17.72632598876953 (17.66955343882243) + | > loss_duration: 1.4878606796264648 (1.4451342970132828) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.071773529052734 (30.323081970214844) + | > grad_norm_1: tensor(181.4892, device='cuda:0') (tensor(230.1037, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.3467 (0.3418051501115163) + | > loader_time: 0.0047 (0.005080064137776693) + + + --> TIME: 2025-04-19 01:59:45 -- STEP: 49/142 -- GLOBAL_STEP: 405175 + | > loss_disc: 2.473836660385132 (2.4497403563285367) + | > loss_disc_real_0: 0.1489344835281372 (0.12180352210998535) + | > loss_disc_real_1: 0.22642453014850616 (0.20669278411232694) + | > loss_disc_real_2: 0.2067105919122696 (0.2251397079350997) + | > loss_disc_real_3: 0.2451072484254837 (0.22560555472665902) + | > loss_disc_real_4: 0.2606924772262573 (0.22515859196380694) + | > loss_disc_real_5: 0.2558426856994629 (0.23367636483542772) + | > loss_0: 2.473836660385132 (2.4497403563285367) + | > grad_norm_0: tensor(5.8474, device='cuda:0') (tensor(7.9235, device='cuda:0')) + | > loss_gen: 2.278895139694214 (2.358626443512586) + | > loss_kl: 1.8913161754608154 (1.8390455051344268) + | > loss_feat: 6.859730243682861 (7.062066379858523) + | > loss_mel: 17.46867561340332 (17.758032701453388) + | > loss_duration: 1.4720642566680908 (1.4520528924708465) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.97068214416504 (30.469823798354792) + | > grad_norm_1: tensor(93.4092, device='cuda:0') (tensor(173.2842, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.3694 (0.34993943389581167) + | > loader_time: 0.0056 (0.0053692496552759286) + + + --> TIME: 2025-04-19 01:59:55 -- STEP: 74/142 -- GLOBAL_STEP: 405200 + | > loss_disc: 2.551431894302368 (2.46228127866178) + | > loss_disc_real_0: 0.1526011973619461 (0.12378037116817527) + | > loss_disc_real_1: 0.27931445837020874 (0.20803103515425247) + | > loss_disc_real_2: 0.21323610842227936 (0.2240899958320566) + | > loss_disc_real_3: 0.22688652575016022 (0.22603524697793498) + | > loss_disc_real_4: 0.23434115946292877 (0.22832318757836878) + | > loss_disc_real_5: 0.2669162452220917 (0.23457617332806457) + | > loss_0: 2.551431894302368 (2.46228127866178) + | > grad_norm_0: tensor(5.9496, device='cuda:0') (tensor(7.4113, device='cuda:0')) + | > loss_gen: 2.3409957885742188 (2.352987643834707) + | > loss_kl: 1.8434367179870605 (1.8640953817883053) + | > loss_feat: 6.931501388549805 (7.051829898679578) + | > loss_mel: 17.604496002197266 (17.81194511619774) + | > loss_duration: 1.4930821657180786 (1.4639579927599107) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.213512420654297 (30.544815939825934) + | > grad_norm_1: tensor(73.7019, device='cuda:0') (tensor(149.9601, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.3865 (0.3604744737212722) + | > loader_time: 0.0068 (0.005634485064326106) + + + --> TIME: 2025-04-19 02:00:05 -- STEP: 99/142 -- GLOBAL_STEP: 405225 + | > loss_disc: 2.404728412628174 (2.4720964383597326) + | > loss_disc_real_0: 0.11984771490097046 (0.1254910308905322) + | > loss_disc_real_1: 0.17164702713489532 (0.20913277942724906) + | > loss_disc_real_2: 0.2262466698884964 (0.2250194334321552) + | > loss_disc_real_3: 0.24460704624652863 (0.22671355006068644) + | > loss_disc_real_4: 0.21796149015426636 (0.22951626250840193) + | > loss_disc_real_5: 0.23516294360160828 (0.2349067029326853) + | > loss_0: 2.404728412628174 (2.4720964383597326) + | > grad_norm_0: tensor(11.6361, device='cuda:0') (tensor(7.6113, device='cuda:0')) + | > loss_gen: 2.3657336235046387 (2.333713851793848) + | > loss_kl: 2.038222551345825 (1.8805050681335758) + | > loss_feat: 7.457863807678223 (6.97073333431976) + | > loss_mel: 18.523075103759766 (17.800748834706315) + | > loss_duration: 1.5017719268798828 (1.4750272119888148) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.886667251586914 (30.460728211836383) + | > grad_norm_1: tensor(283.5902, device='cuda:0') (tensor(155.3400, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.4025 (0.37024941829719926) + | > loader_time: 0.0068 (0.005890954624522814) + + + --> TIME: 2025-04-19 02:00:16 -- STEP: 124/142 -- GLOBAL_STEP: 405250 + | > loss_disc: 2.602370500564575 (2.4735003921293446) + | > loss_disc_real_0: 0.11072730273008347 (0.12673738040029997) + | > loss_disc_real_1: 0.19614337384700775 (0.20896459731363484) + | > loss_disc_real_2: 0.23214836418628693 (0.22494948583264504) + | > loss_disc_real_3: 0.24102665483951569 (0.22579879193536698) + | > loss_disc_real_4: 0.2586821913719177 (0.22996638859471966) + | > loss_disc_real_5: 0.25488030910491943 (0.2338713518073482) + | > loss_0: 2.602370500564575 (2.4735003921293446) + | > grad_norm_0: tensor(12.5089, device='cuda:0') (tensor(7.6894, device='cuda:0')) + | > loss_gen: 2.2029342651367188 (2.3323124204912493) + | > loss_kl: 2.142195701599121 (1.894806904177512) + | > loss_feat: 6.0646867752075195 (6.935304091822717) + | > loss_mel: 16.643970489501953 (17.789882898330692) + | > loss_duration: 1.544956088066101 (1.4818184462285808) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.598743438720703 (30.43412466972105) + | > grad_norm_1: tensor(122.3249, device='cuda:0') (tensor(156.0224, device='cuda:0')) + | > current_lr_0: 0.000195501087985768 + | > current_lr_1: 0.000195501087985768 + | > step_time: 0.4419 (0.38163939791340984) + | > loader_time: 0.0076 (0.006150643671712566) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.2774813175201416 (2.2774813175201416) + | > loss_disc_real_0: 0.11983642727136612 (0.11983642727136612) + | > loss_disc_real_1: 0.21090762317180634 (0.21090762317180634) + | > loss_disc_real_2: 0.2263101190328598 (0.2263101190328598) + | > loss_disc_real_3: 0.28391706943511963 (0.28391706943511963) + | > loss_disc_real_4: 0.19507139921188354 (0.19507139921188354) + | > loss_disc_real_5: 0.1927470564842224 (0.1927470564842224) + | > loss_0: 2.2774813175201416 (2.2774813175201416) + | > loss_gen: 2.56485652923584 (2.56485652923584) + | > loss_kl: 2.2506930828094482 (2.2506930828094482) + | > loss_feat: 7.493756294250488 (7.493756294250488) + | > loss_mel: 19.808441162109375 (19.808441162109375) + | > loss_duration: 1.6789674758911133 (1.6789674758911133) + | > loss_1: 33.796714782714844 (33.796714782714844) + + --> STEP: 1 + | > loss_disc: 2.453186511993408 (2.453186511993408) + | > loss_disc_real_0: 0.1298677921295166 (0.1298677921295166) + | > loss_disc_real_1: 0.2060491442680359 (0.2060491442680359) + | > loss_disc_real_2: 0.23403723537921906 (0.23403723537921906) + | > loss_disc_real_3: 0.24591372907161713 (0.24591372907161713) + | > loss_disc_real_4: 0.21151864528656006 (0.21151864528656006) + | > loss_disc_real_5: 0.19945256412029266 (0.19945256412029266) + | > loss_0: 2.453186511993408 (2.453186511993408) + | > loss_gen: 2.2425904273986816 (2.2425904273986816) + | > loss_kl: 2.5481410026550293 (2.5481410026550293) + | > loss_feat: 7.438446998596191 (7.438446998596191) + | > loss_mel: 18.58270835876465 (18.58270835876465) + | > loss_duration: 1.6852679252624512 (1.6852679252624512) + | > loss_1: 32.497154235839844 (32.497154235839844) + + --> STEP: 2 + | > loss_disc: 2.4728329181671143 (2.4630097150802612) + | > loss_disc_real_0: 0.2619267702102661 (0.19589728116989136) + | > loss_disc_real_1: 0.21230930089950562 (0.20917922258377075) + | > loss_disc_real_2: 0.2312377542257309 (0.23263749480247498) + | > loss_disc_real_3: 0.23624789714813232 (0.24108081310987473) + | > loss_disc_real_4: 0.2043190598487854 (0.20791885256767273) + | > loss_disc_real_5: 0.17668354511260986 (0.18806805461645126) + | > loss_0: 2.4728329181671143 (2.4630097150802612) + | > loss_gen: 2.425562858581543 (2.3340766429901123) + | > loss_kl: 2.243544340133667 (2.395842671394348) + | > loss_feat: 6.15851354598999 (6.798480272293091) + | > loss_mel: 15.983492851257324 (17.283100605010986) + | > loss_duration: 1.6597551107406616 (1.6725115180015564) + | > loss_1: 28.470870971679688 (30.484012603759766) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00393223762512207 (+0.0003228187561035156) + | > avg_loss_disc: 2.4630097150802612 (-0.0980154275894165) + | > avg_loss_disc_real_0: 0.19589728116989136 (+0.04519806057214737) + | > avg_loss_disc_real_1: 0.20917922258377075 (+0.007804781198501587) + | > avg_loss_disc_real_2: 0.23263749480247498 (+0.019577153027057648) + | > avg_loss_disc_real_3: 0.24108081310987473 (+0.018401920795440674) + | > avg_loss_disc_real_4: 0.20791885256767273 (-0.021449275314807892) + | > avg_loss_disc_real_5: 0.18806805461645126 (-0.05329224467277527) + | > avg_loss_0: 2.4630097150802612 (-0.0980154275894165) + | > avg_loss_gen: 2.3340766429901123 (+0.16855132579803467) + | > avg_loss_kl: 2.395842671394348 (-0.10884547233581543) + | > avg_loss_feat: 6.798480272293091 (+0.47086501121520996) + | > avg_loss_mel: 17.283100605010986 (-0.8818268775939941) + | > avg_loss_duration: 1.6725115180015564 (-0.0247231125831604) + | > avg_loss_1: 30.484012603759766 (-0.3759775161743164) + + + > EPOCH: 183/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:00:26)  + + --> TIME: 2025-04-19 02:00:29 -- STEP: 7/142 -- GLOBAL_STEP: 405275 + | > loss_disc: 2.438518762588501 (2.42517352104187) + | > loss_disc_real_0: 0.13519909977912903 (0.1222873723932675) + | > loss_disc_real_1: 0.2291041910648346 (0.20264277926513127) + | > loss_disc_real_2: 0.2326761782169342 (0.2251695202929633) + | > loss_disc_real_3: 0.20833751559257507 (0.21712840029171535) + | > loss_disc_real_4: 0.23809194564819336 (0.23575486881392343) + | > loss_disc_real_5: 0.24148137867450714 (0.22326712310314178) + | > loss_0: 2.438518762588501 (2.42517352104187) + | > grad_norm_0: tensor(6.2500, device='cuda:0') (tensor(6.9169, device='cuda:0')) + | > loss_gen: 2.410144329071045 (2.394441400255476) + | > loss_kl: 1.7494990825653076 (1.8050788811274938) + | > loss_feat: 7.525940418243408 (7.266306468418667) + | > loss_mel: 18.485614776611328 (18.089063371930802) + | > loss_duration: 1.401198148727417 (1.4318607364382063) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.572397232055664 (30.986751011439733) + | > grad_norm_1: tensor(184.4502, device='cuda:0') (tensor(125.5209, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.3243 (0.32549565179007395) + | > loader_time: 0.0042 (0.004365648542131696) + + + --> TIME: 2025-04-19 02:00:38 -- STEP: 32/142 -- GLOBAL_STEP: 405300 + | > loss_disc: 2.483320951461792 (2.453705981373787) + | > loss_disc_real_0: 0.12498404830694199 (0.12603506119921798) + | > loss_disc_real_1: 0.20504130423069 (0.2044708663597703) + | > loss_disc_real_2: 0.23721949756145477 (0.22479717759415507) + | > loss_disc_real_3: 0.2177969515323639 (0.22738361405208707) + | > loss_disc_real_4: 0.21137471497058868 (0.22967367293313146) + | > loss_disc_real_5: 0.1974346935749054 (0.23095289105549455) + | > loss_0: 2.483320951461792 (2.453705981373787) + | > grad_norm_0: tensor(10.8193, device='cuda:0') (tensor(7.1077, device='cuda:0')) + | > loss_gen: 2.3139383792877197 (2.3619478568434706) + | > loss_kl: 1.9648030996322632 (1.826512262225151) + | > loss_feat: 7.662734031677246 (7.254735097289085) + | > loss_mel: 19.041833877563477 (17.923040449619297) + | > loss_duration: 1.4387407302856445 (1.441999897360802) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.42205047607422 (30.808235704898834) + | > grad_norm_1: tensor(244.3598, device='cuda:0') (tensor(160.2045, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.354 (0.33793663978576655) + | > loader_time: 0.005 (0.004973895847797394) + + + --> TIME: 2025-04-19 02:00:47 -- STEP: 57/142 -- GLOBAL_STEP: 405325 + | > loss_disc: 2.4344687461853027 (2.452270637478745) + | > loss_disc_real_0: 0.10400670021772385 (0.12486228882743595) + | > loss_disc_real_1: 0.18868029117584229 (0.2044741561015447) + | > loss_disc_real_2: 0.21228088438510895 (0.22524026663679825) + | > loss_disc_real_3: 0.2077968567609787 (0.22823971745214963) + | > loss_disc_real_4: 0.21163013577461243 (0.2296423744737056) + | > loss_disc_real_5: 0.20829303562641144 (0.2310334799582498) + | > loss_0: 2.4344687461853027 (2.452270637478745) + | > grad_norm_0: tensor(5.4393, device='cuda:0') (tensor(7.4306, device='cuda:0')) + | > loss_gen: 2.145758867263794 (2.3594510053333475) + | > loss_kl: 1.74186372756958 (1.8665044914212143) + | > loss_feat: 6.738029956817627 (7.178964991318552) + | > loss_mel: 17.334579467773438 (17.920750065853724) + | > loss_duration: 1.5069952011108398 (1.4525930379566394) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.467227935791016 (30.77826356051261) + | > grad_norm_1: tensor(79.4016, device='cuda:0') (tensor(168.4050, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.3741 (0.3532464336930659) + | > loader_time: 0.0058 (0.005231577053404691) + + + --> TIME: 2025-04-19 02:00:57 -- STEP: 82/142 -- GLOBAL_STEP: 405350 + | > loss_disc: 2.5436551570892334 (2.4543425222722495) + | > loss_disc_real_0: 0.1024278774857521 (0.12567513808608052) + | > loss_disc_real_1: 0.26033657789230347 (0.2063430709446349) + | > loss_disc_real_2: 0.2272769808769226 (0.22481613842452444) + | > loss_disc_real_3: 0.2506333589553833 (0.22744783922666456) + | > loss_disc_real_4: 0.2475927770137787 (0.22964489060204205) + | > loss_disc_real_5: 0.27982908487319946 (0.23211299391781412) + | > loss_0: 2.5436551570892334 (2.4543425222722495) + | > grad_norm_0: tensor(5.6927, device='cuda:0') (tensor(7.0851, device='cuda:0')) + | > loss_gen: 2.289368152618408 (2.3566043813054147) + | > loss_kl: 2.002598285675049 (1.8822911352646061) + | > loss_feat: 5.663406848907471 (7.1054997734907195) + | > loss_mel: 16.962270736694336 (17.94087166902496) + | > loss_duration: 1.5442872047424316 (1.4650545774436576) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.461931228637695 (30.750321504546374) + | > grad_norm_1: tensor(70.7680, device='cuda:0') (tensor(154.6776, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.3962 (0.3643621467962497) + | > loader_time: 0.0067 (0.0055672947953387016) + + + --> TIME: 2025-04-19 02:01:08 -- STEP: 107/142 -- GLOBAL_STEP: 405375 + | > loss_disc: 2.466216564178467 (2.4554557220958104) + | > loss_disc_real_0: 0.07959792017936707 (0.1270290026876414) + | > loss_disc_real_1: 0.17732635140419006 (0.20551071002661625) + | > loss_disc_real_2: 0.23300573229789734 (0.22464032805411616) + | > loss_disc_real_3: 0.21403200924396515 (0.2279095085703324) + | > loss_disc_real_4: 0.26338115334510803 (0.22982267394801167) + | > loss_disc_real_5: 0.21178793907165527 (0.23180524726337362) + | > loss_0: 2.466216564178467 (2.4554557220958104) + | > grad_norm_0: tensor(13.5547, device='cuda:0') (tensor(7.0622, device='cuda:0')) + | > loss_gen: 2.396083354949951 (2.3536408549157253) + | > loss_kl: 1.9385191202163696 (1.8879196365302968) + | > loss_feat: 6.935403823852539 (7.110633645102243) + | > loss_mel: 17.67868423461914 (17.99887775706355) + | > loss_duration: 1.4783456325531006 (1.4746916082417854) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.427038192749023 (30.825763541961386) + | > grad_norm_1: tensor(297.0090, device='cuda:0') (tensor(150.1093, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.416 (0.37413499065648725) + | > loader_time: 0.007 (0.00582917605605081) + + + --> TIME: 2025-04-19 02:01:19 -- STEP: 132/142 -- GLOBAL_STEP: 405400 + | > loss_disc: 2.5265002250671387 (2.4620157606673962) + | > loss_disc_real_0: 0.11127722263336182 (0.1292187123361862) + | > loss_disc_real_1: 0.2122495323419571 (0.20621123763196397) + | > loss_disc_real_2: 0.2359849065542221 (0.22530050496711876) + | > loss_disc_real_3: 0.24021095037460327 (0.22801633551716805) + | > loss_disc_real_4: 0.22005213797092438 (0.22972171324672122) + | > loss_disc_real_5: 0.24435386061668396 (0.232577503969272) + | > loss_0: 2.5265002250671387 (2.4620157606673962) + | > grad_norm_0: tensor(15.3251, device='cuda:0') (tensor(8.2035, device='cuda:0')) + | > loss_gen: 2.231698989868164 (2.351090945980765) + | > loss_kl: 1.9175834655761719 (1.8978012122891166) + | > loss_feat: 6.318902492523193 (7.043511228127913) + | > loss_mel: 16.903470993041992 (17.850113283504154) + | > loss_duration: 1.4856783151626587 (1.4806729780905175) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.85733413696289 (30.623189680504076) + | > grad_norm_1: tensor(281.8746, device='cuda:0') (tensor(184.1422, device='cuda:0')) + | > current_lr_0: 0.00019547665034976976 + | > current_lr_1: 0.00019547665034976976 + | > step_time: 0.4457 (0.38560548695650965) + | > loader_time: 0.0072 (0.006108076283425997) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.614901065826416 (2.614901065826416) + | > loss_disc_real_0: 0.18884089589118958 (0.18884089589118958) + | > loss_disc_real_1: 0.2674538493156433 (0.2674538493156433) + | > loss_disc_real_2: 0.24242420494556427 (0.24242420494556427) + | > loss_disc_real_3: 0.25666844844818115 (0.25666844844818115) + | > loss_disc_real_4: 0.2599450349807739 (0.2599450349807739) + | > loss_disc_real_5: 0.243045374751091 (0.243045374751091) + | > loss_0: 2.614901065826416 (2.614901065826416) + | > loss_gen: 2.318955898284912 (2.318955898284912) + | > loss_kl: 2.162433624267578 (2.162433624267578) + | > loss_feat: 5.655187129974365 (5.655187129974365) + | > loss_mel: 16.463191986083984 (16.463191986083984) + | > loss_duration: 1.6812762022018433 (1.6812762022018433) + | > loss_1: 28.28104591369629 (28.28104591369629) + + --> STEP: 1 + | > loss_disc: 2.4378747940063477 (2.4378747940063477) + | > loss_disc_real_0: 0.12586751580238342 (0.12586751580238342) + | > loss_disc_real_1: 0.2321127951145172 (0.2321127951145172) + | > loss_disc_real_2: 0.21353919804096222 (0.21353919804096222) + | > loss_disc_real_3: 0.2480171173810959 (0.2480171173810959) + | > loss_disc_real_4: 0.23540887236595154 (0.23540887236595154) + | > loss_disc_real_5: 0.2527555227279663 (0.2527555227279663) + | > loss_0: 2.4378747940063477 (2.4378747940063477) + | > loss_gen: 2.3904504776000977 (2.3904504776000977) + | > loss_kl: 2.5599493980407715 (2.5599493980407715) + | > loss_feat: 6.327627182006836 (6.327627182006836) + | > loss_mel: 18.910852432250977 (18.910852432250977) + | > loss_duration: 1.707497477531433 (1.707497477531433) + | > loss_1: 31.896377563476562 (31.896377563476562) + + --> STEP: 2 + | > loss_disc: 2.4909000396728516 (2.4643874168395996) + | > loss_disc_real_0: 0.12723523378372192 (0.12655137479305267) + | > loss_disc_real_1: 0.22963570058345795 (0.23087424784898758) + | > loss_disc_real_2: 0.2298492044210434 (0.2216942012310028) + | > loss_disc_real_3: 0.24328188598155975 (0.24564950168132782) + | > loss_disc_real_4: 0.23158466815948486 (0.2334967702627182) + | > loss_disc_real_5: 0.22272805869579315 (0.23774179071187973) + | > loss_0: 2.4909000396728516 (2.4643874168395996) + | > loss_gen: 2.2967376708984375 (2.3435940742492676) + | > loss_kl: 2.205871343612671 (2.382910370826721) + | > loss_feat: 6.610838413238525 (6.469232797622681) + | > loss_mel: 16.910140991210938 (17.910496711730957) + | > loss_duration: 1.642024278640747 (1.67476087808609) + | > loss_1: 29.665613174438477 (30.78099536895752) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035109519958496094 (-0.00042128562927246094) + | > avg_loss_disc: 2.4643874168395996 (+0.001377701759338379) + | > avg_loss_disc_real_0: 0.12655137479305267 (-0.06934590637683868) + | > avg_loss_disc_real_1: 0.23087424784898758 (+0.021695025265216827) + | > avg_loss_disc_real_2: 0.2216942012310028 (-0.010943293571472168) + | > avg_loss_disc_real_3: 0.24564950168132782 (+0.0045686885714530945) + | > avg_loss_disc_real_4: 0.2334967702627182 (+0.02557791769504547) + | > avg_loss_disc_real_5: 0.23774179071187973 (+0.04967373609542847) + | > avg_loss_0: 2.4643874168395996 (+0.001377701759338379) + | > avg_loss_gen: 2.3435940742492676 (+0.009517431259155273) + | > avg_loss_kl: 2.382910370826721 (-0.012932300567626953) + | > avg_loss_feat: 6.469232797622681 (-0.32924747467041016) + | > avg_loss_mel: 17.910496711730957 (+0.6273961067199707) + | > avg_loss_duration: 1.67476087808609 (+0.0022493600845336914) + | > avg_loss_1: 30.78099536895752 (+0.2969827651977539) + + + > EPOCH: 184/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:01:25)  + + --> TIME: 2025-04-19 02:01:30 -- STEP: 15/142 -- GLOBAL_STEP: 405425 + | > loss_disc: 2.4338860511779785 (2.4402320861816404) + | > loss_disc_real_0: 0.06562810391187668 (0.10915851394335428) + | > loss_disc_real_1: 0.18411961197853088 (0.20321928858757018) + | > loss_disc_real_2: 0.1681029349565506 (0.22059070567289987) + | > loss_disc_real_3: 0.19959549605846405 (0.22983926633993784) + | > loss_disc_real_4: 0.2060793936252594 (0.23408489922682443) + | > loss_disc_real_5: 0.21749362349510193 (0.23385697603225708) + | > loss_0: 2.4338860511779785 (2.4402320861816404) + | > grad_norm_0: tensor(22.9512, device='cuda:0') (tensor(15.5320, device='cuda:0')) + | > loss_gen: 2.5133883953094482 (2.3883347352345785) + | > loss_kl: 1.736929178237915 (1.8378181060155232) + | > loss_feat: 7.806684494018555 (7.337953376770019) + | > loss_mel: 18.090534210205078 (18.02309684753418) + | > loss_duration: 1.442721962928772 (1.4334649403889974) + | > amp_scaler: 256.0 (307.2) + | > loss_1: 31.59025764465332 (31.02066815694173) + | > grad_norm_1: tensor(589.3793, device='cuda:0') (tensor(312.9029, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.3356 (0.3309092839558919) + | > loader_time: 0.0051 (0.004612096150716146) + + + --> TIME: 2025-04-19 02:01:39 -- STEP: 40/142 -- GLOBAL_STEP: 405450 + | > loss_disc: 2.4519405364990234 (2.4482938349246988) + | > loss_disc_real_0: 0.21422742307186127 (0.11924007255584002) + | > loss_disc_real_1: 0.21461579203605652 (0.20635511241853238) + | > loss_disc_real_2: 0.22698824107646942 (0.22682574279606343) + | > loss_disc_real_3: 0.2181726098060608 (0.2285701885819435) + | > loss_disc_real_4: 0.23396344482898712 (0.22932271398603915) + | > loss_disc_real_5: 0.23245836794376373 (0.23479650765657425) + | > loss_0: 2.4519405364990234 (2.4482938349246988) + | > grad_norm_0: tensor(23.6771, device='cuda:0') (tensor(18.0513, device='cuda:0')) + | > loss_gen: 2.3747899532318115 (2.427475732564926) + | > loss_kl: 1.7645127773284912 (1.8498097091913224) + | > loss_feat: 6.547055244445801 (7.280344808101654) + | > loss_mel: 17.111446380615234 (17.972398424148558) + | > loss_duration: 1.428093671798706 (1.4455474734306337) + | > amp_scaler: 256.0 (275.20000000000005) + | > loss_1: 29.22589874267578 (30.97557611465454) + | > grad_norm_1: tensor(293.1830, device='cuda:0') (tensor(325.6236, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.3759 (0.3455830097198487) + | > loader_time: 0.0089 (0.005363160371780395) + + + --> TIME: 2025-04-19 02:01:49 -- STEP: 65/142 -- GLOBAL_STEP: 405475 + | > loss_disc: 2.5122807025909424 (2.4474329178149885) + | > loss_disc_real_0: 0.1197342500090599 (0.11934372616501955) + | > loss_disc_real_1: 0.24652090668678284 (0.20478225923501528) + | > loss_disc_real_2: 0.2210090160369873 (0.2257991841206184) + | > loss_disc_real_3: 0.22668857872486115 (0.22844081154236426) + | > loss_disc_real_4: 0.2382175773382187 (0.22865026478583997) + | > loss_disc_real_5: 0.22983142733573914 (0.2330039008305623) + | > loss_0: 2.5122807025909424 (2.4474329178149885) + | > grad_norm_0: tensor(10.9572, device='cuda:0') (tensor(16.4592, device='cuda:0')) + | > loss_gen: 2.404721260070801 (2.398562523034903) + | > loss_kl: 1.8423837423324585 (1.8585799767420843) + | > loss_feat: 7.16166353225708 (7.198926815619836) + | > loss_mel: 17.809711456298828 (17.951036409231335) + | > loss_duration: 1.4636739492416382 (1.4579671401243948) + | > amp_scaler: 256.0 (267.81538461538463) + | > loss_1: 30.682153701782227 (30.865072925274188) + | > grad_norm_1: tensor(136.2469, device='cuda:0') (tensor(317.3601, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.3887 (0.3559245402996357) + | > loader_time: 0.0063 (0.0056163604442889875) + + + --> TIME: 2025-04-19 02:01:59 -- STEP: 90/142 -- GLOBAL_STEP: 405500 + | > loss_disc: 2.479564905166626 (2.4489841514163553) + | > loss_disc_real_0: 0.12587617337703705 (0.12224244198037518) + | > loss_disc_real_1: 0.19177231192588806 (0.20428768578502868) + | > loss_disc_real_2: 0.21307772397994995 (0.225181097123358) + | > loss_disc_real_3: 0.24400535225868225 (0.2284624280201064) + | > loss_disc_real_4: 0.22497433423995972 (0.22933090445068147) + | > loss_disc_real_5: 0.23852844536304474 (0.2326528153485722) + | > loss_0: 2.479564905166626 (2.4489841514163553) + | > grad_norm_0: tensor(6.9783, device='cuda:0') (tensor(14.3092, device='cuda:0')) + | > loss_gen: 2.2923572063446045 (2.3784625768661494) + | > loss_kl: 2.0431113243103027 (1.874067278703054) + | > loss_feat: 6.677000999450684 (7.122822184032864) + | > loss_mel: 16.57939338684082 (17.81028941472371) + | > loss_duration: 1.5186123847961426 (1.4701521502600783) + | > amp_scaler: 256.0 (264.5333333333332) + | > loss_1: 29.1104736328125 (30.655793635050454) + | > grad_norm_1: tensor(63.0940, device='cuda:0') (tensor(282.1722, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.4032 (0.3662868791156347) + | > loader_time: 0.0063 (0.005866485171847875) + + + --> TIME: 2025-04-19 02:02:10 -- STEP: 115/142 -- GLOBAL_STEP: 405525 + | > loss_disc: 2.347466468811035 (2.4532591840495246) + | > loss_disc_real_0: 0.1358635425567627 (0.12395263803393944) + | > loss_disc_real_1: 0.21463283896446228 (0.20398262531861014) + | > loss_disc_real_2: 0.21631677448749542 (0.2258422887843588) + | > loss_disc_real_3: 0.20117390155792236 (0.22825372309788414) + | > loss_disc_real_4: 0.24392208456993103 (0.22938346253788988) + | > loss_disc_real_5: 0.19474799931049347 (0.23214559010837388) + | > loss_0: 2.347466468811035 (2.4532591840495246) + | > grad_norm_0: tensor(5.6372, device='cuda:0') (tensor(13.2081, device='cuda:0')) + | > loss_gen: 2.4879558086395264 (2.367121370978977) + | > loss_kl: 1.917680025100708 (1.882298841683761) + | > loss_feat: 7.631030559539795 (7.101815223693848) + | > loss_mel: 18.467674255371094 (17.779391006801436) + | > loss_duration: 1.5122289657592773 (1.476926658464515) + | > amp_scaler: 256.0 (262.6782608695652) + | > loss_1: 32.01656723022461 (30.60755310058594) + | > grad_norm_1: tensor(79.1046, device='cuda:0') (tensor(261.6655, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.4239 (0.376582023371821) + | > loader_time: 0.0075 (0.006108078749283501) + + + --> TIME: 2025-04-19 02:02:21 -- STEP: 140/142 -- GLOBAL_STEP: 405550 + | > loss_disc: 2.655296564102173 (2.461496276514872) + | > loss_disc_real_0: 0.12391191720962524 (0.1262971241293209) + | > loss_disc_real_1: 0.2044588327407837 (0.2051452671842916) + | > loss_disc_real_2: 0.19585053622722626 (0.22575539659176558) + | > loss_disc_real_3: 0.21247093379497528 (0.22819148610745157) + | > loss_disc_real_4: 0.1895514875650406 (0.22881044798663686) + | > loss_disc_real_5: 0.21100227534770966 (0.23248125314712526) + | > loss_0: 2.655296564102173 (2.461496276514872) + | > grad_norm_0: tensor(6.7188, device='cuda:0') (tensor(12.1051, device='cuda:0')) + | > loss_gen: 2.169994592666626 (2.34860394682203) + | > loss_kl: 1.8288260698318481 (1.885901835986546) + | > loss_feat: 6.05599308013916 (7.019111817223684) + | > loss_mel: 15.557684898376465 (17.71996966089521) + | > loss_duration: 1.548593521118164 (1.4833642550877166) + | > amp_scaler: 256.0 (261.4857142857142) + | > loss_1: 27.16109275817871 (30.45695153645107) + | > grad_norm_1: tensor(38.3790, device='cuda:0') (tensor(231.6538, device='cuda:0')) + | > current_lr_0: 0.00019545221576847604 + | > current_lr_1: 0.00019545221576847604 + | > step_time: 0.4558 (0.38848277500697553) + | > loader_time: 0.0074 (0.006337877682277136) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5368101596832275 (2.5368101596832275) + | > loss_disc_real_0: 0.23029585182666779 (0.23029585182666779) + | > loss_disc_real_1: 0.19628314673900604 (0.19628314673900604) + | > loss_disc_real_2: 0.24808847904205322 (0.24808847904205322) + | > loss_disc_real_3: 0.2870292067527771 (0.2870292067527771) + | > loss_disc_real_4: 0.23678012192249298 (0.23678012192249298) + | > loss_disc_real_5: 0.27495262026786804 (0.27495262026786804) + | > loss_0: 2.5368101596832275 (2.5368101596832275) + | > loss_gen: 2.679438352584839 (2.679438352584839) + | > loss_kl: 2.307511329650879 (2.307511329650879) + | > loss_feat: 6.734271049499512 (6.734271049499512) + | > loss_mel: 18.034143447875977 (18.034143447875977) + | > loss_duration: 1.6524494886398315 (1.6524494886398315) + | > loss_1: 31.407812118530273 (31.407812118530273) + + --> STEP: 1 + | > loss_disc: 2.494891405105591 (2.494891405105591) + | > loss_disc_real_0: 0.23927675187587738 (0.23927675187587738) + | > loss_disc_real_1: 0.25351759791374207 (0.25351759791374207) + | > loss_disc_real_2: 0.24278177320957184 (0.24278177320957184) + | > loss_disc_real_3: 0.22506281733512878 (0.22506281733512878) + | > loss_disc_real_4: 0.2220219373703003 (0.2220219373703003) + | > loss_disc_real_5: 0.19967904686927795 (0.19967904686927795) + | > loss_0: 2.494891405105591 (2.494891405105591) + | > loss_gen: 2.725274085998535 (2.725274085998535) + | > loss_kl: 2.674504280090332 (2.674504280090332) + | > loss_feat: 8.088976860046387 (8.088976860046387) + | > loss_mel: 19.661109924316406 (19.661109924316406) + | > loss_duration: 1.6854000091552734 (1.6854000091552734) + | > loss_1: 34.83526611328125 (34.83526611328125) + + --> STEP: 2 + | > loss_disc: 2.2681386470794678 (2.3815150260925293) + | > loss_disc_real_0: 0.19543901085853577 (0.21735788136720657) + | > loss_disc_real_1: 0.2328440397977829 (0.24318081885576248) + | > loss_disc_real_2: 0.2406752109527588 (0.2417284920811653) + | > loss_disc_real_3: 0.2547838091850281 (0.23992331326007843) + | > loss_disc_real_4: 0.24513670802116394 (0.23357932269573212) + | > loss_disc_real_5: 0.23432119190692902 (0.21700011938810349) + | > loss_0: 2.2681386470794678 (2.3815150260925293) + | > loss_gen: 2.948310613632202 (2.8367923498153687) + | > loss_kl: 2.2189061641693115 (2.4467052221298218) + | > loss_feat: 7.764088153839111 (7.926532506942749) + | > loss_mel: 19.33384895324707 (19.49747943878174) + | > loss_duration: 1.6580373048782349 (1.6717186570167542) + | > loss_1: 33.92319107055664 (34.379228591918945) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004959583282470703 (+0.0014486312866210938) + | > avg_loss_disc: 2.3815150260925293 (-0.08287239074707031) + | > avg_loss_disc_real_0: 0.21735788136720657 (+0.0908065065741539) + | > avg_loss_disc_real_1: 0.24318081885576248 (+0.012306571006774902) + | > avg_loss_disc_real_2: 0.2417284920811653 (+0.020034290850162506) + | > avg_loss_disc_real_3: 0.23992331326007843 (-0.00572618842124939) + | > avg_loss_disc_real_4: 0.23357932269573212 (+8.255243301391602e-05) + | > avg_loss_disc_real_5: 0.21700011938810349 (-0.020741671323776245) + | > avg_loss_0: 2.3815150260925293 (-0.08287239074707031) + | > avg_loss_gen: 2.8367923498153687 (+0.4931982755661011) + | > avg_loss_kl: 2.4467052221298218 (+0.06379485130310059) + | > avg_loss_feat: 7.926532506942749 (+1.4572997093200684) + | > avg_loss_mel: 19.49747943878174 (+1.5869827270507812) + | > avg_loss_duration: 1.6717186570167542 (-0.0030422210693359375) + | > avg_loss_1: 34.379228591918945 (+3.598233222961426) + + + > EPOCH: 185/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:02:23)  + + --> TIME: 2025-04-19 02:02:32 -- STEP: 23/142 -- GLOBAL_STEP: 405575 + | > loss_disc: 2.4570846557617188 (2.4634811256242837) + | > loss_disc_real_0: 0.0747099444270134 (0.1279442870746488) + | > loss_disc_real_1: 0.19938349723815918 (0.2053765699915264) + | > loss_disc_real_2: 0.23304374516010284 (0.22702091608358466) + | > loss_disc_real_3: 0.2268735021352768 (0.22434833062731702) + | > loss_disc_real_4: 0.21298842132091522 (0.23102920146092124) + | > loss_disc_real_5: 0.22019748389720917 (0.22935656037019647) + | > loss_0: 2.4570846557617188 (2.4634811256242837) + | > grad_norm_0: tensor(8.8851, device='cuda:0') (tensor(7.4395, device='cuda:0')) + | > loss_gen: 2.3375580310821533 (2.342453935871953) + | > loss_kl: 1.9568743705749512 (1.8232405704000723) + | > loss_feat: 7.272421836853027 (7.116376482922098) + | > loss_mel: 18.069883346557617 (17.955684247224227) + | > loss_duration: 1.4784364700317383 (1.4423451890116152) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.11517333984375 (30.680099901945695) + | > grad_norm_1: tensor(246.8119, device='cuda:0') (tensor(111.1206, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.3461 (0.33764593497566553) + | > loader_time: 0.005 (0.004769563674926758) + + + --> TIME: 2025-04-19 02:02:41 -- STEP: 48/142 -- GLOBAL_STEP: 405600 + | > loss_disc: 2.302339553833008 (2.4645289480686188) + | > loss_disc_real_0: 0.10567457973957062 (0.12924658351888252) + | > loss_disc_real_1: 0.2432672530412674 (0.2057298111418883) + | > loss_disc_real_2: 0.22771941125392914 (0.22712873698522648) + | > loss_disc_real_3: 0.23113732039928436 (0.22726782908042273) + | > loss_disc_real_4: 0.19960688054561615 (0.22850456523398557) + | > loss_disc_real_5: 0.21026542782783508 (0.23236387005696693) + | > loss_0: 2.302339553833008 (2.4645289480686188) + | > grad_norm_0: tensor(5.3681, device='cuda:0') (tensor(8.3556, device='cuda:0')) + | > loss_gen: 2.50842022895813 (2.3467431912819547) + | > loss_kl: 1.8283616304397583 (1.841314231355985) + | > loss_feat: 7.658275604248047 (7.12117259701093) + | > loss_mel: 18.72307777404785 (17.904986302057903) + | > loss_duration: 1.4628610610961914 (1.4498693570494652) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.18099594116211 (30.664085507392883) + | > grad_norm_1: tensor(97.4553, device='cuda:0') (tensor(143.3933, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.3637 (0.34816668927669525) + | > loader_time: 0.0058 (0.0052069177230199175) + + + --> TIME: 2025-04-19 02:02:51 -- STEP: 73/142 -- GLOBAL_STEP: 405625 + | > loss_disc: 2.4228451251983643 (2.4619493092576117) + | > loss_disc_real_0: 0.08022003620862961 (0.12802754128224225) + | > loss_disc_real_1: 0.2403714507818222 (0.20607471180288758) + | > loss_disc_real_2: 0.23784130811691284 (0.22587538030866075) + | > loss_disc_real_3: 0.22790682315826416 (0.22778839587348781) + | > loss_disc_real_4: 0.23680931329727173 (0.22993294915107831) + | > loss_disc_real_5: 0.22450053691864014 (0.2322718474554689) + | > loss_0: 2.4228451251983643 (2.4619493092576117) + | > grad_norm_0: tensor(17.0011, device='cuda:0') (tensor(8.2469, device='cuda:0')) + | > loss_gen: 2.4741203784942627 (2.3518858217213254) + | > loss_kl: 1.934776782989502 (1.8693173421572333) + | > loss_feat: 7.673763751983643 (7.136174371797744) + | > loss_mel: 18.851940155029297 (17.96108829811827) + | > loss_duration: 1.5140291452407837 (1.4619708730749885) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.448631286621094 (30.78043671176858) + | > grad_norm_1: tensor(421.4815, device='cuda:0') (tensor(151.8052, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.3939 (0.35914391360870773) + | > loader_time: 0.0065 (0.005529106479801544) + + + --> TIME: 2025-04-19 02:03:01 -- STEP: 98/142 -- GLOBAL_STEP: 405650 + | > loss_disc: 2.4468278884887695 (2.4691045381584944) + | > loss_disc_real_0: 0.13078512251377106 (0.12932413436320353) + | > loss_disc_real_1: 0.21137705445289612 (0.20615046091225683) + | > loss_disc_real_2: 0.22648341953754425 (0.2265302259094861) + | > loss_disc_real_3: 0.2182849645614624 (0.22813686226703683) + | > loss_disc_real_4: 0.20982052385807037 (0.22975740855445667) + | > loss_disc_real_5: 0.22918854653835297 (0.23361650793528072) + | > loss_0: 2.4468278884887695 (2.4691045381584944) + | > grad_norm_0: tensor(7.0010, device='cuda:0') (tensor(8.9959, device='cuda:0')) + | > loss_gen: 2.3454625606536865 (2.3397495284372445) + | > loss_kl: 1.8879594802856445 (1.8758565807829097) + | > loss_feat: 6.880388259887695 (7.040530175578837) + | > loss_mel: 16.531219482421875 (17.873748360847937) + | > loss_duration: 1.4741853475570679 (1.4723285254167051) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.119216918945312 (30.602213275675872) + | > grad_norm_1: tensor(104.3966, device='cuda:0') (tensor(182.2882, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.4087 (0.36939515629593206) + | > loader_time: 0.0073 (0.005851122797751913) + + + --> TIME: 2025-04-19 02:03:12 -- STEP: 123/142 -- GLOBAL_STEP: 405675 + | > loss_disc: 2.4161217212677 (2.4724086358295225) + | > loss_disc_real_0: 0.11825928837060928 (0.12976469341816935) + | > loss_disc_real_1: 0.1841479241847992 (0.20683918735845302) + | > loss_disc_real_2: 0.19688838720321655 (0.2263457948357109) + | > loss_disc_real_3: 0.2290855199098587 (0.22827629829809917) + | > loss_disc_real_4: 0.22538980841636658 (0.22981192086770283) + | > loss_disc_real_5: 0.2112303227186203 (0.2331704144797674) + | > loss_0: 2.4161217212677 (2.4724086358295225) + | > grad_norm_0: tensor(6.0067, device='cuda:0') (tensor(9.5600, device='cuda:0')) + | > loss_gen: 2.341542959213257 (2.3314209003758624) + | > loss_kl: 1.8412587642669678 (1.8815677960713704) + | > loss_feat: 6.890471935272217 (6.9894124666849775) + | > loss_mel: 16.5037841796875 (17.78331689136785) + | > loss_duration: 1.4830833673477173 (1.4788545292567432) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.060142517089844 (30.464572751425145) + | > grad_norm_1: tensor(122.6531, device='cuda:0') (tensor(200.8377, device='cuda:0')) + | > current_lr_0: 0.00019542778424150499 + | > current_lr_1: 0.00019542778424150499 + | > step_time: 0.4403 (0.3805736875146385) + | > loader_time: 0.0068 (0.006099741633345441) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.705833911895752 (2.705833911895752) + | > loss_disc_real_0: 0.2363867312669754 (0.2363867312669754) + | > loss_disc_real_1: 0.22776752710342407 (0.22776752710342407) + | > loss_disc_real_2: 0.22688497602939606 (0.22688497602939606) + | > loss_disc_real_3: 0.2533811628818512 (0.2533811628818512) + | > loss_disc_real_4: 0.28693661093711853 (0.28693661093711853) + | > loss_disc_real_5: 0.2413388341665268 (0.2413388341665268) + | > loss_0: 2.705833911895752 (2.705833911895752) + | > loss_gen: 2.225029230117798 (2.225029230117798) + | > loss_kl: 2.1100454330444336 (2.1100454330444336) + | > loss_feat: 5.728565216064453 (5.728565216064453) + | > loss_mel: 14.82832145690918 (14.82832145690918) + | > loss_duration: 1.679102897644043 (1.679102897644043) + | > loss_1: 26.571063995361328 (26.571063995361328) + + --> STEP: 1 + | > loss_disc: 2.516003131866455 (2.516003131866455) + | > loss_disc_real_0: 0.19777058064937592 (0.19777058064937592) + | > loss_disc_real_1: 0.21749535202980042 (0.21749535202980042) + | > loss_disc_real_2: 0.220002681016922 (0.220002681016922) + | > loss_disc_real_3: 0.2192196249961853 (0.2192196249961853) + | > loss_disc_real_4: 0.2785809636116028 (0.2785809636116028) + | > loss_disc_real_5: 0.2058233916759491 (0.2058233916759491) + | > loss_0: 2.516003131866455 (2.516003131866455) + | > loss_gen: 2.4316537380218506 (2.4316537380218506) + | > loss_kl: 2.4640305042266846 (2.4640305042266846) + | > loss_feat: 6.840358257293701 (6.840358257293701) + | > loss_mel: 18.69776153564453 (18.69776153564453) + | > loss_duration: 1.690945029258728 (1.690945029258728) + | > loss_1: 32.124752044677734 (32.124752044677734) + + --> STEP: 2 + | > loss_disc: 2.4215524196624756 (2.4687777757644653) + | > loss_disc_real_0: 0.18681778013706207 (0.192294180393219) + | > loss_disc_real_1: 0.21161386370658875 (0.21455460786819458) + | > loss_disc_real_2: 0.24309933185577393 (0.23155100643634796) + | > loss_disc_real_3: 0.260602205991745 (0.23991091549396515) + | > loss_disc_real_4: 0.2954672574996948 (0.2870241105556488) + | > loss_disc_real_5: 0.24645642936229706 (0.22613991051912308) + | > loss_0: 2.4215524196624756 (2.4687777757644653) + | > loss_gen: 2.680100202560425 (2.5558769702911377) + | > loss_kl: 2.211247444152832 (2.3376389741897583) + | > loss_feat: 7.463051795959473 (7.151705026626587) + | > loss_mel: 18.04868507385254 (18.373223304748535) + | > loss_duration: 1.6584744453430176 (1.6747097373008728) + | > loss_1: 32.06155776977539 (32.09315490722656) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00334930419921875 (-0.0016102790832519531) + | > avg_loss_disc: 2.4687777757644653 (+0.08726274967193604) + | > avg_loss_disc_real_0: 0.192294180393219 (-0.02506370097398758) + | > avg_loss_disc_real_1: 0.21455460786819458 (-0.0286262109875679) + | > avg_loss_disc_real_2: 0.23155100643634796 (-0.010177485644817352) + | > avg_loss_disc_real_3: 0.23991091549396515 (-1.239776611328125e-05) + | > avg_loss_disc_real_4: 0.2870241105556488 (+0.05344478785991669) + | > avg_loss_disc_real_5: 0.22613991051912308 (+0.009139791131019592) + | > avg_loss_0: 2.4687777757644653 (+0.08726274967193604) + | > avg_loss_gen: 2.5558769702911377 (-0.28091537952423096) + | > avg_loss_kl: 2.3376389741897583 (-0.10906624794006348) + | > avg_loss_feat: 7.151705026626587 (-0.7748274803161621) + | > avg_loss_mel: 18.373223304748535 (-1.1242561340332031) + | > avg_loss_duration: 1.6747097373008728 (+0.0029910802841186523) + | > avg_loss_1: 32.09315490722656 (-2.286073684692383) + + + > EPOCH: 186/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:03:22)  + + --> TIME: 2025-04-19 02:03:25 -- STEP: 6/142 -- GLOBAL_STEP: 405700 + | > loss_disc: 2.553701877593994 (2.459310293197632) + | > loss_disc_real_0: 0.10412726551294327 (0.11555380622545879) + | > loss_disc_real_1: 0.19589778780937195 (0.20614429066578546) + | > loss_disc_real_2: 0.2606050670146942 (0.2328550467888514) + | > loss_disc_real_3: 0.22815844416618347 (0.23466169834136963) + | > loss_disc_real_4: 0.21970605850219727 (0.23454022655884424) + | > loss_disc_real_5: 0.24348494410514832 (0.2387954369187355) + | > loss_0: 2.553701877593994 (2.459310293197632) + | > grad_norm_0: tensor(8.4396, device='cuda:0') (tensor(9.9814, device='cuda:0')) + | > loss_gen: 2.2462456226348877 (2.397309501965841) + | > loss_kl: 1.6435832977294922 (1.8519393801689148) + | > loss_feat: 6.787688732147217 (7.303814013799031) + | > loss_mel: 15.94079303741455 (17.677717049916584) + | > loss_duration: 1.418116569519043 (1.4411713480949402) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.03643035888672 (30.67195192972819) + | > grad_norm_1: tensor(220.5786, device='cuda:0') (tensor(261.3055, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.3278 (0.32774917284647626) + | > loader_time: 0.0041 (0.004337867101033528) + + + --> TIME: 2025-04-19 02:03:33 -- STEP: 31/142 -- GLOBAL_STEP: 405725 + | > loss_disc: 2.490971803665161 (2.4578789357216126) + | > loss_disc_real_0: 0.12071829289197922 (0.11799434091775649) + | > loss_disc_real_1: 0.21272766590118408 (0.20661438328604545) + | > loss_disc_real_2: 0.23176556825637817 (0.22832969119471888) + | > loss_disc_real_3: 0.23983173072338104 (0.2304247983040348) + | > loss_disc_real_4: 0.2589753568172455 (0.23281603042156465) + | > loss_disc_real_5: 0.26753589510917664 (0.23439523433485338) + | > loss_0: 2.490971803665161 (2.4578789357216126) + | > grad_norm_0: tensor(21.2487, device='cuda:0') (tensor(11.8059, device='cuda:0')) + | > loss_gen: 2.280944585800171 (2.375584271646315) + | > loss_kl: 1.786617398262024 (1.8269301076089182) + | > loss_feat: 7.758545398712158 (7.08965850645496) + | > loss_mel: 18.021514892578125 (17.86598104046237) + | > loss_duration: 1.4204955101013184 (1.4438645147508191) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.268117904663086 (30.602018479377993) + | > grad_norm_1: tensor(443.6736, device='cuda:0') (tensor(288.1820, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.3505 (0.3380832979756016) + | > loader_time: 0.0054 (0.004971304247456212) + + + --> TIME: 2025-04-19 02:03:43 -- STEP: 56/142 -- GLOBAL_STEP: 405750 + | > loss_disc: 2.4667246341705322 (2.4513077395302907) + | > loss_disc_real_0: 0.14368592202663422 (0.1212207672319242) + | > loss_disc_real_1: 0.2014499455690384 (0.2059573573725564) + | > loss_disc_real_2: 0.2712172269821167 (0.22516910758401668) + | > loss_disc_real_3: 0.2377450317144394 (0.2290685661137104) + | > loss_disc_real_4: 0.2396714985370636 (0.22884120605885983) + | > loss_disc_real_5: 0.22559970617294312 (0.23328178083258017) + | > loss_0: 2.4667246341705322 (2.4513077395302907) + | > grad_norm_0: tensor(15.2829, device='cuda:0') (tensor(12.8302, device='cuda:0')) + | > loss_gen: 2.377389430999756 (2.3641537513051705) + | > loss_kl: 1.7562477588653564 (1.8407203746693475) + | > loss_feat: 6.6747236251831055 (7.02715060540608) + | > loss_mel: 17.898143768310547 (17.833871517862594) + | > loss_duration: 1.488219976425171 (1.4523289310080665) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.194725036621094 (30.51822522708348) + | > grad_norm_1: tensor(372.4128, device='cuda:0') (tensor(305.8000, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.3877 (0.3507406626428876) + | > loader_time: 0.0081 (0.005423349993569511) + + + --> TIME: 2025-04-19 02:03:53 -- STEP: 81/142 -- GLOBAL_STEP: 405775 + | > loss_disc: 2.6032841205596924 (2.4530878802876415) + | > loss_disc_real_0: 0.13717959821224213 (0.12084983611180458) + | > loss_disc_real_1: 0.2207949161529541 (0.20727094768741985) + | > loss_disc_real_2: 0.2205224633216858 (0.2248250108074259) + | > loss_disc_real_3: 0.25408273935317993 (0.2292676838459792) + | > loss_disc_real_4: 0.2688816487789154 (0.2297740626482316) + | > loss_disc_real_5: 0.24752821028232574 (0.23250615541581754) + | > loss_0: 2.6032841205596924 (2.4530878802876415) + | > grad_norm_0: tensor(7.2789, device='cuda:0') (tensor(13.0612, device='cuda:0')) + | > loss_gen: 2.3053195476531982 (2.371212991667381) + | > loss_kl: 1.7818599939346313 (1.8533848612396806) + | > loss_feat: 6.466558933258057 (7.075804663293155) + | > loss_mel: 17.098731994628906 (17.921771638187366) + | > loss_duration: 1.4677445888519287 (1.4636168671242988) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.120214462280273 (30.685791109815057) + | > grad_norm_1: tensor(205.8018, device='cuda:0') (tensor(304.3075, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.3937 (0.36241045115906506) + | > loader_time: 0.0065 (0.005777441425088011) + + + --> TIME: 2025-04-19 02:04:03 -- STEP: 106/142 -- GLOBAL_STEP: 405800 + | > loss_disc: 2.4745795726776123 (2.465791742756682) + | > loss_disc_real_0: 0.19408749043941498 (0.12354687945741527) + | > loss_disc_real_1: 0.2126021832227707 (0.20800869391774232) + | > loss_disc_real_2: 0.22074781358242035 (0.22626782250854205) + | > loss_disc_real_3: 0.2514309883117676 (0.22947920465244437) + | > loss_disc_real_4: 0.19814787805080414 (0.23118557064038403) + | > loss_disc_real_5: 0.23187151551246643 (0.23314495454981643) + | > loss_0: 2.4745795726776123 (2.465791742756682) + | > grad_norm_0: tensor(22.8845, device='cuda:0') (tensor(13.0808, device='cuda:0')) + | > loss_gen: 2.407705307006836 (2.353448285246793) + | > loss_kl: 1.9145081043243408 (1.866216242313385) + | > loss_feat: 6.846529960632324 (6.978823086000839) + | > loss_mel: 17.678781509399414 (17.868333501635863) + | > loss_duration: 1.507468581199646 (1.4747565366187185) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35499382019043 (30.541577753031028) + | > grad_norm_1: tensor(339.3002, device='cuda:0') (tensor(302.4995, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.4093 (0.37272181600894566) + | > loader_time: 0.007 (0.00601490263668996) + + + --> TIME: 2025-04-19 02:04:14 -- STEP: 131/142 -- GLOBAL_STEP: 405825 + | > loss_disc: 2.41068696975708 (2.46309368118985) + | > loss_disc_real_0: 0.11627357453107834 (0.1239855757537689) + | > loss_disc_real_1: 0.2028549611568451 (0.20826398404954954) + | > loss_disc_real_2: 0.21616870164871216 (0.2260707193658552) + | > loss_disc_real_3: 0.18575462698936462 (0.228579132270267) + | > loss_disc_real_4: 0.23806075751781464 (0.23017327942466007) + | > loss_disc_real_5: 0.2188798040151596 (0.23135785371747636) + | > loss_0: 2.41068696975708 (2.46309368118985) + | > grad_norm_0: tensor(13.9748, device='cuda:0') (tensor(12.9036, device='cuda:0')) + | > loss_gen: 2.42440128326416 (2.349295321311658) + | > loss_kl: 1.8549381494522095 (1.8738990758211558) + | > loss_feat: 7.046448707580566 (6.947490164341818) + | > loss_mel: 17.75843048095703 (17.814801150605884) + | > loss_duration: 1.4814121723175049 (1.4807181176338486) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.565629959106445 (30.466203922533808) + | > grad_norm_1: tensor(416.2724, device='cuda:0') (tensor(302.1057, device='cuda:0')) + | > current_lr_0: 0.0001954033557684748 + | > current_lr_1: 0.0001954033557684748 + | > step_time: 0.4478 (0.38437997657834116) + | > loader_time: 0.0078 (0.006287631187730163) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.432705879211426 (2.432705879211426) + | > loss_disc_real_0: 0.15721221268177032 (0.15721221268177032) + | > loss_disc_real_1: 0.2436400055885315 (0.2436400055885315) + | > loss_disc_real_2: 0.284451425075531 (0.284451425075531) + | > loss_disc_real_3: 0.2691808342933655 (0.2691808342933655) + | > loss_disc_real_4: 0.20663326978683472 (0.20663326978683472) + | > loss_disc_real_5: 0.24733209609985352 (0.24733209609985352) + | > loss_0: 2.432705879211426 (2.432705879211426) + | > loss_gen: 2.67043137550354 (2.67043137550354) + | > loss_kl: 2.272541046142578 (2.272541046142578) + | > loss_feat: 7.712100028991699 (7.712100028991699) + | > loss_mel: 19.842008590698242 (19.842008590698242) + | > loss_duration: 1.6746668815612793 (1.6746668815612793) + | > loss_1: 34.171749114990234 (34.171749114990234) + + --> STEP: 1 + | > loss_disc: 2.580467939376831 (2.580467939376831) + | > loss_disc_real_0: 0.17826306819915771 (0.17826306819915771) + | > loss_disc_real_1: 0.2535541355609894 (0.2535541355609894) + | > loss_disc_real_2: 0.2679840624332428 (0.2679840624332428) + | > loss_disc_real_3: 0.2805773913860321 (0.2805773913860321) + | > loss_disc_real_4: 0.22497090697288513 (0.22497090697288513) + | > loss_disc_real_5: 0.29901817440986633 (0.29901817440986633) + | > loss_0: 2.580467939376831 (2.580467939376831) + | > loss_gen: 2.455610513687134 (2.455610513687134) + | > loss_kl: 2.4080922603607178 (2.4080922603607178) + | > loss_feat: 6.935318470001221 (6.935318470001221) + | > loss_mel: 19.063114166259766 (19.063114166259766) + | > loss_duration: 1.6943037509918213 (1.6943037509918213) + | > loss_1: 32.55643844604492 (32.55643844604492) + + --> STEP: 2 + | > loss_disc: 2.4741899967193604 (2.5273289680480957) + | > loss_disc_real_0: 0.2590818405151367 (0.21867245435714722) + | > loss_disc_real_1: 0.24485716223716736 (0.24920564889907837) + | > loss_disc_real_2: 0.26014676690101624 (0.2640654146671295) + | > loss_disc_real_3: 0.2623975872993469 (0.2714874893426895) + | > loss_disc_real_4: 0.19674593210220337 (0.21085841953754425) + | > loss_disc_real_5: 0.2741686701774597 (0.286593422293663) + | > loss_0: 2.4741899967193604 (2.5273289680480957) + | > loss_gen: 2.708536148071289 (2.5820733308792114) + | > loss_kl: 2.361722946166992 (2.384907603263855) + | > loss_feat: 7.881220817565918 (7.408269643783569) + | > loss_mel: 19.20124053955078 (19.132177352905273) + | > loss_duration: 1.6413087844848633 (1.6678062677383423) + | > loss_1: 33.79402542114258 (33.17523193359375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003340601921081543 (-8.702278137207031e-06) + | > avg_loss_disc: 2.5273289680480957 (+0.05855119228363037) + | > avg_loss_disc_real_0: 0.21867245435714722 (+0.026378273963928223) + | > avg_loss_disc_real_1: 0.24920564889907837 (+0.03465104103088379) + | > avg_loss_disc_real_2: 0.2640654146671295 (+0.032514408230781555) + | > avg_loss_disc_real_3: 0.2714874893426895 (+0.031576573848724365) + | > avg_loss_disc_real_4: 0.21085841953754425 (-0.07616569101810455) + | > avg_loss_disc_real_5: 0.286593422293663 (+0.06045351177453995) + | > avg_loss_0: 2.5273289680480957 (+0.05855119228363037) + | > avg_loss_gen: 2.5820733308792114 (+0.02619636058807373) + | > avg_loss_kl: 2.384907603263855 (+0.04726862907409668) + | > avg_loss_feat: 7.408269643783569 (+0.2565646171569824) + | > avg_loss_mel: 19.132177352905273 (+0.7589540481567383) + | > avg_loss_duration: 1.6678062677383423 (-0.006903469562530518) + | > avg_loss_1: 33.17523193359375 (+1.0820770263671875) + + + > EPOCH: 187/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:04:21)  + + --> TIME: 2025-04-19 02:04:26 -- STEP: 14/142 -- GLOBAL_STEP: 405850 + | > loss_disc: 2.4201738834381104 (2.4662824869155884) + | > loss_disc_real_0: 0.09950863569974899 (0.12942586466670034) + | > loss_disc_real_1: 0.1529981791973114 (0.20398762289966857) + | > loss_disc_real_2: 0.2185196578502655 (0.22444155918700354) + | > loss_disc_real_3: 0.22531987726688385 (0.23389912396669388) + | > loss_disc_real_4: 0.2346414476633072 (0.2280281281897) + | > loss_disc_real_5: 0.20650714635849 (0.23893857428005763) + | > loss_0: 2.4201738834381104 (2.4662824869155884) + | > grad_norm_0: tensor(13.3185, device='cuda:0') (tensor(8.7110, device='cuda:0')) + | > loss_gen: 2.299778461456299 (2.3546430553708757) + | > loss_kl: 1.8797369003295898 (1.8285508240972246) + | > loss_feat: 8.17501163482666 (7.224186999457223) + | > loss_mel: 17.43332290649414 (17.97519029889788) + | > loss_duration: 1.4383153915405273 (1.4345739143235343) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.226165771484375 (30.817144802638463) + | > grad_norm_1: tensor(296.6254, device='cuda:0') (tensor(159.7155, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.3322 (0.3274031366620745) + | > loader_time: 0.0045 (0.004581144877842495) + + + --> TIME: 2025-04-19 02:04:35 -- STEP: 39/142 -- GLOBAL_STEP: 405875 + | > loss_disc: 2.376708984375 (2.4696405545259132) + | > loss_disc_real_0: 0.1257198601961136 (0.12663716001388353) + | > loss_disc_real_1: 0.19394759833812714 (0.20338985132865417) + | > loss_disc_real_2: 0.2229352593421936 (0.22696499526500702) + | > loss_disc_real_3: 0.23598678410053253 (0.23254738900906) + | > loss_disc_real_4: 0.25554394721984863 (0.22987109499099928) + | > loss_disc_real_5: 0.23459084331989288 (0.2338702304240985) + | > loss_0: 2.376708984375 (2.4696405545259132) + | > grad_norm_0: tensor(4.9231, device='cuda:0') (tensor(8.9489, device='cuda:0')) + | > loss_gen: 2.2870590686798096 (2.3510635510469093) + | > loss_kl: 2.0082709789276123 (1.8449898683107817) + | > loss_feat: 6.840778350830078 (7.1001476263388605) + | > loss_mel: 17.170265197753906 (17.820871671040855) + | > loss_duration: 1.4718925952911377 (1.445742937234732) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.77826690673828 (30.56281544612004) + | > grad_norm_1: tensor(91.6364, device='cuda:0') (tensor(205.3795, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.3569 (0.34046083841568386) + | > loader_time: 0.0055 (0.005047082901000977) + + + --> TIME: 2025-04-19 02:04:45 -- STEP: 64/142 -- GLOBAL_STEP: 405900 + | > loss_disc: 2.4458441734313965 (2.4694751724600796) + | > loss_disc_real_0: 0.128268763422966 (0.12868564866948878) + | > loss_disc_real_1: 0.18450546264648438 (0.20534380548633635) + | > loss_disc_real_2: 0.2172352820634842 (0.2269010073505342) + | > loss_disc_real_3: 0.19724006950855255 (0.23084590630605817) + | > loss_disc_real_4: 0.22002559900283813 (0.22998249460943043) + | > loss_disc_real_5: 0.21128255128860474 (0.2321102493442595) + | > loss_0: 2.4458441734313965 (2.4694751724600796) + | > grad_norm_0: tensor(4.6781, device='cuda:0') (tensor(9.2088, device='cuda:0')) + | > loss_gen: 2.1869590282440186 (2.3443999029695997) + | > loss_kl: 1.8847750425338745 (1.8597842939198017) + | > loss_feat: 7.118232727050781 (7.060660056769848) + | > loss_mel: 18.922090530395508 (17.87736590206623) + | > loss_duration: 1.5067503452301025 (1.4570824149996042) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.618806838989258 (30.59929248690605) + | > grad_norm_1: tensor(146.5904, device='cuda:0') (tensor(204.5219, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.3821 (0.3529885597527027) + | > loader_time: 0.0064 (0.005438167601823807) + + + --> TIME: 2025-04-19 02:04:55 -- STEP: 89/142 -- GLOBAL_STEP: 405925 + | > loss_disc: 2.5206525325775146 (2.4800715821512638) + | > loss_disc_real_0: 0.1381237953901291 (0.12996319087033864) + | > loss_disc_real_1: 0.22148814797401428 (0.20647936372944478) + | > loss_disc_real_2: 0.26135265827178955 (0.2282198677571972) + | > loss_disc_real_3: 0.23987753689289093 (0.2306813294633051) + | > loss_disc_real_4: 0.2195540964603424 (0.2297651000906912) + | > loss_disc_real_5: 0.22442419826984406 (0.23304143347097248) + | > loss_0: 2.5206525325775146 (2.4800715821512638) + | > grad_norm_0: tensor(17.7294, device='cuda:0') (tensor(10.5049, device='cuda:0')) + | > loss_gen: 2.3463644981384277 (2.3370796937621043) + | > loss_kl: 1.8910425901412964 (1.871300191022037) + | > loss_feat: 7.485852241516113 (7.026332319452521) + | > loss_mel: 18.20253562927246 (17.873671531677243) + | > loss_duration: 1.487226963043213 (1.4689717868740635) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.413021087646484 (30.577355481265634) + | > grad_norm_1: tensor(314.0529, device='cuda:0') (tensor(232.2193, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.4004 (0.3642912500360039) + | > loader_time: 0.0066 (0.005710130327203301) + + + --> TIME: 2025-04-19 02:05:05 -- STEP: 114/142 -- GLOBAL_STEP: 405950 + | > loss_disc: 2.3514816761016846 (2.479544543383416) + | > loss_disc_real_0: 0.08339957147836685 (0.1306836245614186) + | > loss_disc_real_1: 0.21760809421539307 (0.20604197236529567) + | > loss_disc_real_2: 0.19841940701007843 (0.22805864376979962) + | > loss_disc_real_3: 0.22609372437000275 (0.23094576950135984) + | > loss_disc_real_4: 0.2057802826166153 (0.2291377367157685) + | > loss_disc_real_5: 0.19418582320213318 (0.23126044137436047) + | > loss_0: 2.3514816761016846 (2.479544543383416) + | > grad_norm_0: tensor(17.2412, device='cuda:0') (tensor(11.7545, device='cuda:0')) + | > loss_gen: 2.386446237564087 (2.3428544893599392) + | > loss_kl: 1.8490358591079712 (1.8795627177807324) + | > loss_feat: 7.259796142578125 (7.035687814679062) + | > loss_mel: 18.74786376953125 (17.828300325494062) + | > loss_duration: 1.5169446468353271 (1.4761545009780346) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.760087966918945 (30.562559797052753) + | > grad_norm_1: tensor(415.6925, device='cuda:0') (tensor(258.6237, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.4197 (0.3749059417791534) + | > loader_time: 0.0071 (0.00596342170447634) + + + --> TIME: 2025-04-19 02:05:17 -- STEP: 139/142 -- GLOBAL_STEP: 405975 + | > loss_disc: 2.558412551879883 (2.48674356337074) + | > loss_disc_real_0: 0.14084522426128387 (0.13277701470706102) + | > loss_disc_real_1: 0.22224128246307373 (0.20703336435685055) + | > loss_disc_real_2: 0.24445775151252747 (0.22865895090772093) + | > loss_disc_real_3: 0.21307231485843658 (0.2307687642977392) + | > loss_disc_real_4: 0.23108842968940735 (0.2294013287952478) + | > loss_disc_real_5: 0.2093430757522583 (0.23205036900455145) + | > loss_0: 2.558412551879883 (2.48674356337074) + | > grad_norm_0: tensor(15.5645, device='cuda:0') (tensor(12.8651, device='cuda:0')) + | > loss_gen: 2.1887848377227783 (2.338746545983733) + | > loss_kl: 1.8933525085449219 (1.8851378538625703) + | > loss_feat: 7.148285388946533 (6.991603391633617) + | > loss_mel: 17.79416275024414 (17.785226760150717) + | > loss_duration: 1.511871337890625 (1.48307243439791) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.536457061767578 (30.483786946577993) + | > grad_norm_1: tensor(345.3421, device='cuda:0') (tensor(264.6587, device='cuda:0')) + | > current_lr_0: 0.0001953789303490037 + | > current_lr_1: 0.0001953789303490037 + | > step_time: 0.4546 (0.38702574393732087) + | > loader_time: 0.0076 (0.00622124466107046) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.449875593185425 (2.449875593185425) + | > loss_disc_real_0: 0.17018729448318481 (0.17018729448318481) + | > loss_disc_real_1: 0.22194364666938782 (0.22194364666938782) + | > loss_disc_real_2: 0.2593192160129547 (0.2593192160129547) + | > loss_disc_real_3: 0.23357555270195007 (0.23357555270195007) + | > loss_disc_real_4: 0.25028955936431885 (0.25028955936431885) + | > loss_disc_real_5: 0.21492905914783478 (0.21492905914783478) + | > loss_0: 2.449875593185425 (2.449875593185425) + | > loss_gen: 2.6773922443389893 (2.6773922443389893) + | > loss_kl: 2.4077155590057373 (2.4077155590057373) + | > loss_feat: 6.445546627044678 (6.445546627044678) + | > loss_mel: 17.814992904663086 (17.814992904663086) + | > loss_duration: 1.6796073913574219 (1.6796073913574219) + | > loss_1: 31.02525520324707 (31.02525520324707) + + --> STEP: 1 + | > loss_disc: 2.6692209243774414 (2.6692209243774414) + | > loss_disc_real_0: 0.1742548942565918 (0.1742548942565918) + | > loss_disc_real_1: 0.21765682101249695 (0.21765682101249695) + | > loss_disc_real_2: 0.25920701026916504 (0.25920701026916504) + | > loss_disc_real_3: 0.25453025102615356 (0.25453025102615356) + | > loss_disc_real_4: 0.26167410612106323 (0.26167410612106323) + | > loss_disc_real_5: 0.2649097740650177 (0.2649097740650177) + | > loss_0: 2.6692209243774414 (2.6692209243774414) + | > loss_gen: 2.227954626083374 (2.227954626083374) + | > loss_kl: 2.2694876194000244 (2.2694876194000244) + | > loss_feat: 6.632552623748779 (6.632552623748779) + | > loss_mel: 18.5410099029541 (18.5410099029541) + | > loss_duration: 1.6764600276947021 (1.6764600276947021) + | > loss_1: 31.34746551513672 (31.34746551513672) + + --> STEP: 2 + | > loss_disc: 2.5750324726104736 (2.6221266984939575) + | > loss_disc_real_0: 0.14864864945411682 (0.1614517718553543) + | > loss_disc_real_1: 0.3061105012893677 (0.2618836611509323) + | > loss_disc_real_2: 0.2437971830368042 (0.2515020966529846) + | > loss_disc_real_3: 0.2735242247581482 (0.2640272378921509) + | > loss_disc_real_4: 0.21325041353702545 (0.23746225982904434) + | > loss_disc_real_5: 0.22542065382003784 (0.24516521394252777) + | > loss_0: 2.5750324726104736 (2.6221266984939575) + | > loss_gen: 2.347571611404419 (2.2877631187438965) + | > loss_kl: 2.2150728702545166 (2.2422802448272705) + | > loss_feat: 7.267333507537842 (6.9499430656433105) + | > loss_mel: 18.63473129272461 (18.587870597839355) + | > loss_duration: 1.6531988382339478 (1.664829432964325) + | > loss_1: 32.1179084777832 (31.73268699645996) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034182071685791016 (+7.76052474975586e-05) + | > avg_loss_disc: 2.6221266984939575 (+0.09479773044586182) + | > avg_loss_disc_real_0: 0.1614517718553543 (-0.05722068250179291) + | > avg_loss_disc_real_1: 0.2618836611509323 (+0.012678012251853943) + | > avg_loss_disc_real_2: 0.2515020966529846 (-0.012563318014144897) + | > avg_loss_disc_real_3: 0.2640272378921509 (-0.007460251450538635) + | > avg_loss_disc_real_4: 0.23746225982904434 (+0.02660384029150009) + | > avg_loss_disc_real_5: 0.24516521394252777 (-0.041428208351135254) + | > avg_loss_0: 2.6221266984939575 (+0.09479773044586182) + | > avg_loss_gen: 2.2877631187438965 (-0.29431021213531494) + | > avg_loss_kl: 2.2422802448272705 (-0.14262735843658447) + | > avg_loss_feat: 6.9499430656433105 (-0.4583265781402588) + | > avg_loss_mel: 18.587870597839355 (-0.544306755065918) + | > avg_loss_duration: 1.664829432964325 (-0.002976834774017334) + | > avg_loss_1: 31.73268699645996 (-1.442544937133789) + + + > EPOCH: 188/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:05:19)  + + --> TIME: 2025-04-19 02:05:27 -- STEP: 22/142 -- GLOBAL_STEP: 406000 + | > loss_disc: 2.4313926696777344 (2.4608130888505415) + | > loss_disc_real_0: 0.14052216708660126 (0.12807253612713376) + | > loss_disc_real_1: 0.23954278230667114 (0.20895278724757108) + | > loss_disc_real_2: 0.23763832449913025 (0.226809677075256) + | > loss_disc_real_3: 0.232680544257164 (0.22837551276792178) + | > loss_disc_real_4: 0.22600869834423065 (0.2319540401751345) + | > loss_disc_real_5: 0.23192918300628662 (0.23375774581323971) + | > loss_0: 2.4313926696777344 (2.4608130888505415) + | > grad_norm_0: tensor(7.7666, device='cuda:0') (tensor(9.5907, device='cuda:0')) + | > loss_gen: 2.4072372913360596 (2.355267145416953) + | > loss_kl: 1.8900359869003296 (1.8371115489439531) + | > loss_feat: 6.6111016273498535 (7.022936517542059) + | > loss_mel: 17.569568634033203 (17.818637761202726) + | > loss_duration: 1.4236836433410645 (1.4363018003377048) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.901626586914062 (30.47025524486195) + | > grad_norm_1: tensor(97.9254, device='cuda:0') (tensor(193.9490, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.3531 (0.33209681510925293) + | > loader_time: 0.0055 (0.004827315157110041) + + + --> TIME: 2025-04-19 02:05:37 -- STEP: 47/142 -- GLOBAL_STEP: 406025 + | > loss_disc: 2.3837571144104004 (2.4632850758572853) + | > loss_disc_real_0: 0.12153372913599014 (0.12474081601868287) + | > loss_disc_real_1: 0.1840578019618988 (0.20543001718977663) + | > loss_disc_real_2: 0.21694573760032654 (0.22711031646170515) + | > loss_disc_real_3: 0.19448629021644592 (0.230425718616932) + | > loss_disc_real_4: 0.22280436754226685 (0.23216338614200024) + | > loss_disc_real_5: 0.18648895621299744 (0.23298972306099344) + | > loss_0: 2.3837571144104004 (2.4632850758572853) + | > grad_norm_0: tensor(6.0458, device='cuda:0') (tensor(8.7414, device='cuda:0')) + | > loss_gen: 2.295506477355957 (2.3435357377884234) + | > loss_kl: 1.8483176231384277 (1.8393462794892332) + | > loss_feat: 6.689474105834961 (7.064791638800439) + | > loss_mel: 17.8691349029541 (17.94270383550766) + | > loss_duration: 1.4227564334869385 (1.4474870727417317) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.12519073486328 (30.637864782455118) + | > grad_norm_1: tensor(218.3242, device='cuda:0') (tensor(184.7257, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.5593 (0.35275086950748524) + | > loader_time: 0.006 (0.005296098425033244) + + + --> TIME: 2025-04-19 02:05:47 -- STEP: 72/142 -- GLOBAL_STEP: 406050 + | > loss_disc: 2.4783897399902344 (2.4539299276140016) + | > loss_disc_real_0: 0.11822821199893951 (0.12594663713955218) + | > loss_disc_real_1: 0.20987211167812347 (0.204631048358149) + | > loss_disc_real_2: 0.2527335584163666 (0.22574138103259933) + | > loss_disc_real_3: 0.2449614256620407 (0.22894534551435047) + | > loss_disc_real_4: 0.20874595642089844 (0.22926288988027307) + | > loss_disc_real_5: 0.2456737756729126 (0.23166683005789915) + | > loss_0: 2.4783897399902344 (2.4539299276140016) + | > grad_norm_0: tensor(11.1925, device='cuda:0') (tensor(8.7856, device='cuda:0')) + | > loss_gen: 2.281888008117676 (2.3427855471769963) + | > loss_kl: 1.931138515472412 (1.8641491830348969) + | > loss_feat: 7.001494407653809 (7.051858160230848) + | > loss_mel: 17.668947219848633 (17.77714566389721) + | > loss_duration: 1.472664475440979 (1.460986336072286) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35613250732422 (30.49692498313056) + | > grad_norm_1: tensor(286.5158, device='cuda:0') (tensor(191.1355, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.3939 (0.36228880948490566) + | > loader_time: 0.0063 (0.005583643913269042) + + + --> TIME: 2025-04-19 02:05:57 -- STEP: 97/142 -- GLOBAL_STEP: 406075 + | > loss_disc: 2.4627344608306885 (2.462820198118073) + | > loss_disc_real_0: 0.13044373691082 (0.1263983642163965) + | > loss_disc_real_1: 0.22942493855953217 (0.20634859861786833) + | > loss_disc_real_2: 0.22490845620632172 (0.2265123029037849) + | > loss_disc_real_3: 0.2542021572589874 (0.2299898305197352) + | > loss_disc_real_4: 0.24623049795627594 (0.23058811168080753) + | > loss_disc_real_5: 0.23514382541179657 (0.23150192723446286) + | > loss_0: 2.4627344608306885 (2.462820198118073) + | > grad_norm_0: tensor(8.9316, device='cuda:0') (tensor(9.4446, device='cuda:0')) + | > loss_gen: 2.253896713256836 (2.3370015572026834) + | > loss_kl: 1.9534448385238647 (1.8698945659952066) + | > loss_feat: 6.305114269256592 (7.037299662521205) + | > loss_mel: 15.761346817016602 (17.812078918378393) + | > loss_duration: 1.4882912635803223 (1.4724218280045027) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.76209259033203 (30.528696650082303) + | > grad_norm_1: tensor(353.5887, device='cuda:0') (tensor(205.0647, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.4097 (0.37283195908536615) + | > loader_time: 0.0069 (0.005922371579199722) + + + --> TIME: 2025-04-19 02:06:08 -- STEP: 122/142 -- GLOBAL_STEP: 406100 + | > loss_disc: 2.4800920486450195 (2.45411521489503) + | > loss_disc_real_0: 0.16955307126045227 (0.1259793539638402) + | > loss_disc_real_1: 0.22139737010002136 (0.20620443879580888) + | > loss_disc_real_2: 0.2150828242301941 (0.22505951110945374) + | > loss_disc_real_3: 0.20989373326301575 (0.2281753646790004) + | > loss_disc_real_4: 0.2175186723470688 (0.22919207335006994) + | > loss_disc_real_5: 0.19246912002563477 (0.2296099640795442) + | > loss_0: 2.4800920486450195 (2.45411521489503) + | > grad_norm_0: tensor(11.0050, device='cuda:0') (tensor(9.8039, device='cuda:0')) + | > loss_gen: 2.4743688106536865 (2.3419076145672415) + | > loss_kl: 1.849886178970337 (1.877094491583402) + | > loss_feat: 6.906128883361816 (7.046134784573415) + | > loss_mel: 17.283132553100586 (17.7982237768955) + | > loss_duration: 1.485225796699524 (1.4791867361694087) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.998741149902344 (30.54254747609623) + | > grad_norm_1: tensor(304.9081, device='cuda:0') (tensor(214.9642, device='cuda:0')) + | > current_lr_0: 0.00019535450798271008 + | > current_lr_1: 0.00019535450798271008 + | > step_time: 0.4499 (0.3832529880961434) + | > loader_time: 0.0076 (0.006191294701372991) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3946661949157715 (2.3946661949157715) + | > loss_disc_real_0: 0.1213606521487236 (0.1213606521487236) + | > loss_disc_real_1: 0.23987416923046112 (0.23987416923046112) + | > loss_disc_real_2: 0.23833800852298737 (0.23833800852298737) + | > loss_disc_real_3: 0.21729299426078796 (0.21729299426078796) + | > loss_disc_real_4: 0.2604079246520996 (0.2604079246520996) + | > loss_disc_real_5: 0.24787697196006775 (0.24787697196006775) + | > loss_0: 2.3946661949157715 (2.3946661949157715) + | > loss_gen: 2.4632492065429688 (2.4632492065429688) + | > loss_kl: 2.5091629028320312 (2.5091629028320312) + | > loss_feat: 7.863101959228516 (7.863101959228516) + | > loss_mel: 19.70384407043457 (19.70384407043457) + | > loss_duration: 1.664691686630249 (1.664691686630249) + | > loss_1: 34.20405197143555 (34.20405197143555) + + --> STEP: 1 + | > loss_disc: 2.600862741470337 (2.600862741470337) + | > loss_disc_real_0: 0.18859313428401947 (0.18859313428401947) + | > loss_disc_real_1: 0.22026601433753967 (0.22026601433753967) + | > loss_disc_real_2: 0.271706223487854 (0.271706223487854) + | > loss_disc_real_3: 0.22263102233409882 (0.22263102233409882) + | > loss_disc_real_4: 0.2651665508747101 (0.2651665508747101) + | > loss_disc_real_5: 0.21089807152748108 (0.21089807152748108) + | > loss_0: 2.600862741470337 (2.600862741470337) + | > loss_gen: 2.277761697769165 (2.277761697769165) + | > loss_kl: 2.688105344772339 (2.688105344772339) + | > loss_feat: 6.891946315765381 (6.891946315765381) + | > loss_mel: 18.221158981323242 (18.221158981323242) + | > loss_duration: 1.6816480159759521 (1.6816480159759521) + | > loss_1: 31.7606201171875 (31.7606201171875) + + --> STEP: 2 + | > loss_disc: 2.3853938579559326 (2.4931282997131348) + | > loss_disc_real_0: 0.11550344526767731 (0.1520482897758484) + | > loss_disc_real_1: 0.24496285617351532 (0.2326144352555275) + | > loss_disc_real_2: 0.23073191940784454 (0.2512190714478493) + | > loss_disc_real_3: 0.2100624144077301 (0.21634671837091446) + | > loss_disc_real_4: 0.22325149178504944 (0.24420902132987976) + | > loss_disc_real_5: 0.23172232508659363 (0.22131019830703735) + | > loss_0: 2.3853938579559326 (2.4931282997131348) + | > loss_gen: 2.3404183387756348 (2.3090900182724) + | > loss_kl: 2.2633769512176514 (2.475741147994995) + | > loss_feat: 7.490272045135498 (7.1911091804504395) + | > loss_mel: 18.575820922851562 (18.398489952087402) + | > loss_duration: 1.6404742002487183 (1.6610611081123352) + | > loss_1: 32.310359954833984 (32.03549003601074) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034073591232299805 (-1.0848045349121094e-05) + | > avg_loss_disc: 2.4931282997131348 (-0.12899839878082275) + | > avg_loss_disc_real_0: 0.1520482897758484 (-0.00940348207950592) + | > avg_loss_disc_real_1: 0.2326144352555275 (-0.029269225895404816) + | > avg_loss_disc_real_2: 0.2512190714478493 (-0.00028302520513534546) + | > avg_loss_disc_real_3: 0.21634671837091446 (-0.04768051952123642) + | > avg_loss_disc_real_4: 0.24420902132987976 (+0.006746761500835419) + | > avg_loss_disc_real_5: 0.22131019830703735 (-0.023855015635490417) + | > avg_loss_0: 2.4931282997131348 (-0.12899839878082275) + | > avg_loss_gen: 2.3090900182724 (+0.021326899528503418) + | > avg_loss_kl: 2.475741147994995 (+0.2334609031677246) + | > avg_loss_feat: 7.1911091804504395 (+0.2411661148071289) + | > avg_loss_mel: 18.398489952087402 (-0.18938064575195312) + | > avg_loss_duration: 1.6610611081123352 (-0.003768324851989746) + | > avg_loss_1: 32.03549003601074 (+0.30280303955078125) + + + > EPOCH: 189/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:06:18)  + + --> TIME: 2025-04-19 02:06:21 -- STEP: 5/142 -- GLOBAL_STEP: 406125 + | > loss_disc: 2.446572780609131 (2.427949094772339) + | > loss_disc_real_0: 0.08413015305995941 (0.10785207152366638) + | > loss_disc_real_1: 0.1967955231666565 (0.19304182827472688) + | > loss_disc_real_2: 0.2385837435722351 (0.22875453531742096) + | > loss_disc_real_3: 0.25138798356056213 (0.23033366203308106) + | > loss_disc_real_4: 0.21212826669216156 (0.230913645029068) + | > loss_disc_real_5: 0.21655422449111938 (0.23343584835529327) + | > loss_0: 2.446572780609131 (2.427949094772339) + | > grad_norm_0: tensor(12.1386, device='cuda:0') (tensor(10.3720, device='cuda:0')) + | > loss_gen: 2.362818479537964 (2.3810266494750976) + | > loss_kl: 1.815374732017517 (1.8563350200653077) + | > loss_feat: 6.708810806274414 (7.365517330169678) + | > loss_mel: 16.49203109741211 (18.06814079284668) + | > loss_duration: 1.4483916759490967 (1.4460705041885376) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.827425003051758 (31.11708984375) + | > grad_norm_1: tensor(319.7068, device='cuda:0') (tensor(301.0290, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.3233 (0.32431812286376954) + | > loader_time: 0.0042 (0.004625368118286133) + + + --> TIME: 2025-04-19 02:06:29 -- STEP: 30/142 -- GLOBAL_STEP: 406150 + | > loss_disc: 2.3883302211761475 (2.426401909192403) + | > loss_disc_real_0: 0.15788163244724274 (0.11740592022736868) + | > loss_disc_real_1: 0.183878555893898 (0.20406246532996494) + | > loss_disc_real_2: 0.21074596047401428 (0.2264061709245046) + | > loss_disc_real_3: 0.2135046124458313 (0.22689712643623353) + | > loss_disc_real_4: 0.23960910737514496 (0.23098081250985464) + | > loss_disc_real_5: 0.2244742065668106 (0.22682479868332545) + | > loss_0: 2.3883302211761475 (2.426401909192403) + | > grad_norm_0: tensor(7.8648, device='cuda:0') (tensor(9.4917, device='cuda:0')) + | > loss_gen: 2.4245450496673584 (2.380286431312561) + | > loss_kl: 1.7706174850463867 (1.81967689593633) + | > loss_feat: 7.2758660316467285 (7.254145908355713) + | > loss_mel: 17.486299514770508 (17.898880068461104) + | > loss_duration: 1.4431055784225464 (1.445714815457662) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.400434494018555 (30.798704020182292) + | > grad_norm_1: tensor(199.1857, device='cuda:0') (tensor(239.6734, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.3484 (0.33559641043345134) + | > loader_time: 0.0057 (0.004979332288106282) + + + --> TIME: 2025-04-19 02:06:39 -- STEP: 55/142 -- GLOBAL_STEP: 406175 + | > loss_disc: 2.5025510787963867 (2.446695748242465) + | > loss_disc_real_0: 0.13756462931632996 (0.12463906339623711) + | > loss_disc_real_1: 0.25472092628479004 (0.20618458010933616) + | > loss_disc_real_2: 0.26137223839759827 (0.2252435039390217) + | > loss_disc_real_3: 0.27117788791656494 (0.22841610583392055) + | > loss_disc_real_4: 0.23978154361248016 (0.22829843163490296) + | > loss_disc_real_5: 0.2623814344406128 (0.2299540408632972) + | > loss_0: 2.5025510787963867 (2.446695748242465) + | > grad_norm_0: tensor(9.2904, device='cuda:0') (tensor(9.1721, device='cuda:0')) + | > loss_gen: 2.357564687728882 (2.3609737743030896) + | > loss_kl: 1.9274204969406128 (1.834129751812328) + | > loss_feat: 6.880964756011963 (7.160832387750799) + | > loss_mel: 17.218055725097656 (17.74172255776145) + | > loss_duration: 1.5057424306869507 (1.4527635769410565) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.889747619628906 (30.550422009554776) + | > grad_norm_1: tensor(230.1137, device='cuda:0') (tensor(209.1345, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.3767 (0.3478585893457586) + | > loader_time: 0.0061 (0.005291596325961025) + + + --> TIME: 2025-04-19 02:06:48 -- STEP: 80/142 -- GLOBAL_STEP: 406200 + | > loss_disc: 2.513120174407959 (2.450904220342637) + | > loss_disc_real_0: 0.09650633484125137 (0.12425094060599803) + | > loss_disc_real_1: 0.1760367453098297 (0.20549162551760677) + | > loss_disc_real_2: 0.21294595301151276 (0.2249185021966696) + | > loss_disc_real_3: 0.23952150344848633 (0.22893330901861192) + | > loss_disc_real_4: 0.23124860227108002 (0.22875644285231828) + | > loss_disc_real_5: 0.22708235681056976 (0.23097102008759976) + | > loss_0: 2.513120174407959 (2.450904220342637) + | > grad_norm_0: tensor(16.1458, device='cuda:0') (tensor(9.5540, device='cuda:0')) + | > loss_gen: 2.3550894260406494 (2.363615700602531) + | > loss_kl: 1.910905122756958 (1.8590798944234848) + | > loss_feat: 8.047807693481445 (7.173016923666) + | > loss_mel: 19.19391441345215 (17.811884975433347) + | > loss_duration: 1.4863746166229248 (1.4641225993633271) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.99409103393555 (30.67171995639801) + | > grad_norm_1: tensor(565.7841, device='cuda:0') (tensor(229.2821, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.4069 (0.36028409302234654) + | > loader_time: 0.0064 (0.005618590116500853) + + + --> TIME: 2025-04-19 02:06:59 -- STEP: 105/142 -- GLOBAL_STEP: 406225 + | > loss_disc: 2.4965405464172363 (2.4578309240795324) + | > loss_disc_real_0: 0.11166378110647202 (0.12630368006371312) + | > loss_disc_real_1: 0.20198199152946472 (0.20733610845747452) + | > loss_disc_real_2: 0.20037870109081268 (0.22447031438350679) + | > loss_disc_real_3: 0.22850532829761505 (0.22827250247909908) + | > loss_disc_real_4: 0.2386973649263382 (0.22849183196113224) + | > loss_disc_real_5: 0.23205602169036865 (0.23031127665724074) + | > loss_0: 2.4965405464172363 (2.4578309240795324) + | > grad_norm_0: tensor(13.3571, device='cuda:0') (tensor(11.2803, device='cuda:0')) + | > loss_gen: 2.532503128051758 (2.365110642569405) + | > loss_kl: 1.924597978591919 (1.8835655973071144) + | > loss_feat: 6.773637294769287 (7.113763014475505) + | > loss_mel: 16.552947998046875 (17.802316901797337) + | > loss_duration: 1.5211397409439087 (1.4754694529942103) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.304824829101562 (30.64022556486584) + | > grad_norm_1: tensor(484.5119, device='cuda:0') (tensor(247.6196, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.4143 (0.3709196681068057) + | > loader_time: 0.0068 (0.005870714641752696) + + + --> TIME: 2025-04-19 02:07:10 -- STEP: 130/142 -- GLOBAL_STEP: 406250 + | > loss_disc: 2.5134358406066895 (2.4622435716482314) + | > loss_disc_real_0: 0.09704223275184631 (0.12597167769876802) + | > loss_disc_real_1: 0.19562160968780518 (0.20852575084337824) + | > loss_disc_real_2: 0.22459451854228973 (0.22625201860299476) + | > loss_disc_real_3: 0.21082013845443726 (0.2281019790814473) + | > loss_disc_real_4: 0.25215044617652893 (0.2293991832779004) + | > loss_disc_real_5: 0.2249353528022766 (0.23008145334628913) + | > loss_0: 2.5134358406066895 (2.4622435716482314) + | > grad_norm_0: tensor(13.4175, device='cuda:0') (tensor(12.3558, device='cuda:0')) + | > loss_gen: 2.4131534099578857 (2.3704235407022325) + | > loss_kl: 1.897113561630249 (1.88763426725681) + | > loss_feat: 5.531737804412842 (7.076784874842717) + | > loss_mel: 15.146915435791016 (17.744701781639687) + | > loss_duration: 1.489976167678833 (1.4819580738361064) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.478897094726562 (30.561502544696513) + | > grad_norm_1: tensor(421.5272, device='cuda:0') (tensor(262.1703, device='cuda:0')) + | > current_lr_0: 0.00019533008866921223 + | > current_lr_1: 0.00019533008866921223 + | > step_time: 0.4497 (0.3826184419485239) + | > loader_time: 0.0075 (0.006127353814932015) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5194191932678223 (2.5194191932678223) + | > loss_disc_real_0: 0.13222408294677734 (0.13222408294677734) + | > loss_disc_real_1: 0.1705174297094345 (0.1705174297094345) + | > loss_disc_real_2: 0.21472790837287903 (0.21472790837287903) + | > loss_disc_real_3: 0.29732266068458557 (0.29732266068458557) + | > loss_disc_real_4: 0.19897110760211945 (0.19897110760211945) + | > loss_disc_real_5: 0.205439493060112 (0.205439493060112) + | > loss_0: 2.5194191932678223 (2.5194191932678223) + | > loss_gen: 2.0744800567626953 (2.0744800567626953) + | > loss_kl: 2.277235507965088 (2.277235507965088) + | > loss_feat: 6.544018745422363 (6.544018745422363) + | > loss_mel: 17.686479568481445 (17.686479568481445) + | > loss_duration: 1.6904114484786987 (1.6904114484786987) + | > loss_1: 30.272624969482422 (30.272624969482422) + + --> STEP: 1 + | > loss_disc: 2.5673437118530273 (2.5673437118530273) + | > loss_disc_real_0: 0.15909937024116516 (0.15909937024116516) + | > loss_disc_real_1: 0.21848513185977936 (0.21848513185977936) + | > loss_disc_real_2: 0.229145348072052 (0.229145348072052) + | > loss_disc_real_3: 0.27077966928482056 (0.27077966928482056) + | > loss_disc_real_4: 0.22067216038703918 (0.22067216038703918) + | > loss_disc_real_5: 0.2614714801311493 (0.2614714801311493) + | > loss_0: 2.5673437118530273 (2.5673437118530273) + | > loss_gen: 2.1972274780273438 (2.1972274780273438) + | > loss_kl: 2.5520660877227783 (2.5520660877227783) + | > loss_feat: 5.716858386993408 (5.716858386993408) + | > loss_mel: 15.864202499389648 (15.864202499389648) + | > loss_duration: 1.7056759595870972 (1.7056759595870972) + | > loss_1: 28.036029815673828 (28.036029815673828) + + --> STEP: 2 + | > loss_disc: 2.6544277667999268 (2.610885739326477) + | > loss_disc_real_0: 0.17168587446212769 (0.16539262235164642) + | > loss_disc_real_1: 0.21020828187465668 (0.21434670686721802) + | > loss_disc_real_2: 0.2496693879365921 (0.23940736800432205) + | > loss_disc_real_3: 0.2479190081357956 (0.2593493387103081) + | > loss_disc_real_4: 0.21551823616027832 (0.21809519827365875) + | > loss_disc_real_5: 0.2575047016143799 (0.2594880908727646) + | > loss_0: 2.6544277667999268 (2.610885739326477) + | > loss_gen: 2.1325557231903076 (2.1648916006088257) + | > loss_kl: 2.1883833408355713 (2.370224714279175) + | > loss_feat: 7.6176533699035645 (6.667255878448486) + | > loss_mel: 18.542694091796875 (17.20344829559326) + | > loss_duration: 1.659250259399414 (1.6824631094932556) + | > loss_1: 32.14053726196289 (30.08828353881836) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003882765769958496 (+0.0004754066467285156) + | > avg_loss_disc: 2.610885739326477 (+0.11775743961334229) + | > avg_loss_disc_real_0: 0.16539262235164642 (+0.013344332575798035) + | > avg_loss_disc_real_1: 0.21434670686721802 (-0.01826772838830948) + | > avg_loss_disc_real_2: 0.23940736800432205 (-0.011811703443527222) + | > avg_loss_disc_real_3: 0.2593493387103081 (+0.043002620339393616) + | > avg_loss_disc_real_4: 0.21809519827365875 (-0.02611382305622101) + | > avg_loss_disc_real_5: 0.2594880908727646 (+0.038177892565727234) + | > avg_loss_0: 2.610885739326477 (+0.11775743961334229) + | > avg_loss_gen: 2.1648916006088257 (-0.14419841766357422) + | > avg_loss_kl: 2.370224714279175 (-0.10551643371582031) + | > avg_loss_feat: 6.667255878448486 (-0.5238533020019531) + | > avg_loss_mel: 17.20344829559326 (-1.1950416564941406) + | > avg_loss_duration: 1.6824631094932556 (+0.02140200138092041) + | > avg_loss_1: 30.08828353881836 (-1.9472064971923828) + + + > EPOCH: 190/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:07:17)  + + --> TIME: 2025-04-19 02:07:22 -- STEP: 13/142 -- GLOBAL_STEP: 406275 + | > loss_disc: 2.509660243988037 (2.5158548171703634) + | > loss_disc_real_0: 0.12660180032253265 (0.14561389443966058) + | > loss_disc_real_1: 0.19958990812301636 (0.209639494235699) + | > loss_disc_real_2: 0.23429179191589355 (0.2306494529430683) + | > loss_disc_real_3: 0.21635515987873077 (0.22405891005809492) + | > loss_disc_real_4: 0.23740875720977783 (0.232688946219591) + | > loss_disc_real_5: 0.21445661783218384 (0.2271042913198471) + | > loss_0: 2.509660243988037 (2.5158548171703634) + | > grad_norm_0: tensor(17.5967, device='cuda:0') (tensor(14.0354, device='cuda:0')) + | > loss_gen: 2.2747929096221924 (2.298531458928035) + | > loss_kl: 1.7996751070022583 (1.836784188564007) + | > loss_feat: 6.9070258140563965 (6.978989381056565) + | > loss_mel: 17.983409881591797 (18.081068479097805) + | > loss_duration: 1.4250431060791016 (1.428005080956679) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.38994789123535 (30.623378313504734) + | > grad_norm_1: tensor(56.2502, device='cuda:0') (tensor(273.0847, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.3294 (0.3301814886239859) + | > loader_time: 0.0049 (0.004601276837862455) + + + --> TIME: 2025-04-19 02:07:31 -- STEP: 38/142 -- GLOBAL_STEP: 406300 + | > loss_disc: 2.4874539375305176 (2.468266951410394) + | > loss_disc_real_0: 0.15989021956920624 (0.1294674745907909) + | > loss_disc_real_1: 0.23542863130569458 (0.20658307287253833) + | > loss_disc_real_2: 0.2107468694448471 (0.2261409077205156) + | > loss_disc_real_3: 0.23469915986061096 (0.22669274830504468) + | > loss_disc_real_4: 0.23155611753463745 (0.23018565852391093) + | > loss_disc_real_5: 0.21694566309452057 (0.22960027856262108) + | > loss_0: 2.4874539375305176 (2.468266951410394) + | > grad_norm_0: tensor(4.4494, device='cuda:0') (tensor(10.6082, device='cuda:0')) + | > loss_gen: 2.2662158012390137 (2.325453055532355) + | > loss_kl: 2.0135903358459473 (1.8477222166563336) + | > loss_feat: 6.523388385772705 (6.994004186831023) + | > loss_mel: 18.44588279724121 (17.961876041010807) + | > loss_duration: 1.453899621963501 (1.4434692796907929) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.70297622680664 (30.57252477344714) + | > grad_norm_1: tensor(127.6431, device='cuda:0') (tensor(260.5085, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.3526 (0.3469274985162835) + | > loader_time: 0.0059 (0.0051335723776566354) + + + --> TIME: 2025-04-19 02:07:40 -- STEP: 63/142 -- GLOBAL_STEP: 406325 + | > loss_disc: 2.4638423919677734 (2.4647881833333813) + | > loss_disc_real_0: 0.1406058669090271 (0.12949061334606202) + | > loss_disc_real_1: 0.21060968935489655 (0.20655748815763564) + | > loss_disc_real_2: 0.24124787747859955 (0.2265522163065653) + | > loss_disc_real_3: 0.24636881053447723 (0.22647382129752447) + | > loss_disc_real_4: 0.2359522134065628 (0.2304199536641439) + | > loss_disc_real_5: 0.18867190182209015 (0.23103445650093138) + | > loss_0: 2.4638423919677734 (2.4647881833333813) + | > grad_norm_0: tensor(12.3233, device='cuda:0') (tensor(10.1526, device='cuda:0')) + | > loss_gen: 2.249485969543457 (2.3349282741546626) + | > loss_kl: 1.9202473163604736 (1.865265236960517) + | > loss_feat: 6.562305450439453 (7.033962007552859) + | > loss_mel: 17.74262809753418 (17.872773140195807) + | > loss_duration: 1.5200419425964355 (1.4581121187361459) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.994709014892578 (30.565040739755784) + | > grad_norm_1: tensor(315.0900, device='cuda:0') (tensor(245.4227, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.3915 (0.3572970004308792) + | > loader_time: 0.0062 (0.005493421403188554) + + + --> TIME: 2025-04-19 02:07:50 -- STEP: 88/142 -- GLOBAL_STEP: 406350 + | > loss_disc: 2.5291335582733154 (2.474281292070042) + | > loss_disc_real_0: 0.12090916931629181 (0.12981006960299882) + | > loss_disc_real_1: 0.20148922502994537 (0.20679440738802607) + | > loss_disc_real_2: 0.2219914197921753 (0.22704173929312013) + | > loss_disc_real_3: 0.25870802998542786 (0.22749473882669752) + | > loss_disc_real_4: 0.22041739523410797 (0.23024063388055022) + | > loss_disc_real_5: 0.25273698568344116 (0.23107655820521442) + | > loss_0: 2.5291335582733154 (2.474281292070042) + | > grad_norm_0: tensor(10.8250, device='cuda:0') (tensor(10.7697, device='cuda:0')) + | > loss_gen: 2.305845260620117 (2.3299793384291894) + | > loss_kl: 2.004854202270508 (1.8804545145143161) + | > loss_feat: 6.836420059204102 (7.025416748090223) + | > loss_mel: 18.937538146972656 (17.886983123692595) + | > loss_duration: 1.5450255870819092 (1.4704093418338076) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.629682540893555 (30.593242905356668) + | > grad_norm_1: tensor(137.1077, device='cuda:0') (tensor(255.0672, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.3992 (0.3673237318342382) + | > loader_time: 0.0068 (0.0057458416982130575) + + + --> TIME: 2025-04-19 02:08:01 -- STEP: 113/142 -- GLOBAL_STEP: 406375 + | > loss_disc: 2.4673562049865723 (2.4762256504160103) + | > loss_disc_real_0: 0.16699980199337006 (0.13055203175913965) + | > loss_disc_real_1: 0.2115297168493271 (0.20762834295762322) + | > loss_disc_real_2: 0.2675861120223999 (0.22692612552009853) + | > loss_disc_real_3: 0.227059468626976 (0.22852020242572885) + | > loss_disc_real_4: 0.22732262313365936 (0.2306802093455222) + | > loss_disc_real_5: 0.21765483915805817 (0.23132029755980568) + | > loss_0: 2.4673562049865723 (2.4762256504160103) + | > grad_norm_0: tensor(10.8491, device='cuda:0') (tensor(10.8630, device='cuda:0')) + | > loss_gen: 2.2527313232421875 (2.3239217374177086) + | > loss_kl: 1.7606803178787231 (1.8882936576826383) + | > loss_feat: 5.943543910980225 (6.962080482888011) + | > loss_mel: 16.240564346313477 (17.776419158530445) + | > loss_duration: 1.5207016468048096 (1.477684707768195) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.71822166442871 (30.428399617693067) + | > grad_norm_1: tensor(333.7790, device='cuda:0') (tensor(254.8455, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.4281 (0.37872855009230894) + | > loader_time: 0.0071 (0.0060369187751702505) + + + --> TIME: 2025-04-19 02:08:12 -- STEP: 138/142 -- GLOBAL_STEP: 406400 + | > loss_disc: 2.501206636428833 (2.4769953696624083) + | > loss_disc_real_0: 0.1235383078455925 (0.1304470488558644) + | > loss_disc_real_1: 0.20203569531440735 (0.2073403784762258) + | > loss_disc_real_2: 0.27086523175239563 (0.22753564657076544) + | > loss_disc_real_3: 0.23208002746105194 (0.22849121301070507) + | > loss_disc_real_4: 0.26215603947639465 (0.230706209073896) + | > loss_disc_real_5: 0.27680930495262146 (0.23162387818961905) + | > loss_0: 2.501206636428833 (2.4769953696624083) + | > grad_norm_0: tensor(16.5573, device='cuda:0') (tensor(10.7863, device='cuda:0')) + | > loss_gen: 2.447000503540039 (2.3248673884764943) + | > loss_kl: 2.0657224655151367 (1.8951713779698247) + | > loss_feat: 7.263949871063232 (6.930973941001339) + | > loss_mel: 20.08193588256836 (17.734992545583975) + | > loss_duration: 1.5439834594726562 (1.4838379362355099) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.402591705322266 (30.369843054508816) + | > grad_norm_1: tensor(411.0493, device='cuda:0') (tensor(252.8180, device='cuda:0')) + | > current_lr_0: 0.00019530567240812858 + | > current_lr_1: 0.00019530567240812858 + | > step_time: 0.4501 (0.38988909859588183) + | > loader_time: 0.0077 (0.00630740497423255) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5422768592834473 (2.5422768592834473) + | > loss_disc_real_0: 0.14352884888648987 (0.14352884888648987) + | > loss_disc_real_1: 0.22146962583065033 (0.22146962583065033) + | > loss_disc_real_2: 0.22883586585521698 (0.22883586585521698) + | > loss_disc_real_3: 0.24736420810222626 (0.24736420810222626) + | > loss_disc_real_4: 0.29449892044067383 (0.29449892044067383) + | > loss_disc_real_5: 0.3186362385749817 (0.3186362385749817) + | > loss_0: 2.5422768592834473 (2.5422768592834473) + | > loss_gen: 2.372135877609253 (2.372135877609253) + | > loss_kl: 2.214707851409912 (2.214707851409912) + | > loss_feat: 7.023036956787109 (7.023036956787109) + | > loss_mel: 20.055105209350586 (20.055105209350586) + | > loss_duration: 1.693829894065857 (1.693829894065857) + | > loss_1: 33.35881423950195 (33.35881423950195) + + --> STEP: 1 + | > loss_disc: 2.4510600566864014 (2.4510600566864014) + | > loss_disc_real_0: 0.14407090842723846 (0.14407090842723846) + | > loss_disc_real_1: 0.24807068705558777 (0.24807068705558777) + | > loss_disc_real_2: 0.2454967200756073 (0.2454967200756073) + | > loss_disc_real_3: 0.23607343435287476 (0.23607343435287476) + | > loss_disc_real_4: 0.2091827690601349 (0.2091827690601349) + | > loss_disc_real_5: 0.18682578206062317 (0.18682578206062317) + | > loss_0: 2.4510600566864014 (2.4510600566864014) + | > loss_gen: 2.2605276107788086 (2.2605276107788086) + | > loss_kl: 2.6749942302703857 (2.6749942302703857) + | > loss_feat: 6.32509183883667 (6.32509183883667) + | > loss_mel: 17.442079544067383 (17.442079544067383) + | > loss_duration: 1.7218780517578125 (1.7218780517578125) + | > loss_1: 30.424571990966797 (30.424571990966797) + + --> STEP: 2 + | > loss_disc: 2.5190751552581787 (2.48506760597229) + | > loss_disc_real_0: 0.13877049088478088 (0.14142069965600967) + | > loss_disc_real_1: 0.2725372910499573 (0.2603039890527725) + | > loss_disc_real_2: 0.21877506375312805 (0.23213589191436768) + | > loss_disc_real_3: 0.22773373126983643 (0.2319035828113556) + | > loss_disc_real_4: 0.1956174224615097 (0.2024000957608223) + | > loss_disc_real_5: 0.212356835603714 (0.19959130883216858) + | > loss_0: 2.5190751552581787 (2.48506760597229) + | > loss_gen: 2.1384620666503906 (2.1994948387145996) + | > loss_kl: 2.292595863342285 (2.4837950468063354) + | > loss_feat: 6.33362340927124 (6.329357624053955) + | > loss_mel: 18.163307189941406 (17.802693367004395) + | > loss_duration: 1.651823878288269 (1.6868509650230408) + | > loss_1: 30.579811096191406 (30.5021915435791) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00324094295501709 (-0.0006418228149414062) + | > avg_loss_disc: 2.48506760597229 (-0.125818133354187) + | > avg_loss_disc_real_0: 0.14142069965600967 (-0.02397192269563675) + | > avg_loss_disc_real_1: 0.2603039890527725 (+0.045957282185554504) + | > avg_loss_disc_real_2: 0.23213589191436768 (-0.007271476089954376) + | > avg_loss_disc_real_3: 0.2319035828113556 (-0.027445755898952484) + | > avg_loss_disc_real_4: 0.2024000957608223 (-0.015695102512836456) + | > avg_loss_disc_real_5: 0.19959130883216858 (-0.05989678204059601) + | > avg_loss_0: 2.48506760597229 (-0.125818133354187) + | > avg_loss_gen: 2.1994948387145996 (+0.034603238105773926) + | > avg_loss_kl: 2.4837950468063354 (+0.11357033252716064) + | > avg_loss_feat: 6.329357624053955 (-0.33789825439453125) + | > avg_loss_mel: 17.802693367004395 (+0.5992450714111328) + | > avg_loss_duration: 1.6868509650230408 (+0.004387855529785156) + | > avg_loss_1: 30.5021915435791 (+0.4139080047607422) + + + > EPOCH: 191/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:08:16)  + + --> TIME: 2025-04-19 02:08:24 -- STEP: 21/142 -- GLOBAL_STEP: 406425 + | > loss_disc: 2.560472011566162 (2.4590704214005243) + | > loss_disc_real_0: 0.14307287335395813 (0.12659916991279238) + | > loss_disc_real_1: 0.217862069606781 (0.20770629885650815) + | > loss_disc_real_2: 0.2063225358724594 (0.23224692472389766) + | > loss_disc_real_3: 0.20540907979011536 (0.22493543795176915) + | > loss_disc_real_4: 0.2242535501718521 (0.23023390628042675) + | > loss_disc_real_5: 0.23418809473514557 (0.2313864209822246) + | > loss_0: 2.560472011566162 (2.4590704214005243) + | > grad_norm_0: tensor(14.3049, device='cuda:0') (tensor(9.1665, device='cuda:0')) + | > loss_gen: 2.102696418762207 (2.3635881741841636) + | > loss_kl: 1.924217939376831 (1.8260786760421026) + | > loss_feat: 6.009005546569824 (7.039566607702346) + | > loss_mel: 15.63494873046875 (17.83120318821498) + | > loss_duration: 1.433483362197876 (1.4370131265549433) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.104351043701172 (30.497449784051803) + | > grad_norm_1: tensor(254.0622, device='cuda:0') (tensor(181.7752, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.3434 (0.3350989137377058) + | > loader_time: 0.0052 (0.004651478358677456) + + + --> TIME: 2025-04-19 02:08:33 -- STEP: 46/142 -- GLOBAL_STEP: 406450 + | > loss_disc: 2.485654830932617 (2.441512263339499) + | > loss_disc_real_0: 0.1499594748020172 (0.12121393256213354) + | > loss_disc_real_1: 0.2050779014825821 (0.20722849537496982) + | > loss_disc_real_2: 0.2575587332248688 (0.22675508876209674) + | > loss_disc_real_3: 0.23175354301929474 (0.2257935602379882) + | > loss_disc_real_4: 0.23521310091018677 (0.22756112593671549) + | > loss_disc_real_5: 0.21500010788440704 (0.22922506021416705) + | > loss_0: 2.485654830932617 (2.441512263339499) + | > grad_norm_0: tensor(13.5745, device='cuda:0') (tensor(11.0001, device='cuda:0')) + | > loss_gen: 2.4173011779785156 (2.3738076324048247) + | > loss_kl: 1.8702195882797241 (1.8374716380368108) + | > loss_feat: 7.163312911987305 (7.0927678709444795) + | > loss_mel: 17.303836822509766 (17.863313550534457) + | > loss_duration: 1.4386171102523804 (1.4476081910340681) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.193286895751953 (30.614969046219535) + | > grad_norm_1: tensor(381.1996, device='cuda:0') (tensor(261.0257, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.3653 (0.34586295355921204) + | > loader_time: 0.006 (0.005096725795579993) + + + --> TIME: 2025-04-19 02:08:42 -- STEP: 71/142 -- GLOBAL_STEP: 406475 + | > loss_disc: 2.434650421142578 (2.4542370144750048) + | > loss_disc_real_0: 0.11465751379728317 (0.1237522649093413) + | > loss_disc_real_1: 0.21786171197891235 (0.20902515276217123) + | > loss_disc_real_2: 0.2691197693347931 (0.2270807684727118) + | > loss_disc_real_3: 0.2075345516204834 (0.22798720421925397) + | > loss_disc_real_4: 0.24992991983890533 (0.2283010808095126) + | > loss_disc_real_5: 0.20673581957817078 (0.23025444569722028) + | > loss_0: 2.434650421142578 (2.4542370144750048) + | > grad_norm_0: tensor(26.8511, device='cuda:0') (tensor(13.1293, device='cuda:0')) + | > loss_gen: 2.487722635269165 (2.377409626060808) + | > loss_kl: 1.8608851432800293 (1.857450545673639) + | > loss_feat: 7.612560272216797 (7.110088523005096) + | > loss_mel: 18.371997833251953 (17.849794683322102) + | > loss_duration: 1.5033810138702393 (1.460423128705629) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.8365478515625 (30.655166652840627) + | > grad_norm_1: tensor(510.9325, device='cuda:0') (tensor(290.8481, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.3863 (0.35750057999516877) + | > loader_time: 0.0064 (0.005450890097819584) + + + --> TIME: 2025-04-19 02:08:52 -- STEP: 96/142 -- GLOBAL_STEP: 406500 + | > loss_disc: 2.5212056636810303 (2.455231748521329) + | > loss_disc_real_0: 0.10874681919813156 (0.12206667033024132) + | > loss_disc_real_1: 0.19600579142570496 (0.21021350162724653) + | > loss_disc_real_2: 0.19973723590373993 (0.22643391586219272) + | > loss_disc_real_3: 0.1979777067899704 (0.22790039516985416) + | > loss_disc_real_4: 0.23852676153182983 (0.2284408725487689) + | > loss_disc_real_5: 0.2441060096025467 (0.2309462440510591) + | > loss_0: 2.5212056636810303 (2.455231748521329) + | > grad_norm_0: tensor(3.1848, device='cuda:0') (tensor(12.6628, device='cuda:0')) + | > loss_gen: 2.1918962001800537 (2.3845875461896253) + | > loss_kl: 1.8929791450500488 (1.8646471897761028) + | > loss_feat: 7.031299114227295 (7.068678696950276) + | > loss_mel: 17.409320831298828 (17.85049813985824) + | > loss_duration: 1.4911425113677979 (1.4726361818611622) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.016637802124023 (30.64104773600896) + | > grad_norm_1: tensor(103.2048, device='cuda:0') (tensor(285.6643, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.4072 (0.3679319644967714) + | > loader_time: 0.0065 (0.005737364292144775) + + + --> TIME: 2025-04-19 02:09:03 -- STEP: 121/142 -- GLOBAL_STEP: 406525 + | > loss_disc: 2.5730812549591064 (2.462224026356847) + | > loss_disc_real_0: 0.2414393275976181 (0.12486710867359618) + | > loss_disc_real_1: 0.22462859749794006 (0.208742686416492) + | > loss_disc_real_2: 0.2505747675895691 (0.22709360447796909) + | > loss_disc_real_3: 0.2659027874469757 (0.228473387597021) + | > loss_disc_real_4: 0.23694242537021637 (0.22869327196404954) + | > loss_disc_real_5: 0.2529606521129608 (0.23109964841653494) + | > loss_0: 2.5730812549591064 (2.462224026356847) + | > grad_norm_0: tensor(7.8298, device='cuda:0') (tensor(11.9236, device='cuda:0')) + | > loss_gen: 2.3658201694488525 (2.3693612488833335) + | > loss_kl: 2.0155863761901855 (1.8724841431152721) + | > loss_feat: 5.958874702453613 (7.003594158109554) + | > loss_mel: 17.01338005065918 (17.77804659220797) + | > loss_duration: 1.5417423248291016 (1.480042417187336) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.895402908325195 (30.503528531917855) + | > grad_norm_1: tensor(116.9092, device='cuda:0') (tensor(265.7086, device='cuda:0')) + | > current_lr_0: 0.00019528125919907757 + | > current_lr_1: 0.00019528125919907757 + | > step_time: 0.4426 (0.3788528718239019) + | > loader_time: 0.0075 (0.005988288516840659) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.535374641418457 (2.535374641418457) + | > loss_disc_real_0: 0.19880908727645874 (0.19880908727645874) + | > loss_disc_real_1: 0.2331397533416748 (0.2331397533416748) + | > loss_disc_real_2: 0.2514917254447937 (0.2514917254447937) + | > loss_disc_real_3: 0.2634391486644745 (0.2634391486644745) + | > loss_disc_real_4: 0.24596324563026428 (0.24596324563026428) + | > loss_disc_real_5: 0.26525822281837463 (0.26525822281837463) + | > loss_0: 2.535374641418457 (2.535374641418457) + | > loss_gen: 2.626539707183838 (2.626539707183838) + | > loss_kl: 2.265509605407715 (2.265509605407715) + | > loss_feat: 7.013993740081787 (7.013993740081787) + | > loss_mel: 19.578502655029297 (19.578502655029297) + | > loss_duration: 1.686780571937561 (1.686780571937561) + | > loss_1: 33.17132568359375 (33.17132568359375) + + --> STEP: 1 + | > loss_disc: 2.5157663822174072 (2.5157663822174072) + | > loss_disc_real_0: 0.19289198517799377 (0.19289198517799377) + | > loss_disc_real_1: 0.31180059909820557 (0.31180059909820557) + | > loss_disc_real_2: 0.2546364963054657 (0.2546364963054657) + | > loss_disc_real_3: 0.24746546149253845 (0.24746546149253845) + | > loss_disc_real_4: 0.26673033833503723 (0.26673033833503723) + | > loss_disc_real_5: 0.24094831943511963 (0.24094831943511963) + | > loss_0: 2.5157663822174072 (2.5157663822174072) + | > loss_gen: 2.764613389968872 (2.764613389968872) + | > loss_kl: 2.727971076965332 (2.727971076965332) + | > loss_feat: 6.620084762573242 (6.620084762573242) + | > loss_mel: 17.76163101196289 (17.76163101196289) + | > loss_duration: 1.6992924213409424 (1.6992924213409424) + | > loss_1: 31.57359504699707 (31.57359504699707) + + --> STEP: 2 + | > loss_disc: 2.6000008583068848 (2.557883620262146) + | > loss_disc_real_0: 0.20768894255161285 (0.20029046386480331) + | > loss_disc_real_1: 0.264488160610199 (0.28814437985420227) + | > loss_disc_real_2: 0.2398051917552948 (0.24722084403038025) + | > loss_disc_real_3: 0.23174454271793365 (0.23960500210523605) + | > loss_disc_real_4: 0.21266886591911316 (0.2396996021270752) + | > loss_disc_real_5: 0.29548266530036926 (0.26821549236774445) + | > loss_0: 2.6000008583068848 (2.557883620262146) + | > loss_gen: 2.464141845703125 (2.6143776178359985) + | > loss_kl: 2.3361034393310547 (2.5320372581481934) + | > loss_feat: 6.965525150299072 (6.792804956436157) + | > loss_mel: 18.476882934570312 (18.1192569732666) + | > loss_duration: 1.6584523916244507 (1.6788724064826965) + | > loss_1: 31.901105880737305 (31.737350463867188) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003364086151123047 (+0.00012314319610595703) + | > avg_loss_disc: 2.557883620262146 (+0.07281601428985596) + | > avg_loss_disc_real_0: 0.20029046386480331 (+0.05886976420879364) + | > avg_loss_disc_real_1: 0.28814437985420227 (+0.02784039080142975) + | > avg_loss_disc_real_2: 0.24722084403038025 (+0.015084952116012573) + | > avg_loss_disc_real_3: 0.23960500210523605 (+0.007701419293880463) + | > avg_loss_disc_real_4: 0.2396996021270752 (+0.0372995063662529) + | > avg_loss_disc_real_5: 0.26821549236774445 (+0.06862418353557587) + | > avg_loss_0: 2.557883620262146 (+0.07281601428985596) + | > avg_loss_gen: 2.6143776178359985 (+0.4148827791213989) + | > avg_loss_kl: 2.5320372581481934 (+0.04824221134185791) + | > avg_loss_feat: 6.792804956436157 (+0.46344733238220215) + | > avg_loss_mel: 18.1192569732666 (+0.31656360626220703) + | > avg_loss_duration: 1.6788724064826965 (-0.007978558540344238) + | > avg_loss_1: 31.737350463867188 (+1.235158920288086) + + + > EPOCH: 192/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:09:14)  + + --> TIME: 2025-04-19 02:09:16 -- STEP: 4/142 -- GLOBAL_STEP: 406550 + | > loss_disc: 2.385312557220459 (2.5224855542182922) + | > loss_disc_real_0: 0.0965660810470581 (0.11231199279427528) + | > loss_disc_real_1: 0.18502207100391388 (0.20260611176490784) + | > loss_disc_real_2: 0.1954130083322525 (0.2429719753563404) + | > loss_disc_real_3: 0.20515534281730652 (0.23882488161325455) + | > loss_disc_real_4: 0.23989856243133545 (0.2535257004201412) + | > loss_disc_real_5: 0.2709120810031891 (0.26877228915691376) + | > loss_0: 2.385312557220459 (2.5224855542182922) + | > grad_norm_0: tensor(9.2306, device='cuda:0') (tensor(11.5027, device='cuda:0')) + | > loss_gen: 2.5448334217071533 (2.2980816066265106) + | > loss_kl: 2.068760633468628 (1.9089077711105347) + | > loss_feat: 7.365223407745361 (6.709416151046753) + | > loss_mel: 17.8569393157959 (17.178629159927368) + | > loss_duration: 1.470810055732727 (1.4483403265476227) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.306568145751953 (29.54337453842163) + | > grad_norm_1: tensor(88.0996, device='cuda:0') (tensor(183.3030, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.3191 (0.3232901096343994) + | > loader_time: 0.0049 (0.00426250696182251) + + + --> TIME: 2025-04-19 02:09:25 -- STEP: 29/142 -- GLOBAL_STEP: 406575 + | > loss_disc: 2.4695658683776855 (2.4808608499066582) + | > loss_disc_real_0: 0.1316455453634262 (0.11936126700763044) + | > loss_disc_real_1: 0.20570434629917145 (0.20839131186748372) + | > loss_disc_real_2: 0.25405940413475037 (0.2292391524232667) + | > loss_disc_real_3: 0.24142159521579742 (0.22947047496664114) + | > loss_disc_real_4: 0.24064064025878906 (0.2331438717143289) + | > loss_disc_real_5: 0.21743252873420715 (0.23838805278827405) + | > loss_0: 2.4695658683776855 (2.4808608499066582) + | > grad_norm_0: tensor(7.3762, device='cuda:0') (tensor(7.5995, device='cuda:0')) + | > loss_gen: 2.3435699939727783 (2.33561856582247) + | > loss_kl: 1.8313279151916504 (1.865277783624057) + | > loss_feat: 7.284913063049316 (6.994917606485301) + | > loss_mel: 18.015777587890625 (17.555795176275847) + | > loss_duration: 1.4360527992248535 (1.447950161736587) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.91164207458496 (30.199559441928205) + | > grad_norm_1: tensor(232.0376, device='cuda:0') (tensor(164.2391, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.3466 (0.3352481907811658) + | > loader_time: 0.0052 (0.004652409717954437) + + + --> TIME: 2025-04-19 02:09:34 -- STEP: 54/142 -- GLOBAL_STEP: 406600 + | > loss_disc: 2.4748497009277344 (2.47686759630839) + | > loss_disc_real_0: 0.14024567604064941 (0.12007796281465778) + | > loss_disc_real_1: 0.2351701259613037 (0.2099146889867606) + | > loss_disc_real_2: 0.22725887596607208 (0.22830735754083703) + | > loss_disc_real_3: 0.2512193024158478 (0.22951412118143505) + | > loss_disc_real_4: 0.2385597676038742 (0.23186007666367073) + | > loss_disc_real_5: 0.24599051475524902 (0.23790451359969597) + | > loss_0: 2.4748497009277344 (2.47686759630839) + | > grad_norm_0: tensor(13.4581, device='cuda:0') (tensor(7.4702, device='cuda:0')) + | > loss_gen: 2.4129507541656494 (2.3403449257214857) + | > loss_kl: 1.8838481903076172 (1.8591525267671656) + | > loss_feat: 7.305239677429199 (7.024675987384938) + | > loss_mel: 18.138944625854492 (17.666670887558546) + | > loss_duration: 1.488653302192688 (1.4541514714558923) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.229637145996094 (30.34499595783375) + | > grad_norm_1: tensor(419.2484, device='cuda:0') (tensor(150.2884, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.3709 (0.34757164231053106) + | > loader_time: 0.0056 (0.005032936731974284) + + + --> TIME: 2025-04-19 02:09:44 -- STEP: 79/142 -- GLOBAL_STEP: 406625 + | > loss_disc: 2.727480411529541 (2.483652664136282) + | > loss_disc_real_0: 0.20188835263252258 (0.12295533377158491) + | > loss_disc_real_1: 0.2250855714082718 (0.20957352164425427) + | > loss_disc_real_2: 0.28598231077194214 (0.2285112676363957) + | > loss_disc_real_3: 0.26886916160583496 (0.22969183080558536) + | > loss_disc_real_4: 0.24876642227172852 (0.23162082724178892) + | > loss_disc_real_5: 0.23897327482700348 (0.2359372903651829) + | > loss_0: 2.727480411529541 (2.483652664136282) + | > grad_norm_0: tensor(5.5258, device='cuda:0') (tensor(7.1696, device='cuda:0')) + | > loss_gen: 2.125666856765747 (2.3261752143690853) + | > loss_kl: 1.7943533658981323 (1.8589186879652966) + | > loss_feat: 5.10068941116333 (6.936371911930133) + | > loss_mel: 15.45811653137207 (17.6338655737382) + | > loss_duration: 1.5286595821380615 (1.4658524039425427) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.00748634338379 (30.22118384928643) + | > grad_norm_1: tensor(132.9000, device='cuda:0') (tensor(140.4869, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.3856 (0.3598859883561919) + | > loader_time: 0.0058 (0.005348223674146437) + + + --> TIME: 2025-04-19 02:09:54 -- STEP: 104/142 -- GLOBAL_STEP: 406650 + | > loss_disc: 2.4367191791534424 (2.4885464975467095) + | > loss_disc_real_0: 0.20967738330364227 (0.12599294618345222) + | > loss_disc_real_1: 0.1813681721687317 (0.21047698520123959) + | > loss_disc_real_2: 0.2031479924917221 (0.2275355469722014) + | > loss_disc_real_3: 0.1941402107477188 (0.22985157642800075) + | > loss_disc_real_4: 0.20479756593704224 (0.23279862464047396) + | > loss_disc_real_5: 0.2277594655752182 (0.23707312245208484) + | > loss_0: 2.4367191791534424 (2.4885464975467095) + | > grad_norm_0: tensor(27.3839, device='cuda:0') (tensor(9.3726, device='cuda:0')) + | > loss_gen: 2.3317487239837646 (2.3246431660193667) + | > loss_kl: 1.9691518545150757 (1.8654513657093048) + | > loss_feat: 6.821379661560059 (6.913235691877512) + | > loss_mel: 17.91876220703125 (17.681680477582496) + | > loss_duration: 1.524052619934082 (1.4752884381092513) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.565093994140625 (30.26029922411992) + | > grad_norm_1: tensor(257.6716, device='cuda:0') (tensor(189.5073, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.4082 (0.3705839973229628) + | > loader_time: 0.0066 (0.0056413710117340105) + + + --> TIME: 2025-04-19 02:10:05 -- STEP: 129/142 -- GLOBAL_STEP: 406675 + | > loss_disc: 2.4471893310546875 (2.482911562734796) + | > loss_disc_real_0: 0.08429504185914993 (0.12664191806039155) + | > loss_disc_real_1: 0.21917743980884552 (0.20974991487902264) + | > loss_disc_real_2: 0.20641887187957764 (0.22675808605759643) + | > loss_disc_real_3: 0.20400582253932953 (0.2287972924090171) + | > loss_disc_real_4: 0.21224555373191833 (0.23113876165345656) + | > loss_disc_real_5: 0.2115803360939026 (0.23450650401817735) + | > loss_0: 2.4471893310546875 (2.482911562734796) + | > grad_norm_0: tensor(17.4868, device='cuda:0') (tensor(11.2126, device='cuda:0')) + | > loss_gen: 2.4593207836151123 (2.3361924572508475) + | > loss_kl: 1.8635519742965698 (1.8744786561921585) + | > loss_feat: 8.096600532531738 (6.920745339504508) + | > loss_mel: 18.419445037841797 (17.67755682893502) + | > loss_duration: 1.4920594692230225 (1.4816010330998621) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.33097457885742 (30.29057433254035) + | > grad_norm_1: tensor(473.1377, device='cuda:0') (tensor(218.5748, device='cuda:0')) + | > current_lr_0: 0.00019525684904167766 + | > current_lr_1: 0.00019525684904167766 + | > step_time: 0.4444 (0.382797030515449) + | > loader_time: 0.0073 (0.005992449531259465) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.505730152130127 (2.505730152130127) + | > loss_disc_real_0: 0.08514607697725296 (0.08514607697725296) + | > loss_disc_real_1: 0.19099579751491547 (0.19099579751491547) + | > loss_disc_real_2: 0.23052115738391876 (0.23052115738391876) + | > loss_disc_real_3: 0.17611877620220184 (0.17611877620220184) + | > loss_disc_real_4: 0.20325809717178345 (0.20325809717178345) + | > loss_disc_real_5: 0.1833106279373169 (0.1833106279373169) + | > loss_0: 2.505730152130127 (2.505730152130127) + | > loss_gen: 1.897006630897522 (1.897006630897522) + | > loss_kl: 2.399672269821167 (2.399672269821167) + | > loss_feat: 6.306274890899658 (6.306274890899658) + | > loss_mel: 17.105161666870117 (17.105161666870117) + | > loss_duration: 1.6794679164886475 (1.6794679164886475) + | > loss_1: 29.387582778930664 (29.387582778930664) + + --> STEP: 1 + | > loss_disc: 2.5513439178466797 (2.5513439178466797) + | > loss_disc_real_0: 0.08548574894666672 (0.08548574894666672) + | > loss_disc_real_1: 0.20029088854789734 (0.20029088854789734) + | > loss_disc_real_2: 0.22051826119422913 (0.22051826119422913) + | > loss_disc_real_3: 0.24390403926372528 (0.24390403926372528) + | > loss_disc_real_4: 0.1935650259256363 (0.1935650259256363) + | > loss_disc_real_5: 0.26144179701805115 (0.26144179701805115) + | > loss_0: 2.5513439178466797 (2.5513439178466797) + | > loss_gen: 1.9706212282180786 (1.9706212282180786) + | > loss_kl: 2.5450546741485596 (2.5450546741485596) + | > loss_feat: 6.611940860748291 (6.611940860748291) + | > loss_mel: 17.708526611328125 (17.708526611328125) + | > loss_duration: 1.690466046333313 (1.690466046333313) + | > loss_1: 30.526609420776367 (30.526609420776367) + + --> STEP: 2 + | > loss_disc: 2.429136276245117 (2.4902400970458984) + | > loss_disc_real_0: 0.06838391721248627 (0.07693483307957649) + | > loss_disc_real_1: 0.17189785838127136 (0.18609437346458435) + | > loss_disc_real_2: 0.19452446699142456 (0.20752136409282684) + | > loss_disc_real_3: 0.20291806757450104 (0.22341105341911316) + | > loss_disc_real_4: 0.18059863150119781 (0.18708182871341705) + | > loss_disc_real_5: 0.22723643481731415 (0.24433911591768265) + | > loss_0: 2.429136276245117 (2.4902400970458984) + | > loss_gen: 1.984250545501709 (1.9774358868598938) + | > loss_kl: 2.096524715423584 (2.3207896947860718) + | > loss_feat: 6.595724105834961 (6.603832483291626) + | > loss_mel: 18.42424774169922 (18.066387176513672) + | > loss_duration: 1.6362683773040771 (1.663367211818695) + | > loss_1: 30.737016677856445 (30.631813049316406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00365447998046875 (+0.0002903938293457031) + | > avg_loss_disc: 2.4902400970458984 (-0.06764352321624756) + | > avg_loss_disc_real_0: 0.07693483307957649 (-0.12335563078522682) + | > avg_loss_disc_real_1: 0.18609437346458435 (-0.10205000638961792) + | > avg_loss_disc_real_2: 0.20752136409282684 (-0.039699479937553406) + | > avg_loss_disc_real_3: 0.22341105341911316 (-0.016193948686122894) + | > avg_loss_disc_real_4: 0.18708182871341705 (-0.05261777341365814) + | > avg_loss_disc_real_5: 0.24433911591768265 (-0.023876376450061798) + | > avg_loss_0: 2.4902400970458984 (-0.06764352321624756) + | > avg_loss_gen: 1.9774358868598938 (-0.6369417309761047) + | > avg_loss_kl: 2.3207896947860718 (-0.21124756336212158) + | > avg_loss_feat: 6.603832483291626 (-0.18897247314453125) + | > avg_loss_mel: 18.066387176513672 (-0.05286979675292969) + | > avg_loss_duration: 1.663367211818695 (-0.015505194664001465) + | > avg_loss_1: 30.631813049316406 (-1.1055374145507812) + + + > EPOCH: 193/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:10:13)  + + --> TIME: 2025-04-19 02:10:17 -- STEP: 12/142 -- GLOBAL_STEP: 406700 + | > loss_disc: 2.4113388061523438 (2.466874122619629) + | > loss_disc_real_0: 0.04570719227194786 (0.13180796212206283) + | > loss_disc_real_1: 0.20607085525989532 (0.20990955581267676) + | > loss_disc_real_2: 0.2602856457233429 (0.22702002276976904) + | > loss_disc_real_3: 0.25576648116111755 (0.22942345092693964) + | > loss_disc_real_4: 0.24092541635036469 (0.22545419509212175) + | > loss_disc_real_5: 0.2341514676809311 (0.22743004436294237) + | > loss_0: 2.4113388061523438 (2.466874122619629) + | > grad_norm_0: tensor(24.9712, device='cuda:0') (tensor(19.6771, device='cuda:0')) + | > loss_gen: 2.3896560668945312 (2.3988004326820374) + | > loss_kl: 1.908206582069397 (1.8146692713101704) + | > loss_feat: 7.712851524353027 (7.145269870758057) + | > loss_mel: 19.10272789001465 (17.620692094167072) + | > loss_duration: 1.4560350179672241 (1.425765484571457) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.56947708129883 (30.405197143554688) + | > grad_norm_1: tensor(464.2746, device='cuda:0') (tensor(341.1216, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.3301 (0.32662057876586914) + | > loader_time: 0.0046 (0.004495879014333089) + + + --> TIME: 2025-04-19 02:10:26 -- STEP: 37/142 -- GLOBAL_STEP: 406725 + | > loss_disc: 2.3934781551361084 (2.4580719857602507) + | > loss_disc_real_0: 0.09961307048797607 (0.13002078819113808) + | > loss_disc_real_1: 0.17466388642787933 (0.207079207574999) + | > loss_disc_real_2: 0.19677920639514923 (0.22528164572006948) + | > loss_disc_real_3: 0.24026882648468018 (0.22498620684082443) + | > loss_disc_real_4: 0.23396705090999603 (0.22802139617301323) + | > loss_disc_real_5: 0.23858320713043213 (0.23318751840978055) + | > loss_0: 2.3934781551361084 (2.4580719857602507) + | > grad_norm_0: tensor(4.1187, device='cuda:0') (tensor(15.9394, device='cuda:0')) + | > loss_gen: 2.270340919494629 (2.376127326810682) + | > loss_kl: 1.8386598825454712 (1.8316712927173924) + | > loss_feat: 7.332182884216309 (7.045812361949199) + | > loss_mel: 18.392881393432617 (17.51750345487852) + | > loss_duration: 1.479245662689209 (1.4442674533740893) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.313310623168945 (30.215381828514307) + | > grad_norm_1: tensor(78.7055, device='cuda:0') (tensor(319.0858, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.3592 (0.34284783054042506) + | > loader_time: 0.0055 (0.004906770345327015) + + + --> TIME: 2025-04-19 02:10:36 -- STEP: 62/142 -- GLOBAL_STEP: 406750 + | > loss_disc: 2.586003303527832 (2.4551995069749895) + | > loss_disc_real_0: 0.12208070605993271 (0.12891251509708743) + | > loss_disc_real_1: 0.21757705509662628 (0.20649639925649088) + | > loss_disc_real_2: 0.23459956049919128 (0.22227762615488422) + | > loss_disc_real_3: 0.23939932882785797 (0.22564244582768408) + | > loss_disc_real_4: 0.2453264743089676 (0.2268675630611758) + | > loss_disc_real_5: 0.23307935893535614 (0.2331472858786583) + | > loss_0: 2.586003303527832 (2.4551995069749895) + | > grad_norm_0: tensor(7.3795, device='cuda:0') (tensor(13.3796, device='cuda:0')) + | > loss_gen: 2.145887851715088 (2.3573247655745475) + | > loss_kl: 1.9426134824752808 (1.8395665403335326) + | > loss_feat: 6.978355407714844 (7.0120110511779785) + | > loss_mel: 17.272661209106445 (17.481322565386357) + | > loss_duration: 1.510423183441162 (1.4557638187562265) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.84994125366211 (30.145988679701283) + | > grad_norm_1: tensor(47.5055, device='cuda:0') (tensor(274.5833, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.3959 (0.3569151393828854) + | > loader_time: 0.0067 (0.005273399814482657) + + + --> TIME: 2025-04-19 02:10:46 -- STEP: 87/142 -- GLOBAL_STEP: 406775 + | > loss_disc: 2.4634382724761963 (2.4691555856288163) + | > loss_disc_real_0: 0.1010173112154007 (0.1302713615627124) + | > loss_disc_real_1: 0.20910127460956573 (0.20636762986923085) + | > loss_disc_real_2: 0.2328251600265503 (0.22580463961623182) + | > loss_disc_real_3: 0.290847510099411 (0.22716616436667825) + | > loss_disc_real_4: 0.25821250677108765 (0.2289516793927927) + | > loss_disc_real_5: 0.2583467960357666 (0.23187416911810294) + | > loss_0: 2.4634382724761963 (2.4691555856288163) + | > grad_norm_0: tensor(8.8624, device='cuda:0') (tensor(11.9356, device='cuda:0')) + | > loss_gen: 2.4556496143341064 (2.3386757099765463) + | > loss_kl: 1.928576946258545 (1.8572477294110703) + | > loss_feat: 6.920441150665283 (6.930544962828186) + | > loss_mel: 18.44386100769043 (17.547212578784446) + | > loss_duration: 1.5218505859375 (1.4685573139409902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.2703800201416 (30.142238288090148) + | > grad_norm_1: tensor(272.1499, device='cuda:0') (tensor(244.9972, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.4137 (0.3706011525515852) + | > loader_time: 0.0068 (0.005619610863170403) + + + --> TIME: 2025-04-19 02:10:57 -- STEP: 112/142 -- GLOBAL_STEP: 406800 + | > loss_disc: 2.4521355628967285 (2.4731451783861442) + | > loss_disc_real_0: 0.147078275680542 (0.13015903519200425) + | > loss_disc_real_1: 0.17453604936599731 (0.2052788699844054) + | > loss_disc_real_2: 0.19765710830688477 (0.22723841933267458) + | > loss_disc_real_3: 0.1920417845249176 (0.22783636301755905) + | > loss_disc_real_4: 0.22715148329734802 (0.22943373237337386) + | > loss_disc_real_5: 0.21232858300209045 (0.23226324109626667) + | > loss_0: 2.4521355628967285 (2.4731451783861442) + | > grad_norm_0: tensor(15.9118, device='cuda:0') (tensor(11.2847, device='cuda:0')) + | > loss_gen: 2.2892487049102783 (2.3287785989897594) + | > loss_kl: 1.8823691606521606 (1.8665509521961212) + | > loss_feat: 6.971659183502197 (6.893144130706787) + | > loss_mel: 17.35207748413086 (17.528277201311933) + | > loss_duration: 1.4673757553100586 (1.4762170538306232) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.962730407714844 (30.09296793597085) + | > grad_norm_1: tensor(361.3330, device='cuda:0') (tensor(236.9664, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.4406 (0.38260544410773684) + | > loader_time: 0.0069 (0.0059381340231214235) + + + --> TIME: 2025-04-19 02:11:09 -- STEP: 137/142 -- GLOBAL_STEP: 406825 + | > loss_disc: 2.532531499862671 (2.470685675196405) + | > loss_disc_real_0: 0.1315852254629135 (0.12966334694710963) + | > loss_disc_real_1: 0.2366732805967331 (0.20654028121137272) + | > loss_disc_real_2: 0.2256370186805725 (0.22700296918841173) + | > loss_disc_real_3: 0.204362154006958 (0.22749284953966628) + | > loss_disc_real_4: 0.23615410923957825 (0.22879457441124604) + | > loss_disc_real_5: 0.2105734497308731 (0.2319610051430055) + | > loss_0: 2.532531499862671 (2.470685675196405) + | > grad_norm_0: tensor(14.3688, device='cuda:0') (tensor(11.5013, device='cuda:0')) + | > loss_gen: 2.2403078079223633 (2.3326681948056187) + | > loss_kl: 1.8651546239852905 (1.8716041606708165) + | > loss_feat: 6.562623023986816 (6.900637671895271) + | > loss_mel: 16.966522216796875 (17.56037301390711) + | > loss_duration: 1.5157880783081055 (1.4823773420640147) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.150394439697266 (30.147660359849024) + | > grad_norm_1: tensor(328.4826, device='cuda:0') (tensor(246.5347, device='cuda:0')) + | > current_lr_0: 0.00019523244193554745 + | > current_lr_1: 0.00019523244193554745 + | > step_time: 0.4673 (0.39531924428730986) + | > loader_time: 0.0077 (0.006208188342351982) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.412611961364746 (2.412611961364746) + | > loss_disc_real_0: 0.1153472438454628 (0.1153472438454628) + | > loss_disc_real_1: 0.2652920186519623 (0.2652920186519623) + | > loss_disc_real_2: 0.22686493396759033 (0.22686493396759033) + | > loss_disc_real_3: 0.2201978713274002 (0.2201978713274002) + | > loss_disc_real_4: 0.23641280829906464 (0.23641280829906464) + | > loss_disc_real_5: 0.2468114197254181 (0.2468114197254181) + | > loss_0: 2.412611961364746 (2.412611961364746) + | > loss_gen: 2.4055497646331787 (2.4055497646331787) + | > loss_kl: 2.4850878715515137 (2.4850878715515137) + | > loss_feat: 7.329710483551025 (7.329710483551025) + | > loss_mel: 18.89280128479004 (18.89280128479004) + | > loss_duration: 1.6815825700759888 (1.6815825700759888) + | > loss_1: 32.79473114013672 (32.79473114013672) + + --> STEP: 1 + | > loss_disc: 2.4704689979553223 (2.4704689979553223) + | > loss_disc_real_0: 0.09742676466703415 (0.09742676466703415) + | > loss_disc_real_1: 0.2797738015651703 (0.2797738015651703) + | > loss_disc_real_2: 0.26183077692985535 (0.26183077692985535) + | > loss_disc_real_3: 0.23551426827907562 (0.23551426827907562) + | > loss_disc_real_4: 0.2564132809638977 (0.2564132809638977) + | > loss_disc_real_5: 0.2059554010629654 (0.2059554010629654) + | > loss_0: 2.4704689979553223 (2.4704689979553223) + | > loss_gen: 2.367046594619751 (2.367046594619751) + | > loss_kl: 2.5349225997924805 (2.5349225997924805) + | > loss_feat: 7.080422401428223 (7.080422401428223) + | > loss_mel: 19.44376564025879 (19.44376564025879) + | > loss_duration: 1.7089509963989258 (1.7089509963989258) + | > loss_1: 33.135108947753906 (33.135108947753906) + + --> STEP: 2 + | > loss_disc: 2.6131207942962646 (2.5417948961257935) + | > loss_disc_real_0: 0.14934095740318298 (0.12338386103510857) + | > loss_disc_real_1: 0.28870972990989685 (0.28424176573753357) + | > loss_disc_real_2: 0.20645198225975037 (0.23414137959480286) + | > loss_disc_real_3: 0.24805349111557007 (0.24178387969732285) + | > loss_disc_real_4: 0.25252246856689453 (0.2544678747653961) + | > loss_disc_real_5: 0.24028818309307098 (0.2231217920780182) + | > loss_0: 2.6131207942962646 (2.5417948961257935) + | > loss_gen: 2.183030128479004 (2.2750383615493774) + | > loss_kl: 1.906190037727356 (2.220556318759918) + | > loss_feat: 5.5944671630859375 (6.33744478225708) + | > loss_mel: 14.331830024719238 (16.887797832489014) + | > loss_duration: 1.6521005630493164 (1.680525779724121) + | > loss_1: 25.667617797851562 (29.401363372802734) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037239789962768555 (+6.949901580810547e-05) + | > avg_loss_disc: 2.5417948961257935 (+0.05155479907989502) + | > avg_loss_disc_real_0: 0.12338386103510857 (+0.046449027955532074) + | > avg_loss_disc_real_1: 0.28424176573753357 (+0.09814739227294922) + | > avg_loss_disc_real_2: 0.23414137959480286 (+0.026620015501976013) + | > avg_loss_disc_real_3: 0.24178387969732285 (+0.018372826278209686) + | > avg_loss_disc_real_4: 0.2544678747653961 (+0.06738604605197906) + | > avg_loss_disc_real_5: 0.2231217920780182 (-0.02121732383966446) + | > avg_loss_0: 2.5417948961257935 (+0.05155479907989502) + | > avg_loss_gen: 2.2750383615493774 (+0.29760247468948364) + | > avg_loss_kl: 2.220556318759918 (-0.10023337602615356) + | > avg_loss_feat: 6.33744478225708 (-0.2663877010345459) + | > avg_loss_mel: 16.887797832489014 (-1.1785893440246582) + | > avg_loss_duration: 1.680525779724121 (+0.017158567905426025) + | > avg_loss_1: 29.401363372802734 (-1.2304496765136719) + + + > EPOCH: 194/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:11:13)  + + --> TIME: 2025-04-19 02:11:20 -- STEP: 20/142 -- GLOBAL_STEP: 406850 + | > loss_disc: 2.3697097301483154 (2.455153918266297) + | > loss_disc_real_0: 0.09722699224948883 (0.12222821675240995) + | > loss_disc_real_1: 0.20737476646900177 (0.21099725365638733) + | > loss_disc_real_2: 0.2399481236934662 (0.2218460850417614) + | > loss_disc_real_3: 0.2541622519493103 (0.22685198560357095) + | > loss_disc_real_4: 0.24961067736148834 (0.2300403319299221) + | > loss_disc_real_5: 0.22814130783081055 (0.23071735724806786) + | > loss_0: 2.3697097301483154 (2.455153918266297) + | > grad_norm_0: tensor(8.5823, device='cuda:0') (tensor(13.3158, device='cuda:0')) + | > loss_gen: 2.557124137878418 (2.3708805441856384) + | > loss_kl: 1.8337279558181763 (1.8596077740192414) + | > loss_feat: 6.543941974639893 (7.188800811767578) + | > loss_mel: 17.63282012939453 (18.120405197143555) + | > loss_duration: 1.4989354610443115 (1.4382165193557739) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.066551208496094 (30.97791109085083) + | > grad_norm_1: tensor(209.6234, device='cuda:0') (tensor(278.0438, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.3427 (0.3301758527755737) + | > loader_time: 0.0049 (0.004689478874206543) + + + --> TIME: 2025-04-19 02:11:29 -- STEP: 45/142 -- GLOBAL_STEP: 406875 + | > loss_disc: 2.402833938598633 (2.452655055787828) + | > loss_disc_real_0: 0.12810413539409637 (0.12228469765848583) + | > loss_disc_real_1: 0.17797142267227173 (0.20926398502455817) + | > loss_disc_real_2: 0.20832833647727966 (0.22263759308391148) + | > loss_disc_real_3: 0.2126663774251938 (0.22719563477569157) + | > loss_disc_real_4: 0.2015942931175232 (0.22613219519456226) + | > loss_disc_real_5: 0.20899386703968048 (0.23540405697292752) + | > loss_0: 2.402833938598633 (2.452655055787828) + | > grad_norm_0: tensor(5.2864, device='cuda:0') (tensor(11.5631, device='cuda:0')) + | > loss_gen: 2.241079807281494 (2.36372324095832) + | > loss_kl: 1.8857600688934326 (1.8759824011060926) + | > loss_feat: 6.808243751525879 (7.134769100613064) + | > loss_mel: 18.109357833862305 (18.08677618238661) + | > loss_duration: 1.493270754814148 (1.4514098246892293) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.53771209716797 (30.912660768296984) + | > grad_norm_1: tensor(277.6020, device='cuda:0') (tensor(263.8768, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.3679 (0.3434046427408854) + | > loader_time: 0.0058 (0.005050733354356554) + + + --> TIME: 2025-04-19 02:11:39 -- STEP: 70/142 -- GLOBAL_STEP: 406900 + | > loss_disc: 2.450650691986084 (2.4589469807488586) + | > loss_disc_real_0: 0.1100691556930542 (0.122483905511243) + | > loss_disc_real_1: 0.19096079468727112 (0.20922096648386546) + | > loss_disc_real_2: 0.25549402832984924 (0.22489352268832072) + | > loss_disc_real_3: 0.24257658421993256 (0.2275029840213912) + | > loss_disc_real_4: 0.25808417797088623 (0.22705792316368648) + | > loss_disc_real_5: 0.24123865365982056 (0.23320126831531524) + | > loss_0: 2.450650691986084 (2.4589469807488586) + | > grad_norm_0: tensor(5.5947, device='cuda:0') (tensor(10.3571, device='cuda:0')) + | > loss_gen: 2.3725602626800537 (2.3552706956863405) + | > loss_kl: 1.950299620628357 (1.8905428103038242) + | > loss_feat: 7.283413410186768 (7.114287798745291) + | > loss_mel: 17.85175895690918 (18.059841850825723) + | > loss_duration: 1.499132752418518 (1.4603163310459681) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.957164764404297 (30.88025940486363) + | > grad_norm_1: tensor(252.2739, device='cuda:0') (tensor(233.1183, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.3901 (0.35519498416355677) + | > loader_time: 0.0062 (0.005315119879586357) + + + --> TIME: 2025-04-19 02:11:49 -- STEP: 95/142 -- GLOBAL_STEP: 406925 + | > loss_disc: 2.4873735904693604 (2.455253325010601) + | > loss_disc_real_0: 0.11775314807891846 (0.12285561883135845) + | > loss_disc_real_1: 0.18765367567539215 (0.20744273301802182) + | > loss_disc_real_2: 0.22526736557483673 (0.22397842909160412) + | > loss_disc_real_3: 0.20808273553848267 (0.22736175232811978) + | > loss_disc_real_4: 0.21872715651988983 (0.22763126116049917) + | > loss_disc_real_5: 0.21882450580596924 (0.23434385861221113) + | > loss_0: 2.4873735904693604 (2.455253325010601) + | > grad_norm_0: tensor(7.9949, device='cuda:0') (tensor(10.8676, device='cuda:0')) + | > loss_gen: 2.2067384719848633 (2.3527279828724104) + | > loss_kl: 1.931760549545288 (1.904989100757398) + | > loss_feat: 6.630985736846924 (7.107894987809031) + | > loss_mel: 17.242158889770508 (18.03181587018465) + | > loss_duration: 1.5144848823547363 (1.4717456641950106) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.526126861572266 (30.869173531783254) + | > grad_norm_1: tensor(122.5992, device='cuda:0') (tensor(251.7600, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.403 (0.3661594842609608) + | > loader_time: 0.0066 (0.005626364758140164) + + + --> TIME: 2025-04-19 02:12:00 -- STEP: 120/142 -- GLOBAL_STEP: 406950 + | > loss_disc: 2.398076057434082 (2.460365893443426) + | > loss_disc_real_0: 0.10746804624795914 (0.12508711398889624) + | > loss_disc_real_1: 0.18624253571033478 (0.20691347358127435) + | > loss_disc_real_2: 0.2347182035446167 (0.22458411393066247) + | > loss_disc_real_3: 0.2565333843231201 (0.22806873979667822) + | > loss_disc_real_4: 0.2553270757198334 (0.22766114870707194) + | > loss_disc_real_5: 0.22578543424606323 (0.23426520228385925) + | > loss_0: 2.398076057434082 (2.460365893443426) + | > grad_norm_0: tensor(7.9806, device='cuda:0') (tensor(12.2122, device='cuda:0')) + | > loss_gen: 2.6723415851593018 (2.3564963400363923) + | > loss_kl: 1.9480035305023193 (1.9124445378780366) + | > loss_feat: 7.445258140563965 (7.093565893173218) + | > loss_mel: 18.312002182006836 (17.98988244533539) + | > loss_duration: 1.5222091674804688 (1.478769021232923) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.899816513061523 (30.831158272425334) + | > grad_norm_1: tensor(209.2033, device='cuda:0') (tensor(265.9236, device='cuda:0')) + | > current_lr_0: 0.0001952080378803055 + | > current_lr_1: 0.0001952080378803055 + | > step_time: 0.4414 (0.377936081091563) + | > loader_time: 0.007 (0.005913557608922325) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.462062120437622 (2.462062120437622) + | > loss_disc_real_0: 0.16286104917526245 (0.16286104917526245) + | > loss_disc_real_1: 0.2355862259864807 (0.2355862259864807) + | > loss_disc_real_2: 0.27871909737586975 (0.27871909737586975) + | > loss_disc_real_3: 0.2538644075393677 (0.2538644075393677) + | > loss_disc_real_4: 0.2567596733570099 (0.2567596733570099) + | > loss_disc_real_5: 0.22457075119018555 (0.22457075119018555) + | > loss_0: 2.462062120437622 (2.462062120437622) + | > loss_gen: 2.369860887527466 (2.369860887527466) + | > loss_kl: 2.31891131401062 (2.31891131401062) + | > loss_feat: 6.549821853637695 (6.549821853637695) + | > loss_mel: 18.218093872070312 (18.218093872070312) + | > loss_duration: 1.7029314041137695 (1.7029314041137695) + | > loss_1: 31.159618377685547 (31.159618377685547) + + --> STEP: 1 + | > loss_disc: 2.465754508972168 (2.465754508972168) + | > loss_disc_real_0: 0.1133669838309288 (0.1133669838309288) + | > loss_disc_real_1: 0.25723910331726074 (0.25723910331726074) + | > loss_disc_real_2: 0.2607816457748413 (0.2607816457748413) + | > loss_disc_real_3: 0.23765821754932404 (0.23765821754932404) + | > loss_disc_real_4: 0.28022515773773193 (0.28022515773773193) + | > loss_disc_real_5: 0.2099325954914093 (0.2099325954914093) + | > loss_0: 2.465754508972168 (2.465754508972168) + | > loss_gen: 2.3481791019439697 (2.3481791019439697) + | > loss_kl: 2.59346342086792 (2.59346342086792) + | > loss_feat: 7.487616539001465 (7.487616539001465) + | > loss_mel: 19.26056671142578 (19.26056671142578) + | > loss_duration: 1.7089622020721436 (1.7089622020721436) + | > loss_1: 33.39878845214844 (33.39878845214844) + + --> STEP: 2 + | > loss_disc: 2.6574039459228516 (2.5615792274475098) + | > loss_disc_real_0: 0.21603423357009888 (0.16470060870051384) + | > loss_disc_real_1: 0.22496894001960754 (0.24110402166843414) + | > loss_disc_real_2: 0.2741164565086365 (0.2674490511417389) + | > loss_disc_real_3: 0.2605622708797455 (0.24911024421453476) + | > loss_disc_real_4: 0.29746171832084656 (0.28884343802928925) + | > loss_disc_real_5: 0.24198392033576965 (0.22595825791358948) + | > loss_0: 2.6574039459228516 (2.5615792274475098) + | > loss_gen: 2.212968349456787 (2.2805737257003784) + | > loss_kl: 2.025547981262207 (2.3095057010650635) + | > loss_feat: 4.9735107421875 (6.230563640594482) + | > loss_mel: 14.578689575195312 (16.919628143310547) + | > loss_duration: 1.6519842147827148 (1.6804732084274292) + | > loss_1: 25.442699432373047 (29.420743942260742) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033963918685913086 (-0.0003275871276855469) + | > avg_loss_disc: 2.5615792274475098 (+0.01978433132171631) + | > avg_loss_disc_real_0: 0.16470060870051384 (+0.04131674766540527) + | > avg_loss_disc_real_1: 0.24110402166843414 (-0.043137744069099426) + | > avg_loss_disc_real_2: 0.2674490511417389 (+0.033307671546936035) + | > avg_loss_disc_real_3: 0.24911024421453476 (+0.007326364517211914) + | > avg_loss_disc_real_4: 0.28884343802928925 (+0.03437556326389313) + | > avg_loss_disc_real_5: 0.22595825791358948 (+0.002836465835571289) + | > avg_loss_0: 2.5615792274475098 (+0.01978433132171631) + | > avg_loss_gen: 2.2805737257003784 (+0.0055353641510009766) + | > avg_loss_kl: 2.3095057010650635 (+0.08894938230514526) + | > avg_loss_feat: 6.230563640594482 (-0.10688114166259766) + | > avg_loss_mel: 16.919628143310547 (+0.0318303108215332) + | > avg_loss_duration: 1.6804732084274292 (-5.257129669189453e-05) + | > avg_loss_1: 29.420743942260742 (+0.019380569458007812) + + + > EPOCH: 195/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:12:11)  + + --> TIME: 2025-04-19 02:12:13 -- STEP: 3/142 -- GLOBAL_STEP: 406975 + | > loss_disc: 2.456894636154175 (2.4865907033284507) + | > loss_disc_real_0: 0.13817191123962402 (0.16905942559242249) + | > loss_disc_real_1: 0.1950816810131073 (0.20862633486588797) + | > loss_disc_real_2: 0.23351918160915375 (0.25661601126194) + | > loss_disc_real_3: 0.24858218431472778 (0.2378526578346888) + | > loss_disc_real_4: 0.24964354932308197 (0.2517421692609787) + | > loss_disc_real_5: 0.26733314990997314 (0.25013985733191174) + | > loss_0: 2.456894636154175 (2.4865907033284507) + | > grad_norm_0: tensor(15.6924, device='cuda:0') (tensor(15.4200, device='cuda:0')) + | > loss_gen: 2.311115264892578 (2.4072282314300537) + | > loss_kl: 1.8118016719818115 (1.7164678573608398) + | > loss_feat: 6.256513595581055 (7.0612508455912275) + | > loss_mel: 16.85177993774414 (17.38235346476237) + | > loss_duration: 1.4510955810546875 (1.4458812872568767) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.68230628967285 (30.013182322184246) + | > grad_norm_1: tensor(162.5151, device='cuda:0') (tensor(216.3921, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.324 (0.32445796330769855) + | > loader_time: 0.0043 (0.004210869471232097) + + + --> TIME: 2025-04-19 02:12:22 -- STEP: 28/142 -- GLOBAL_STEP: 407000 + | > loss_disc: 2.4840877056121826 (2.426663543496813) + | > loss_disc_real_0: 0.0858473926782608 (0.11853053207908358) + | > loss_disc_real_1: 0.19126617908477783 (0.2017024883202144) + | > loss_disc_real_2: 0.1946001797914505 (0.22727966947214945) + | > loss_disc_real_3: 0.20511925220489502 (0.22763207448380335) + | > loss_disc_real_4: 0.2265661358833313 (0.22722149853195464) + | > loss_disc_real_5: 0.2268817126750946 (0.23022321345550673) + | > loss_0: 2.4840877056121826 (2.426663543496813) + | > grad_norm_0: tensor(16.6518, device='cuda:0') (tensor(10.1549, device='cuda:0')) + | > loss_gen: 2.293480396270752 (2.387996086052486) + | > loss_kl: 1.8849024772644043 (1.8290621723447527) + | > loss_feat: 7.710136890411377 (7.228373714855739) + | > loss_mel: 19.567731857299805 (17.94554632050651) + | > loss_duration: 1.4688198566436768 (1.4407624687467302) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.925071716308594 (30.831740515572683) + | > grad_norm_1: tensor(266.5215, device='cuda:0') (tensor(182.3022, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.3436 (0.3342922500201634) + | > loader_time: 0.0052 (0.0047588518687656945) + + + --> TIME: 2025-04-19 02:12:31 -- STEP: 53/142 -- GLOBAL_STEP: 407025 + | > loss_disc: 2.3888933658599854 (2.4296013229298143) + | > loss_disc_real_0: 0.1351124346256256 (0.12060887127552393) + | > loss_disc_real_1: 0.23314373195171356 (0.20538481760699795) + | > loss_disc_real_2: 0.21165326237678528 (0.22441281462615392) + | > loss_disc_real_3: 0.21674861013889313 (0.22581837087307335) + | > loss_disc_real_4: 0.2545875608921051 (0.22637610896578375) + | > loss_disc_real_5: 0.23359158635139465 (0.23205483578286082) + | > loss_0: 2.3888933658599854 (2.4296013229298143) + | > grad_norm_0: tensor(6.9368, device='cuda:0') (tensor(9.8094, device='cuda:0')) + | > loss_gen: 2.426757335662842 (2.374392828851376) + | > loss_kl: 1.8423446416854858 (1.8541837165940482) + | > loss_feat: 7.42837381362915 (7.162091920960624) + | > loss_mel: 17.943161010742188 (17.874299373266826) + | > loss_duration: 1.470376968383789 (1.4500107225382104) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.111013412475586 (30.714978595949567) + | > grad_norm_1: tensor(56.5569, device='cuda:0') (tensor(170.0180, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.3658 (0.34682039944630744) + | > loader_time: 0.0059 (0.005183822703811357) + + + --> TIME: 2025-04-19 02:12:41 -- STEP: 78/142 -- GLOBAL_STEP: 407050 + | > loss_disc: 2.4658467769622803 (2.4445066604858785) + | > loss_disc_real_0: 0.1328604519367218 (0.12367158574171555) + | > loss_disc_real_1: 0.2124735414981842 (0.2054135635113105) + | > loss_disc_real_2: 0.2186664491891861 (0.22563986900525215) + | > loss_disc_real_3: 0.23906439542770386 (0.22678862378383294) + | > loss_disc_real_4: 0.2327999770641327 (0.22712082797900224) + | > loss_disc_real_5: 0.2202683389186859 (0.23120913960230657) + | > loss_0: 2.4658467769622803 (2.4445066604858785) + | > grad_norm_0: tensor(3.8092, device='cuda:0') (tensor(8.9803, device='cuda:0')) + | > loss_gen: 2.319248676300049 (2.3547832140555744) + | > loss_kl: 1.998151183128357 (1.8762332949882898) + | > loss_feat: 7.181028366088867 (7.082729859229846) + | > loss_mel: 18.698379516601562 (17.793265110407123) + | > loss_duration: 1.5107841491699219 (1.4621645365005884) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.707592010498047 (30.56917606255947) + | > grad_norm_1: tensor(82.7697, device='cuda:0') (tensor(162.5841, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.3943 (0.35947419129885155) + | > loader_time: 0.0063 (0.005509694417317707) + + + --> TIME: 2025-04-19 02:12:51 -- STEP: 103/142 -- GLOBAL_STEP: 407075 + | > loss_disc: 2.5265984535217285 (2.4520137286880668) + | > loss_disc_real_0: 0.1467311531305313 (0.12467523989746872) + | > loss_disc_real_1: 0.19032277166843414 (0.20595968840191664) + | > loss_disc_real_2: 0.2312900573015213 (0.22644903604845398) + | > loss_disc_real_3: 0.22988632321357727 (0.22765233678725158) + | > loss_disc_real_4: 0.22183476388454437 (0.22828257040491381) + | > loss_disc_real_5: 0.27773475646972656 (0.2337608800351041) + | > loss_0: 2.5265984535217285 (2.4520137286880668) + | > grad_norm_0: tensor(4.3709, device='cuda:0') (tensor(8.5939, device='cuda:0')) + | > loss_gen: 2.220407247543335 (2.3491210358814123) + | > loss_kl: 1.9968262910842896 (1.8964545611038948) + | > loss_feat: 6.256963729858398 (7.025224468083057) + | > loss_mel: 17.250316619873047 (17.77372480596153) + | > loss_duration: 1.547456979751587 (1.4736289121572255) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.271970748901367 (30.518153810964048) + | > grad_norm_1: tensor(59.4414, device='cuda:0') (tensor(159.1307, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.4189 (0.37085171579157256) + | > loader_time: 0.0071 (0.005808652025982014) + + + --> TIME: 2025-04-19 02:13:02 -- STEP: 128/142 -- GLOBAL_STEP: 407100 + | > loss_disc: 2.51660418510437 (2.455359520390632) + | > loss_disc_real_0: 0.1195741519331932 (0.12576756230555475) + | > loss_disc_real_1: 0.1782490313053131 (0.2055232310667634) + | > loss_disc_real_2: 0.2261923998594284 (0.2263156621484086) + | > loss_disc_real_3: 0.2093888372182846 (0.22765075811184943) + | > loss_disc_real_4: 0.21524402499198914 (0.22849370853509754) + | > loss_disc_real_5: 0.2381109595298767 (0.23225328186526895) + | > loss_0: 2.51660418510437 (2.455359520390632) + | > grad_norm_0: tensor(7.3151, device='cuda:0') (tensor(8.4954, device='cuda:0')) + | > loss_gen: 2.2681498527526855 (2.340017480775714) + | > loss_kl: 1.8525851964950562 (1.9050414320081472) + | > loss_feat: 6.877902030944824 (6.9879756309092045) + | > loss_mel: 17.735361099243164 (17.752491123974316) + | > loss_duration: 1.5028765201568604 (1.480497445911169) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.236873626708984 (30.466023072600365) + | > grad_norm_1: tensor(96.4818, device='cuda:0') (tensor(159.6293, device='cuda:0')) + | > current_lr_0: 0.00019518363687557043 + | > current_lr_1: 0.00019518363687557043 + | > step_time: 0.4455 (0.38402783870697016) + | > loader_time: 0.0078 (0.006109660491347313) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.688389301300049 (2.688389301300049) + | > loss_disc_real_0: 0.140695258975029 (0.140695258975029) + | > loss_disc_real_1: 0.22835415601730347 (0.22835415601730347) + | > loss_disc_real_2: 0.23837745189666748 (0.23837745189666748) + | > loss_disc_real_3: 0.25800538063049316 (0.25800538063049316) + | > loss_disc_real_4: 0.22842128574848175 (0.22842128574848175) + | > loss_disc_real_5: 0.29893508553504944 (0.29893508553504944) + | > loss_0: 2.688389301300049 (2.688389301300049) + | > loss_gen: 2.0864648818969727 (2.0864648818969727) + | > loss_kl: 2.4903953075408936 (2.4903953075408936) + | > loss_feat: 5.987948417663574 (5.987948417663574) + | > loss_mel: 17.406177520751953 (17.406177520751953) + | > loss_duration: 1.6798617839813232 (1.6798617839813232) + | > loss_1: 29.650846481323242 (29.650846481323242) + + --> STEP: 1 + | > loss_disc: 2.526045799255371 (2.526045799255371) + | > loss_disc_real_0: 0.19138967990875244 (0.19138967990875244) + | > loss_disc_real_1: 0.27438437938690186 (0.27438437938690186) + | > loss_disc_real_2: 0.23975926637649536 (0.23975926637649536) + | > loss_disc_real_3: 0.2245539128780365 (0.2245539128780365) + | > loss_disc_real_4: 0.2094048112630844 (0.2094048112630844) + | > loss_disc_real_5: 0.23413681983947754 (0.23413681983947754) + | > loss_0: 2.526045799255371 (2.526045799255371) + | > loss_gen: 2.358715772628784 (2.358715772628784) + | > loss_kl: 2.665910005569458 (2.665910005569458) + | > loss_feat: 6.286370754241943 (6.286370754241943) + | > loss_mel: 16.0339412689209 (16.0339412689209) + | > loss_duration: 1.703798532485962 (1.703798532485962) + | > loss_1: 29.048734664916992 (29.048734664916992) + + --> STEP: 2 + | > loss_disc: 2.6770517826080322 (2.6015487909317017) + | > loss_disc_real_0: 0.1270737498998642 (0.15923171490430832) + | > loss_disc_real_1: 0.27377861738204956 (0.2740814983844757) + | > loss_disc_real_2: 0.2444899082183838 (0.24212458729743958) + | > loss_disc_real_3: 0.24864044785499573 (0.2365971803665161) + | > loss_disc_real_4: 0.21276132762432098 (0.2110830694437027) + | > loss_disc_real_5: 0.26201897859573364 (0.2480778992176056) + | > loss_0: 2.6770517826080322 (2.6015487909317017) + | > loss_gen: 2.0476813316345215 (2.203198552131653) + | > loss_kl: 1.9324220418930054 (2.2991660237312317) + | > loss_feat: 6.007741451263428 (6.1470561027526855) + | > loss_mel: 16.768413543701172 (16.401177406311035) + | > loss_duration: 1.640209674835205 (1.6720041036605835) + | > loss_1: 28.396467208862305 (28.72260093688965) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003730177879333496 (+0.0003337860107421875) + | > avg_loss_disc: 2.6015487909317017 (+0.039969563484191895) + | > avg_loss_disc_real_0: 0.15923171490430832 (-0.005468893796205521) + | > avg_loss_disc_real_1: 0.2740814983844757 (+0.032977476716041565) + | > avg_loss_disc_real_2: 0.24212458729743958 (-0.025324463844299316) + | > avg_loss_disc_real_3: 0.2365971803665161 (-0.012513063848018646) + | > avg_loss_disc_real_4: 0.2110830694437027 (-0.07776036858558655) + | > avg_loss_disc_real_5: 0.2480778992176056 (+0.022119641304016113) + | > avg_loss_0: 2.6015487909317017 (+0.039969563484191895) + | > avg_loss_gen: 2.203198552131653 (-0.07737517356872559) + | > avg_loss_kl: 2.2991660237312317 (-0.010339677333831787) + | > avg_loss_feat: 6.1470561027526855 (-0.08350753784179688) + | > avg_loss_mel: 16.401177406311035 (-0.5184507369995117) + | > avg_loss_duration: 1.6720041036605835 (-0.008469104766845703) + | > avg_loss_1: 28.72260093688965 (-0.6981430053710938) + + + > EPOCH: 196/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:13:10)  + + --> TIME: 2025-04-19 02:13:15 -- STEP: 11/142 -- GLOBAL_STEP: 407125 + | > loss_disc: 2.514664888381958 (2.459968935359608) + | > loss_disc_real_0: 0.11532089859247208 (0.1253031688657674) + | > loss_disc_real_1: 0.20503059029579163 (0.20915040509267288) + | > loss_disc_real_2: 0.23048239946365356 (0.21983499012210153) + | > loss_disc_real_3: 0.24399006366729736 (0.23083223673430356) + | > loss_disc_real_4: 0.20891913771629333 (0.22618179429661145) + | > loss_disc_real_5: 0.25950121879577637 (0.230452685193582) + | > loss_0: 2.514664888381958 (2.459968935359608) + | > grad_norm_0: tensor(12.1121, device='cuda:0') (tensor(9.9176, device='cuda:0')) + | > loss_gen: 2.3907277584075928 (2.3922293836420234) + | > loss_kl: 1.841339349746704 (1.8501136628064243) + | > loss_feat: 7.074819564819336 (7.254354910417036) + | > loss_mel: 18.287742614746094 (18.03434146534313) + | > loss_duration: 1.407252311706543 (1.4272071448239414) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.001880645751953 (30.95824657786976) + | > grad_norm_1: tensor(161.6482, device='cuda:0') (tensor(188.4758, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.3323 (0.32984807274558325) + | > loader_time: 0.0048 (0.004402854225852273) + + + --> TIME: 2025-04-19 02:13:24 -- STEP: 36/142 -- GLOBAL_STEP: 407150 + | > loss_disc: 2.412782907485962 (2.4476233787006803) + | > loss_disc_real_0: 0.14959996938705444 (0.12090922498868571) + | > loss_disc_real_1: 0.20152202248573303 (0.20753023525079092) + | > loss_disc_real_2: 0.23443232476711273 (0.22713984259300762) + | > loss_disc_real_3: 0.24391721189022064 (0.22785097319218847) + | > loss_disc_real_4: 0.23118995130062103 (0.22583029460575846) + | > loss_disc_real_5: 0.2189757227897644 (0.22759819734427664) + | > loss_0: 2.412782907485962 (2.4476233787006803) + | > grad_norm_0: tensor(5.6227, device='cuda:0') (tensor(9.3719, device='cuda:0')) + | > loss_gen: 2.4240877628326416 (2.374419887860616) + | > loss_kl: 1.8274263143539429 (1.8367659118440416) + | > loss_feat: 6.934543132781982 (7.123587833510505) + | > loss_mel: 17.623441696166992 (17.779294702741836) + | > loss_duration: 1.4379055500030518 (1.4402308861414592) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.247404098510742 (30.55429924858941) + | > grad_norm_1: tensor(160.8040, device='cuda:0') (tensor(221.2074, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.3566 (0.34092898501290214) + | > loader_time: 0.0053 (0.004800081253051758) + + + --> TIME: 2025-04-19 02:13:33 -- STEP: 61/142 -- GLOBAL_STEP: 407175 + | > loss_disc: 2.504430055618286 (2.4576220082455) + | > loss_disc_real_0: 0.1403825879096985 (0.12337580933922628) + | > loss_disc_real_1: 0.21065230667591095 (0.20861173360074153) + | > loss_disc_real_2: 0.2664109468460083 (0.22630906666888564) + | > loss_disc_real_3: 0.2690017521381378 (0.22916194723277797) + | > loss_disc_real_4: 0.24135218560695648 (0.2275266188089965) + | > loss_disc_real_5: 0.2661477327346802 (0.22985949452783241) + | > loss_0: 2.504430055618286 (2.4576220082455) + | > grad_norm_0: tensor(20.8079, device='cuda:0') (tensor(10.0140, device='cuda:0')) + | > loss_gen: 2.338808536529541 (2.3586507273502035) + | > loss_kl: 1.8948906660079956 (1.855543718963373) + | > loss_feat: 6.865553379058838 (7.078947778607978) + | > loss_mel: 17.9715518951416 (17.815779216953967) + | > loss_duration: 1.466156005859375 (1.4538724305199795) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.536958694458008 (30.562793856761495) + | > grad_norm_1: tensor(389.5708, device='cuda:0') (tensor(241.0431, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.3813 (0.35149657140012647) + | > loader_time: 0.0057 (0.005230970070010327) + + + --> TIME: 2025-04-19 02:13:43 -- STEP: 86/142 -- GLOBAL_STEP: 407200 + | > loss_disc: 2.4499032497406006 (2.4633391596550176) + | > loss_disc_real_0: 0.19759824872016907 (0.12543078347347503) + | > loss_disc_real_1: 0.18672245740890503 (0.20878854186035867) + | > loss_disc_real_2: 0.21400660276412964 (0.22641995445240376) + | > loss_disc_real_3: 0.22228044271469116 (0.2280443012714386) + | > loss_disc_real_4: 0.22160354256629944 (0.22811133899660999) + | > loss_disc_real_5: 0.2289295345544815 (0.2314727822708529) + | > loss_0: 2.4499032497406006 (2.4633391596550176) + | > grad_norm_0: tensor(19.0229, device='cuda:0') (tensor(12.0313, device='cuda:0')) + | > loss_gen: 2.383192539215088 (2.370233167049496) + | > loss_kl: 2.0484018325805664 (1.8719344291576119) + | > loss_feat: 7.097139358520508 (7.065737424894821) + | > loss_mel: 18.407760620117188 (17.877605981604994) + | > loss_duration: 1.5173054933547974 (1.4662055830622827) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.453800201416016 (30.651716609333835) + | > grad_norm_1: tensor(414.9135, device='cuda:0') (tensor(270.5957, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.4018 (0.3637636151424674) + | > loader_time: 0.0068 (0.005598744680715162) + + + --> TIME: 2025-04-19 02:13:54 -- STEP: 111/142 -- GLOBAL_STEP: 407225 + | > loss_disc: 2.4857096672058105 (2.4683330144968134) + | > loss_disc_real_0: 0.1598459929227829 (0.12504536168532326) + | > loss_disc_real_1: 0.20291073620319366 (0.20943630211525135) + | > loss_disc_real_2: 0.22850435972213745 (0.22727581643843436) + | > loss_disc_real_3: 0.2392914593219757 (0.22862118442316312) + | > loss_disc_real_4: 0.2628522515296936 (0.22903242613281216) + | > loss_disc_real_5: 0.24228088557720184 (0.2301441255184981) + | > loss_0: 2.4857096672058105 (2.4683330144968134) + | > grad_norm_0: tensor(20.5652, device='cuda:0') (tensor(12.1124, device='cuda:0')) + | > loss_gen: 2.2787792682647705 (2.3602688870988446) + | > loss_kl: 1.9379944801330566 (1.8923150913135425) + | > loss_feat: 6.706165790557861 (7.029698737032779) + | > loss_mel: 18.653545379638672 (17.841106028170202) + | > loss_duration: 1.5271592140197754 (1.4747254794782347) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.10364532470703 (30.59811427142169) + | > grad_norm_1: tensor(375.5050, device='cuda:0') (tensor(275.2821, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.4298 (0.3751940039901046) + | > loader_time: 0.0069 (0.005872024072183146) + + + --> TIME: 2025-04-19 02:14:05 -- STEP: 136/142 -- GLOBAL_STEP: 407250 + | > loss_disc: 2.4984869956970215 (2.47097882628441) + | > loss_disc_real_0: 0.16429801285266876 (0.1274380292195607) + | > loss_disc_real_1: 0.19563868641853333 (0.20882165563457153) + | > loss_disc_real_2: 0.21876434981822968 (0.22734249492778497) + | > loss_disc_real_3: 0.22956383228302002 (0.22854911305886857) + | > loss_disc_real_4: 0.20864726603031158 (0.22850021029658177) + | > loss_disc_real_5: 0.2189444601535797 (0.23043718813535044) + | > loss_0: 2.4984869956970215 (2.47097882628441) + | > grad_norm_0: tensor(29.4340, device='cuda:0') (tensor(13.3383, device='cuda:0')) + | > loss_gen: 2.201404094696045 (2.353364732335595) + | > loss_kl: 1.9985848665237427 (1.8984561387230368) + | > loss_feat: 7.3120293617248535 (6.995105161386378) + | > loss_mel: 17.451011657714844 (17.8288003767238) + | > loss_duration: 1.521369218826294 (1.4810356383814531) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.48440170288086 (30.556762162376852) + | > grad_norm_1: tensor(261.1830, device='cuda:0') (tensor(285.1407, device='cuda:0')) + | > current_lr_0: 0.00019515923892096098 + | > current_lr_1: 0.00019515923892096098 + | > step_time: 0.4609 (0.38847744640182047) + | > loader_time: 0.0078 (0.006141592474544751) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6616015434265137 (2.6616015434265137) + | > loss_disc_real_0: 0.24271026253700256 (0.24271026253700256) + | > loss_disc_real_1: 0.22985194623470306 (0.22985194623470306) + | > loss_disc_real_2: 0.2433885782957077 (0.2433885782957077) + | > loss_disc_real_3: 0.23136261105537415 (0.23136261105537415) + | > loss_disc_real_4: 0.23965150117874146 (0.23965150117874146) + | > loss_disc_real_5: 0.25815123319625854 (0.25815123319625854) + | > loss_0: 2.6616015434265137 (2.6616015434265137) + | > loss_gen: 2.350557804107666 (2.350557804107666) + | > loss_kl: 2.417620897293091 (2.417620897293091) + | > loss_feat: 5.478157043457031 (5.478157043457031) + | > loss_mel: 15.859766006469727 (15.859766006469727) + | > loss_duration: 1.6816949844360352 (1.6816949844360352) + | > loss_1: 27.787796020507812 (27.787796020507812) + + --> STEP: 1 + | > loss_disc: 2.72124981880188 (2.72124981880188) + | > loss_disc_real_0: 0.32541361451148987 (0.32541361451148987) + | > loss_disc_real_1: 0.2284490019083023 (0.2284490019083023) + | > loss_disc_real_2: 0.25611406564712524 (0.25611406564712524) + | > loss_disc_real_3: 0.22305536270141602 (0.22305536270141602) + | > loss_disc_real_4: 0.24167867004871368 (0.24167867004871368) + | > loss_disc_real_5: 0.2648024260997772 (0.2648024260997772) + | > loss_0: 2.72124981880188 (2.72124981880188) + | > loss_gen: 2.533886194229126 (2.533886194229126) + | > loss_kl: 2.6684746742248535 (2.6684746742248535) + | > loss_feat: 6.516218185424805 (6.516218185424805) + | > loss_mel: 17.394580841064453 (17.394580841064453) + | > loss_duration: 1.7059614658355713 (1.7059614658355713) + | > loss_1: 30.819120407104492 (30.819120407104492) + + --> STEP: 2 + | > loss_disc: 2.523994207382202 (2.622622013092041) + | > loss_disc_real_0: 0.32781052589416504 (0.32661207020282745) + | > loss_disc_real_1: 0.18175821006298065 (0.20510360598564148) + | > loss_disc_real_2: 0.22414755821228027 (0.24013081192970276) + | > loss_disc_real_3: 0.21718811988830566 (0.22012174129486084) + | > loss_disc_real_4: 0.25589826703071594 (0.2487884685397148) + | > loss_disc_real_5: 0.19966045022010803 (0.23223143815994263) + | > loss_0: 2.523994207382202 (2.622622013092041) + | > loss_gen: 2.757441997528076 (2.645664095878601) + | > loss_kl: 2.2831778526306152 (2.4758262634277344) + | > loss_feat: 6.801580905914307 (6.658899545669556) + | > loss_mel: 16.416349411010742 (16.905465126037598) + | > loss_duration: 1.662061333656311 (1.6840113997459412) + | > loss_1: 29.920612335205078 (30.369866371154785) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035789012908935547 (-0.0001512765884399414) + | > avg_loss_disc: 2.622622013092041 (+0.021073222160339355) + | > avg_loss_disc_real_0: 0.32661207020282745 (+0.16738035529851913) + | > avg_loss_disc_real_1: 0.20510360598564148 (-0.06897789239883423) + | > avg_loss_disc_real_2: 0.24013081192970276 (-0.0019937753677368164) + | > avg_loss_disc_real_3: 0.22012174129486084 (-0.016475439071655273) + | > avg_loss_disc_real_4: 0.2487884685397148 (+0.037705399096012115) + | > avg_loss_disc_real_5: 0.23223143815994263 (-0.015846461057662964) + | > avg_loss_0: 2.622622013092041 (+0.021073222160339355) + | > avg_loss_gen: 2.645664095878601 (+0.44246554374694824) + | > avg_loss_kl: 2.4758262634277344 (+0.17666023969650269) + | > avg_loss_feat: 6.658899545669556 (+0.5118434429168701) + | > avg_loss_mel: 16.905465126037598 (+0.5042877197265625) + | > avg_loss_duration: 1.6840113997459412 (+0.012007296085357666) + | > avg_loss_1: 30.369866371154785 (+1.6472654342651367) + + + > EPOCH: 197/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:14:09)  + + --> TIME: 2025-04-19 02:14:17 -- STEP: 19/142 -- GLOBAL_STEP: 407275 + | > loss_disc: 2.438688278198242 (2.4809684627934505) + | > loss_disc_real_0: 0.10666636377573013 (0.12899131171013178) + | > loss_disc_real_1: 0.19584016501903534 (0.20212535952266894) + | > loss_disc_real_2: 0.19720037281513214 (0.22542758364426463) + | > loss_disc_real_3: 0.20055022835731506 (0.22761820335137217) + | > loss_disc_real_4: 0.19242939352989197 (0.22890937328338623) + | > loss_disc_real_5: 0.2276657372713089 (0.23449563274258062) + | > loss_0: 2.438688278198242 (2.4809684627934505) + | > grad_norm_0: tensor(25.0321, device='cuda:0') (tensor(19.6129, device='cuda:0')) + | > loss_gen: 2.234584093093872 (2.3732860213831852) + | > loss_kl: 1.84783136844635 (1.839848141921194) + | > loss_feat: 7.318222522735596 (7.127475362075002) + | > loss_mel: 17.969715118408203 (17.833916864897077) + | > loss_duration: 1.4264235496520996 (1.4318272628282245) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.796777725219727 (30.606353458605316) + | > grad_norm_1: tensor(446.2478, device='cuda:0') (tensor(284.6404, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.3378 (0.337028327741121) + | > loader_time: 0.0048 (0.00477633978191175) + + + --> TIME: 2025-04-19 02:14:26 -- STEP: 44/142 -- GLOBAL_STEP: 407300 + | > loss_disc: 2.4474196434020996 (2.450875553217801) + | > loss_disc_real_0: 0.10026460140943527 (0.12336879799311809) + | > loss_disc_real_1: 0.22502757608890533 (0.20397736064412378) + | > loss_disc_real_2: 0.21755893528461456 (0.22204578498547728) + | > loss_disc_real_3: 0.23770903050899506 (0.22963791476054626) + | > loss_disc_real_4: 0.23026369512081146 (0.22762179205363448) + | > loss_disc_real_5: 0.23236015439033508 (0.23476378111676735) + | > loss_0: 2.4474196434020996 (2.450875553217801) + | > grad_norm_0: tensor(5.8853, device='cuda:0') (tensor(14.3056, device='cuda:0')) + | > loss_gen: 2.2660973072052 (2.363832641731609) + | > loss_kl: 1.912706732749939 (1.8440569612112911) + | > loss_feat: 7.428032398223877 (7.0869632525877515) + | > loss_mel: 17.640657424926758 (17.598660165613342) + | > loss_duration: 1.4828256368637085 (1.4462802816521034) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.73031997680664 (30.339793118563566) + | > grad_norm_1: tensor(85.0169, device='cuda:0') (tensor(268.3567, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.3664 (0.34990044073625043) + | > loader_time: 0.0059 (0.005220662463795055) + + + --> TIME: 2025-04-19 02:14:36 -- STEP: 69/142 -- GLOBAL_STEP: 407325 + | > loss_disc: 2.496509313583374 (2.4611758978470513) + | > loss_disc_real_0: 0.10956454277038574 (0.12390140940745671) + | > loss_disc_real_1: 0.19617171585559845 (0.20645506723203522) + | > loss_disc_real_2: 0.25018465518951416 (0.22430602191151053) + | > loss_disc_real_3: 0.2528090178966522 (0.22911147039005722) + | > loss_disc_real_4: 0.2555207908153534 (0.2301692636548609) + | > loss_disc_real_5: 0.23222064971923828 (0.23290412603081137) + | > loss_0: 2.496509313583374 (2.4611758978470513) + | > grad_norm_0: tensor(3.8571, device='cuda:0') (tensor(12.0111, device='cuda:0')) + | > loss_gen: 2.4439923763275146 (2.3537218847136567) + | > loss_kl: 1.8740462064743042 (1.8573645474254221) + | > loss_feat: 7.6660895347595215 (7.068895153377367) + | > loss_mel: 19.537513732910156 (17.729253851849098) + | > loss_duration: 1.5248165130615234 (1.4586246428282361) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.04645538330078 (30.467859903971355) + | > grad_norm_1: tensor(134.7407, device='cuda:0') (tensor(244.8593, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.3943 (0.36347297309101495) + | > loader_time: 0.0062 (0.0055429486260897875) + + + --> TIME: 2025-04-19 02:14:46 -- STEP: 94/142 -- GLOBAL_STEP: 407350 + | > loss_disc: 2.5617501735687256 (2.476433195966354) + | > loss_disc_real_0: 0.1677805632352829 (0.1281181740316939) + | > loss_disc_real_1: 0.20988163352012634 (0.20673025383594187) + | > loss_disc_real_2: 0.22127652168273926 (0.22570108051629775) + | > loss_disc_real_3: 0.2435433715581894 (0.22975253805201104) + | > loss_disc_real_4: 0.23741595447063446 (0.23203755471300572) + | > loss_disc_real_5: 0.2135593146085739 (0.23467807344933775) + | > loss_0: 2.5617501735687256 (2.476433195966354) + | > grad_norm_0: tensor(5.7347, device='cuda:0') (tensor(11.2590, device='cuda:0')) + | > loss_gen: 2.191633701324463 (2.338562640737979) + | > loss_kl: 1.831747055053711 (1.872085306238621) + | > loss_feat: 6.251621246337891 (7.009940005363302) + | > loss_mel: 16.76889419555664 (17.75384555978977) + | > loss_duration: 1.5059791803359985 (1.470205511184449) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.549875259399414 (30.444638901568474) + | > grad_norm_1: tensor(108.7664, device='cuda:0') (tensor(220.1252, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.3979 (0.37231056740943436) + | > loader_time: 0.007 (0.005857911515743173) + + + --> TIME: 2025-04-19 02:14:56 -- STEP: 119/142 -- GLOBAL_STEP: 407375 + | > loss_disc: 2.612133264541626 (2.4777273029840283) + | > loss_disc_real_0: 0.1716248095035553 (0.13005531579256063) + | > loss_disc_real_1: 0.17839646339416504 (0.2070307248279828) + | > loss_disc_real_2: 0.21474865078926086 (0.225534259271221) + | > loss_disc_real_3: 0.23787344992160797 (0.22969116283064128) + | > loss_disc_real_4: 0.2128092646598816 (0.23117041212170064) + | > loss_disc_real_5: 0.2790163457393646 (0.23444762738311992) + | > loss_0: 2.612133264541626 (2.4777273029840283) + | > grad_norm_0: tensor(8.5468, device='cuda:0') (tensor(11.1387, device='cuda:0')) + | > loss_gen: 2.130474328994751 (2.3317355949337744) + | > loss_kl: 1.9030601978302002 (1.8884755423089035) + | > loss_feat: 5.551858901977539 (6.940830887866621) + | > loss_mel: 15.681052207946777 (17.673513043828372) + | > loss_duration: 1.558782935142517 (1.4774038981990654) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.825227737426758 (30.311958841916894) + | > grad_norm_1: tensor(194.9640, device='cuda:0') (tensor(218.3449, device='cuda:0')) + | > current_lr_0: 0.00019513484401609586 + | > current_lr_1: 0.00019513484401609586 + | > step_time: 0.427 (0.3822935328764075) + | > loader_time: 0.0071 (0.006142203547373539) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.156254291534424 (2.156254291534424) + | > loss_disc_real_0: 0.08468326926231384 (0.08468326926231384) + | > loss_disc_real_1: 0.16721592843532562 (0.16721592843532562) + | > loss_disc_real_2: 0.20386303961277008 (0.20386303961277008) + | > loss_disc_real_3: 0.18982484936714172 (0.18982484936714172) + | > loss_disc_real_4: 0.17959420382976532 (0.17959420382976532) + | > loss_disc_real_5: 0.1952466517686844 (0.1952466517686844) + | > loss_0: 2.156254291534424 (2.156254291534424) + | > loss_gen: 2.3832876682281494 (2.3832876682281494) + | > loss_kl: 2.2150344848632812 (2.2150344848632812) + | > loss_feat: 8.069035530090332 (8.069035530090332) + | > loss_mel: 20.814239501953125 (20.814239501953125) + | > loss_duration: 1.7052924633026123 (1.7052924633026123) + | > loss_1: 35.1868896484375 (35.1868896484375) + + --> STEP: 1 + | > loss_disc: 2.492600440979004 (2.492600440979004) + | > loss_disc_real_0: 0.13460223376750946 (0.13460223376750946) + | > loss_disc_real_1: 0.24297988414764404 (0.24297988414764404) + | > loss_disc_real_2: 0.25497007369995117 (0.25497007369995117) + | > loss_disc_real_3: 0.20736801624298096 (0.20736801624298096) + | > loss_disc_real_4: 0.18379518389701843 (0.18379518389701843) + | > loss_disc_real_5: 0.17458751797676086 (0.17458751797676086) + | > loss_0: 2.492600440979004 (2.492600440979004) + | > loss_gen: 2.081343650817871 (2.081343650817871) + | > loss_kl: 2.518543243408203 (2.518543243408203) + | > loss_feat: 7.180339813232422 (7.180339813232422) + | > loss_mel: 18.6678466796875 (18.6678466796875) + | > loss_duration: 1.7086101770401 (1.7086101770401) + | > loss_1: 32.15668487548828 (32.15668487548828) + + --> STEP: 2 + | > loss_disc: 2.320119619369507 (2.4063600301742554) + | > loss_disc_real_0: 0.15942415595054626 (0.14701319485902786) + | > loss_disc_real_1: 0.25667819380760193 (0.24982903897762299) + | > loss_disc_real_2: 0.22946900129318237 (0.24221953749656677) + | > loss_disc_real_3: 0.1913536936044693 (0.19936085492372513) + | > loss_disc_real_4: 0.18170969188213348 (0.18275243788957596) + | > loss_disc_real_5: 0.18959836661815643 (0.18209294229745865) + | > loss_0: 2.320119619369507 (2.4063600301742554) + | > loss_gen: 2.435664653778076 (2.2585041522979736) + | > loss_kl: 2.097471237182617 (2.30800724029541) + | > loss_feat: 7.606435775756836 (7.393387794494629) + | > loss_mel: 17.948612213134766 (18.308229446411133) + | > loss_duration: 1.6556739807128906 (1.6821420788764954) + | > loss_1: 31.743858337402344 (31.950271606445312) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003298521041870117 (-0.0002803802490234375) + | > avg_loss_disc: 2.4063600301742554 (-0.21626198291778564) + | > avg_loss_disc_real_0: 0.14701319485902786 (-0.1795988753437996) + | > avg_loss_disc_real_1: 0.24982903897762299 (+0.044725432991981506) + | > avg_loss_disc_real_2: 0.24221953749656677 (+0.0020887255668640137) + | > avg_loss_disc_real_3: 0.19936085492372513 (-0.02076088637113571) + | > avg_loss_disc_real_4: 0.18275243788957596 (-0.06603603065013885) + | > avg_loss_disc_real_5: 0.18209294229745865 (-0.05013849586248398) + | > avg_loss_0: 2.4063600301742554 (-0.21626198291778564) + | > avg_loss_gen: 2.2585041522979736 (-0.38715994358062744) + | > avg_loss_kl: 2.30800724029541 (-0.16781902313232422) + | > avg_loss_feat: 7.393387794494629 (+0.7344882488250732) + | > avg_loss_mel: 18.308229446411133 (+1.4027643203735352) + | > avg_loss_duration: 1.6821420788764954 (-0.0018693208694458008) + | > avg_loss_1: 31.950271606445312 (+1.5804052352905273) + + + > EPOCH: 198/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:15:08)  + + --> TIME: 2025-04-19 02:15:10 -- STEP: 2/142 -- GLOBAL_STEP: 407400 + | > loss_disc: 2.56396484375 (2.535443902015686) + | > loss_disc_real_0: 0.18431618809700012 (0.16800504177808762) + | > loss_disc_real_1: 0.24303419888019562 (0.23658493906259537) + | > loss_disc_real_2: 0.236612930893898 (0.24625452607870102) + | > loss_disc_real_3: 0.2736566364765167 (0.2426494061946869) + | > loss_disc_real_4: 0.2400980293750763 (0.22731589525938034) + | > loss_disc_real_5: 0.23517023026943207 (0.2208385467529297) + | > loss_0: 2.56396484375 (2.535443902015686) + | > grad_norm_0: tensor(6.1347, device='cuda:0') (tensor(6.3527, device='cuda:0')) + | > loss_gen: 2.3473563194274902 (2.389647126197815) + | > loss_kl: 1.8628824949264526 (1.8436248302459717) + | > loss_feat: 7.098923206329346 (7.058374881744385) + | > loss_mel: 16.60132598876953 (16.477011680603027) + | > loss_duration: 1.4553526639938354 (1.4310796856880188) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.365840911865234 (29.19973850250244) + | > grad_norm_1: tensor(33.0917, device='cuda:0') (tensor(48.8424, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.3246 (0.32623815536499023) + | > loader_time: 0.0044 (0.004301190376281738) + + + --> TIME: 2025-04-19 02:15:18 -- STEP: 27/142 -- GLOBAL_STEP: 407425 + | > loss_disc: 2.4739978313446045 (2.481647429642854) + | > loss_disc_real_0: 0.13119851052761078 (0.12407499468988842) + | > loss_disc_real_1: 0.20361362397670746 (0.20967982764597293) + | > loss_disc_real_2: 0.24670995771884918 (0.22987843498035712) + | > loss_disc_real_3: 0.22729091346263885 (0.23082121047708723) + | > loss_disc_real_4: 0.2022613137960434 (0.2365412794881397) + | > loss_disc_real_5: 0.22424206137657166 (0.23518028579376363) + | > loss_0: 2.4739978313446045 (2.481647429642854) + | > grad_norm_0: tensor(12.9640, device='cuda:0') (tensor(7.6277, device='cuda:0')) + | > loss_gen: 2.2451069355010986 (2.343384521978873) + | > loss_kl: 1.7599596977233887 (1.8470714886983235) + | > loss_feat: 7.005555629730225 (7.0576483585216385) + | > loss_mel: 17.567474365234375 (17.835378929420752) + | > loss_duration: 1.4452763795852661 (1.4400856803964686) + | > amp_scaler: 512.0 (284.44444444444446) + | > loss_1: 30.023372650146484 (30.523569107055664) + | > grad_norm_1: tensor(358.0468, device='cuda:0') (tensor(145.8628, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.3481 (0.3358857719986527) + | > loader_time: 0.0052 (0.005060814045093677) + + + --> TIME: 2025-04-19 02:15:28 -- STEP: 52/142 -- GLOBAL_STEP: 407450 + | > loss_disc: 2.532404661178589 (2.463466185789841) + | > loss_disc_real_0: 0.1425413191318512 (0.12411809311463283) + | > loss_disc_real_1: 0.2169228196144104 (0.2082488276064396) + | > loss_disc_real_2: 0.2603278160095215 (0.22793703039105123) + | > loss_disc_real_3: 0.22748444974422455 (0.22942121240955132) + | > loss_disc_real_4: 0.26058319211006165 (0.23186288831325677) + | > loss_disc_real_5: 0.20925958454608917 (0.23074783270175642) + | > loss_0: 2.532404661178589 (2.463466185789841) + | > grad_norm_0: tensor(4.7322, device='cuda:0') (tensor(6.7909, device='cuda:0')) + | > loss_gen: 2.3299710750579834 (2.345876019734603) + | > loss_kl: 1.8114995956420898 (1.8531876114698558) + | > loss_feat: 6.064524173736572 (7.026374147488521) + | > loss_mel: 17.069660186767578 (17.7490066748399) + | > loss_duration: 1.508164644241333 (1.4465168897922223) + | > amp_scaler: 512.0 (393.84615384615387) + | > loss_1: 28.78382110595703 (30.4209613433251) + | > grad_norm_1: tensor(66.3450, device='cuda:0') (tensor(122.2737, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.3641 (0.3473486579381503) + | > loader_time: 0.0058 (0.005354890456566444) + + + --> TIME: 2025-04-19 02:15:37 -- STEP: 77/142 -- GLOBAL_STEP: 407475 + | > loss_disc: 2.5426580905914307 (2.4645134164141362) + | > loss_disc_real_0: 0.11991914361715317 (0.12660426414245132) + | > loss_disc_real_1: 0.24364939332008362 (0.2088507464179745) + | > loss_disc_real_2: 0.21389468014240265 (0.22691195119510998) + | > loss_disc_real_3: 0.20776857435703278 (0.22906034488182564) + | > loss_disc_real_4: 0.2036714404821396 (0.23102359512409607) + | > loss_disc_real_5: 0.23470045626163483 (0.23065334519782624) + | > loss_0: 2.5426580905914307 (2.4645134164141362) + | > grad_norm_0: tensor(10.4764, device='cuda:0') (tensor(7.0185, device='cuda:0')) + | > loss_gen: 2.2463300228118896 (2.345118082963027) + | > loss_kl: 1.9263724088668823 (1.8754276997083192) + | > loss_feat: 6.719484806060791 (7.046190038903967) + | > loss_mel: 17.331769943237305 (17.813444806383803) + | > loss_duration: 1.4818031787872314 (1.457821556500026) + | > amp_scaler: 512.0 (432.2077922077923) + | > loss_1: 29.705760955810547 (30.538002212326248) + | > grad_norm_1: tensor(182.9044, device='cuda:0') (tensor(137.1790, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.3869 (0.35912501657163937) + | > loader_time: 0.0061 (0.00561869918525993) + + + --> TIME: 2025-04-19 02:15:48 -- STEP: 102/142 -- GLOBAL_STEP: 407500 + | > loss_disc: 2.5973100662231445 (2.474576454536587) + | > loss_disc_real_0: 0.13419109582901 (0.1272265442153986) + | > loss_disc_real_1: 0.2341080754995346 (0.20898383402941273) + | > loss_disc_real_2: 0.2535105049610138 (0.22732695542714176) + | > loss_disc_real_3: 0.2544850707054138 (0.22977317197650088) + | > loss_disc_real_4: 0.25122615694999695 (0.23199385083189197) + | > loss_disc_real_5: 0.27344560623168945 (0.23204554953411513) + | > loss_0: 2.5973100662231445 (2.474576454536587) + | > grad_norm_0: tensor(12.1164, device='cuda:0') (tensor(8.0361, device='cuda:0')) + | > loss_gen: 2.2023072242736816 (2.3353671560100473) + | > loss_kl: 2.059396505355835 (1.8911480027086593) + | > loss_feat: 6.501624584197998 (6.998321729547837) + | > loss_mel: 17.30176544189453 (17.818974532333076) + | > loss_duration: 1.4898475408554077 (1.4694412490900826) + | > amp_scaler: 512.0 (451.764705882353) + | > loss_1: 29.55493927001953 (30.513252707088697) + | > grad_norm_1: tensor(335.9612, device='cuda:0') (tensor(174.8514, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.4194 (0.3697156882753558) + | > loader_time: 0.0065 (0.0058410167694091806) + + + --> TIME: 2025-04-19 02:15:59 -- STEP: 127/142 -- GLOBAL_STEP: 407525 + | > loss_disc: 2.4305782318115234 (2.4723869946997934) + | > loss_disc_real_0: 0.10296323150396347 (0.1286740263146678) + | > loss_disc_real_1: 0.2083410918712616 (0.20811003939373288) + | > loss_disc_real_2: 0.20376954972743988 (0.22621072394641364) + | > loss_disc_real_3: 0.22592297196388245 (0.22927069546669487) + | > loss_disc_real_4: 0.23445433378219604 (0.23105432726736144) + | > loss_disc_real_5: 0.23899731040000916 (0.23266582261389634) + | > loss_0: 2.4305782318115234 (2.4723869946997934) + | > grad_norm_0: tensor(14.6397, device='cuda:0') (tensor(9.6724, device='cuda:0')) + | > loss_gen: 2.4796345233917236 (2.340330003753421) + | > loss_kl: 1.9123306274414062 (1.8911899910198422) + | > loss_feat: 7.384866714477539 (6.967319890270083) + | > loss_mel: 16.90553092956543 (17.751333829924814) + | > loss_duration: 1.5034077167510986 (1.476591817975983) + | > amp_scaler: 256.0 (439.4330708661418) + | > loss_1: 30.185771942138672 (30.426765607097956) + | > grad_norm_1: tensor(465.3156, device='cuda:0') (tensor(200.5894, device='cuda:0')) + | > current_lr_0: 0.00019511045216059385 + | > current_lr_1: 0.00019511045216059385 + | > step_time: 0.4399 (0.38139153277780113) + | > loader_time: 0.0072 (0.0061302579294039514) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5373740196228027 (2.5373740196228027) + | > loss_disc_real_0: 0.2506621479988098 (0.2506621479988098) + | > loss_disc_real_1: 0.2768995761871338 (0.2768995761871338) + | > loss_disc_real_2: 0.24648398160934448 (0.24648398160934448) + | > loss_disc_real_3: 0.2823749780654907 (0.2823749780654907) + | > loss_disc_real_4: 0.21966552734375 (0.21966552734375) + | > loss_disc_real_5: 0.21075451374053955 (0.21075451374053955) + | > loss_0: 2.5373740196228027 (2.5373740196228027) + | > loss_gen: 2.6390247344970703 (2.6390247344970703) + | > loss_kl: 2.2179384231567383 (2.2179384231567383) + | > loss_feat: 6.6481451988220215 (6.6481451988220215) + | > loss_mel: 17.776823043823242 (17.776823043823242) + | > loss_duration: 1.6727373600006104 (1.6727373600006104) + | > loss_1: 30.954668045043945 (30.954668045043945) + + --> STEP: 1 + | > loss_disc: 2.644449472427368 (2.644449472427368) + | > loss_disc_real_0: 0.25106555223464966 (0.25106555223464966) + | > loss_disc_real_1: 0.24354104697704315 (0.24354104697704315) + | > loss_disc_real_2: 0.22604496777057648 (0.22604496777057648) + | > loss_disc_real_3: 0.2758024334907532 (0.2758024334907532) + | > loss_disc_real_4: 0.26510217785835266 (0.26510217785835266) + | > loss_disc_real_5: 0.26811468601226807 (0.26811468601226807) + | > loss_0: 2.644449472427368 (2.644449472427368) + | > loss_gen: 2.4064841270446777 (2.4064841270446777) + | > loss_kl: 2.3844590187072754 (2.3844590187072754) + | > loss_feat: 6.395456790924072 (6.395456790924072) + | > loss_mel: 18.205223083496094 (18.205223083496094) + | > loss_duration: 1.7103922367095947 (1.7103922367095947) + | > loss_1: 31.102014541625977 (31.102014541625977) + + --> STEP: 2 + | > loss_disc: 2.5592172145843506 (2.6018333435058594) + | > loss_disc_real_0: 0.2622588276863098 (0.25666218996047974) + | > loss_disc_real_1: 0.22836615145206451 (0.23595359921455383) + | > loss_disc_real_2: 0.207954540848732 (0.21699975430965424) + | > loss_disc_real_3: 0.291475385427475 (0.2836389094591141) + | > loss_disc_real_4: 0.19251753389835358 (0.22880985587835312) + | > loss_disc_real_5: 0.26665034890174866 (0.26738251745700836) + | > loss_0: 2.5592172145843506 (2.6018333435058594) + | > loss_gen: 2.608290433883667 (2.5073872804641724) + | > loss_kl: 2.194164752960205 (2.2893118858337402) + | > loss_feat: 7.569428443908691 (6.982442617416382) + | > loss_mel: 19.019119262695312 (18.612171173095703) + | > loss_duration: 1.6750763654708862 (1.6927343010902405) + | > loss_1: 33.066078186035156 (32.084046363830566) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037255287170410156 (+0.00042700767517089844) + | > avg_loss_disc: 2.6018333435058594 (+0.195473313331604) + | > avg_loss_disc_real_0: 0.25666218996047974 (+0.10964899510145187) + | > avg_loss_disc_real_1: 0.23595359921455383 (-0.013875439763069153) + | > avg_loss_disc_real_2: 0.21699975430965424 (-0.025219783186912537) + | > avg_loss_disc_real_3: 0.2836389094591141 (+0.08427805453538895) + | > avg_loss_disc_real_4: 0.22880985587835312 (+0.04605741798877716) + | > avg_loss_disc_real_5: 0.26738251745700836 (+0.08528957515954971) + | > avg_loss_0: 2.6018333435058594 (+0.195473313331604) + | > avg_loss_gen: 2.5073872804641724 (+0.24888312816619873) + | > avg_loss_kl: 2.2893118858337402 (-0.018695354461669922) + | > avg_loss_feat: 6.982442617416382 (-0.41094517707824707) + | > avg_loss_mel: 18.612171173095703 (+0.3039417266845703) + | > avg_loss_duration: 1.6927343010902405 (+0.010592222213745117) + | > avg_loss_1: 32.084046363830566 (+0.1337747573852539) + + + > EPOCH: 199/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:16:07)  + + --> TIME: 2025-04-19 02:16:11 -- STEP: 10/142 -- GLOBAL_STEP: 407550 + | > loss_disc: 2.5888941287994385 (2.5206531047821046) + | > loss_disc_real_0: 0.13510428369045258 (0.12615702152252198) + | > loss_disc_real_1: 0.1889955848455429 (0.21461160182952882) + | > loss_disc_real_2: 0.25464335083961487 (0.24059569090604782) + | > loss_disc_real_3: 0.2407035529613495 (0.23533043414354324) + | > loss_disc_real_4: 0.2778192460536957 (0.25139830708503724) + | > loss_disc_real_5: 0.20420721173286438 (0.23311284482479094) + | > loss_0: 2.5888941287994385 (2.5206531047821046) + | > grad_norm_0: tensor(18.6461, device='cuda:0') (tensor(16.7925, device='cuda:0')) + | > loss_gen: 2.16231632232666 (2.3332509279251097) + | > loss_kl: 1.9370554685592651 (1.8689472436904908) + | > loss_feat: 6.614844799041748 (7.002597379684448) + | > loss_mel: 16.56835174560547 (17.797340202331544) + | > loss_duration: 1.4137589931488037 (1.4212949752807618) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.696325302124023 (30.42343044281006) + | > grad_norm_1: tensor(289.7295, device='cuda:0') (tensor(303.8885, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.3373 (0.3259379148483276) + | > loader_time: 0.0049 (0.004603147506713867) + + + --> TIME: 2025-04-19 02:16:20 -- STEP: 35/142 -- GLOBAL_STEP: 407575 + | > loss_disc: 2.40986967086792 (2.4666902473994665) + | > loss_disc_real_0: 0.11671413481235504 (0.119108875308718) + | > loss_disc_real_1: 0.22206060588359833 (0.2084226186786379) + | > loss_disc_real_2: 0.2220430076122284 (0.227958835874285) + | > loss_disc_real_3: 0.24631740152835846 (0.23178836873599462) + | > loss_disc_real_4: 0.23293963074684143 (0.23612050669533866) + | > loss_disc_real_5: 0.2226046472787857 (0.2318781601531165) + | > loss_0: 2.40986967086792 (2.4666902473994665) + | > grad_norm_0: tensor(7.2959, device='cuda:0') (tensor(10.7856, device='cuda:0')) + | > loss_gen: 2.3866844177246094 (2.3502791132245746) + | > loss_kl: 1.8770198822021484 (1.8619023254939489) + | > loss_feat: 8.130603790283203 (7.1481204577854704) + | > loss_mel: 19.304645538330078 (17.928868430001394) + | > loss_duration: 1.4611876010894775 (1.4406177282333374) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.16014099121094 (30.729787935529437) + | > grad_norm_1: tensor(339.1748, device='cuda:0') (tensor(238.9922, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.3747 (0.3414531912122454) + | > loader_time: 0.0056 (0.005168206351143975) + + + --> TIME: 2025-04-19 02:16:29 -- STEP: 60/142 -- GLOBAL_STEP: 407600 + | > loss_disc: 2.460453748703003 (2.4616352756818127) + | > loss_disc_real_0: 0.11745177954435349 (0.12097978939612707) + | > loss_disc_real_1: 0.18099647760391235 (0.20822316507498423) + | > loss_disc_real_2: 0.23984354734420776 (0.22538725112875302) + | > loss_disc_real_3: 0.20657874643802643 (0.23026312390963236) + | > loss_disc_real_4: 0.20412321388721466 (0.23319323882460594) + | > loss_disc_real_5: 0.19145743548870087 (0.23001681566238402) + | > loss_0: 2.460453748703003 (2.4616352756818127) + | > grad_norm_0: tensor(7.8380, device='cuda:0') (tensor(9.2233, device='cuda:0')) + | > loss_gen: 2.265169620513916 (2.3474630435307824) + | > loss_kl: 1.980266809463501 (1.8728169222672781) + | > loss_feat: 7.227426052093506 (7.089218616485596) + | > loss_mel: 17.50341796875 (17.811069202423095) + | > loss_duration: 1.5039384365081787 (1.4525332808494567) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.4802188873291 (30.573100980122884) + | > grad_norm_1: tensor(152.4964, device='cuda:0') (tensor(218.2980, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.3763 (0.3531156897544861) + | > loader_time: 0.0065 (0.005476804574330649) + + + --> TIME: 2025-04-19 02:16:39 -- STEP: 85/142 -- GLOBAL_STEP: 407625 + | > loss_disc: 2.4634926319122314 (2.462116190966436) + | > loss_disc_real_0: 0.1458999365568161 (0.12461726612904493) + | > loss_disc_real_1: 0.21644069254398346 (0.20773802890497095) + | > loss_disc_real_2: 0.22898539900779724 (0.22636063169030582) + | > loss_disc_real_3: 0.231948584318161 (0.2302051240906996) + | > loss_disc_real_4: 0.19360263645648956 (0.23068654151523815) + | > loss_disc_real_5: 0.2289784997701645 (0.23004260343663832) + | > loss_0: 2.4634926319122314 (2.462116190966436) + | > grad_norm_0: tensor(9.2108, device='cuda:0') (tensor(9.1470, device='cuda:0')) + | > loss_gen: 2.349393367767334 (2.3468017073238605) + | > loss_kl: 1.8155536651611328 (1.8809415003832648) + | > loss_feat: 6.589774131774902 (7.042100782955394) + | > loss_mel: 17.20768165588379 (17.76125771017636) + | > loss_duration: 1.4860732555389404 (1.4643163386513203) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.448476791381836 (30.4954179427203) + | > grad_norm_1: tensor(344.8026, device='cuda:0') (tensor(217.7759, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.3959 (0.36399278079762176) + | > loader_time: 0.0063 (0.005800625857184915) + + + --> TIME: 2025-04-19 02:16:50 -- STEP: 110/142 -- GLOBAL_STEP: 407650 + | > loss_disc: 2.527367353439331 (2.4719238736412743) + | > loss_disc_real_0: 0.11968191713094711 (0.1257790077139031) + | > loss_disc_real_1: 0.20398254692554474 (0.20897639705376192) + | > loss_disc_real_2: 0.20735406875610352 (0.22701892920515754) + | > loss_disc_real_3: 0.22760802507400513 (0.2295001756061207) + | > loss_disc_real_4: 0.2497219741344452 (0.23162897743962027) + | > loss_disc_real_5: 0.2152201384305954 (0.23092755431478673) + | > loss_0: 2.527367353439331 (2.4719238736412743) + | > grad_norm_0: tensor(7.4894, device='cuda:0') (tensor(9.6372, device='cuda:0')) + | > loss_gen: 2.2847681045532227 (2.337989804961464) + | > loss_kl: 1.9960229396820068 (1.891606523773887) + | > loss_feat: 6.384723663330078 (7.0130839998071846) + | > loss_mel: 17.268220901489258 (17.761654732444075) + | > loss_duration: 1.503629207611084 (1.4731456117196517) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.43736457824707 (30.477480576255104) + | > grad_norm_1: tensor(213.6027, device='cuda:0') (tensor(228.3914, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.4305 (0.3745346654545177) + | > loader_time: 0.0072 (0.006047491593794389) + + + --> TIME: 2025-04-19 02:17:01 -- STEP: 135/142 -- GLOBAL_STEP: 407675 + | > loss_disc: 2.459667921066284 (2.4680679533216687) + | > loss_disc_real_0: 0.14147347211837769 (0.12657110486869463) + | > loss_disc_real_1: 0.25214093923568726 (0.20930926932228935) + | > loss_disc_real_2: 0.24163269996643066 (0.22689735293388366) + | > loss_disc_real_3: 0.2276705652475357 (0.22867417269282866) + | > loss_disc_real_4: 0.2307238131761551 (0.23089337547620137) + | > loss_disc_real_5: 0.27512189745903015 (0.23001719768400544) + | > loss_0: 2.459667921066284 (2.4680679533216687) + | > grad_norm_0: tensor(13.8732, device='cuda:0') (tensor(10.2219, device='cuda:0')) + | > loss_gen: 2.344086170196533 (2.3395044980225737) + | > loss_kl: 1.9166885614395142 (1.8986507247995448) + | > loss_feat: 6.895463943481445 (6.983787257583053) + | > loss_mel: 18.333513259887695 (17.705752111364298) + | > loss_duration: 1.4746077060699463 (1.4795318797782615) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.964359283447266 (30.407226378829392) + | > grad_norm_1: tensor(191.3498, device='cuda:0') (tensor(237.9062, device='cuda:0')) + | > current_lr_0: 0.00019508606335407377 + | > current_lr_1: 0.00019508606335407377 + | > step_time: 0.455 (0.38614486588372127) + | > loader_time: 0.0076 (0.006317532504046405) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4414656162261963 (2.4414656162261963) + | > loss_disc_real_0: 0.1334524154663086 (0.1334524154663086) + | > loss_disc_real_1: 0.23663979768753052 (0.23663979768753052) + | > loss_disc_real_2: 0.2805587649345398 (0.2805587649345398) + | > loss_disc_real_3: 0.26172947883605957 (0.26172947883605957) + | > loss_disc_real_4: 0.2951042652130127 (0.2951042652130127) + | > loss_disc_real_5: 0.25916096568107605 (0.25916096568107605) + | > loss_0: 2.4414656162261963 (2.4414656162261963) + | > loss_gen: 2.653801679611206 (2.653801679611206) + | > loss_kl: 2.3022758960723877 (2.3022758960723877) + | > loss_feat: 6.49597692489624 (6.49597692489624) + | > loss_mel: 18.630746841430664 (18.630746841430664) + | > loss_duration: 1.6733455657958984 (1.6733455657958984) + | > loss_1: 31.756147384643555 (31.756147384643555) + + --> STEP: 1 + | > loss_disc: 2.5840699672698975 (2.5840699672698975) + | > loss_disc_real_0: 0.1969265192747116 (0.1969265192747116) + | > loss_disc_real_1: 0.2573980689048767 (0.2573980689048767) + | > loss_disc_real_2: 0.29659444093704224 (0.29659444093704224) + | > loss_disc_real_3: 0.29683414101600647 (0.29683414101600647) + | > loss_disc_real_4: 0.23819586634635925 (0.23819586634635925) + | > loss_disc_real_5: 0.275147020816803 (0.275147020816803) + | > loss_0: 2.5840699672698975 (2.5840699672698975) + | > loss_gen: 2.5811586380004883 (2.5811586380004883) + | > loss_kl: 2.7465665340423584 (2.7465665340423584) + | > loss_feat: 7.195449352264404 (7.195449352264404) + | > loss_mel: 18.53131866455078 (18.53131866455078) + | > loss_duration: 1.7000707387924194 (1.7000707387924194) + | > loss_1: 32.75456237792969 (32.75456237792969) + + --> STEP: 2 + | > loss_disc: 2.5292844772338867 (2.556677222251892) + | > loss_disc_real_0: 0.17980021238327026 (0.18836336582899094) + | > loss_disc_real_1: 0.2887452244758606 (0.27307164669036865) + | > loss_disc_real_2: 0.2718791961669922 (0.2842368185520172) + | > loss_disc_real_3: 0.26557812094688416 (0.2812061309814453) + | > loss_disc_real_4: 0.2813156545162201 (0.2597557604312897) + | > loss_disc_real_5: 0.24144265055656433 (0.25829483568668365) + | > loss_0: 2.5292844772338867 (2.556677222251892) + | > loss_gen: 2.531294345855713 (2.5562264919281006) + | > loss_kl: 2.0703697204589844 (2.4084681272506714) + | > loss_feat: 7.279134273529053 (7.2372918128967285) + | > loss_mel: 19.19582748413086 (18.86357307434082) + | > loss_duration: 1.6544466018676758 (1.6772586703300476) + | > loss_1: 32.73107147216797 (32.74281692504883) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034732818603515625 (-0.0002522468566894531) + | > avg_loss_disc: 2.556677222251892 (-0.045156121253967285) + | > avg_loss_disc_real_0: 0.18836336582899094 (-0.0682988241314888) + | > avg_loss_disc_real_1: 0.27307164669036865 (+0.03711804747581482) + | > avg_loss_disc_real_2: 0.2842368185520172 (+0.06723706424236298) + | > avg_loss_disc_real_3: 0.2812061309814453 (-0.002432778477668762) + | > avg_loss_disc_real_4: 0.2597557604312897 (+0.030945904552936554) + | > avg_loss_disc_real_5: 0.25829483568668365 (-0.009087681770324707) + | > avg_loss_0: 2.556677222251892 (-0.045156121253967285) + | > avg_loss_gen: 2.5562264919281006 (+0.04883921146392822) + | > avg_loss_kl: 2.4084681272506714 (+0.11915624141693115) + | > avg_loss_feat: 7.2372918128967285 (+0.2548491954803467) + | > avg_loss_mel: 18.86357307434082 (+0.2514019012451172) + | > avg_loss_duration: 1.6772586703300476 (-0.015475630760192871) + | > avg_loss_1: 32.74281692504883 (+0.6587705612182617) + + + > EPOCH: 200/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:17:06)  + + --> TIME: 2025-04-19 02:17:12 -- STEP: 18/142 -- GLOBAL_STEP: 407700 + | > loss_disc: 2.265990972518921 (2.4426876836352878) + | > loss_disc_real_0: 0.09021544456481934 (0.1160837858915329) + | > loss_disc_real_1: 0.20695467293262482 (0.2081441374288665) + | > loss_disc_real_2: 0.19231553375720978 (0.23060466680261824) + | > loss_disc_real_3: 0.21371591091156006 (0.22503140651517445) + | > loss_disc_real_4: 0.22396449744701385 (0.22440587646431392) + | > loss_disc_real_5: 0.22089092433452606 (0.23408929424153435) + | > loss_0: 2.265990972518921 (2.4426876836352878) + | > grad_norm_0: tensor(18.9282, device='cuda:0') (tensor(13.1487, device='cuda:0')) + | > loss_gen: 2.545548439025879 (2.3802686002519398) + | > loss_kl: 1.7793869972229004 (1.7888492080900404) + | > loss_feat: 8.006885528564453 (7.2673656940460205) + | > loss_mel: 18.265779495239258 (17.9370330174764) + | > loss_duration: 1.4412479400634766 (1.4280521207385592) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.038848876953125 (30.801569090949165) + | > grad_norm_1: tensor(571.4719, device='cuda:0') (tensor(321.0478, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.3354 (0.32928261491987443) + | > loader_time: 0.0049 (0.0048298173480563695) + + + --> TIME: 2025-04-19 02:17:22 -- STEP: 43/142 -- GLOBAL_STEP: 407725 + | > loss_disc: 2.397688388824463 (2.4540521155956188) + | > loss_disc_real_0: 0.22382186353206635 (0.12495878997237184) + | > loss_disc_real_1: 0.20842908322811127 (0.20700878008853557) + | > loss_disc_real_2: 0.2191859483718872 (0.2270187486049741) + | > loss_disc_real_3: 0.19869782030582428 (0.2270608741876691) + | > loss_disc_real_4: 0.2140895277261734 (0.2242258645074312) + | > loss_disc_real_5: 0.2242310345172882 (0.2335496738899586) + | > loss_0: 2.397688388824463 (2.4540521155956188) + | > grad_norm_0: tensor(26.2979, device='cuda:0') (tensor(18.5607, device='cuda:0')) + | > loss_gen: 2.579414129257202 (2.4060091417889264) + | > loss_kl: 1.809880256652832 (1.8185423196748245) + | > loss_feat: 7.595980644226074 (7.246765657912853) + | > loss_mel: 19.035768508911133 (17.844651200050524) + | > loss_duration: 1.4589800834655762 (1.4429050129513405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.48002243041992 (30.758873385052347) + | > grad_norm_1: tensor(258.4391, device='cuda:0') (tensor(349.1325, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.374 (0.34686152879581905) + | > loader_time: 0.0058 (0.005283610765324083) + + + --> TIME: 2025-04-19 02:17:31 -- STEP: 68/142 -- GLOBAL_STEP: 407750 + | > loss_disc: 2.3963088989257812 (2.4580614146064317) + | > loss_disc_real_0: 0.19313453137874603 (0.1255004568682874) + | > loss_disc_real_1: 0.18065665662288666 (0.20739066118703167) + | > loss_disc_real_2: 0.21613632142543793 (0.2275845741524416) + | > loss_disc_real_3: 0.19560670852661133 (0.22787328217835986) + | > loss_disc_real_4: 0.19746820628643036 (0.22439496153417757) + | > loss_disc_real_5: 0.21172550320625305 (0.23097313830957694) + | > loss_0: 2.3963088989257812 (2.4580614146064317) + | > grad_norm_0: tensor(16.0078, device='cuda:0') (tensor(19.1578, device='cuda:0')) + | > loss_gen: 2.4175405502319336 (2.3976462623652295) + | > loss_kl: 1.8405965566635132 (1.8371326309793137) + | > loss_feat: 6.842300891876221 (7.192816068144405) + | > loss_mel: 16.653196334838867 (17.829865567824413) + | > loss_duration: 1.4775669574737549 (1.4544060826301572) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.231203079223633 (30.71186649098116) + | > grad_norm_1: tensor(216.4523, device='cuda:0') (tensor(355.5545, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.3986 (0.3609455964144539) + | > loader_time: 0.0071 (0.005634213195127601) + + + --> TIME: 2025-04-19 02:17:42 -- STEP: 93/142 -- GLOBAL_STEP: 407775 + | > loss_disc: 2.5155954360961914 (2.460978487486481) + | > loss_disc_real_0: 0.13896524906158447 (0.12539427563227631) + | > loss_disc_real_1: 0.22817255556583405 (0.2088835777774934) + | > loss_disc_real_2: 0.21092644333839417 (0.22809301428897408) + | > loss_disc_real_3: 0.22518248856067657 (0.22722797256003144) + | > loss_disc_real_4: 0.25732704997062683 (0.22592382469484884) + | > loss_disc_real_5: 0.24826538562774658 (0.23259732255371668) + | > loss_0: 2.5155954360961914 (2.460978487486481) + | > grad_norm_0: tensor(14.0514, device='cuda:0') (tensor(17.3253, device='cuda:0')) + | > loss_gen: 2.2684736251831055 (2.3773782765993494) + | > loss_kl: 1.9277420043945312 (1.8567090252394318) + | > loss_feat: 6.344740390777588 (7.137926429830571) + | > loss_mel: 16.978605270385742 (17.843213706888164) + | > loss_duration: 1.4955785274505615 (1.4670739161070954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.015140533447266 (30.682301182900705) + | > grad_norm_1: tensor(250.5079, device='cuda:0') (tensor(332.3661, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.4126 (0.3738096555074056) + | > loader_time: 0.0065 (0.005903082509194651) + + + --> TIME: 2025-04-19 02:17:53 -- STEP: 118/142 -- GLOBAL_STEP: 407800 + | > loss_disc: 2.461928606033325 (2.46767795085907) + | > loss_disc_real_0: 0.1835193634033203 (0.1278603189009226) + | > loss_disc_real_1: 0.2074948102235794 (0.20828359220492637) + | > loss_disc_real_2: 0.2498357892036438 (0.22867210461931714) + | > loss_disc_real_3: 0.23399753868579865 (0.22761297642679537) + | > loss_disc_real_4: 0.2444610595703125 (0.227768590515953) + | > loss_disc_real_5: 0.24225996434688568 (0.2325039370837858) + | > loss_0: 2.461928606033325 (2.46767795085907) + | > grad_norm_0: tensor(7.3662, device='cuda:0') (tensor(15.4512, device='cuda:0')) + | > loss_gen: 2.249419927597046 (2.363042768785507) + | > loss_kl: 1.8611292839050293 (1.8656874339459306) + | > loss_feat: 7.0493974685668945 (7.062226970317) + | > loss_mel: 19.202899932861328 (17.827838372375993) + | > loss_duration: 1.4668467044830322 (1.47425414748111) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.829692840576172 (30.593049598952472) + | > grad_norm_1: tensor(193.0766, device='cuda:0') (tensor(287.9173, device='cuda:0')) + | > current_lr_0: 0.00019506167759615451 + | > current_lr_1: 0.00019506167759615451 + | > step_time: 0.4548 (0.3861893617500693) + | > loader_time: 0.0074 (0.006154426073623916) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5252485275268555 (2.5252485275268555) + | > loss_disc_real_0: 0.16951225697994232 (0.16951225697994232) + | > loss_disc_real_1: 0.317056804895401 (0.317056804895401) + | > loss_disc_real_2: 0.21257373690605164 (0.21257373690605164) + | > loss_disc_real_3: 0.2339712679386139 (0.2339712679386139) + | > loss_disc_real_4: 0.24014239013195038 (0.24014239013195038) + | > loss_disc_real_5: 0.18671143054962158 (0.18671143054962158) + | > loss_0: 2.5252485275268555 (2.5252485275268555) + | > loss_gen: 2.442608118057251 (2.442608118057251) + | > loss_kl: 2.2232701778411865 (2.2232701778411865) + | > loss_feat: 7.298070907592773 (7.298070907592773) + | > loss_mel: 18.540403366088867 (18.540403366088867) + | > loss_duration: 1.6963281631469727 (1.6963281631469727) + | > loss_1: 32.200679779052734 (32.200679779052734) + + --> STEP: 1 + | > loss_disc: 2.274819850921631 (2.274819850921631) + | > loss_disc_real_0: 0.14904123544692993 (0.14904123544692993) + | > loss_disc_real_1: 0.22562609612941742 (0.22562609612941742) + | > loss_disc_real_2: 0.18674251437187195 (0.18674251437187195) + | > loss_disc_real_3: 0.2202591598033905 (0.2202591598033905) + | > loss_disc_real_4: 0.20611684024333954 (0.20611684024333954) + | > loss_disc_real_5: 0.1676872819662094 (0.1676872819662094) + | > loss_0: 2.274819850921631 (2.274819850921631) + | > loss_gen: 2.4436163902282715 (2.4436163902282715) + | > loss_kl: 2.4592888355255127 (2.4592888355255127) + | > loss_feat: 8.111214637756348 (8.111214637756348) + | > loss_mel: 19.706483840942383 (19.706483840942383) + | > loss_duration: 1.710853934288025 (1.710853934288025) + | > loss_1: 34.43145751953125 (34.43145751953125) + + --> STEP: 2 + | > loss_disc: 2.516922950744629 (2.39587140083313) + | > loss_disc_real_0: 0.15345057845115662 (0.15124590694904327) + | > loss_disc_real_1: 0.2554527521133423 (0.24053942412137985) + | > loss_disc_real_2: 0.2158622443675995 (0.20130237936973572) + | > loss_disc_real_3: 0.23829959332942963 (0.22927937656641006) + | > loss_disc_real_4: 0.22367285192012787 (0.2148948460817337) + | > loss_disc_real_5: 0.1908172219991684 (0.1792522519826889) + | > loss_0: 2.516922950744629 (2.39587140083313) + | > loss_gen: 2.2309083938598633 (2.3372623920440674) + | > loss_kl: 2.0172500610351562 (2.2382694482803345) + | > loss_feat: 6.937933921813965 (7.524574279785156) + | > loss_mel: 19.25849723815918 (19.48249053955078) + | > loss_duration: 1.6531561613082886 (1.6820050477981567) + | > loss_1: 32.09774398803711 (33.26460075378418) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00364840030670166 (+0.00017511844635009766) + | > avg_loss_disc: 2.39587140083313 (-0.1608058214187622) + | > avg_loss_disc_real_0: 0.15124590694904327 (-0.03711745887994766) + | > avg_loss_disc_real_1: 0.24053942412137985 (-0.0325322225689888) + | > avg_loss_disc_real_2: 0.20130237936973572 (-0.0829344391822815) + | > avg_loss_disc_real_3: 0.22927937656641006 (-0.05192675441503525) + | > avg_loss_disc_real_4: 0.2148948460817337 (-0.04486091434955597) + | > avg_loss_disc_real_5: 0.1792522519826889 (-0.07904258370399475) + | > avg_loss_0: 2.39587140083313 (-0.1608058214187622) + | > avg_loss_gen: 2.3372623920440674 (-0.2189640998840332) + | > avg_loss_kl: 2.2382694482803345 (-0.17019867897033691) + | > avg_loss_feat: 7.524574279785156 (+0.28728246688842773) + | > avg_loss_mel: 19.48249053955078 (+0.6189174652099609) + | > avg_loss_duration: 1.6820050477981567 (+0.004746377468109131) + | > avg_loss_1: 33.26460075378418 (+0.5217838287353516) + + + > EPOCH: 201/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:18:06)  + + --> TIME: 2025-04-19 02:18:07 -- STEP: 1/142 -- GLOBAL_STEP: 407825 + | > loss_disc: 2.376485824584961 (2.376485824584961) + | > loss_disc_real_0: 0.16382963955402374 (0.16382963955402374) + | > loss_disc_real_1: 0.21936096251010895 (0.21936096251010895) + | > loss_disc_real_2: 0.22381822764873505 (0.22381822764873505) + | > loss_disc_real_3: 0.2112693041563034 (0.2112693041563034) + | > loss_disc_real_4: 0.20895016193389893 (0.20895016193389893) + | > loss_disc_real_5: 0.1722627431154251 (0.1722627431154251) + | > loss_0: 2.376485824584961 (2.376485824584961) + | > grad_norm_0: tensor(8.3525, device='cuda:0') (tensor(8.3525, device='cuda:0')) + | > loss_gen: 2.3440775871276855 (2.3440775871276855) + | > loss_kl: 1.7897018194198608 (1.7897018194198608) + | > loss_feat: 6.925872802734375 (6.925872802734375) + | > loss_mel: 17.086702346801758 (17.086702346801758) + | > loss_duration: 1.3926557302474976 (1.3926557302474976) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.539011001586914 (29.539011001586914) + | > grad_norm_1: tensor(106.5827, device='cuda:0') (tensor(106.5827, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.3253 (0.32526087760925293) + | > loader_time: 0.0049 (0.004912137985229492) + + + --> TIME: 2025-04-19 02:18:15 -- STEP: 26/142 -- GLOBAL_STEP: 407850 + | > loss_disc: 2.4914145469665527 (2.463485442675077) + | > loss_disc_real_0: 0.10804565995931625 (0.13135490136650893) + | > loss_disc_real_1: 0.21390461921691895 (0.21050193103460166) + | > loss_disc_real_2: 0.21328206360340118 (0.22503134436332262) + | > loss_disc_real_3: 0.20736147463321686 (0.2247008538016906) + | > loss_disc_real_4: 0.24514040350914001 (0.22822080437953657) + | > loss_disc_real_5: 0.23231498897075653 (0.22635504259512976) + | > loss_0: 2.4914145469665527 (2.463485442675077) + | > grad_norm_0: tensor(14.3138, device='cuda:0') (tensor(9.1615, device='cuda:0')) + | > loss_gen: 2.305537700653076 (2.367448467474717) + | > loss_kl: 1.8484948873519897 (1.7930670243043165) + | > loss_feat: 6.504739284515381 (7.124086801822369) + | > loss_mel: 17.53621482849121 (17.998752850752606) + | > loss_duration: 1.4522225856781006 (1.434684625038734) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.6472110748291 (30.71804009951078) + | > grad_norm_1: tensor(305.6174, device='cuda:0') (tensor(194.9715, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.3472 (0.3332607104228093) + | > loader_time: 0.005 (0.004930881353525015) + + + --> TIME: 2025-04-19 02:18:24 -- STEP: 51/142 -- GLOBAL_STEP: 407875 + | > loss_disc: 2.554502487182617 (2.450767400217991) + | > loss_disc_real_0: 0.11908667534589767 (0.1263043142709077) + | > loss_disc_real_1: 0.22969867289066315 (0.20846070290780536) + | > loss_disc_real_2: 0.204514279961586 (0.22429349551013872) + | > loss_disc_real_3: 0.2312440723180771 (0.22560110425247865) + | > loss_disc_real_4: 0.25262483954429626 (0.22593109455763125) + | > loss_disc_real_5: 0.22058263421058655 (0.2301061515714608) + | > loss_0: 2.554502487182617 (2.450767400217991) + | > grad_norm_0: tensor(5.6885, device='cuda:0') (tensor(9.8403, device='cuda:0')) + | > loss_gen: 2.43565034866333 (2.363364775975546) + | > loss_kl: 1.9140115976333618 (1.822642494650448) + | > loss_feat: 7.741186618804932 (7.107797510483685) + | > loss_mel: 19.19477081298828 (17.97361327152626) + | > loss_duration: 1.4639263153076172 (1.4454805383495255) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.749542236328125 (30.71289862838446) + | > grad_norm_1: tensor(113.3333, device='cuda:0') (tensor(230.0352, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.3687 (0.34601416774824556) + | > loader_time: 0.0057 (0.005319057726392559) + + + --> TIME: 2025-04-19 02:18:34 -- STEP: 76/142 -- GLOBAL_STEP: 407900 + | > loss_disc: 2.4399657249450684 (2.4709486992735608) + | > loss_disc_real_0: 0.1406450867652893 (0.12832874638077452) + | > loss_disc_real_1: 0.22118283808231354 (0.21034481494050275) + | > loss_disc_real_2: 0.2460261732339859 (0.22553935490156474) + | > loss_disc_real_3: 0.2568901479244232 (0.2268159793395745) + | > loss_disc_real_4: 0.23653292655944824 (0.22857821144555746) + | > loss_disc_real_5: 0.23858249187469482 (0.23394057880106725) + | > loss_0: 2.4399657249450684 (2.4709486992735608) + | > grad_norm_0: tensor(9.8242, device='cuda:0') (tensor(10.3688, device='cuda:0')) + | > loss_gen: 2.368669033050537 (2.3546960353851323) + | > loss_kl: 1.8768296241760254 (1.847671284487373) + | > loss_feat: 6.773041725158691 (7.043108714254279) + | > loss_mel: 18.106767654418945 (17.894489853005663) + | > loss_duration: 1.489987850189209 (1.4580169069139581) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.615297317504883 (30.597982858356676) + | > grad_norm_1: tensor(306.3785, device='cuda:0') (tensor(245.7047, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.3977 (0.3581439162555493) + | > loader_time: 0.0064 (0.005593585340600263) + + + --> TIME: 2025-04-19 02:18:45 -- STEP: 101/142 -- GLOBAL_STEP: 407925 + | > loss_disc: 2.485543966293335 (2.471545287878205) + | > loss_disc_real_0: 0.09038063138723373 (0.12726359150492303) + | > loss_disc_real_1: 0.1844691038131714 (0.20910405212699776) + | > loss_disc_real_2: 0.24693921208381653 (0.22659095148048777) + | > loss_disc_real_3: 0.24062184989452362 (0.2273454894818882) + | > loss_disc_real_4: 0.23334679007530212 (0.2288008450281502) + | > loss_disc_real_5: 0.22744716703891754 (0.23351408024825673) + | > loss_0: 2.485543966293335 (2.471545287878205) + | > grad_norm_0: tensor(11.8356, device='cuda:0') (tensor(11.4103, device='cuda:0')) + | > loss_gen: 2.2748184204101562 (2.352252188295421) + | > loss_kl: 2.0023481845855713 (1.8641930311033041) + | > loss_feat: 6.370081424713135 (7.02105703448305) + | > loss_mel: 17.875425338745117 (17.913011701980437) + | > loss_duration: 1.5342375040054321 (1.469476600684742) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05691146850586 (30.619990584873918) + | > grad_norm_1: tensor(208.1371, device='cuda:0') (tensor(267.9540, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.4173 (0.37005652059422844) + | > loader_time: 0.0068 (0.005879059876545819) + + + --> TIME: 2025-04-19 02:18:56 -- STEP: 126/142 -- GLOBAL_STEP: 407950 + | > loss_disc: 2.3836922645568848 (2.4705445388006777) + | > loss_disc_real_0: 0.1225741058588028 (0.12656939591443722) + | > loss_disc_real_1: 0.2186540812253952 (0.20855523302914605) + | > loss_disc_real_2: 0.21296225488185883 (0.22619758581831342) + | > loss_disc_real_3: 0.22447431087493896 (0.22795393140543074) + | > loss_disc_real_4: 0.23141580820083618 (0.2290366929438379) + | > loss_disc_real_5: 0.2413252741098404 (0.23380148044181248) + | > loss_0: 2.3836922645568848 (2.4705445388006777) + | > grad_norm_0: tensor(16.0669, device='cuda:0') (tensor(11.1146, device='cuda:0')) + | > loss_gen: 2.3749425411224365 (2.3463169241708424) + | > loss_kl: 1.9030054807662964 (1.881354690544189) + | > loss_feat: 7.776908874511719 (6.986978530883789) + | > loss_mel: 19.09672737121582 (17.857210568019326) + | > loss_duration: 1.5099513530731201 (1.4765508837170072) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.661537170410156 (30.548411626664418) + | > grad_norm_1: tensor(235.1470, device='cuda:0') (tensor(265.7093, device='cuda:0')) + | > current_lr_0: 0.000195037294886455 + | > current_lr_1: 0.000195037294886455 + | > step_time: 0.4467 (0.3826575279235839) + | > loader_time: 0.0072 (0.006144733656020389) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4375579357147217 (2.4375579357147217) + | > loss_disc_real_0: 0.13340270519256592 (0.13340270519256592) + | > loss_disc_real_1: 0.20148953795433044 (0.20148953795433044) + | > loss_disc_real_2: 0.21972890198230743 (0.21972890198230743) + | > loss_disc_real_3: 0.2519717812538147 (0.2519717812538147) + | > loss_disc_real_4: 0.23662324249744415 (0.23662324249744415) + | > loss_disc_real_5: 0.23769131302833557 (0.23769131302833557) + | > loss_0: 2.4375579357147217 (2.4375579357147217) + | > loss_gen: 2.401508331298828 (2.401508331298828) + | > loss_kl: 2.307802677154541 (2.307802677154541) + | > loss_feat: 7.191214084625244 (7.191214084625244) + | > loss_mel: 19.71002769470215 (19.71002769470215) + | > loss_duration: 1.684122085571289 (1.684122085571289) + | > loss_1: 33.294677734375 (33.294677734375) + + --> STEP: 1 + | > loss_disc: 2.3726422786712646 (2.3726422786712646) + | > loss_disc_real_0: 0.12261593341827393 (0.12261593341827393) + | > loss_disc_real_1: 0.23350603878498077 (0.23350603878498077) + | > loss_disc_real_2: 0.2613140046596527 (0.2613140046596527) + | > loss_disc_real_3: 0.22432628273963928 (0.22432628273963928) + | > loss_disc_real_4: 0.23002766072750092 (0.23002766072750092) + | > loss_disc_real_5: 0.19373977184295654 (0.19373977184295654) + | > loss_0: 2.3726422786712646 (2.3726422786712646) + | > loss_gen: 2.456908702850342 (2.456908702850342) + | > loss_kl: 2.806108236312866 (2.806108236312866) + | > loss_feat: 7.407299995422363 (7.407299995422363) + | > loss_mel: 17.73587989807129 (17.73587989807129) + | > loss_duration: 1.7201602458953857 (1.7201602458953857) + | > loss_1: 32.12635803222656 (32.12635803222656) + + --> STEP: 2 + | > loss_disc: 2.2541537284851074 (2.313398003578186) + | > loss_disc_real_0: 0.12278271466493607 (0.122699324041605) + | > loss_disc_real_1: 0.22708971798419952 (0.23029787838459015) + | > loss_disc_real_2: 0.23853102326393127 (0.249922513961792) + | > loss_disc_real_3: 0.2376619130373001 (0.2309940978884697) + | > loss_disc_real_4: 0.19687439501285553 (0.21345102787017822) + | > loss_disc_real_5: 0.18496011197566986 (0.1893499419093132) + | > loss_0: 2.2541537284851074 (2.313398003578186) + | > loss_gen: 2.5085229873657227 (2.4827158451080322) + | > loss_kl: 2.1428792476654053 (2.4744937419891357) + | > loss_feat: 7.377124786376953 (7.392212390899658) + | > loss_mel: 17.841135025024414 (17.78850746154785) + | > loss_duration: 1.6669228076934814 (1.6935415267944336) + | > loss_1: 31.53658676147461 (31.831472396850586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036995410919189453 (+5.1140785217285156e-05) + | > avg_loss_disc: 2.313398003578186 (-0.08247339725494385) + | > avg_loss_disc_real_0: 0.122699324041605 (-0.028546582907438278) + | > avg_loss_disc_real_1: 0.23029787838459015 (-0.010241545736789703) + | > avg_loss_disc_real_2: 0.249922513961792 (+0.048620134592056274) + | > avg_loss_disc_real_3: 0.2309940978884697 (+0.0017147213220596313) + | > avg_loss_disc_real_4: 0.21345102787017822 (-0.001443818211555481) + | > avg_loss_disc_real_5: 0.1893499419093132 (+0.010097689926624298) + | > avg_loss_0: 2.313398003578186 (-0.08247339725494385) + | > avg_loss_gen: 2.4827158451080322 (+0.14545345306396484) + | > avg_loss_kl: 2.4744937419891357 (+0.23622429370880127) + | > avg_loss_feat: 7.392212390899658 (-0.13236188888549805) + | > avg_loss_mel: 17.78850746154785 (-1.6939830780029297) + | > avg_loss_duration: 1.6935415267944336 (+0.011536478996276855) + | > avg_loss_1: 31.831472396850586 (-1.4331283569335938) + + + > EPOCH: 202/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:19:05)  + + --> TIME: 2025-04-19 02:19:08 -- STEP: 9/142 -- GLOBAL_STEP: 407975 + | > loss_disc: 2.3737757205963135 (2.3991869025760226) + | > loss_disc_real_0: 0.14586690068244934 (0.11817328217956755) + | > loss_disc_real_1: 0.24654431641101837 (0.2158011860317654) + | > loss_disc_real_2: 0.23543187975883484 (0.22085672285821703) + | > loss_disc_real_3: 0.2023470550775528 (0.21941416462262472) + | > loss_disc_real_4: 0.21543581783771515 (0.215349480509758) + | > loss_disc_real_5: 0.23228169977664948 (0.22928960786925423) + | > loss_0: 2.3737757205963135 (2.3991869025760226) + | > grad_norm_0: tensor(15.9481, device='cuda:0') (tensor(9.7118, device='cuda:0')) + | > loss_gen: 2.5445327758789062 (2.4553023444281683) + | > loss_kl: 1.8318294286727905 (1.8588087028927274) + | > loss_feat: 7.752443313598633 (7.418429427676731) + | > loss_mel: 18.56577491760254 (18.09147686428494) + | > loss_duration: 1.4175469875335693 (1.435649077097575) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.112125396728516 (31.259666230943466) + | > grad_norm_1: tensor(274.6969, device='cuda:0') (tensor(231.5654, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.3248 (0.3240230878194173) + | > loader_time: 0.0054 (0.004570722579956055) + + + --> TIME: 2025-04-19 02:19:17 -- STEP: 34/142 -- GLOBAL_STEP: 408000 + | > loss_disc: 2.456045627593994 (2.471487122423508) + | > loss_disc_real_0: 0.10757637023925781 (0.12200012430548668) + | > loss_disc_real_1: 0.18926608562469482 (0.21026866225635304) + | > loss_disc_real_2: 0.22016854584217072 (0.22710908467278762) + | > loss_disc_real_3: 0.2570468485355377 (0.2309440768816892) + | > loss_disc_real_4: 0.23036836087703705 (0.2307877260095933) + | > loss_disc_real_5: 0.24118830263614655 (0.23559386721428702) + | > loss_0: 2.456045627593994 (2.471487122423508) + | > grad_norm_0: tensor(11.3467, device='cuda:0') (tensor(12.7339, device='cuda:0')) + | > loss_gen: 2.40691876411438 (2.3810660067726586) + | > loss_kl: 1.9024534225463867 (1.8889048555318046) + | > loss_feat: 7.266242504119873 (7.240328003378475) + | > loss_mel: 18.368606567382812 (18.084010797388416) + | > loss_duration: 1.449569821357727 (1.4413466734044693) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.39379119873047 (31.035656031440286) + | > grad_norm_1: tensor(513.2172, device='cuda:0') (tensor(278.6936, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.3682 (0.342049142893623) + | > loader_time: 0.0062 (0.005247698110692641) + + + --> TIME: 2025-04-19 02:19:27 -- STEP: 59/142 -- GLOBAL_STEP: 408025 + | > loss_disc: 2.5281524658203125 (2.4548107607889986) + | > loss_disc_real_0: 0.16152052581310272 (0.12008294488413859) + | > loss_disc_real_1: 0.21586403250694275 (0.20811768027685457) + | > loss_disc_real_2: 0.22061921656131744 (0.22424743261377691) + | > loss_disc_real_3: 0.25462576746940613 (0.23128497903629885) + | > loss_disc_real_4: 0.21568448841571808 (0.22871575441400885) + | > loss_disc_real_5: 0.2595074474811554 (0.2357762905500703) + | > loss_0: 2.5281524658203125 (2.4548107607889986) + | > grad_norm_0: tensor(25.3885, device='cuda:0') (tensor(15.2996, device='cuda:0')) + | > loss_gen: 2.3818976879119873 (2.3943085710881116) + | > loss_kl: 1.948368787765503 (1.891611551834365) + | > loss_feat: 7.2168049812316895 (7.215280088327699) + | > loss_mel: 17.744739532470703 (17.92364175440901) + | > loss_duration: 1.4718561172485352 (1.4533354427854892) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.76366424560547 (30.878177190231064) + | > grad_norm_1: tensor(205.8785, device='cuda:0') (tensor(319.9087, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.3857 (0.3564018960726464) + | > loader_time: 0.0061 (0.005565712007425599) + + + --> TIME: 2025-04-19 02:19:37 -- STEP: 84/142 -- GLOBAL_STEP: 408050 + | > loss_disc: 2.524707317352295 (2.468848750704811) + | > loss_disc_real_0: 0.23431268334388733 (0.12444332685498964) + | > loss_disc_real_1: 0.22669661045074463 (0.20803805884151233) + | > loss_disc_real_2: 0.22028639912605286 (0.22703576141170093) + | > loss_disc_real_3: 0.20665472745895386 (0.22975686458604677) + | > loss_disc_real_4: 0.2346588522195816 (0.2291062649871622) + | > loss_disc_real_5: 0.22643272578716278 (0.2354696731837023) + | > loss_0: 2.524707317352295 (2.468848750704811) + | > grad_norm_0: tensor(23.2934, device='cuda:0') (tensor(15.4026, device='cuda:0')) + | > loss_gen: 2.13592267036438 (2.369556287924447) + | > loss_kl: 1.8917450904846191 (1.9064383861564456) + | > loss_feat: 6.062014102935791 (7.024174769719441) + | > loss_mel: 16.085481643676758 (17.690792038327174) + | > loss_duration: 1.5358390808105469 (1.466131403332665) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.71100425720215 (30.457092739286878) + | > grad_norm_1: tensor(294.3581, device='cuda:0') (tensor(315.3514, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.4068 (0.3696172067097256) + | > loader_time: 0.0072 (0.00590347959881737) + + + --> TIME: 2025-04-19 02:19:48 -- STEP: 109/142 -- GLOBAL_STEP: 408075 + | > loss_disc: 2.4333860874176025 (2.4772824059932614) + | > loss_disc_real_0: 0.09822572022676468 (0.12437410235678384) + | > loss_disc_real_1: 0.19156984984874725 (0.20871569366630063) + | > loss_disc_real_2: 0.22295120358467102 (0.2278296460799121) + | > loss_disc_real_3: 0.2138414978981018 (0.2294341537657134) + | > loss_disc_real_4: 0.2245268076658249 (0.22996269357860635) + | > loss_disc_real_5: 0.24760112166404724 (0.23607956392502566) + | > loss_0: 2.4333860874176025 (2.4772824059932614) + | > grad_norm_0: tensor(12.5141, device='cuda:0') (tensor(14.8108, device='cuda:0')) + | > loss_gen: 2.300715923309326 (2.351649124687965) + | > loss_kl: 1.8671787977218628 (1.9192195463618007) + | > loss_feat: 7.004777431488037 (6.974725351421111) + | > loss_mel: 17.59793472290039 (17.65300936217703) + | > loss_duration: 1.5041652917861938 (1.4750391135521985) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.27477264404297 (30.37364242273733) + | > grad_norm_1: tensor(382.8260, device='cuda:0') (tensor(319.4175, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.4263 (0.3812371372082913) + | > loader_time: 0.007 (0.006204268254271341) + + + --> TIME: 2025-04-19 02:19:59 -- STEP: 134/142 -- GLOBAL_STEP: 408100 + | > loss_disc: 2.5405397415161133 (2.4750743944253504) + | > loss_disc_real_0: 0.10842566937208176 (0.12460099149550964) + | > loss_disc_real_1: 0.2216738611459732 (0.20836243900790144) + | > loss_disc_real_2: 0.23013615608215332 (0.22738038192489254) + | > loss_disc_real_3: 0.23524285852909088 (0.22978665890978345) + | > loss_disc_real_4: 0.21266226470470428 (0.2303397374144241) + | > loss_disc_real_5: 0.2289854884147644 (0.2357322654839772) + | > loss_0: 2.5405397415161133 (2.4750743944253504) + | > grad_norm_0: tensor(6.1981, device='cuda:0') (tensor(14.4675, device='cuda:0')) + | > loss_gen: 2.269735336303711 (2.345231684286203) + | > loss_kl: 1.9783120155334473 (1.926240951267641) + | > loss_feat: 6.998042106628418 (6.9512149042157985) + | > loss_mel: 17.92198371887207 (17.646155072681946) + | > loss_duration: 1.5508229732513428 (1.4816728643516996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.718896865844727 (30.3505154652382) + | > grad_norm_1: tensor(79.9158, device='cuda:0') (tensor(317.4219, device='cuda:0')) + | > current_lr_0: 0.00019501291522459419 + | > current_lr_1: 0.00019501291522459419 + | > step_time: 0.4491 (0.39375351969875516) + | > loader_time: 0.0078 (0.006453047937421657) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.624049663543701 (2.624049663543701) + | > loss_disc_real_0: 0.17474986612796783 (0.17474986612796783) + | > loss_disc_real_1: 0.2509779930114746 (0.2509779930114746) + | > loss_disc_real_2: 0.2519527077674866 (0.2519527077674866) + | > loss_disc_real_3: 0.281233549118042 (0.281233549118042) + | > loss_disc_real_4: 0.29535192251205444 (0.29535192251205444) + | > loss_disc_real_5: 0.24476659297943115 (0.24476659297943115) + | > loss_0: 2.624049663543701 (2.624049663543701) + | > loss_gen: 2.242365837097168 (2.242365837097168) + | > loss_kl: 2.271207571029663 (2.271207571029663) + | > loss_feat: 5.141961574554443 (5.141961574554443) + | > loss_mel: 15.705336570739746 (15.705336570739746) + | > loss_duration: 1.6939903497695923 (1.6939903497695923) + | > loss_1: 27.054861068725586 (27.054861068725586) + + --> STEP: 1 + | > loss_disc: 2.5187430381774902 (2.5187430381774902) + | > loss_disc_real_0: 0.20674796402454376 (0.20674796402454376) + | > loss_disc_real_1: 0.25435787439346313 (0.25435787439346313) + | > loss_disc_real_2: 0.24013619124889374 (0.24013619124889374) + | > loss_disc_real_3: 0.22037196159362793 (0.22037196159362793) + | > loss_disc_real_4: 0.2275126427412033 (0.2275126427412033) + | > loss_disc_real_5: 0.22618597745895386 (0.22618597745895386) + | > loss_0: 2.5187430381774902 (2.5187430381774902) + | > loss_gen: 2.478353500366211 (2.478353500366211) + | > loss_kl: 2.463819980621338 (2.463819980621338) + | > loss_feat: 7.095135688781738 (7.095135688781738) + | > loss_mel: 18.079200744628906 (18.079200744628906) + | > loss_duration: 1.7295876741409302 (1.7295876741409302) + | > loss_1: 31.846097946166992 (31.846097946166992) + + --> STEP: 2 + | > loss_disc: 2.4503250122070312 (2.4845340251922607) + | > loss_disc_real_0: 0.12545296549797058 (0.16610046476125717) + | > loss_disc_real_1: 0.24559302628040314 (0.24997545033693314) + | > loss_disc_real_2: 0.22947342693805695 (0.23480480909347534) + | > loss_disc_real_3: 0.25878649950027466 (0.2395792305469513) + | > loss_disc_real_4: 0.2505686581134796 (0.23904065042734146) + | > loss_disc_real_5: 0.21157877147197723 (0.21888237446546555) + | > loss_0: 2.4503250122070312 (2.4845340251922607) + | > loss_gen: 2.3825230598449707 (2.430438280105591) + | > loss_kl: 2.248361825942993 (2.3560909032821655) + | > loss_feat: 5.977498531341553 (6.5363171100616455) + | > loss_mel: 16.97544288635254 (17.527321815490723) + | > loss_duration: 1.661008358001709 (1.6952980160713196) + | > loss_1: 29.244834899902344 (30.545466423034668) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003568291664123535 (-0.00013124942779541016) + | > avg_loss_disc: 2.4845340251922607 (+0.1711360216140747) + | > avg_loss_disc_real_0: 0.16610046476125717 (+0.043401140719652176) + | > avg_loss_disc_real_1: 0.24997545033693314 (+0.019677571952342987) + | > avg_loss_disc_real_2: 0.23480480909347534 (-0.01511770486831665) + | > avg_loss_disc_real_3: 0.2395792305469513 (+0.008585132658481598) + | > avg_loss_disc_real_4: 0.23904065042734146 (+0.02558962255716324) + | > avg_loss_disc_real_5: 0.21888237446546555 (+0.029532432556152344) + | > avg_loss_0: 2.4845340251922607 (+0.1711360216140747) + | > avg_loss_gen: 2.430438280105591 (-0.052277565002441406) + | > avg_loss_kl: 2.3560909032821655 (-0.11840283870697021) + | > avg_loss_feat: 6.5363171100616455 (-0.8558952808380127) + | > avg_loss_mel: 17.527321815490723 (-0.2611856460571289) + | > avg_loss_duration: 1.6952980160713196 (+0.0017564892768859863) + | > avg_loss_1: 30.545466423034668 (-1.286005973815918) + + + > EPOCH: 203/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:20:04)  + + --> TIME: 2025-04-19 02:20:11 -- STEP: 17/142 -- GLOBAL_STEP: 408125 + | > loss_disc: 2.6451168060302734 (2.4562853504629696) + | > loss_disc_real_0: 0.17106758058071136 (0.12521827878320918) + | > loss_disc_real_1: 0.2491530478000641 (0.20590372208286734) + | > loss_disc_real_2: 0.22563767433166504 (0.22311298023251927) + | > loss_disc_real_3: 0.237167626619339 (0.22738799540435567) + | > loss_disc_real_4: 0.2798756957054138 (0.23836574747281916) + | > loss_disc_real_5: 0.26002874970436096 (0.22704754594494314) + | > loss_0: 2.6451168060302734 (2.4562853504629696) + | > grad_norm_0: tensor(12.9692, device='cuda:0') (tensor(9.8813, device='cuda:0')) + | > loss_gen: 2.250544309616089 (2.373540625852697) + | > loss_kl: 2.0476009845733643 (1.8705847263336182) + | > loss_feat: 7.1600422859191895 (7.137130961698644) + | > loss_mel: 18.12286376953125 (17.654319763183594) + | > loss_duration: 1.4294204711914062 (1.4306887808968038) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.01047134399414 (30.466264612534466) + | > grad_norm_1: tensor(162.4164, device='cuda:0') (tensor(222.2448, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.336 (0.3292541083167581) + | > loader_time: 0.0047 (0.004767025218290442) + + + --> TIME: 2025-04-19 02:20:20 -- STEP: 42/142 -- GLOBAL_STEP: 408150 + | > loss_disc: 2.389925479888916 (2.4790896801721485) + | > loss_disc_real_0: 0.11653514206409454 (0.12697379007225945) + | > loss_disc_real_1: 0.2153034210205078 (0.2094849564489864) + | > loss_disc_real_2: 0.24606876075267792 (0.22598394893464588) + | > loss_disc_real_3: 0.19345621764659882 (0.22850490042141505) + | > loss_disc_real_4: 0.22247514128684998 (0.23191557966527485) + | > loss_disc_real_5: 0.20488406717777252 (0.23196319242318472) + | > loss_0: 2.389925479888916 (2.4790896801721485) + | > grad_norm_0: tensor(19.5322, device='cuda:0') (tensor(11.8380, device='cuda:0')) + | > loss_gen: 2.357605457305908 (2.3336133616311203) + | > loss_kl: 1.9353591203689575 (1.8811589167231606) + | > loss_feat: 6.787929534912109 (7.028962714331491) + | > loss_mel: 18.53646469116211 (17.876470815567746) + | > loss_duration: 1.4945615530014038 (1.4435284336407979) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.111921310424805 (30.563733963739303) + | > grad_norm_1: tensor(278.6128, device='cuda:0') (tensor(261.8368, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.3631 (0.34189577897389734) + | > loader_time: 0.0063 (0.00510651157015846) + + + --> TIME: 2025-04-19 02:20:29 -- STEP: 67/142 -- GLOBAL_STEP: 408175 + | > loss_disc: 2.500030040740967 (2.474663289625253) + | > loss_disc_real_0: 0.13966384530067444 (0.12571651594185115) + | > loss_disc_real_1: 0.22603526711463928 (0.20942438916483921) + | > loss_disc_real_2: 0.21102306246757507 (0.22642552318857678) + | > loss_disc_real_3: 0.25129127502441406 (0.22937920867507136) + | > loss_disc_real_4: 0.23449084162712097 (0.23072939555146801) + | > loss_disc_real_5: 0.258242130279541 (0.23118855503957664) + | > loss_0: 2.500030040740967 (2.474663289625253) + | > grad_norm_0: tensor(13.4292, device='cuda:0') (tensor(12.5714, device='cuda:0')) + | > loss_gen: 2.2874274253845215 (2.3401773331770257) + | > loss_kl: 1.9470843076705933 (1.8767860772004767) + | > loss_feat: 6.045590400695801 (7.016311595689005) + | > loss_mel: 17.034652709960938 (17.79607008464302) + | > loss_duration: 1.4111472368240356 (1.4552274248493249) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.72590446472168 (30.48457251022111) + | > grad_norm_1: tensor(101.0065, device='cuda:0') (tensor(282.5302, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.3845 (0.3541893745536235) + | > loader_time: 0.0062 (0.0054568461517789475) + + + --> TIME: 2025-04-19 02:20:40 -- STEP: 92/142 -- GLOBAL_STEP: 408200 + | > loss_disc: 2.4791152477264404 (2.470772683620452) + | > loss_disc_real_0: 0.13694646954536438 (0.12675627543712434) + | > loss_disc_real_1: 0.2217486947774887 (0.20732686458074528) + | > loss_disc_real_2: 0.26315030455589294 (0.2266612932409929) + | > loss_disc_real_3: 0.24829165637493134 (0.22863140021977219) + | > loss_disc_real_4: 0.23201251029968262 (0.22969780586983846) + | > loss_disc_real_5: 0.23542141914367676 (0.23194405809044838) + | > loss_0: 2.4791152477264404 (2.470772683620452) + | > grad_norm_0: tensor(15.8267, device='cuda:0') (tensor(12.1588, device='cuda:0')) + | > loss_gen: 2.3581814765930176 (2.3414864306864533) + | > loss_kl: 1.8644579648971558 (1.8853859383126963) + | > loss_feat: 6.704248905181885 (7.005102085030598) + | > loss_mel: 18.188405990600586 (17.755431216696046) + | > loss_duration: 1.461712121963501 (1.467404194500135) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.57700538635254 (30.454809831536334) + | > grad_norm_1: tensor(482.1503, device='cuda:0') (tensor(283.3660, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.414 (0.3660752358643905) + | > loader_time: 0.0072 (0.005763214567433233) + + + --> TIME: 2025-04-19 02:20:51 -- STEP: 117/142 -- GLOBAL_STEP: 408225 + | > loss_disc: 2.5904552936553955 (2.4720473289489737) + | > loss_disc_real_0: 0.128170445561409 (0.12577481430947268) + | > loss_disc_real_1: 0.1776222437620163 (0.20615011071547484) + | > loss_disc_real_2: 0.270472913980484 (0.22768156612530732) + | > loss_disc_real_3: 0.2696775794029236 (0.22922632314710537) + | > loss_disc_real_4: 0.29909783601760864 (0.23067867717681786) + | > loss_disc_real_5: 0.27020779252052307 (0.23217319092179975) + | > loss_0: 2.5904552936553955 (2.4720473289489737) + | > grad_norm_0: tensor(15.7800, device='cuda:0') (tensor(12.6633, device='cuda:0')) + | > loss_gen: 2.370572090148926 (2.3422361418732205) + | > loss_kl: 1.6482393741607666 (1.8932137896871974) + | > loss_feat: 6.269519805908203 (6.9978873668572845) + | > loss_mel: 17.347795486450195 (17.753351692460555) + | > loss_duration: 1.4713516235351562 (1.473801405001909) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.107479095458984 (30.460490365313667) + | > grad_norm_1: tensor(427.3541, device='cuda:0') (tensor(297.7956, device='cuda:0')) + | > current_lr_0: 0.0001949885386101911 + | > current_lr_1: 0.0001949885386101911 + | > step_time: 0.4383 (0.37911665745270556) + | > loader_time: 0.007 (0.006068728927873138) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.432422161102295 (2.432422161102295) + | > loss_disc_real_0: 0.11831917613744736 (0.11831917613744736) + | > loss_disc_real_1: 0.24017754197120667 (0.24017754197120667) + | > loss_disc_real_2: 0.23976430296897888 (0.23976430296897888) + | > loss_disc_real_3: 0.2649538218975067 (0.2649538218975067) + | > loss_disc_real_4: 0.2384217083454132 (0.2384217083454132) + | > loss_disc_real_5: 0.2571207582950592 (0.2571207582950592) + | > loss_0: 2.432422161102295 (2.432422161102295) + | > loss_gen: 2.461324453353882 (2.461324453353882) + | > loss_kl: 2.229606866836548 (2.229606866836548) + | > loss_feat: 6.826982498168945 (6.826982498168945) + | > loss_mel: 18.178909301757812 (18.178909301757812) + | > loss_duration: 1.6503103971481323 (1.6503103971481323) + | > loss_1: 31.34713363647461 (31.34713363647461) + + --> STEP: 1 + | > loss_disc: 2.414792060852051 (2.414792060852051) + | > loss_disc_real_0: 0.12803418934345245 (0.12803418934345245) + | > loss_disc_real_1: 0.25631266832351685 (0.25631266832351685) + | > loss_disc_real_2: 0.22331462800502777 (0.22331462800502777) + | > loss_disc_real_3: 0.22644676268100739 (0.22644676268100739) + | > loss_disc_real_4: 0.2392188161611557 (0.2392188161611557) + | > loss_disc_real_5: 0.17702752351760864 (0.17702752351760864) + | > loss_0: 2.414792060852051 (2.414792060852051) + | > loss_gen: 2.366785764694214 (2.366785764694214) + | > loss_kl: 2.3993566036224365 (2.3993566036224365) + | > loss_feat: 7.369868755340576 (7.369868755340576) + | > loss_mel: 19.40389633178711 (19.40389633178711) + | > loss_duration: 1.6741358041763306 (1.6741358041763306) + | > loss_1: 33.21404266357422 (33.21404266357422) + + --> STEP: 2 + | > loss_disc: 2.330643653869629 (2.37271785736084) + | > loss_disc_real_0: 0.14460235834121704 (0.13631827384233475) + | > loss_disc_real_1: 0.24873042106628418 (0.2525215446949005) + | > loss_disc_real_2: 0.25163987278938293 (0.23747725039720535) + | > loss_disc_real_3: 0.23573540151119232 (0.23109108209609985) + | > loss_disc_real_4: 0.2256307750940323 (0.232424795627594) + | > loss_disc_real_5: 0.23101840913295746 (0.20402296632528305) + | > loss_0: 2.330643653869629 (2.37271785736084) + | > loss_gen: 2.6494171619415283 (2.508101463317871) + | > loss_kl: 2.16164231300354 (2.2804994583129883) + | > loss_feat: 6.969264030456543 (7.16956639289856) + | > loss_mel: 18.53807258605957 (18.97098445892334) + | > loss_duration: 1.6490588188171387 (1.6615973114967346) + | > loss_1: 31.967456817626953 (32.590749740600586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003225564956665039 (-0.0003427267074584961) + | > avg_loss_disc: 2.37271785736084 (-0.1118161678314209) + | > avg_loss_disc_real_0: 0.13631827384233475 (-0.029782190918922424) + | > avg_loss_disc_real_1: 0.2525215446949005 (+0.0025460943579673767) + | > avg_loss_disc_real_2: 0.23747725039720535 (+0.002672441303730011) + | > avg_loss_disc_real_3: 0.23109108209609985 (-0.00848814845085144) + | > avg_loss_disc_real_4: 0.232424795627594 (-0.006615854799747467) + | > avg_loss_disc_real_5: 0.20402296632528305 (-0.014859408140182495) + | > avg_loss_0: 2.37271785736084 (-0.1118161678314209) + | > avg_loss_gen: 2.508101463317871 (+0.07766318321228027) + | > avg_loss_kl: 2.2804994583129883 (-0.07559144496917725) + | > avg_loss_feat: 7.16956639289856 (+0.6332492828369141) + | > avg_loss_mel: 18.97098445892334 (+1.4436626434326172) + | > avg_loss_duration: 1.6615973114967346 (-0.03370070457458496) + | > avg_loss_1: 32.590749740600586 (+2.045283317565918) + + + > EPOCH: 204/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:21:03)  + + --> TIME: 2025-04-19 02:21:04 -- STEP: 0/142 -- GLOBAL_STEP: 408250 + | > loss_disc: 2.4741857051849365 (2.4741857051849365) + | > loss_disc_real_0: 0.1048474833369255 (0.1048474833369255) + | > loss_disc_real_1: 0.24064968526363373 (0.24064968526363373) + | > loss_disc_real_2: 0.24640384316444397 (0.24640384316444397) + | > loss_disc_real_3: 0.21746893227100372 (0.21746893227100372) + | > loss_disc_real_4: 0.21586991846561432 (0.21586991846561432) + | > loss_disc_real_5: 0.20160849392414093 (0.20160849392414093) + | > loss_0: 2.4741857051849365 (2.4741857051849365) + | > grad_norm_0: tensor(13.9971, device='cuda:0') (tensor(13.9971, device='cuda:0')) + | > loss_gen: 2.3793253898620605 (2.3793253898620605) + | > loss_kl: 1.8187531232833862 (1.8187531232833862) + | > loss_feat: 6.066533088684082 (6.066533088684082) + | > loss_mel: 16.861509323120117 (16.861509323120117) + | > loss_duration: 1.467067003250122 (1.467067003250122) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.593189239501953 (28.593189239501953) + | > grad_norm_1: tensor(304.9214, device='cuda:0') (tensor(304.9214, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.3548 (0.3547520637512207) + | > loader_time: 0.3389 (0.33887505531311035) + + + --> TIME: 2025-04-19 02:21:13 -- STEP: 25/142 -- GLOBAL_STEP: 408275 + | > loss_disc: 2.428049325942993 (2.4424302959442143) + | > loss_disc_real_0: 0.15030376613140106 (0.126412495970726) + | > loss_disc_real_1: 0.2104932963848114 (0.20900906205177308) + | > loss_disc_real_2: 0.22106578946113586 (0.22396620273590087) + | > loss_disc_real_3: 0.203215092420578 (0.2292141157388687) + | > loss_disc_real_4: 0.22820286452770233 (0.2296062457561493) + | > loss_disc_real_5: 0.24204404652118683 (0.2285091120004654) + | > loss_0: 2.428049325942993 (2.4424302959442143) + | > grad_norm_0: tensor(16.1182, device='cuda:0') (tensor(12.9336, device='cuda:0')) + | > loss_gen: 2.279510498046875 (2.360760612487793) + | > loss_kl: 1.8953044414520264 (1.8517707252502442) + | > loss_feat: 7.136280059814453 (7.104148178100586) + | > loss_mel: 17.865604400634766 (17.8747131729126) + | > loss_duration: 1.4661829471588135 (1.4364196968078613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.64288330078125 (30.6278125) + | > grad_norm_1: tensor(255.1297, device='cuda:0') (tensor(274.0661, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.3426 (0.3350839900970459) + | > loader_time: 0.0058 (0.0049908065795898435) + + + --> TIME: 2025-04-19 02:21:22 -- STEP: 50/142 -- GLOBAL_STEP: 408300 + | > loss_disc: 2.396235704421997 (2.4238980674743664) + | > loss_disc_real_0: 0.08436787873506546 (0.12171399906277656) + | > loss_disc_real_1: 0.1919858604669571 (0.20530178248882294) + | > loss_disc_real_2: 0.25761082768440247 (0.22246304780244827) + | > loss_disc_real_3: 0.24200519919395447 (0.22827718704938887) + | > loss_disc_real_4: 0.22798965871334076 (0.22634831070899963) + | > loss_disc_real_5: 0.23461808264255524 (0.22816752076148986) + | > loss_0: 2.396235704421997 (2.4238980674743664) + | > grad_norm_0: tensor(5.4678, device='cuda:0') (tensor(11.5363, device='cuda:0')) + | > loss_gen: 2.3220746517181396 (2.3822881126403805) + | > loss_kl: 1.9438972473144531 (1.8679208612442018) + | > loss_feat: 6.985315322875977 (7.1909241771698) + | > loss_mel: 17.923233032226562 (17.904267101287846) + | > loss_duration: 1.4837757349014282 (1.446272301673889) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.658296585083008 (30.791672668457032) + | > grad_norm_1: tensor(106.4254, device='cuda:0') (tensor(270.7512, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.3809 (0.3506871509552002) + | > loader_time: 0.0066 (0.005448188781738281) + + + --> TIME: 2025-04-19 02:21:32 -- STEP: 75/142 -- GLOBAL_STEP: 408325 + | > loss_disc: 2.4652352333068848 (2.454350471496583) + | > loss_disc_real_0: 0.15789952874183655 (0.12528588751951852) + | > loss_disc_real_1: 0.21628494560718536 (0.2071807916959127) + | > loss_disc_real_2: 0.23991574347019196 (0.22559263249238332) + | > loss_disc_real_3: 0.22272436320781708 (0.2284408845504125) + | > loss_disc_real_4: 0.20033779740333557 (0.22886605322360992) + | > loss_disc_real_5: 0.21555475890636444 (0.23085660020510357) + | > loss_0: 2.4652352333068848 (2.454350471496583) + | > grad_norm_0: tensor(9.4868, device='cuda:0') (tensor(10.8891, device='cuda:0')) + | > loss_gen: 2.3262858390808105 (2.359771502812703) + | > loss_kl: 1.9142804145812988 (1.883888365427653) + | > loss_feat: 7.312917232513428 (7.131081428527832) + | > loss_mel: 17.840639114379883 (17.87145769755046) + | > loss_duration: 1.4425733089447021 (1.457827175458272) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.83669662475586 (30.704026260375976) + | > grad_norm_1: tensor(281.3515, device='cuda:0') (tensor(235.9538, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.3989 (0.36471109072367347) + | > loader_time: 0.0065 (0.005723218917846679) + + + --> TIME: 2025-04-19 02:21:42 -- STEP: 100/142 -- GLOBAL_STEP: 408350 + | > loss_disc: 2.5894830226898193 (2.4656422305107117) + | > loss_disc_real_0: 0.10809459537267685 (0.12589311353862284) + | > loss_disc_real_1: 0.2309505045413971 (0.20946938097476958) + | > loss_disc_real_2: 0.19951727986335754 (0.22551037207245828) + | > loss_disc_real_3: 0.20688016712665558 (0.22824161872267723) + | > loss_disc_real_4: 0.21034659445285797 (0.2294612258672714) + | > loss_disc_real_5: 0.2177729606628418 (0.23158617913722992) + | > loss_0: 2.5894830226898193 (2.4656422305107117) + | > grad_norm_0: tensor(7.2142, device='cuda:0') (tensor(10.7776, device='cuda:0')) + | > loss_gen: 2.1547000408172607 (2.343345351219178) + | > loss_kl: 2.0721113681793213 (1.8989664924144745) + | > loss_feat: 6.146664142608643 (7.0534094715118405) + | > loss_mel: 14.920206069946289 (17.8380489063263) + | > loss_duration: 1.4909257888793945 (1.4684993863105773) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.78460693359375 (30.602269649505615) + | > grad_norm_1: tensor(100.6377, device='cuda:0') (tensor(237.3195, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.4262 (0.376565682888031) + | > loader_time: 0.0077 (0.005986669063568114) + + + --> TIME: 2025-04-19 02:21:54 -- STEP: 125/142 -- GLOBAL_STEP: 408375 + | > loss_disc: 2.5893290042877197 (2.4765655841827394) + | > loss_disc_real_0: 0.1849818080663681 (0.12765145307779313) + | > loss_disc_real_1: 0.2002795934677124 (0.20881577956676484) + | > loss_disc_real_2: 0.22001507878303528 (0.22654905247688292) + | > loss_disc_real_3: 0.24593819677829742 (0.22934193205833436) + | > loss_disc_real_4: 0.23318524658679962 (0.2304221112728119) + | > loss_disc_real_5: 0.26474493741989136 (0.23316582143306733) + | > loss_0: 2.5893290042877197 (2.4765655841827394) + | > grad_norm_0: tensor(19.1435, device='cuda:0') (tensor(10.7342, device='cuda:0')) + | > loss_gen: 2.193044662475586 (2.332514043807983) + | > loss_kl: 1.9153164625167847 (1.9072672443389893) + | > loss_feat: 5.092621803283691 (6.988727073669434) + | > loss_mel: 15.194156646728516 (17.79017037200929) + | > loss_duration: 1.5459654331207275 (1.4751900682449341) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.941104888916016 (30.493868789672852) + | > grad_norm_1: tensor(315.4144, device='cuda:0') (tensor(239.5276, device='cuda:0')) + | > current_lr_0: 0.00019496416504286482 + | > current_lr_1: 0.00019496416504286482 + | > step_time: 0.4514 (0.3893270034790039) + | > loader_time: 0.0074 (0.006251199722290038) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.798527240753174 (2.798527240753174) + | > loss_disc_real_0: 0.14079928398132324 (0.14079928398132324) + | > loss_disc_real_1: 0.30877968668937683 (0.30877968668937683) + | > loss_disc_real_2: 0.2530662715435028 (0.2530662715435028) + | > loss_disc_real_3: 0.23654264211654663 (0.23654264211654663) + | > loss_disc_real_4: 0.27647897601127625 (0.27647897601127625) + | > loss_disc_real_5: 0.2944720685482025 (0.2944720685482025) + | > loss_0: 2.798527240753174 (2.798527240753174) + | > loss_gen: 2.168386220932007 (2.168386220932007) + | > loss_kl: 2.462270736694336 (2.462270736694336) + | > loss_feat: 5.465178489685059 (5.465178489685059) + | > loss_mel: 17.8590087890625 (17.8590087890625) + | > loss_duration: 1.6944888830184937 (1.6944888830184937) + | > loss_1: 29.64933204650879 (29.64933204650879) + + --> STEP: 1 + | > loss_disc: 2.3731071949005127 (2.3731071949005127) + | > loss_disc_real_0: 0.11127588897943497 (0.11127588897943497) + | > loss_disc_real_1: 0.25309500098228455 (0.25309500098228455) + | > loss_disc_real_2: 0.22380343079566956 (0.22380343079566956) + | > loss_disc_real_3: 0.2082890123128891 (0.2082890123128891) + | > loss_disc_real_4: 0.23177890479564667 (0.23177890479564667) + | > loss_disc_real_5: 0.2559128701686859 (0.2559128701686859) + | > loss_0: 2.3731071949005127 (2.3731071949005127) + | > loss_gen: 2.415787935256958 (2.415787935256958) + | > loss_kl: 2.563772439956665 (2.563772439956665) + | > loss_feat: 7.320496082305908 (7.320496082305908) + | > loss_mel: 18.75214195251465 (18.75214195251465) + | > loss_duration: 1.7091097831726074 (1.7091097831726074) + | > loss_1: 32.76130676269531 (32.76130676269531) + + --> STEP: 2 + | > loss_disc: 2.248485565185547 (2.31079638004303) + | > loss_disc_real_0: 0.11005853116512299 (0.11066721007227898) + | > loss_disc_real_1: 0.23307465016841888 (0.24308482557535172) + | > loss_disc_real_2: 0.21814291179180145 (0.2209731712937355) + | > loss_disc_real_3: 0.16595458984375 (0.18712180107831955) + | > loss_disc_real_4: 0.2188885360956192 (0.22533372044563293) + | > loss_disc_real_5: 0.19752471148967743 (0.22671879082918167) + | > loss_0: 2.248485565185547 (2.31079638004303) + | > loss_gen: 2.4990406036376953 (2.4574142694473267) + | > loss_kl: 2.2944328784942627 (2.429102659225464) + | > loss_feat: 7.713879108428955 (7.517187595367432) + | > loss_mel: 17.471450805664062 (18.111796379089355) + | > loss_duration: 1.663374900817871 (1.6862423419952393) + | > loss_1: 31.64217758178711 (32.20174217224121) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034962892532348633 (+0.0002707242965698242) + | > avg_loss_disc: 2.31079638004303 (-0.06192147731781006) + | > avg_loss_disc_real_0: 0.11066721007227898 (-0.02565106377005577) + | > avg_loss_disc_real_1: 0.24308482557535172 (-0.009436719119548798) + | > avg_loss_disc_real_2: 0.2209731712937355 (-0.01650407910346985) + | > avg_loss_disc_real_3: 0.18712180107831955 (-0.043969281017780304) + | > avg_loss_disc_real_4: 0.22533372044563293 (-0.00709107518196106) + | > avg_loss_disc_real_5: 0.22671879082918167 (+0.02269582450389862) + | > avg_loss_0: 2.31079638004303 (-0.06192147731781006) + | > avg_loss_gen: 2.4574142694473267 (-0.050687193870544434) + | > avg_loss_kl: 2.429102659225464 (+0.14860320091247559) + | > avg_loss_feat: 7.517187595367432 (+0.34762120246887207) + | > avg_loss_mel: 18.111796379089355 (-0.8591880798339844) + | > avg_loss_duration: 1.6862423419952393 (+0.02464503049850464) + | > avg_loss_1: 32.20174217224121 (-0.389007568359375) + + + > EPOCH: 205/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:22:03)  + + --> TIME: 2025-04-19 02:22:07 -- STEP: 8/142 -- GLOBAL_STEP: 408400 + | > loss_disc: 2.372140407562256 (2.423806458711624) + | > loss_disc_real_0: 0.14131304621696472 (0.13443218730390072) + | > loss_disc_real_1: 0.18487301468849182 (0.2073427438735962) + | > loss_disc_real_2: 0.21190112829208374 (0.22137894295156002) + | > loss_disc_real_3: 0.1865886151790619 (0.2240336537361145) + | > loss_disc_real_4: 0.18680383265018463 (0.22170164436101913) + | > loss_disc_real_5: 0.20173290371894836 (0.24408930726349354) + | > loss_0: 2.372140407562256 (2.423806458711624) + | > grad_norm_0: tensor(12.4384, device='cuda:0') (tensor(14.0303, device='cuda:0')) + | > loss_gen: 2.325463056564331 (2.434861481189728) + | > loss_kl: 1.8448175191879272 (1.8935783356428146) + | > loss_feat: 7.4745025634765625 (7.243397772312164) + | > loss_mel: 17.60838508605957 (17.685644149780273) + | > loss_duration: 1.442431926727295 (1.4317311495542526) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.695598602294922 (30.689212560653687) + | > grad_norm_1: tensor(511.1122, device='cuda:0') (tensor(328.2441, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.325 (0.32717156410217285) + | > loader_time: 0.0046 (0.004561811685562134) + + + --> TIME: 2025-04-19 02:22:15 -- STEP: 33/142 -- GLOBAL_STEP: 408425 + | > loss_disc: 2.3894505500793457 (2.4426269242257783) + | > loss_disc_real_0: 0.08436892181634903 (0.12025202991384448) + | > loss_disc_real_1: 0.19248022139072418 (0.20185502234733466) + | > loss_disc_real_2: 0.21460218727588654 (0.2241790317224734) + | > loss_disc_real_3: 0.2356753945350647 (0.2297133252476201) + | > loss_disc_real_4: 0.25455451011657715 (0.2293038097294894) + | > loss_disc_real_5: 0.21830186247825623 (0.23176588795401834) + | > loss_0: 2.3894505500793457 (2.4426269242257783) + | > grad_norm_0: tensor(14.1749, device='cuda:0') (tensor(14.1939, device='cuda:0')) + | > loss_gen: 2.4644455909729004 (2.389393835356741) + | > loss_kl: 1.6867750883102417 (1.858565344954982) + | > loss_feat: 7.2741570472717285 (7.205224297263405) + | > loss_mel: 18.360422134399414 (17.778269738862008) + | > loss_duration: 1.4489375352859497 (1.4404314973137595) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.234737396240234 (30.67188494133227) + | > grad_norm_1: tensor(504.2442, device='cuda:0') (tensor(345.2856, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.356 (0.33907954620592523) + | > loader_time: 0.0048 (0.004947871872873017) + + + --> TIME: 2025-04-19 02:22:25 -- STEP: 58/142 -- GLOBAL_STEP: 408450 + | > loss_disc: 2.359168529510498 (2.4421362712465484) + | > loss_disc_real_0: 0.11118059605360031 (0.1218822319060564) + | > loss_disc_real_1: 0.18329696357250214 (0.20391808989746818) + | > loss_disc_real_2: 0.2011217474937439 (0.2233834783064908) + | > loss_disc_real_3: 0.24029137194156647 (0.2290764075928721) + | > loss_disc_real_4: 0.22072191536426544 (0.229562782264989) + | > loss_disc_real_5: 0.22214502096176147 (0.23265933271112113) + | > loss_0: 2.359168529510498 (2.4421362712465484) + | > grad_norm_0: tensor(10.9690, device='cuda:0') (tensor(15.6238, device='cuda:0')) + | > loss_gen: 2.3549067974090576 (2.3968949646785345) + | > loss_kl: 1.8593738079071045 (1.8704801304587002) + | > loss_feat: 6.753507614135742 (7.180736697953323) + | > loss_mel: 19.696269989013672 (17.825274960748075) + | > loss_duration: 1.4882177114486694 (1.45064796250442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.15227508544922 (30.72403490132299) + | > grad_norm_1: tensor(391.0693, device='cuda:0') (tensor(360.3023, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.3758 (0.35152602195739746) + | > loader_time: 0.0072 (0.005306367216439083) + + + --> TIME: 2025-04-19 02:22:35 -- STEP: 83/142 -- GLOBAL_STEP: 408475 + | > loss_disc: 2.4211721420288086 (2.453260062688804) + | > loss_disc_real_0: 0.1422511637210846 (0.12270158632512553) + | > loss_disc_real_1: 0.16998781263828278 (0.2064039024962) + | > loss_disc_real_2: 0.20273703336715698 (0.22395865098539605) + | > loss_disc_real_3: 0.21672265231609344 (0.2282645718519946) + | > loss_disc_real_4: 0.2111213058233261 (0.22989176411226572) + | > loss_disc_real_5: 0.20629411935806274 (0.2328530453055738) + | > loss_0: 2.4211721420288086 (2.453260062688804) + | > grad_norm_0: tensor(11.6250, device='cuda:0') (tensor(15.3018, device='cuda:0')) + | > loss_gen: 2.4026196002960205 (2.37694909486426) + | > loss_kl: 1.9367514848709106 (1.882544485919447) + | > loss_feat: 6.86345911026001 (7.106316922658897) + | > loss_mel: 17.863004684448242 (17.868148677320356) + | > loss_duration: 1.5528985261917114 (1.461663060877697) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.61873435974121 (30.695622363722467) + | > grad_norm_1: tensor(149.4722, device='cuda:0') (tensor(344.1881, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.3971 (0.36297257549791445) + | > loader_time: 0.0062 (0.00556947811540351) + + + --> TIME: 2025-04-19 02:22:45 -- STEP: 108/142 -- GLOBAL_STEP: 408500 + | > loss_disc: 2.456996440887451 (2.4618242073942116) + | > loss_disc_real_0: 0.1381211131811142 (0.12449035065731516) + | > loss_disc_real_1: 0.20554356276988983 (0.20588786141188056) + | > loss_disc_real_2: 0.21735092997550964 (0.22537700804295363) + | > loss_disc_real_3: 0.24570806324481964 (0.22910566227855506) + | > loss_disc_real_4: 0.23615683615207672 (0.23038836168470206) + | > loss_disc_real_5: 0.2338513731956482 (0.23287774622440338) + | > loss_0: 2.456996440887451 (2.4618242073942116) + | > grad_norm_0: tensor(9.2903, device='cuda:0') (tensor(15.1518, device='cuda:0')) + | > loss_gen: 2.3679351806640625 (2.3636480172475185) + | > loss_kl: 1.9863286018371582 (1.9034136103259192) + | > loss_feat: 6.965573787689209 (7.083590533998278) + | > loss_mel: 18.211267471313477 (17.85770316477174) + | > loss_duration: 1.51334810256958 (1.4712195330195963) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.044452667236328 (30.679575037073207) + | > grad_norm_1: tensor(289.1724, device='cuda:0') (tensor(336.4431, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.4331 (0.3733962398988229) + | > loader_time: 0.0066 (0.005810585286882191) + + + --> TIME: 2025-04-19 02:22:57 -- STEP: 133/142 -- GLOBAL_STEP: 408525 + | > loss_disc: 2.4128408432006836 (2.463268998870276) + | > loss_disc_real_0: 0.10213210433721542 (0.1247238608959474) + | > loss_disc_real_1: 0.1697339564561844 (0.2063200825587251) + | > loss_disc_real_2: 0.21097593009471893 (0.2256580848891036) + | > loss_disc_real_3: 0.20318949222564697 (0.2288157514163426) + | > loss_disc_real_4: 0.2134491503238678 (0.2302560939600593) + | > loss_disc_real_5: 0.19421949982643127 (0.23280947985953854) + | > loss_0: 2.4128408432006836 (2.463268998870276) + | > grad_norm_0: tensor(6.8738, device='cuda:0') (tensor(13.7964, device='cuda:0')) + | > loss_gen: 2.3974874019622803 (2.3569220779533677) + | > loss_kl: 2.008074998855591 (1.9127772831379022) + | > loss_feat: 7.501579284667969 (7.078188243665193) + | > loss_mel: 18.982845306396484 (17.868837901524124) + | > loss_duration: 1.55037522315979 (1.477661888402208) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.440364837646484 (30.694387536299857) + | > grad_norm_1: tensor(39.7177, device='cuda:0') (tensor(299.8459, device='cuda:0')) + | > current_lr_0: 0.00019493979452223445 + | > current_lr_1: 0.00019493979452223445 + | > step_time: 0.4437 (0.38692896885979444) + | > loader_time: 0.0067 (0.0061024436377044924) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5037026405334473 (2.5037026405334473) + | > loss_disc_real_0: 0.16976487636566162 (0.16976487636566162) + | > loss_disc_real_1: 0.23337069153785706 (0.23337069153785706) + | > loss_disc_real_2: 0.25848859548568726 (0.25848859548568726) + | > loss_disc_real_3: 0.24029718339443207 (0.24029718339443207) + | > loss_disc_real_4: 0.24620628356933594 (0.24620628356933594) + | > loss_disc_real_5: 0.24265524744987488 (0.24265524744987488) + | > loss_0: 2.5037026405334473 (2.5037026405334473) + | > loss_gen: 2.3733572959899902 (2.3733572959899902) + | > loss_kl: 2.535025119781494 (2.535025119781494) + | > loss_feat: 6.51894998550415 (6.51894998550415) + | > loss_mel: 18.08901023864746 (18.08901023864746) + | > loss_duration: 1.6875007152557373 (1.6875007152557373) + | > loss_1: 31.20384407043457 (31.20384407043457) + + --> STEP: 1 + | > loss_disc: 2.487866163253784 (2.487866163253784) + | > loss_disc_real_0: 0.13396190106868744 (0.13396190106868744) + | > loss_disc_real_1: 0.22705726325511932 (0.22705726325511932) + | > loss_disc_real_2: 0.23379401862621307 (0.23379401862621307) + | > loss_disc_real_3: 0.2248605638742447 (0.2248605638742447) + | > loss_disc_real_4: 0.20739983022212982 (0.20739983022212982) + | > loss_disc_real_5: 0.22026197612285614 (0.22026197612285614) + | > loss_0: 2.487866163253784 (2.487866163253784) + | > loss_gen: 2.240945816040039 (2.240945816040039) + | > loss_kl: 2.667236566543579 (2.667236566543579) + | > loss_feat: 6.760678291320801 (6.760678291320801) + | > loss_mel: 19.101165771484375 (19.101165771484375) + | > loss_duration: 1.7217903137207031 (1.7217903137207031) + | > loss_1: 32.49181365966797 (32.49181365966797) + + --> STEP: 2 + | > loss_disc: 2.4225621223449707 (2.4552141427993774) + | > loss_disc_real_0: 0.13756714761257172 (0.13576452434062958) + | > loss_disc_real_1: 0.23154659569263458 (0.22930192947387695) + | > loss_disc_real_2: 0.21008899807929993 (0.2219415083527565) + | > loss_disc_real_3: 0.21441051363945007 (0.21963553875684738) + | > loss_disc_real_4: 0.20491673052310944 (0.20615828037261963) + | > loss_disc_real_5: 0.2914406657218933 (0.2558513209223747) + | > loss_0: 2.4225621223449707 (2.4552141427993774) + | > loss_gen: 2.410374641418457 (2.325660228729248) + | > loss_kl: 2.492837905883789 (2.580037236213684) + | > loss_feat: 7.915368556976318 (7.33802342414856) + | > loss_mel: 19.096920013427734 (19.099042892456055) + | > loss_duration: 1.6864126920700073 (1.7041015028953552) + | > loss_1: 33.60191345214844 (33.0468635559082) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003238677978515625 (-0.0002576112747192383) + | > avg_loss_disc: 2.4552141427993774 (+0.14441776275634766) + | > avg_loss_disc_real_0: 0.13576452434062958 (+0.0250973142683506) + | > avg_loss_disc_real_1: 0.22930192947387695 (-0.013782896101474762) + | > avg_loss_disc_real_2: 0.2219415083527565 (+0.0009683370590209961) + | > avg_loss_disc_real_3: 0.21963553875684738 (+0.03251373767852783) + | > avg_loss_disc_real_4: 0.20615828037261963 (-0.019175440073013306) + | > avg_loss_disc_real_5: 0.2558513209223747 (+0.029132530093193054) + | > avg_loss_0: 2.4552141427993774 (+0.14441776275634766) + | > avg_loss_gen: 2.325660228729248 (-0.1317540407180786) + | > avg_loss_kl: 2.580037236213684 (+0.15093457698822021) + | > avg_loss_feat: 7.33802342414856 (-0.17916417121887207) + | > avg_loss_mel: 19.099042892456055 (+0.9872465133666992) + | > avg_loss_duration: 1.7041015028953552 (+0.017859160900115967) + | > avg_loss_1: 33.0468635559082 (+0.8451213836669922) + + + > EPOCH: 206/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:23:02)  + + --> TIME: 2025-04-19 02:23:08 -- STEP: 16/142 -- GLOBAL_STEP: 408550 + | > loss_disc: 2.444298267364502 (2.4328166097402573) + | > loss_disc_real_0: 0.1282104104757309 (0.12333398032933474) + | > loss_disc_real_1: 0.23953701555728912 (0.21309383027255535) + | > loss_disc_real_2: 0.23754827678203583 (0.2228841232135892) + | > loss_disc_real_3: 0.23049913346767426 (0.22681992407888174) + | > loss_disc_real_4: 0.212992325425148 (0.22132917400449514) + | > loss_disc_real_5: 0.22633227705955505 (0.22051423601806164) + | > loss_0: 2.444298267364502 (2.4328166097402573) + | > grad_norm_0: tensor(11.5066, device='cuda:0') (tensor(7.5056, device='cuda:0')) + | > loss_gen: 2.346555709838867 (2.348088338971138) + | > loss_kl: 1.6514551639556885 (1.8066016286611557) + | > loss_feat: 6.4695048332214355 (6.928379952907562) + | > loss_mel: 16.84945297241211 (17.283698618412018) + | > loss_duration: 1.415198802947998 (1.4329204261302948) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.732168197631836 (29.799689412117004) + | > grad_norm_1: tensor(130.5788, device='cuda:0') (tensor(131.4870, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.3411 (0.32856765389442444) + | > loader_time: 0.0052 (0.004762142896652222) + + + --> TIME: 2025-04-19 02:23:17 -- STEP: 41/142 -- GLOBAL_STEP: 408575 + | > loss_disc: 2.4708590507507324 (2.4431359302706834) + | > loss_disc_real_0: 0.10958228260278702 (0.12398026810913551) + | > loss_disc_real_1: 0.195091113448143 (0.20691405164032448) + | > loss_disc_real_2: 0.19803008437156677 (0.22473299067194868) + | > loss_disc_real_3: 0.24541735649108887 (0.22669945112088832) + | > loss_disc_real_4: 0.1910974085330963 (0.22662413956188573) + | > loss_disc_real_5: 0.18825994431972504 (0.22673219987531987) + | > loss_0: 2.4708590507507324 (2.4431359302706834) + | > grad_norm_0: tensor(11.3665, device='cuda:0') (tensor(7.2847, device='cuda:0')) + | > loss_gen: 2.3014960289001465 (2.356900244224363) + | > loss_kl: 1.8126155138015747 (1.8362715418745832) + | > loss_feat: 6.700625419616699 (7.0450075893867306) + | > loss_mel: 16.63558006286621 (17.550083486045274) + | > loss_duration: 1.4248263835906982 (1.4431680179223783) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.87514305114746 (30.23143084456281) + | > grad_norm_1: tensor(222.8863, device='cuda:0') (tensor(140.9257, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.3713 (0.3455136985313602) + | > loader_time: 0.0059 (0.005285431699054997) + + + --> TIME: 2025-04-19 02:23:27 -- STEP: 66/142 -- GLOBAL_STEP: 408600 + | > loss_disc: 2.46254563331604 (2.455127878622575) + | > loss_disc_real_0: 0.09120713919401169 (0.12642024423588405) + | > loss_disc_real_1: 0.23764704167842865 (0.2086361206390641) + | > loss_disc_real_2: 0.2240191102027893 (0.22750353835748904) + | > loss_disc_real_3: 0.20042195916175842 (0.2273322269320488) + | > loss_disc_real_4: 0.23854808509349823 (0.22927488657561215) + | > loss_disc_real_5: 0.2000715732574463 (0.22748267311941495) + | > loss_0: 2.46254563331604 (2.455127878622575) + | > grad_norm_0: tensor(11.3826, device='cuda:0') (tensor(8.6550, device='cuda:0')) + | > loss_gen: 2.335684061050415 (2.356718012780855) + | > loss_kl: 1.8573497533798218 (1.854500696514592) + | > loss_feat: 6.810853481292725 (7.028153578440349) + | > loss_mel: 18.01962661743164 (17.60426168730765) + | > loss_duration: 1.4640839099884033 (1.4550555240024219) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.487598419189453 (30.298689379836574) + | > grad_norm_1: tensor(276.5828, device='cuda:0') (tensor(187.4147, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.3947 (0.3597379706122659) + | > loader_time: 0.0067 (0.005654468680873062) + + + --> TIME: 2025-04-19 02:23:37 -- STEP: 91/142 -- GLOBAL_STEP: 408625 + | > loss_disc: 2.3796467781066895 (2.459363641319694) + | > loss_disc_real_0: 0.11330743134021759 (0.12548013735603505) + | > loss_disc_real_1: 0.2152617871761322 (0.207709700687901) + | > loss_disc_real_2: 0.21000789105892181 (0.22799311791147506) + | > loss_disc_real_3: 0.1855282187461853 (0.22858056900920448) + | > loss_disc_real_4: 0.21648618578910828 (0.23004047687237078) + | > loss_disc_real_5: 0.2109752744436264 (0.2282858887216547) + | > loss_0: 2.3796467781066895 (2.459363641319694) + | > grad_norm_0: tensor(9.2493, device='cuda:0') (tensor(9.3684, device='cuda:0')) + | > loss_gen: 2.460649013519287 (2.3581712586539147) + | > loss_kl: 1.9552671909332275 (1.875491636140006) + | > loss_feat: 7.503304958343506 (7.027582923134605) + | > loss_mel: 16.916635513305664 (17.62508199503134) + | > loss_duration: 1.4935377836227417 (1.4664513907589751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.329395294189453 (30.35277907402961) + | > grad_norm_1: tensor(51.5692, device='cuda:0') (tensor(212.1864, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.4151 (0.3720070398770847) + | > loader_time: 0.0067 (0.005949046585586044) + + + --> TIME: 2025-04-19 02:23:48 -- STEP: 116/142 -- GLOBAL_STEP: 408650 + | > loss_disc: 2.461012125015259 (2.4645317813445784) + | > loss_disc_real_0: 0.1165914312005043 (0.12784776984360724) + | > loss_disc_real_1: 0.23658718168735504 (0.20859712657743487) + | > loss_disc_real_2: 0.2412486970424652 (0.22768853155189547) + | > loss_disc_real_3: 0.2463551014661789 (0.22830684238980556) + | > loss_disc_real_4: 0.21810518205165863 (0.2304908213173521) + | > loss_disc_real_5: 0.21888047456741333 (0.2276279972545032) + | > loss_0: 2.461012125015259 (2.4645317813445784) + | > grad_norm_0: tensor(11.3532, device='cuda:0') (tensor(9.6798, device='cuda:0')) + | > loss_gen: 2.365265369415283 (2.3450172584632365) + | > loss_kl: 1.8805640935897827 (1.8787490263067443) + | > loss_feat: 6.896814346313477 (6.981313623231033) + | > loss_mel: 17.633094787597656 (17.586306884371) + | > loss_duration: 1.4720163345336914 (1.472590187500263) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.247753143310547 (30.263976820584002) + | > grad_norm_1: tensor(354.0356, device='cuda:0') (tensor(219.4092, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.4411 (0.3839507781226061) + | > loader_time: 0.0075 (0.006206243202604097) + + + --> TIME: 2025-04-19 02:24:00 -- STEP: 141/142 -- GLOBAL_STEP: 408675 + | > loss_disc: 2.3569157123565674 (2.4657835250205182) + | > loss_disc_real_0: 0.10770448297262192 (0.1282660194428256) + | > loss_disc_real_1: 0.18253636360168457 (0.20701192076324568) + | > loss_disc_real_2: 0.20838718116283417 (0.22678405236690605) + | > loss_disc_real_3: 0.21543554961681366 (0.22838985550065413) + | > loss_disc_real_4: 0.20691746473312378 (0.23072092678952724) + | > loss_disc_real_5: 0.2109481543302536 (0.22881551699858185) + | > loss_0: 2.3569157123565674 (2.4657835250205182) + | > grad_norm_0: tensor(14.2185, device='cuda:0') (tensor(10.0945, device='cuda:0')) + | > loss_gen: 2.7024731636047363 (2.346264531426398) + | > loss_kl: 1.7610070705413818 (1.8866730446511126) + | > loss_feat: 8.909072875976562 (6.965983857499792) + | > loss_mel: 18.94330406188965 (17.554368952487373) + | > loss_duration: 1.322969913482666 (1.4779882135120685) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.63882827758789 (30.231278507422047) + | > grad_norm_1: tensor(428.1900, device='cuda:0') (tensor(229.9931, device='cuda:0')) + | > current_lr_0: 0.00019491542704791915 + | > current_lr_1: 0.00019491542704791915 + | > step_time: 0.201 (0.39458107779212054) + | > loader_time: 0.0026 (0.006414344124760188) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4346818923950195 (2.4346818923950195) + | > loss_disc_real_0: 0.17507818341255188 (0.17507818341255188) + | > loss_disc_real_1: 0.252642959356308 (0.252642959356308) + | > loss_disc_real_2: 0.23491370677947998 (0.23491370677947998) + | > loss_disc_real_3: 0.24580536782741547 (0.24580536782741547) + | > loss_disc_real_4: 0.2427801489830017 (0.2427801489830017) + | > loss_disc_real_5: 0.2549480199813843 (0.2549480199813843) + | > loss_0: 2.4346818923950195 (2.4346818923950195) + | > loss_gen: 2.607848644256592 (2.607848644256592) + | > loss_kl: 2.586258888244629 (2.586258888244629) + | > loss_feat: 7.126516819000244 (7.126516819000244) + | > loss_mel: 18.122495651245117 (18.122495651245117) + | > loss_duration: 1.6984968185424805 (1.6984968185424805) + | > loss_1: 32.14161682128906 (32.14161682128906) + + --> STEP: 1 + | > loss_disc: 2.750072956085205 (2.750072956085205) + | > loss_disc_real_0: 0.2754075229167938 (0.2754075229167938) + | > loss_disc_real_1: 0.3091496527194977 (0.3091496527194977) + | > loss_disc_real_2: 0.274448961019516 (0.274448961019516) + | > loss_disc_real_3: 0.2864544987678528 (0.2864544987678528) + | > loss_disc_real_4: 0.2624538242816925 (0.2624538242816925) + | > loss_disc_real_5: 0.30546364188194275 (0.30546364188194275) + | > loss_0: 2.750072956085205 (2.750072956085205) + | > loss_gen: 2.677891492843628 (2.677891492843628) + | > loss_kl: 2.7445766925811768 (2.7445766925811768) + | > loss_feat: 6.1226420402526855 (6.1226420402526855) + | > loss_mel: 17.025911331176758 (17.025911331176758) + | > loss_duration: 1.7166129350662231 (1.7166129350662231) + | > loss_1: 30.287635803222656 (30.287635803222656) + + --> STEP: 2 + | > loss_disc: 2.4327070713043213 (2.591390013694763) + | > loss_disc_real_0: 0.24365893006324768 (0.25953322649002075) + | > loss_disc_real_1: 0.3078738749027252 (0.30851176381111145) + | > loss_disc_real_2: 0.23616084456443787 (0.25530490279197693) + | > loss_disc_real_3: 0.204341322183609 (0.2453979104757309) + | > loss_disc_real_4: 0.2437550276517868 (0.25310442596673965) + | > loss_disc_real_5: 0.19920022785663605 (0.2523319348692894) + | > loss_0: 2.4327070713043213 (2.591390013694763) + | > loss_gen: 2.729978084564209 (2.7039347887039185) + | > loss_kl: 2.008423328399658 (2.3765000104904175) + | > loss_feat: 8.500715255737305 (7.311678647994995) + | > loss_mel: 18.78749656677246 (17.90670394897461) + | > loss_duration: 1.6540687084197998 (1.6853408217430115) + | > loss_1: 33.68067932128906 (31.98415756225586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038018226623535156 (+0.0005631446838378906) + | > avg_loss_disc: 2.591390013694763 (+0.13617587089538574) + | > avg_loss_disc_real_0: 0.25953322649002075 (+0.12376870214939117) + | > avg_loss_disc_real_1: 0.30851176381111145 (+0.0792098343372345) + | > avg_loss_disc_real_2: 0.25530490279197693 (+0.03336339443922043) + | > avg_loss_disc_real_3: 0.2453979104757309 (+0.025762371718883514) + | > avg_loss_disc_real_4: 0.25310442596673965 (+0.046946145594120026) + | > avg_loss_disc_real_5: 0.2523319348692894 (-0.003519386053085327) + | > avg_loss_0: 2.591390013694763 (+0.13617587089538574) + | > avg_loss_gen: 2.7039347887039185 (+0.3782745599746704) + | > avg_loss_kl: 2.3765000104904175 (-0.2035372257232666) + | > avg_loss_feat: 7.311678647994995 (-0.026344776153564453) + | > avg_loss_mel: 17.90670394897461 (-1.1923389434814453) + | > avg_loss_duration: 1.6853408217430115 (-0.01876068115234375) + | > avg_loss_1: 31.98415756225586 (-1.0627059936523438) + + + > EPOCH: 207/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:24:02)  + + --> TIME: 2025-04-19 02:24:11 -- STEP: 24/142 -- GLOBAL_STEP: 408700 + | > loss_disc: 2.5684168338775635 (2.4391562143961587) + | > loss_disc_real_0: 0.18430523574352264 (0.12310273386538029) + | > loss_disc_real_1: 0.22746364772319794 (0.20620037987828255) + | > loss_disc_real_2: 0.24040283262729645 (0.23053056498368582) + | > loss_disc_real_3: 0.264261931180954 (0.22881503279010454) + | > loss_disc_real_4: 0.26549118757247925 (0.23080871192117533) + | > loss_disc_real_5: 0.2704308032989502 (0.2242482528090477) + | > loss_0: 2.5684168338775635 (2.4391562143961587) + | > grad_norm_0: tensor(16.2495, device='cuda:0') (tensor(14.9204, device='cuda:0')) + | > loss_gen: 2.259416103363037 (2.3791155914465585) + | > loss_kl: 1.8927831649780273 (1.8563164969285328) + | > loss_feat: 5.942911624908447 (7.225040217240651) + | > loss_mel: 16.499326705932617 (17.84405215581258) + | > loss_duration: 1.4669554233551025 (1.4339880794286728) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.06139373779297 (30.738512595494587) + | > grad_norm_1: tensor(295.1996, device='cuda:0') (tensor(288.0049, device='cuda:0')) + | > current_lr_0: 0.00019489106261953815 + | > current_lr_1: 0.00019489106261953815 + | > step_time: 0.3444 (0.33366111914316815) + | > loader_time: 0.0056 (0.004919012387593587) + + + --> TIME: 2025-04-19 02:24:20 -- STEP: 49/142 -- GLOBAL_STEP: 408725 + | > loss_disc: 2.4963459968566895 (2.430698277998943) + | > loss_disc_real_0: 0.15544159710407257 (0.11993365141810203) + | > loss_disc_real_1: 0.19763338565826416 (0.20494827688956746) + | > loss_disc_real_2: 0.21036578714847565 (0.22664648507322585) + | > loss_disc_real_3: 0.23474128544330597 (0.22588686523388843) + | > loss_disc_real_4: 0.27857306599617004 (0.2272626447434328) + | > loss_disc_real_5: 0.26167169213294983 (0.22623358545254688) + | > loss_0: 2.4963459968566895 (2.430698277998943) + | > grad_norm_0: tensor(15.8600, device='cuda:0') (tensor(13.7808, device='cuda:0')) + | > loss_gen: 2.2958879470825195 (2.3838945748854647) + | > loss_kl: 2.050811290740967 (1.8658686487042173) + | > loss_feat: 6.488257884979248 (7.218570689765775) + | > loss_mel: 17.39093017578125 (17.863718110687877) + | > loss_duration: 1.480329155921936 (1.4426796144368697) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70621681213379 (30.774731850137517) + | > grad_norm_1: tensor(354.8143, device='cuda:0') (tensor(305.5094, device='cuda:0')) + | > current_lr_0: 0.00019489106261953815 + | > current_lr_1: 0.00019489106261953815 + | > step_time: 0.3715 (0.3456779888698034) + | > loader_time: 0.0063 (0.00523024675797443) + + + --> TIME: 2025-04-19 02:24:30 -- STEP: 74/142 -- GLOBAL_STEP: 408750 + | > loss_disc: 2.5915348529815674 (2.450190089844368) + | > loss_disc_real_0: 0.11648466438055038 (0.12359359049917878) + | > loss_disc_real_1: 0.2001715749502182 (0.2055681762663094) + | > loss_disc_real_2: 0.2447887361049652 (0.22737201424063863) + | > loss_disc_real_3: 0.20775753259658813 (0.22794374763160138) + | > loss_disc_real_4: 0.21957886219024658 (0.228249458243718) + | > loss_disc_real_5: 0.21437053382396698 (0.22905426875159546) + | > loss_0: 2.5915348529815674 (2.450190089844368) + | > grad_norm_0: tensor(7.6470, device='cuda:0') (tensor(13.0804, device='cuda:0')) + | > loss_gen: 2.143035888671875 (2.369831671585907) + | > loss_kl: 2.018375873565674 (1.8846511792492222) + | > loss_feat: 6.1686859130859375 (7.147973878963573) + | > loss_mel: 17.21722412109375 (17.820767247999036) + | > loss_duration: 1.496597170829773 (1.4569022671596428) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.04391860961914 (30.68012642216038) + | > grad_norm_1: tensor(245.3926, device='cuda:0') (tensor(294.0623, device='cuda:0')) + | > current_lr_0: 0.00019489106261953815 + | > current_lr_1: 0.00019489106261953815 + | > step_time: 0.3888 (0.3576763739457001) + | > loader_time: 0.0063 (0.005516864158011772) + + + --> TIME: 2025-04-19 02:24:40 -- STEP: 99/142 -- GLOBAL_STEP: 408775 + | > loss_disc: 2.4708621501922607 (2.4579076188983335) + | > loss_disc_real_0: 0.14059226214885712 (0.12553497979586775) + | > loss_disc_real_1: 0.22665338218212128 (0.2052508694957001) + | > loss_disc_real_2: 0.2520007789134979 (0.22804605298572117) + | > loss_disc_real_3: 0.25104859471321106 (0.22895544978103252) + | > loss_disc_real_4: 0.22771143913269043 (0.22889855985689644) + | > loss_disc_real_5: 0.25381404161453247 (0.23054817078089473) + | > loss_0: 2.4708621501922607 (2.4579076188983335) + | > grad_norm_0: tensor(6.3711, device='cuda:0') (tensor(12.9114, device='cuda:0')) + | > loss_gen: 2.400299072265625 (2.357396556873514) + | > loss_kl: 1.9259847402572632 (1.8957805007395119) + | > loss_feat: 6.842632293701172 (7.028307495695172) + | > loss_mel: 17.7891845703125 (17.750685932660357) + | > loss_duration: 1.508311152458191 (1.4677364741913002) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.46640968322754 (30.499907040836835) + | > grad_norm_1: tensor(333.9157, device='cuda:0') (tensor(293.1630, device='cuda:0')) + | > current_lr_0: 0.00019489106261953815 + | > current_lr_1: 0.00019489106261953815 + | > step_time: 0.409 (0.3689050433611628) + | > loader_time: 0.0066 (0.005752134804773811) + + + --> TIME: 2025-04-19 02:24:51 -- STEP: 124/142 -- GLOBAL_STEP: 408800 + | > loss_disc: 2.4983737468719482 (2.4588353153197993) + | > loss_disc_real_0: 0.07099021226167679 (0.12606504628615037) + | > loss_disc_real_1: 0.20345188677310944 (0.2061419130092667) + | > loss_disc_real_2: 0.2373446375131607 (0.22632033726380718) + | > loss_disc_real_3: 0.2698374390602112 (0.22824196817894135) + | > loss_disc_real_4: 0.21480190753936768 (0.22894783522332868) + | > loss_disc_real_5: 0.25090232491493225 (0.22968339835924487) + | > loss_0: 2.4983737468719482 (2.4588353153197993) + | > grad_norm_0: tensor(20.0257, device='cuda:0') (tensor(13.3233, device='cuda:0')) + | > loss_gen: 2.5021958351135254 (2.3620527425119953) + | > loss_kl: 1.8822983503341675 (1.8986524795332262) + | > loss_feat: 7.070234298706055 (7.02916068800034) + | > loss_mel: 18.234228134155273 (17.717978615914635) + | > loss_duration: 1.5329301357269287 (1.4748610652262168) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.221885681152344 (30.482705654636508) + | > grad_norm_1: tensor(409.5092, device='cuda:0') (tensor(300.7518, device='cuda:0')) + | > current_lr_0: 0.00019489106261953815 + | > current_lr_1: 0.00019489106261953815 + | > step_time: 0.437 (0.38098764804101753) + | > loader_time: 0.0067 (0.00600410469116703) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4427661895751953 (2.4427661895751953) + | > loss_disc_real_0: 0.09431774914264679 (0.09431774914264679) + | > loss_disc_real_1: 0.22768083214759827 (0.22768083214759827) + | > loss_disc_real_2: 0.21726834774017334 (0.21726834774017334) + | > loss_disc_real_3: 0.22950340807437897 (0.22950340807437897) + | > loss_disc_real_4: 0.220144122838974 (0.220144122838974) + | > loss_disc_real_5: 0.23337818682193756 (0.23337818682193756) + | > loss_0: 2.4427661895751953 (2.4427661895751953) + | > loss_gen: 2.334282159805298 (2.334282159805298) + | > loss_kl: 2.066509962081909 (2.066509962081909) + | > loss_feat: 6.576548099517822 (6.576548099517822) + | > loss_mel: 17.430522918701172 (17.430522918701172) + | > loss_duration: 1.647719383239746 (1.647719383239746) + | > loss_1: 30.055583953857422 (30.055583953857422) + + --> STEP: 1 + | > loss_disc: 2.4638864994049072 (2.4638864994049072) + | > loss_disc_real_0: 0.06394775956869125 (0.06394775956869125) + | > loss_disc_real_1: 0.21029679477214813 (0.21029679477214813) + | > loss_disc_real_2: 0.24775435030460358 (0.24775435030460358) + | > loss_disc_real_3: 0.23067452013492584 (0.23067452013492584) + | > loss_disc_real_4: 0.1867193728685379 (0.1867193728685379) + | > loss_disc_real_5: 0.24508138000965118 (0.24508138000965118) + | > loss_0: 2.4638864994049072 (2.4638864994049072) + | > loss_gen: 2.187404155731201 (2.187404155731201) + | > loss_kl: 2.5788848400115967 (2.5788848400115967) + | > loss_feat: 7.467649936676025 (7.467649936676025) + | > loss_mel: 19.105144500732422 (19.105144500732422) + | > loss_duration: 1.711562991142273 (1.711562991142273) + | > loss_1: 33.0506477355957 (33.0506477355957) + + --> STEP: 2 + | > loss_disc: 2.507225275039673 (2.48555588722229) + | > loss_disc_real_0: 0.0744779109954834 (0.06921283528208733) + | > loss_disc_real_1: 0.22825711965560913 (0.21927695721387863) + | > loss_disc_real_2: 0.2332155406475067 (0.24048494547605515) + | > loss_disc_real_3: 0.25644612312316895 (0.2435603216290474) + | > loss_disc_real_4: 0.2209453135728836 (0.20383234322071075) + | > loss_disc_real_5: 0.21005743741989136 (0.22756940871477127) + | > loss_0: 2.507225275039673 (2.48555588722229) + | > loss_gen: 2.1409146785736084 (2.164159417152405) + | > loss_kl: 2.0905556678771973 (2.334720253944397) + | > loss_feat: 8.210909843444824 (7.839279890060425) + | > loss_mel: 18.72980308532715 (18.917473793029785) + | > loss_duration: 1.650122046470642 (1.6808425188064575) + | > loss_1: 32.82230758666992 (32.93647766113281) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0045015811920166016 (+0.0006997585296630859) + | > avg_loss_disc: 2.48555588722229 (-0.10583412647247314) + | > avg_loss_disc_real_0: 0.06921283528208733 (-0.19032039120793343) + | > avg_loss_disc_real_1: 0.21927695721387863 (-0.08923480659723282) + | > avg_loss_disc_real_2: 0.24048494547605515 (-0.014819957315921783) + | > avg_loss_disc_real_3: 0.2435603216290474 (-0.0018375888466835022) + | > avg_loss_disc_real_4: 0.20383234322071075 (-0.0492720827460289) + | > avg_loss_disc_real_5: 0.22756940871477127 (-0.024762526154518127) + | > avg_loss_0: 2.48555588722229 (-0.10583412647247314) + | > avg_loss_gen: 2.164159417152405 (-0.5397753715515137) + | > avg_loss_kl: 2.334720253944397 (-0.04177975654602051) + | > avg_loss_feat: 7.839279890060425 (+0.5276012420654297) + | > avg_loss_mel: 18.917473793029785 (+1.0107698440551758) + | > avg_loss_duration: 1.6808425188064575 (-0.004498302936553955) + | > avg_loss_1: 32.93647766113281 (+0.9523200988769531) + + + > EPOCH: 208/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:25:01)  + + --> TIME: 2025-04-19 02:25:04 -- STEP: 7/142 -- GLOBAL_STEP: 408825 + | > loss_disc: 2.459223985671997 (2.394261121749878) + | > loss_disc_real_0: 0.07675661891698837 (0.10071041967187609) + | > loss_disc_real_1: 0.22932173311710358 (0.20487276571137564) + | > loss_disc_real_2: 0.24910756945610046 (0.2355163757290159) + | > loss_disc_real_3: 0.22531671822071075 (0.21370521826403482) + | > loss_disc_real_4: 0.2044406682252884 (0.21872652428490774) + | > loss_disc_real_5: 0.2460733950138092 (0.2169227408511298) + | > loss_0: 2.459223985671997 (2.394261121749878) + | > grad_norm_0: tensor(12.5765, device='cuda:0') (tensor(17.1245, device='cuda:0')) + | > loss_gen: 2.5037436485290527 (2.4453980241503035) + | > loss_kl: 1.7128115892410278 (1.849221876689366) + | > loss_feat: 7.619916915893555 (7.487909521375384) + | > loss_mel: 17.864791870117188 (17.74886757986886) + | > loss_duration: 1.410378098487854 (1.424244029181344) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.111642837524414 (30.955641882760183) + | > grad_norm_1: tensor(468.8552, device='cuda:0') (tensor(387.7584, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.3229 (0.3250509330204555) + | > loader_time: 0.0042 (0.00447014399937221) + + + --> TIME: 2025-04-19 02:25:13 -- STEP: 32/142 -- GLOBAL_STEP: 408850 + | > loss_disc: 2.4026496410369873 (2.4405114948749542) + | > loss_disc_real_0: 0.1091851219534874 (0.11606396944262087) + | > loss_disc_real_1: 0.226432204246521 (0.20068908389657736) + | > loss_disc_real_2: 0.21245262026786804 (0.22947589354589581) + | > loss_disc_real_3: 0.21262100338935852 (0.22448810981586576) + | > loss_disc_real_4: 0.23779292404651642 (0.23202132200822234) + | > loss_disc_real_5: 0.2207012176513672 (0.22858691308647394) + | > loss_0: 2.4026496410369873 (2.4405114948749542) + | > grad_norm_0: tensor(11.4285, device='cuda:0') (tensor(16.7444, device='cuda:0')) + | > loss_gen: 2.5404891967773438 (2.405735395848751) + | > loss_kl: 1.9571868181228638 (1.8645462915301323) + | > loss_feat: 7.810282230377197 (7.244194120168686) + | > loss_mel: 18.7600154876709 (17.93560069799424) + | > loss_duration: 1.4443511962890625 (1.4379261806607246) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.512325286865234 (30.888003289699554) + | > grad_norm_1: tensor(206.7354, device='cuda:0') (tensor(353.6112, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.3552 (0.3440781608223915) + | > loader_time: 0.0056 (0.005120329558849334) + + + --> TIME: 2025-04-19 02:25:22 -- STEP: 57/142 -- GLOBAL_STEP: 408875 + | > loss_disc: 2.495450973510742 (2.4490273500743664) + | > loss_disc_real_0: 0.13291071355342865 (0.11957088439610966) + | > loss_disc_real_1: 0.21730946004390717 (0.2052268181976519) + | > loss_disc_real_2: 0.24848753213882446 (0.2293008482247068) + | > loss_disc_real_3: 0.2395072728395462 (0.22711179523091568) + | > loss_disc_real_4: 0.27221447229385376 (0.2305252212181426) + | > loss_disc_real_5: 0.2255554497241974 (0.22920477651713186) + | > loss_0: 2.495450973510742 (2.4490273500743664) + | > grad_norm_0: tensor(7.5178, device='cuda:0') (tensor(14.1724, device='cuda:0')) + | > loss_gen: 2.272188663482666 (2.384112655070789) + | > loss_kl: 1.7764408588409424 (1.865527211574086) + | > loss_feat: 6.649909496307373 (7.191256949776097) + | > loss_mel: 17.57023048400879 (17.869290485716714) + | > loss_duration: 1.4995335340499878 (1.4508000633172822) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.76830291748047 (30.760987666615268) + | > grad_norm_1: tensor(136.3716, device='cuda:0') (tensor(313.4977, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.3751 (0.35369460206282766) + | > loader_time: 0.0061 (0.005451984572828861) + + + --> TIME: 2025-04-19 02:25:32 -- STEP: 82/142 -- GLOBAL_STEP: 408900 + | > loss_disc: 2.4880146980285645 (2.460711964746801) + | > loss_disc_real_0: 0.1672462373971939 (0.12270623554543751) + | > loss_disc_real_1: 0.20454055070877075 (0.2058824008194412) + | > loss_disc_real_2: 0.23570507764816284 (0.22861750078637424) + | > loss_disc_real_3: 0.24437476694583893 (0.22770894446024081) + | > loss_disc_real_4: 0.24411381781101227 (0.23100101620685765) + | > loss_disc_real_5: 0.23461131751537323 (0.2306105861576592) + | > loss_0: 2.4880146980285645 (2.460711964746801) + | > grad_norm_0: tensor(10.4635, device='cuda:0') (tensor(13.4072, device='cuda:0')) + | > loss_gen: 2.358022689819336 (2.3671852350234985) + | > loss_kl: 1.9513598680496216 (1.88787138752821) + | > loss_feat: 6.76600456237793 (7.080537627382976) + | > loss_mel: 18.39580535888672 (17.794186266457174) + | > loss_duration: 1.5548298358917236 (1.4624802470207214) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.026023864746094 (30.592260895705806) + | > grad_norm_1: tensor(100.9803, device='cuda:0') (tensor(290.6165, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.3945 (0.36438206347023566) + | > loader_time: 0.0068 (0.005777943425062226) + + + --> TIME: 2025-04-19 02:25:43 -- STEP: 107/142 -- GLOBAL_STEP: 408925 + | > loss_disc: 2.4996845722198486 (2.461725564760583) + | > loss_disc_real_0: 0.09226813912391663 (0.12331039507255376) + | > loss_disc_real_1: 0.18846631050109863 (0.20658347247359912) + | > loss_disc_real_2: 0.2270926982164383 (0.22885439120163428) + | > loss_disc_real_3: 0.23555152118206024 (0.22799401918304302) + | > loss_disc_real_4: 0.22211617231369019 (0.22932918425475327) + | > loss_disc_real_5: 0.23094643652439117 (0.23026898029808687) + | > loss_0: 2.4996845722198486 (2.461725564760583) + | > grad_norm_0: tensor(6.6306, device='cuda:0') (tensor(12.4192, device='cuda:0')) + | > loss_gen: 2.3854012489318848 (2.3544826061926165) + | > loss_kl: 1.7497917413711548 (1.891143796599914) + | > loss_feat: 7.3286027908325195 (7.039481243240499) + | > loss_mel: 19.10781478881836 (17.76573703445007) + | > loss_duration: 1.4830045700073242 (1.4723145337862389) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.05461502075195 (30.52315931231062) + | > grad_norm_1: tensor(54.4120, device='cuda:0') (tensor(267.1377, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.4153 (0.37408527704042793) + | > loader_time: 0.007 (0.006030272100573387) + + + --> TIME: 2025-04-19 02:25:54 -- STEP: 132/142 -- GLOBAL_STEP: 408950 + | > loss_disc: 2.4210240840911865 (2.463845065145782) + | > loss_disc_real_0: 0.17667698860168457 (0.12509150628113386) + | > loss_disc_real_1: 0.18048129975795746 (0.20659960998278679) + | > loss_disc_real_2: 0.25526654720306396 (0.2282898976947322) + | > loss_disc_real_3: 0.2506575882434845 (0.22784260479789792) + | > loss_disc_real_4: 0.24507997930049896 (0.22933267643957428) + | > loss_disc_real_5: 0.22911213338375092 (0.23107362403111023) + | > loss_0: 2.4210240840911865 (2.463845065145782) + | > grad_norm_0: tensor(10.9519, device='cuda:0') (tensor(11.8735, device='cuda:0')) + | > loss_gen: 2.468867301940918 (2.347344788638029) + | > loss_kl: 2.0011403560638428 (1.8989415936397784) + | > loss_feat: 6.7492570877075195 (7.006959886261911) + | > loss_mel: 18.406736373901367 (17.7775374398087) + | > loss_duration: 1.4694515466690063 (1.4782995518409845) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.095455169677734 (30.509083372173887) + | > grad_norm_1: tensor(334.6323, device='cuda:0') (tensor(254.9317, device='cuda:0')) + | > current_lr_0: 0.0001948667012367107 + | > current_lr_1: 0.0001948667012367107 + | > step_time: 0.4464 (0.385684463110837) + | > loader_time: 0.0077 (0.006289202155488909) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.43562388420105 (2.43562388420105) + | > loss_disc_real_0: 0.21946299076080322 (0.21946299076080322) + | > loss_disc_real_1: 0.20534971356391907 (0.20534971356391907) + | > loss_disc_real_2: 0.2741130292415619 (0.2741130292415619) + | > loss_disc_real_3: 0.2325419932603836 (0.2325419932603836) + | > loss_disc_real_4: 0.22593119740486145 (0.22593119740486145) + | > loss_disc_real_5: 0.23938149213790894 (0.23938149213790894) + | > loss_0: 2.43562388420105 (2.43562388420105) + | > loss_gen: 2.662116527557373 (2.662116527557373) + | > loss_kl: 2.3661091327667236 (2.3661091327667236) + | > loss_feat: 6.797050476074219 (6.797050476074219) + | > loss_mel: 17.249982833862305 (17.249982833862305) + | > loss_duration: 1.6752307415008545 (1.6752307415008545) + | > loss_1: 30.750490188598633 (30.750490188598633) + + --> STEP: 1 + | > loss_disc: 2.6033220291137695 (2.6033220291137695) + | > loss_disc_real_0: 0.2868713438510895 (0.2868713438510895) + | > loss_disc_real_1: 0.2217256873846054 (0.2217256873846054) + | > loss_disc_real_2: 0.24636322259902954 (0.24636322259902954) + | > loss_disc_real_3: 0.21233102679252625 (0.21233102679252625) + | > loss_disc_real_4: 0.2793346643447876 (0.2793346643447876) + | > loss_disc_real_5: 0.24351082742214203 (0.24351082742214203) + | > loss_0: 2.6033220291137695 (2.6033220291137695) + | > loss_gen: 2.4555540084838867 (2.4555540084838867) + | > loss_kl: 2.4591472148895264 (2.4591472148895264) + | > loss_feat: 5.925271511077881 (5.925271511077881) + | > loss_mel: 17.401939392089844 (17.401939392089844) + | > loss_duration: 1.6973240375518799 (1.6973240375518799) + | > loss_1: 29.939237594604492 (29.939237594604492) + + --> STEP: 2 + | > loss_disc: 2.4784812927246094 (2.5409016609191895) + | > loss_disc_real_0: 0.22296050190925598 (0.25491592288017273) + | > loss_disc_real_1: 0.19158592820167542 (0.2066558077931404) + | > loss_disc_real_2: 0.22488495707511902 (0.23562408983707428) + | > loss_disc_real_3: 0.18568524718284607 (0.19900813698768616) + | > loss_disc_real_4: 0.19420373439788818 (0.2367691993713379) + | > loss_disc_real_5: 0.15558066964149475 (0.1995457485318184) + | > loss_0: 2.4784812927246094 (2.5409016609191895) + | > loss_gen: 2.2644777297973633 (2.360015869140625) + | > loss_kl: 2.2531895637512207 (2.3561683893203735) + | > loss_feat: 6.678320407867432 (6.301795959472656) + | > loss_mel: 17.09779167175293 (17.249865531921387) + | > loss_duration: 1.6628857851028442 (1.680104911327362) + | > loss_1: 29.956663131713867 (29.94795036315918) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034537315368652344 (-0.0010478496551513672) + | > avg_loss_disc: 2.5409016609191895 (+0.055345773696899414) + | > avg_loss_disc_real_0: 0.25491592288017273 (+0.1857030875980854) + | > avg_loss_disc_real_1: 0.2066558077931404 (-0.01262114942073822) + | > avg_loss_disc_real_2: 0.23562408983707428 (-0.0048608556389808655) + | > avg_loss_disc_real_3: 0.19900813698768616 (-0.04455218464136124) + | > avg_loss_disc_real_4: 0.2367691993713379 (+0.032936856150627136) + | > avg_loss_disc_real_5: 0.1995457485318184 (-0.02802366018295288) + | > avg_loss_0: 2.5409016609191895 (+0.055345773696899414) + | > avg_loss_gen: 2.360015869140625 (+0.19585645198822021) + | > avg_loss_kl: 2.3561683893203735 (+0.021448135375976562) + | > avg_loss_feat: 6.301795959472656 (-1.5374839305877686) + | > avg_loss_mel: 17.249865531921387 (-1.6676082611083984) + | > avg_loss_duration: 1.680104911327362 (-0.000737607479095459) + | > avg_loss_1: 29.94795036315918 (-2.988527297973633) + + + > EPOCH: 209/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:26:00)  + + --> TIME: 2025-04-19 02:26:05 -- STEP: 15/142 -- GLOBAL_STEP: 408975 + | > loss_disc: 2.4457430839538574 (2.426899226506551) + | > loss_disc_real_0: 0.10545772314071655 (0.1221850112080574) + | > loss_disc_real_1: 0.18551534414291382 (0.2111205776532491) + | > loss_disc_real_2: 0.23465244472026825 (0.2192106028397878) + | > loss_disc_real_3: 0.25556427240371704 (0.21934361855189005) + | > loss_disc_real_4: 0.2295694500207901 (0.22373350361982983) + | > loss_disc_real_5: 0.24878592789173126 (0.2271171122789383) + | > loss_0: 2.4457430839538574 (2.426899226506551) + | > grad_norm_0: tensor(6.4672, device='cuda:0') (tensor(8.2868, device='cuda:0')) + | > loss_gen: 2.399198532104492 (2.3765328407287596) + | > loss_kl: 1.8489805459976196 (1.7908854722976684) + | > loss_feat: 7.644911766052246 (7.286880811055501) + | > loss_mel: 18.974254608154297 (18.183283805847168) + | > loss_duration: 1.4406172037124634 (1.4276606639226277) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.30796432495117 (31.06524378458659) + | > grad_norm_1: tensor(149.4244, device='cuda:0') (tensor(215.3215, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.3334 (0.327775764465332) + | > loader_time: 0.0057 (0.004735215504964193) + + + --> TIME: 2025-04-19 02:26:14 -- STEP: 40/142 -- GLOBAL_STEP: 409000 + | > loss_disc: 2.548846960067749 (2.439905279874802) + | > loss_disc_real_0: 0.17906734347343445 (0.12391480747610331) + | > loss_disc_real_1: 0.20535340905189514 (0.20586551241576673) + | > loss_disc_real_2: 0.21241343021392822 (0.22213005498051644) + | > loss_disc_real_3: 0.22856305539608002 (0.22747505381703376) + | > loss_disc_real_4: 0.25430697202682495 (0.22932070381939412) + | > loss_disc_real_5: 0.2438414990901947 (0.22859618216753005) + | > loss_0: 2.548846960067749 (2.439905279874802) + | > grad_norm_0: tensor(8.1852, device='cuda:0') (tensor(7.4283, device='cuda:0')) + | > loss_gen: 2.3304972648620605 (2.358173203468323) + | > loss_kl: 1.8801876306533813 (1.8352185994386674) + | > loss_feat: 6.343504905700684 (7.141692805290222) + | > loss_mel: 16.82896614074707 (17.803986191749573) + | > loss_duration: 1.4242098331451416 (1.4404564023017883) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.80736541748047 (30.579527139663696) + | > grad_norm_1: tensor(240.6962, device='cuda:0') (tensor(181.1597, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.359 (0.34082638621330263) + | > loader_time: 0.0061 (0.005164796113967894) + + + --> TIME: 2025-04-19 02:26:24 -- STEP: 65/142 -- GLOBAL_STEP: 409025 + | > loss_disc: 2.505389928817749 (2.44330784724309) + | > loss_disc_real_0: 0.10335677117109299 (0.12293758971186784) + | > loss_disc_real_1: 0.20960667729377747 (0.2054111434863164) + | > loss_disc_real_2: 0.22738438844680786 (0.22262658706078162) + | > loss_disc_real_3: 0.24925397336483002 (0.22739369594133818) + | > loss_disc_real_4: 0.22432947158813477 (0.22774842862899486) + | > loss_disc_real_5: 0.2448607236146927 (0.22926225203734177) + | > loss_0: 2.505389928817749 (2.44330784724309) + | > grad_norm_0: tensor(6.1929, device='cuda:0') (tensor(8.5366, device='cuda:0')) + | > loss_gen: 2.2740590572357178 (2.362524755184467) + | > loss_kl: 1.797663927078247 (1.8540725744687594) + | > loss_feat: 7.053135871887207 (7.1262528639573315) + | > loss_mel: 17.70427131652832 (17.75695599776048) + | > loss_duration: 1.4611318111419678 (1.452829674574045) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.29026222229004 (30.552635779747597) + | > grad_norm_1: tensor(70.5915, device='cuda:0') (tensor(210.0577, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.3937 (0.3527559683873103) + | > loader_time: 0.0063 (0.005497661003699669) + + + --> TIME: 2025-04-19 02:26:34 -- STEP: 90/142 -- GLOBAL_STEP: 409050 + | > loss_disc: 2.5668296813964844 (2.4491756227281365) + | > loss_disc_real_0: 0.10716874897480011 (0.12427399708992905) + | > loss_disc_real_1: 0.19867409765720367 (0.20681738439533442) + | > loss_disc_real_2: 0.22043296694755554 (0.22421298523743946) + | > loss_disc_real_3: 0.24401195347309113 (0.22763082964552772) + | > loss_disc_real_4: 0.23468095064163208 (0.22873380515310499) + | > loss_disc_real_5: 0.22043894231319427 (0.2285540266169442) + | > loss_0: 2.5668296813964844 (2.4491756227281365) + | > grad_norm_0: tensor(17.2734, device='cuda:0') (tensor(8.8246, device='cuda:0')) + | > loss_gen: 2.148655891418457 (2.36121781402164) + | > loss_kl: 1.8093938827514648 (1.860964630709754) + | > loss_feat: 6.408750534057617 (7.101321204503377) + | > loss_mel: 16.50493621826172 (17.768892372979064) + | > loss_duration: 1.49941086769104 (1.4638425442907548) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.371145248413086 (30.55623853471544) + | > grad_norm_1: tensor(197.6906, device='cuda:0') (tensor(206.2177, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.4013 (0.36418143113454193) + | > loader_time: 0.0065 (0.005760849846733941) + + + --> TIME: 2025-04-19 02:26:44 -- STEP: 115/142 -- GLOBAL_STEP: 409075 + | > loss_disc: 2.4359521865844727 (2.4548198243846064) + | > loss_disc_real_0: 0.09104740619659424 (0.12734622038576918) + | > loss_disc_real_1: 0.21300333738327026 (0.20611730308636372) + | > loss_disc_real_2: 0.26608285307884216 (0.22423470434935197) + | > loss_disc_real_3: 0.2104450911283493 (0.227938073096068) + | > loss_disc_real_4: 0.22238712012767792 (0.22935511184775312) + | > loss_disc_real_5: 0.19733965396881104 (0.22983605511810468) + | > loss_0: 2.4359521865844727 (2.4548198243846064) + | > grad_norm_0: tensor(12.8432, device='cuda:0') (tensor(9.5330, device='cuda:0')) + | > loss_gen: 2.2396068572998047 (2.357427522410517) + | > loss_kl: 1.9084796905517578 (1.8749497538027555) + | > loss_feat: 6.40875244140625 (7.027684671982475) + | > loss_mel: 15.867552757263184 (17.656060981750482) + | > loss_duration: 1.5016825199127197 (1.4711186699245289) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.92607307434082 (30.387241562553076) + | > grad_norm_1: tensor(197.5837, device='cuda:0') (tensor(221.3666, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.4303 (0.3748629963916282) + | > loader_time: 0.008 (0.00598877823871115) + + + --> TIME: 2025-04-19 02:26:56 -- STEP: 140/142 -- GLOBAL_STEP: 409100 + | > loss_disc: 2.542208433151245 (2.4636486223765783) + | > loss_disc_real_0: 0.21158428490161896 (0.1279870703816414) + | > loss_disc_real_1: 0.20417679846286774 (0.20668284595012662) + | > loss_disc_real_2: 0.1948731243610382 (0.22589531870824953) + | > loss_disc_real_3: 0.18610848486423492 (0.22827606605631964) + | > loss_disc_real_4: 0.20393165946006775 (0.22928331483687672) + | > loss_disc_real_5: 0.1761641651391983 (0.22956138604453633) + | > loss_0: 2.542208433151245 (2.4636486223765783) + | > grad_norm_0: tensor(39.0631, device='cuda:0') (tensor(11.0953, device='cuda:0')) + | > loss_gen: 2.2178962230682373 (2.3549182976995198) + | > loss_kl: 1.9124971628189087 (1.8776070211614881) + | > loss_feat: 6.114964485168457 (6.993414047786167) + | > loss_mel: 15.862195014953613 (17.622568055561604) + | > loss_duration: 1.548973560333252 (1.4777913945061822) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.656524658203125 (30.326298795427594) + | > grad_norm_1: tensor(410.9968, device='cuda:0') (tensor(238.2125, device='cuda:0')) + | > current_lr_0: 0.0001948423428990561 + | > current_lr_1: 0.0001948423428990561 + | > step_time: 0.4585 (0.38708979572568636) + | > loader_time: 0.0078 (0.006240934985024589) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3302204608917236 (2.3302204608917236) + | > loss_disc_real_0: 0.15471652150154114 (0.15471652150154114) + | > loss_disc_real_1: 0.23152197897434235 (0.23152197897434235) + | > loss_disc_real_2: 0.1986505687236786 (0.1986505687236786) + | > loss_disc_real_3: 0.2334199696779251 (0.2334199696779251) + | > loss_disc_real_4: 0.2141723334789276 (0.2141723334789276) + | > loss_disc_real_5: 0.255532443523407 (0.255532443523407) + | > loss_0: 2.3302204608917236 (2.3302204608917236) + | > loss_gen: 2.5370304584503174 (2.5370304584503174) + | > loss_kl: 2.4552001953125 (2.4552001953125) + | > loss_feat: 7.918537139892578 (7.918537139892578) + | > loss_mel: 20.389583587646484 (20.389583587646484) + | > loss_duration: 1.6924775838851929 (1.6924775838851929) + | > loss_1: 34.992828369140625 (34.992828369140625) + + --> STEP: 1 + | > loss_disc: 2.3883934020996094 (2.3883934020996094) + | > loss_disc_real_0: 0.1849338859319687 (0.1849338859319687) + | > loss_disc_real_1: 0.24857383966445923 (0.24857383966445923) + | > loss_disc_real_2: 0.19788645207881927 (0.19788645207881927) + | > loss_disc_real_3: 0.23768609762191772 (0.23768609762191772) + | > loss_disc_real_4: 0.24879033863544464 (0.24879033863544464) + | > loss_disc_real_5: 0.2443390190601349 (0.2443390190601349) + | > loss_0: 2.3883934020996094 (2.3883934020996094) + | > loss_gen: 2.507368564605713 (2.507368564605713) + | > loss_kl: 2.5530920028686523 (2.5530920028686523) + | > loss_feat: 7.100802421569824 (7.100802421569824) + | > loss_mel: 18.55499267578125 (18.55499267578125) + | > loss_duration: 1.7111375331878662 (1.7111375331878662) + | > loss_1: 32.427391052246094 (32.427391052246094) + + --> STEP: 2 + | > loss_disc: 2.376168727874756 (2.3822810649871826) + | > loss_disc_real_0: 0.16548985242843628 (0.17521186918020248) + | > loss_disc_real_1: 0.27832311391830444 (0.26344847679138184) + | > loss_disc_real_2: 0.2037447839975357 (0.2008156180381775) + | > loss_disc_real_3: 0.23029887676239014 (0.23399248719215393) + | > loss_disc_real_4: 0.2300737053155899 (0.23943202197551727) + | > loss_disc_real_5: 0.2894516885280609 (0.2668953537940979) + | > loss_0: 2.376168727874756 (2.3822810649871826) + | > loss_gen: 2.624119281768799 (2.565743923187256) + | > loss_kl: 2.3046469688415527 (2.4288694858551025) + | > loss_feat: 7.084907531738281 (7.092854976654053) + | > loss_mel: 17.58441925048828 (18.069705963134766) + | > loss_duration: 1.6655247211456299 (1.688331127166748) + | > loss_1: 31.26361846923828 (31.845504760742188) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003463149070739746 (+9.417533874511719e-06) + | > avg_loss_disc: 2.3822810649871826 (-0.15862059593200684) + | > avg_loss_disc_real_0: 0.17521186918020248 (-0.07970405369997025) + | > avg_loss_disc_real_1: 0.26344847679138184 (+0.056792668998241425) + | > avg_loss_disc_real_2: 0.2008156180381775 (-0.03480847179889679) + | > avg_loss_disc_real_3: 0.23399248719215393 (+0.03498435020446777) + | > avg_loss_disc_real_4: 0.23943202197551727 (+0.0026628226041793823) + | > avg_loss_disc_real_5: 0.2668953537940979 (+0.06734960526227951) + | > avg_loss_0: 2.3822810649871826 (-0.15862059593200684) + | > avg_loss_gen: 2.565743923187256 (+0.20572805404663086) + | > avg_loss_kl: 2.4288694858551025 (+0.072701096534729) + | > avg_loss_feat: 7.092854976654053 (+0.7910590171813965) + | > avg_loss_mel: 18.069705963134766 (+0.8198404312133789) + | > avg_loss_duration: 1.688331127166748 (+0.008226215839385986) + | > avg_loss_1: 31.845504760742188 (+1.8975543975830078) + + + > EPOCH: 210/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:26:58)  + + --> TIME: 2025-04-19 02:27:07 -- STEP: 23/142 -- GLOBAL_STEP: 409125 + | > loss_disc: 2.5250515937805176 (2.412257142688917) + | > loss_disc_real_0: 0.14102061092853546 (0.10904850389646448) + | > loss_disc_real_1: 0.18811756372451782 (0.20359393112037494) + | > loss_disc_real_2: 0.25910836458206177 (0.22689977806547415) + | > loss_disc_real_3: 0.24839000403881073 (0.22843347425046173) + | > loss_disc_real_4: 0.26913759112358093 (0.23109621003918027) + | > loss_disc_real_5: 0.210004523396492 (0.22851079378439032) + | > loss_0: 2.5250515937805176 (2.412257142688917) + | > grad_norm_0: tensor(11.1171, device='cuda:0') (tensor(13.9889, device='cuda:0')) + | > loss_gen: 2.4337778091430664 (2.4569571225539497) + | > loss_kl: 1.7938201427459717 (1.804406824319259) + | > loss_feat: 6.268877029418945 (7.470228298850682) + | > loss_mel: 15.983362197875977 (18.165312062139098) + | > loss_duration: 1.4569462537765503 (1.4335198454234912) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.93678092956543 (31.330424433169156) + | > grad_norm_1: tensor(432.4153, device='cuda:0') (tensor(356.2458, device='cuda:0')) + | > current_lr_0: 0.0001948179876061937 + | > current_lr_1: 0.0001948179876061937 + | > step_time: 0.3537 (0.33798726745273755) + | > loader_time: 0.0079 (0.005397257597550102) + + + --> TIME: 2025-04-19 02:27:16 -- STEP: 48/142 -- GLOBAL_STEP: 409150 + | > loss_disc: 2.385530948638916 (2.407085915406545) + | > loss_disc_real_0: 0.15818572044372559 (0.11332289781421423) + | > loss_disc_real_1: 0.2173124998807907 (0.20454179247220358) + | > loss_disc_real_2: 0.21795614063739777 (0.22541020593295494) + | > loss_disc_real_3: 0.22667360305786133 (0.22729279100894928) + | > loss_disc_real_4: 0.2136378288269043 (0.2276380561913053) + | > loss_disc_real_5: 0.22241203486919403 (0.22539839490006366) + | > loss_0: 2.385530948638916 (2.407085915406545) + | > grad_norm_0: tensor(4.1985, device='cuda:0') (tensor(11.8970, device='cuda:0')) + | > loss_gen: 2.541802406311035 (2.4254559675852456) + | > loss_kl: 1.9710197448730469 (1.8310011699795723) + | > loss_feat: 7.323657512664795 (7.372446795304616) + | > loss_mel: 18.553024291992188 (17.930675725142162) + | > loss_duration: 1.4602622985839844 (1.4447256301840146) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.84976577758789 (31.004305243492126) + | > grad_norm_1: tensor(40.6509, device='cuda:0') (tensor(306.8053, device='cuda:0')) + | > current_lr_0: 0.0001948179876061937 + | > current_lr_1: 0.0001948179876061937 + | > step_time: 0.3637 (0.34818606078624725) + | > loader_time: 0.0063 (0.005526363849639893) + + + --> TIME: 2025-04-19 02:27:26 -- STEP: 73/142 -- GLOBAL_STEP: 409175 + | > loss_disc: 2.5530662536621094 (2.4422074311400115) + | > loss_disc_real_0: 0.15896780788898468 (0.12154974043369293) + | > loss_disc_real_1: 0.21124319732189178 (0.20660691636882417) + | > loss_disc_real_2: 0.20380723476409912 (0.22740950772207078) + | > loss_disc_real_3: 0.2301919311285019 (0.22893025532160718) + | > loss_disc_real_4: 0.2404104769229889 (0.2293943407600873) + | > loss_disc_real_5: 0.23993147909641266 (0.22869756511629444) + | > loss_0: 2.5530662536621094 (2.4422074311400115) + | > grad_norm_0: tensor(4.5208, device='cuda:0') (tensor(10.5111, device='cuda:0')) + | > loss_gen: 2.2319180965423584 (2.383165477073356) + | > loss_kl: 1.994026780128479 (1.8603154649473217) + | > loss_feat: 6.957919120788574 (7.176171694716362) + | > loss_mel: 17.811628341674805 (17.779532105955354) + | > loss_duration: 1.5410585403442383 (1.4576117273879379) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.53655242919922 (30.65679667747184) + | > grad_norm_1: tensor(40.6467, device='cuda:0') (tensor(240.4095, device='cuda:0')) + | > current_lr_0: 0.0001948179876061937 + | > current_lr_1: 0.0001948179876061937 + | > step_time: 0.3923 (0.35936083532359503) + | > loader_time: 0.0063 (0.005724877527315323) + + + --> TIME: 2025-04-19 02:27:36 -- STEP: 98/142 -- GLOBAL_STEP: 409200 + | > loss_disc: 2.3406269550323486 (2.4480464215181312) + | > loss_disc_real_0: 0.12033021450042725 (0.12346999225567798) + | > loss_disc_real_1: 0.1817832589149475 (0.20618647701886236) + | > loss_disc_real_2: 0.19404880702495575 (0.22748569095013094) + | > loss_disc_real_3: 0.20676016807556152 (0.22842871808275886) + | > loss_disc_real_4: 0.19275587797164917 (0.2293135345894463) + | > loss_disc_real_5: 0.241241455078125 (0.2297653035849941) + | > loss_0: 2.3406269550323486 (2.4480464215181312) + | > grad_norm_0: tensor(5.3797, device='cuda:0') (tensor(9.3652, device='cuda:0')) + | > loss_gen: 2.4489238262176514 (2.3704263142177044) + | > loss_kl: 1.9486265182495117 (1.8742878108608478) + | > loss_feat: 7.99212121963501 (7.112155486126335) + | > loss_mel: 18.951229095458984 (17.69800606552435) + | > loss_duration: 1.462512731552124 (1.46811844743028) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.80341339111328 (30.52299437230947) + | > grad_norm_1: tensor(77.3125, device='cuda:0') (tensor(208.8521, device='cuda:0')) + | > current_lr_0: 0.0001948179876061937 + | > current_lr_1: 0.0001948179876061937 + | > step_time: 0.4108 (0.3696492326502897) + | > loader_time: 0.0068 (0.005982459807882504) + + + --> TIME: 2025-04-19 02:27:47 -- STEP: 123/142 -- GLOBAL_STEP: 409225 + | > loss_disc: 2.3711695671081543 (2.4579113188797868) + | > loss_disc_real_0: 0.11642993986606598 (0.12467454004336179) + | > loss_disc_real_1: 0.19999194145202637 (0.2065483061521034) + | > loss_disc_real_2: 0.23043282330036163 (0.2284431475691679) + | > loss_disc_real_3: 0.20791937410831451 (0.22917104748691) + | > loss_disc_real_4: 0.21467933058738708 (0.23078365151475116) + | > loss_disc_real_5: 0.21516261994838715 (0.2304427485640456) + | > loss_0: 2.3711695671081543 (2.4579113188797868) + | > grad_norm_0: tensor(9.8114, device='cuda:0') (tensor(8.9030, device='cuda:0')) + | > loss_gen: 2.4124677181243896 (2.3537048847694715) + | > loss_kl: 1.8589352369308472 (1.8805261297923763) + | > loss_feat: 6.829887390136719 (7.039486179506875) + | > loss_mel: 17.2972412109375 (17.67812387730048) + | > loss_duration: 1.4835108518600464 (1.4754137934708016) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.882043838500977 (30.427255087751682) + | > grad_norm_1: tensor(194.3591, device='cuda:0') (tensor(195.5806, device='cuda:0')) + | > current_lr_0: 0.0001948179876061937 + | > current_lr_1: 0.0001948179876061937 + | > step_time: 0.4385 (0.3808017688068917) + | > loader_time: 0.0072 (0.006242323697097902) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5997204780578613 (2.5997204780578613) + | > loss_disc_real_0: 0.15487149357795715 (0.15487149357795715) + | > loss_disc_real_1: 0.21676132082939148 (0.21676132082939148) + | > loss_disc_real_2: 0.2439715415239334 (0.2439715415239334) + | > loss_disc_real_3: 0.28250062465667725 (0.28250062465667725) + | > loss_disc_real_4: 0.2521289587020874 (0.2521289587020874) + | > loss_disc_real_5: 0.2623851001262665 (0.2623851001262665) + | > loss_0: 2.5997204780578613 (2.5997204780578613) + | > loss_gen: 2.2974181175231934 (2.2974181175231934) + | > loss_kl: 2.436999797821045 (2.436999797821045) + | > loss_feat: 6.70346212387085 (6.70346212387085) + | > loss_mel: 17.902210235595703 (17.902210235595703) + | > loss_duration: 1.667975664138794 (1.667975664138794) + | > loss_1: 31.008066177368164 (31.008066177368164) + + --> STEP: 1 + | > loss_disc: 2.549534320831299 (2.549534320831299) + | > loss_disc_real_0: 0.21697966754436493 (0.21697966754436493) + | > loss_disc_real_1: 0.229233518242836 (0.229233518242836) + | > loss_disc_real_2: 0.25195926427841187 (0.25195926427841187) + | > loss_disc_real_3: 0.24826526641845703 (0.24826526641845703) + | > loss_disc_real_4: 0.25052759051322937 (0.25052759051322937) + | > loss_disc_real_5: 0.24218522012233734 (0.24218522012233734) + | > loss_0: 2.549534320831299 (2.549534320831299) + | > loss_gen: 2.461956262588501 (2.461956262588501) + | > loss_kl: 2.4819462299346924 (2.4819462299346924) + | > loss_feat: 7.157419681549072 (7.157419681549072) + | > loss_mel: 19.167724609375 (19.167724609375) + | > loss_duration: 1.714977741241455 (1.714977741241455) + | > loss_1: 32.98402404785156 (32.98402404785156) + + --> STEP: 2 + | > loss_disc: 2.518073797225952 (2.5338040590286255) + | > loss_disc_real_0: 0.17529365420341492 (0.19613666087388992) + | > loss_disc_real_1: 0.22659729421138763 (0.22791540622711182) + | > loss_disc_real_2: 0.25413423776626587 (0.25304675102233887) + | > loss_disc_real_3: 0.26393774151802063 (0.25610150396823883) + | > loss_disc_real_4: 0.20875629782676697 (0.22964194416999817) + | > loss_disc_real_5: 0.21106477081775665 (0.226624995470047) + | > loss_0: 2.518073797225952 (2.5338040590286255) + | > loss_gen: 2.323495864868164 (2.3927260637283325) + | > loss_kl: 2.100029706954956 (2.290987968444824) + | > loss_feat: 6.358165264129639 (6.7577924728393555) + | > loss_mel: 15.304780960083008 (17.236252784729004) + | > loss_duration: 1.6574010848999023 (1.6861894130706787) + | > loss_1: 27.743873596191406 (30.363948822021484) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035680532455444336 (+0.0001049041748046875) + | > avg_loss_disc: 2.5338040590286255 (+0.15152299404144287) + | > avg_loss_disc_real_0: 0.19613666087388992 (+0.02092479169368744) + | > avg_loss_disc_real_1: 0.22791540622711182 (-0.03553307056427002) + | > avg_loss_disc_real_2: 0.25304675102233887 (+0.05223113298416138) + | > avg_loss_disc_real_3: 0.25610150396823883 (+0.0221090167760849) + | > avg_loss_disc_real_4: 0.22964194416999817 (-0.009790077805519104) + | > avg_loss_disc_real_5: 0.226624995470047 (-0.0402703583240509) + | > avg_loss_0: 2.5338040590286255 (+0.15152299404144287) + | > avg_loss_gen: 2.3927260637283325 (-0.17301785945892334) + | > avg_loss_kl: 2.290987968444824 (-0.13788151741027832) + | > avg_loss_feat: 6.7577924728393555 (-0.33506250381469727) + | > avg_loss_mel: 17.236252784729004 (-0.8334531784057617) + | > avg_loss_duration: 1.6861894130706787 (-0.002141714096069336) + | > avg_loss_1: 30.363948822021484 (-1.4815559387207031) + + + > EPOCH: 211/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:27:57)  + + --> TIME: 2025-04-19 02:28:00 -- STEP: 6/142 -- GLOBAL_STEP: 409250 + | > loss_disc: 2.410982131958008 (2.4510335524876914) + | > loss_disc_real_0: 0.14842410385608673 (0.13675640275081) + | > loss_disc_real_1: 0.19161300361156464 (0.21038712561130524) + | > loss_disc_real_2: 0.21648037433624268 (0.2283344864845276) + | > loss_disc_real_3: 0.18936645984649658 (0.21999437858661017) + | > loss_disc_real_4: 0.21978701651096344 (0.23625769466161728) + | > loss_disc_real_5: 0.23359431326389313 (0.23365379869937897) + | > loss_0: 2.410982131958008 (2.4510335524876914) + | > grad_norm_0: tensor(6.7299, device='cuda:0') (tensor(9.9422, device='cuda:0')) + | > loss_gen: 2.4185261726379395 (2.343372027079264) + | > loss_kl: 1.6722989082336426 (1.8110199371973674) + | > loss_feat: 7.8783369064331055 (7.158986250559489) + | > loss_mel: 18.28542137145996 (17.640056133270264) + | > loss_duration: 1.424931526184082 (1.4440956711769104) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.679515838623047 (30.39752960205078) + | > grad_norm_1: tensor(75.2720, device='cuda:0') (tensor(175.1548, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.3289 (0.32707250118255615) + | > loader_time: 0.0041 (0.004492282867431641) + + + --> TIME: 2025-04-19 02:28:08 -- STEP: 31/142 -- GLOBAL_STEP: 409275 + | > loss_disc: 2.4228196144104004 (2.463055564511207) + | > loss_disc_real_0: 0.12359261512756348 (0.12625686752219356) + | > loss_disc_real_1: 0.20491552352905273 (0.21310864533147506) + | > loss_disc_real_2: 0.2136533558368683 (0.2254078157486454) + | > loss_disc_real_3: 0.21626156568527222 (0.22548456970722444) + | > loss_disc_real_4: 0.21146388351917267 (0.2273400079819464) + | > loss_disc_real_5: 0.21097645163536072 (0.23372925866034724) + | > loss_0: 2.4228196144104004 (2.463055564511207) + | > grad_norm_0: tensor(10.3146, device='cuda:0') (tensor(8.9295, device='cuda:0')) + | > loss_gen: 2.4575297832489014 (2.3495796188231437) + | > loss_kl: 1.8585010766983032 (1.8302418301182408) + | > loss_feat: 7.069343566894531 (7.128111854676278) + | > loss_mel: 17.112958908081055 (17.77105260664417) + | > loss_duration: 1.4180365800857544 (1.443009361144035) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.916370391845703 (30.521995359851466) + | > grad_norm_1: tensor(227.4937, device='cuda:0') (tensor(214.2811, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.3605 (0.33964244011909733) + | > loader_time: 0.0053 (0.0051953484935145225) + + + --> TIME: 2025-04-19 02:28:18 -- STEP: 56/142 -- GLOBAL_STEP: 409300 + | > loss_disc: 2.4502885341644287 (2.447170091526849) + | > loss_disc_real_0: 0.17026753723621368 (0.12653611680226673) + | > loss_disc_real_1: 0.20449209213256836 (0.21171805927796022) + | > loss_disc_real_2: 0.2527956962585449 (0.22589059679635934) + | > loss_disc_real_3: 0.26411929726600647 (0.22478005875434195) + | > loss_disc_real_4: 0.28768888115882874 (0.22799813774015223) + | > loss_disc_real_5: 0.25389403104782104 (0.23008206406874315) + | > loss_0: 2.4502885341644287 (2.447170091526849) + | > grad_norm_0: tensor(16.3351, device='cuda:0') (tensor(9.8360, device='cuda:0')) + | > loss_gen: 2.4391140937805176 (2.3664987214974005) + | > loss_kl: 2.009694814682007 (1.8531990689890725) + | > loss_feat: 7.001506805419922 (7.140003902571542) + | > loss_mel: 18.26669692993164 (17.749530638967247) + | > loss_duration: 1.4728062152862549 (1.4514320130859102) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.1898193359375 (30.560664449419296) + | > grad_norm_1: tensor(504.3149, device='cuda:0') (tensor(239.4185, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.3845 (0.3538983804838998) + | > loader_time: 0.0073 (0.005572591509137835) + + + --> TIME: 2025-04-19 02:28:28 -- STEP: 81/142 -- GLOBAL_STEP: 409325 + | > loss_disc: 2.4427781105041504 (2.464230746398737) + | > loss_disc_real_0: 0.1376279592514038 (0.12686818580568573) + | > loss_disc_real_1: 0.18423235416412354 (0.2108442672976741) + | > loss_disc_real_2: 0.2300867736339569 (0.22656436264514923) + | > loss_disc_real_3: 0.21834012866020203 (0.22724066895467263) + | > loss_disc_real_4: 0.2324995994567871 (0.22845829986495736) + | > loss_disc_real_5: 0.23367193341255188 (0.23107506886676507) + | > loss_0: 2.4427781105041504 (2.464230746398737) + | > grad_norm_0: tensor(15.4314, device='cuda:0') (tensor(9.8691, device='cuda:0')) + | > loss_gen: 2.370840072631836 (2.3530636922812764) + | > loss_kl: 2.0658953189849854 (1.8717203419885518) + | > loss_feat: 6.944804668426514 (7.029289322134889) + | > loss_mel: 18.6664981842041 (17.67852939794093) + | > loss_duration: 1.477062463760376 (1.4621299946749653) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.525100708007812 (30.394732816719713) + | > grad_norm_1: tensor(475.1295, device='cuda:0') (tensor(237.7582, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.4055 (0.367653567113994) + | > loader_time: 0.007 (0.005872629306934498) + + + --> TIME: 2025-04-19 02:28:39 -- STEP: 106/142 -- GLOBAL_STEP: 409350 + | > loss_disc: 2.5289957523345947 (2.4725546859345355) + | > loss_disc_real_0: 0.12735603749752045 (0.12814777977061725) + | > loss_disc_real_1: 0.17607830464839935 (0.21038678190055884) + | > loss_disc_real_2: 0.2295834720134735 (0.22795990348424552) + | > loss_disc_real_3: 0.22967031598091125 (0.22816981754775317) + | > loss_disc_real_4: 0.2304406762123108 (0.230270561463428) + | > loss_disc_real_5: 0.21683838963508606 (0.230593223616762) + | > loss_0: 2.5289957523345947 (2.4725546859345355) + | > grad_norm_0: tensor(25.2367, device='cuda:0') (tensor(11.3635, device='cuda:0')) + | > loss_gen: 2.185964345932007 (2.346904163090688) + | > loss_kl: 1.8750532865524292 (1.8852468099234239) + | > loss_feat: 7.36699104309082 (6.995125397196356) + | > loss_mel: 18.044734954833984 (17.70472599425405) + | > loss_duration: 1.499103307723999 (1.4721483531987891) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.971847534179688 (30.404150836872606) + | > grad_norm_1: tensor(342.4942, device='cuda:0') (tensor(259.6117, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.4081 (0.37924132931907223) + | > loader_time: 0.0069 (0.006148880382753768) + + + --> TIME: 2025-04-19 02:28:50 -- STEP: 131/142 -- GLOBAL_STEP: 409375 + | > loss_disc: 2.4094722270965576 (2.475097696289762) + | > loss_disc_real_0: 0.08360777050256729 (0.13010282780377927) + | > loss_disc_real_1: 0.2567901909351349 (0.20976646153526451) + | > loss_disc_real_2: 0.23341216146945953 (0.22797333648186605) + | > loss_disc_real_3: 0.2372806817293167 (0.2276606616628079) + | > loss_disc_real_4: 0.23733915388584137 (0.22971687505718405) + | > loss_disc_real_5: 0.2560218870639801 (0.23072410562566217) + | > loss_0: 2.4094722270965576 (2.475097696289762) + | > grad_norm_0: tensor(10.8065, device='cuda:0') (tensor(12.9211, device='cuda:0')) + | > loss_gen: 2.4507858753204346 (2.3524713807433617) + | > loss_kl: 1.9298298358917236 (1.8910275907006882) + | > loss_feat: 7.427555084228516 (6.959244633448943) + | > loss_mel: 17.881593704223633 (17.657966118732478) + | > loss_duration: 1.5053013563156128 (1.4778560700307366) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.195066452026367 (30.33856588465567) + | > grad_norm_1: tensor(252.7613, device='cuda:0') (tensor(275.5284, device='cuda:0')) + | > current_lr_0: 0.00019479363535774292 + | > current_lr_1: 0.00019479363535774292 + | > step_time: 0.4475 (0.3898183644272899) + | > loader_time: 0.0075 (0.00639402411366237) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4081597328186035 (2.4081597328186035) + | > loss_disc_real_0: 0.16121739149093628 (0.16121739149093628) + | > loss_disc_real_1: 0.2081817388534546 (0.2081817388534546) + | > loss_disc_real_2: 0.24724316596984863 (0.24724316596984863) + | > loss_disc_real_3: 0.2521156966686249 (0.2521156966686249) + | > loss_disc_real_4: 0.2019297331571579 (0.2019297331571579) + | > loss_disc_real_5: 0.21015360951423645 (0.21015360951423645) + | > loss_0: 2.4081597328186035 (2.4081597328186035) + | > loss_gen: 2.3839523792266846 (2.3839523792266846) + | > loss_kl: 2.5232672691345215 (2.5232672691345215) + | > loss_feat: 7.088547229766846 (7.088547229766846) + | > loss_mel: 18.98592758178711 (18.98592758178711) + | > loss_duration: 1.6865789890289307 (1.6865789890289307) + | > loss_1: 32.66827392578125 (32.66827392578125) + + --> STEP: 1 + | > loss_disc: 2.4051053524017334 (2.4051053524017334) + | > loss_disc_real_0: 0.20733559131622314 (0.20733559131622314) + | > loss_disc_real_1: 0.17935286462306976 (0.17935286462306976) + | > loss_disc_real_2: 0.21608300507068634 (0.21608300507068634) + | > loss_disc_real_3: 0.24699662625789642 (0.24699662625789642) + | > loss_disc_real_4: 0.19815826416015625 (0.19815826416015625) + | > loss_disc_real_5: 0.1932278424501419 (0.1932278424501419) + | > loss_0: 2.4051053524017334 (2.4051053524017334) + | > loss_gen: 2.3978538513183594 (2.3978538513183594) + | > loss_kl: 2.495685338973999 (2.495685338973999) + | > loss_feat: 7.2549967765808105 (7.2549967765808105) + | > loss_mel: 19.74882698059082 (19.74882698059082) + | > loss_duration: 1.7221322059631348 (1.7221322059631348) + | > loss_1: 33.6194953918457 (33.6194953918457) + + --> STEP: 2 + | > loss_disc: 2.6032917499542236 (2.5041985511779785) + | > loss_disc_real_0: 0.17681515216827393 (0.19207537174224854) + | > loss_disc_real_1: 0.19221501052379608 (0.18578393757343292) + | > loss_disc_real_2: 0.25391915440559387 (0.2350010797381401) + | > loss_disc_real_3: 0.2700255811214447 (0.25851110368967056) + | > loss_disc_real_4: 0.27284952998161316 (0.2355038970708847) + | > loss_disc_real_5: 0.23920460045337677 (0.21621622145175934) + | > loss_0: 2.6032917499542236 (2.5041985511779785) + | > loss_gen: 2.393444061279297 (2.395648956298828) + | > loss_kl: 2.2198376655578613 (2.35776150226593) + | > loss_feat: 6.398731708526611 (6.826864242553711) + | > loss_mel: 17.107074737548828 (18.427950859069824) + | > loss_duration: 1.6554797887802124 (1.6888059973716736) + | > loss_1: 29.774566650390625 (31.697031021118164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034608840942382812 (-0.00010716915130615234) + | > avg_loss_disc: 2.5041985511779785 (-0.029605507850646973) + | > avg_loss_disc_real_0: 0.19207537174224854 (-0.004061289131641388) + | > avg_loss_disc_real_1: 0.18578393757343292 (-0.042131468653678894) + | > avg_loss_disc_real_2: 0.2350010797381401 (-0.01804567128419876) + | > avg_loss_disc_real_3: 0.25851110368967056 (+0.002409599721431732) + | > avg_loss_disc_real_4: 0.2355038970708847 (+0.005861952900886536) + | > avg_loss_disc_real_5: 0.21621622145175934 (-0.010408774018287659) + | > avg_loss_0: 2.5041985511779785 (-0.029605507850646973) + | > avg_loss_gen: 2.395648956298828 (+0.0029228925704956055) + | > avg_loss_kl: 2.35776150226593 (+0.06677353382110596) + | > avg_loss_feat: 6.826864242553711 (+0.06907176971435547) + | > avg_loss_mel: 18.427950859069824 (+1.1916980743408203) + | > avg_loss_duration: 1.6888059973716736 (+0.002616584300994873) + | > avg_loss_1: 31.697031021118164 (+1.3330821990966797) + + + > EPOCH: 212/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:28:56)  + + --> TIME: 2025-04-19 02:29:02 -- STEP: 14/142 -- GLOBAL_STEP: 409400 + | > loss_disc: 2.406748056411743 (2.4087711913245067) + | > loss_disc_real_0: 0.10473842918872833 (0.10938759573868342) + | > loss_disc_real_1: 0.21094179153442383 (0.20831624631370818) + | > loss_disc_real_2: 0.22203496098518372 (0.2222311475447246) + | > loss_disc_real_3: 0.17737162113189697 (0.22388707527092525) + | > loss_disc_real_4: 0.20504364371299744 (0.23403074060167586) + | > loss_disc_real_5: 0.21905991435050964 (0.23601011399711883) + | > loss_0: 2.406748056411743 (2.4087711913245067) + | > grad_norm_0: tensor(20.4739, device='cuda:0') (tensor(13.9075, device='cuda:0')) + | > loss_gen: 2.1476891040802 (2.426665561539786) + | > loss_kl: 1.8597732782363892 (1.8453588485717773) + | > loss_feat: 6.473302841186523 (7.445694140025547) + | > loss_mel: 16.430023193359375 (18.166911670139857) + | > loss_duration: 1.444318175315857 (1.430699041911534) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.3551082611084 (31.315329687935964) + | > grad_norm_1: tensor(232.7917, device='cuda:0') (tensor(358.3987, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.3324 (0.326846718788147) + | > loader_time: 0.005 (0.004672936030796596) + + + --> TIME: 2025-04-19 02:29:11 -- STEP: 39/142 -- GLOBAL_STEP: 409425 + | > loss_disc: 2.474283218383789 (2.4273068904876713) + | > loss_disc_real_0: 0.12134845554828644 (0.11576924740503995) + | > loss_disc_real_1: 0.18461374938488007 (0.20369353011632577) + | > loss_disc_real_2: 0.21645265817642212 (0.2255084541363594) + | > loss_disc_real_3: 0.27045199275016785 (0.23129357321140093) + | > loss_disc_real_4: 0.23725765943527222 (0.23080978714502776) + | > loss_disc_real_5: 0.21627900004386902 (0.23120757784598914) + | > loss_0: 2.474283218383789 (2.4273068904876713) + | > grad_norm_0: tensor(16.4639, device='cuda:0') (tensor(11.2744, device='cuda:0')) + | > loss_gen: 2.2511303424835205 (2.396901106223082) + | > loss_kl: 1.9211410284042358 (1.8490488070708055) + | > loss_feat: 6.063802719116211 (7.243308972089719) + | > loss_mel: 16.842754364013672 (17.887831761286808) + | > loss_duration: 1.4570975303649902 (1.4387275041678012) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.535926818847656 (30.815818542089218) + | > grad_norm_1: tensor(194.5988, device='cuda:0') (tensor(284.7603, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.3608 (0.3407753308614095) + | > loader_time: 0.0058 (0.005178262025882036) + + + --> TIME: 2025-04-19 02:29:20 -- STEP: 64/142 -- GLOBAL_STEP: 409450 + | > loss_disc: 2.512256622314453 (2.4444796219468117) + | > loss_disc_real_0: 0.13402080535888672 (0.12120604433584958) + | > loss_disc_real_1: 0.1947833001613617 (0.20423260936513543) + | > loss_disc_real_2: 0.22714142501354218 (0.22565683210268617) + | > loss_disc_real_3: 0.243305504322052 (0.2303381720557809) + | > loss_disc_real_4: 0.24647082388401031 (0.2315440676175058) + | > loss_disc_real_5: 0.19681303203105927 (0.23114292905665934) + | > loss_0: 2.512256622314453 (2.4444796219468117) + | > grad_norm_0: tensor(10.6161, device='cuda:0') (tensor(12.3660, device='cuda:0')) + | > loss_gen: 2.185682773590088 (2.382087863981724) + | > loss_kl: 1.9225306510925293 (1.8711584229022264) + | > loss_feat: 6.122751235961914 (7.181987002491951) + | > loss_mel: 18.10982894897461 (17.954722553491592) + | > loss_duration: 1.5053025484085083 (1.4513324461877344) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.84609603881836 (30.841288626194) + | > grad_norm_1: tensor(237.6468, device='cuda:0') (tensor(280.3433, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.3813 (0.3538676612079143) + | > loader_time: 0.0066 (0.0055743493139743805) + + + --> TIME: 2025-04-19 02:29:30 -- STEP: 89/142 -- GLOBAL_STEP: 409475 + | > loss_disc: 2.4919371604919434 (2.454820394515991) + | > loss_disc_real_0: 0.15074989199638367 (0.12450561379448752) + | > loss_disc_real_1: 0.19005762040615082 (0.20617904706617424) + | > loss_disc_real_2: 0.2262544482946396 (0.22570982558673688) + | > loss_disc_real_3: 0.2402282953262329 (0.23009326267108493) + | > loss_disc_real_4: 0.24166788160800934 (0.23092059908288248) + | > loss_disc_real_5: 0.22587169706821442 (0.23076364163602336) + | > loss_0: 2.4919371604919434 (2.454820394515991) + | > grad_norm_0: tensor(5.9151, device='cuda:0') (tensor(11.1347, device='cuda:0')) + | > loss_gen: 2.2302420139312744 (2.367930021178856) + | > loss_kl: 1.927119255065918 (1.8879271517978626) + | > loss_feat: 6.153922080993652 (7.1096075197284145) + | > loss_mel: 16.302791595458984 (17.9513267667106) + | > loss_duration: 1.4608882665634155 (1.463132224725873) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.074962615966797 (30.779923921220757) + | > grad_norm_1: tensor(72.1609, device='cuda:0') (tensor(251.7387, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.4028 (0.36499918980544854) + | > loader_time: 0.0068 (0.00587638844265027) + + + --> TIME: 2025-04-19 02:29:41 -- STEP: 114/142 -- GLOBAL_STEP: 409500 + | > loss_disc: 2.589615821838379 (2.4564160317705377) + | > loss_disc_real_0: 0.14718452095985413 (0.12582235613413023) + | > loss_disc_real_1: 0.22158437967300415 (0.20685768140512603) + | > loss_disc_real_2: 0.24617984890937805 (0.22491142375950227) + | > loss_disc_real_3: 0.24939653277397156 (0.22961538572583284) + | > loss_disc_real_4: 0.24981637299060822 (0.22963222534510128) + | > loss_disc_real_5: 0.2613389492034912 (0.22997771974718362) + | > loss_0: 2.589615821838379 (2.4564160317705377) + | > grad_norm_0: tensor(5.0665, device='cuda:0') (tensor(10.3502, device='cuda:0')) + | > loss_gen: 2.2430474758148193 (2.3519312540690094) + | > loss_kl: 2.0204031467437744 (1.8987727928579898) + | > loss_feat: 6.549070358276367 (7.04032648237128) + | > loss_mel: 16.781967163085938 (17.86516973428559) + | > loss_duration: 1.517742395401001 (1.4710319836934405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.11223030090332 (30.627232467919065) + | > grad_norm_1: tensor(113.2923, device='cuda:0') (tensor(228.3158, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.4186 (0.3754228458069917) + | > loader_time: 0.0072 (0.0061057680531551965) + + + --> TIME: 2025-04-19 02:29:52 -- STEP: 139/142 -- GLOBAL_STEP: 409525 + | > loss_disc: 2.5076866149902344 (2.4609489115022063) + | > loss_disc_real_0: 0.13784603774547577 (0.12835777743900423) + | > loss_disc_real_1: 0.21971073746681213 (0.2067942460663885) + | > loss_disc_real_2: 0.22958077490329742 (0.22444016448885418) + | > loss_disc_real_3: 0.20303130149841309 (0.22960415000109366) + | > loss_disc_real_4: 0.25565651059150696 (0.2290387016406162) + | > loss_disc_real_5: 0.23101624846458435 (0.23066251205025817) + | > loss_0: 2.5076866149902344 (2.4609489115022063) + | > grad_norm_0: tensor(6.2860, device='cuda:0') (tensor(9.9626, device='cuda:0')) + | > loss_gen: 2.3897452354431152 (2.3467559608624127) + | > loss_kl: 1.836390733718872 (1.9016480574504935) + | > loss_feat: 7.379559516906738 (6.999826290624605) + | > loss_mel: 17.99259376525879 (17.794330960555044) + | > loss_duration: 1.4987256526947021 (1.4772549859053796) + | > amp_scaler: 512.0 (278.10071942446046) + | > loss_1: 31.097015380859375 (30.51981644664737) + | > grad_norm_1: tensor(48.6291, device='cuda:0') (tensor(210.2167, device='cuda:0')) + | > current_lr_0: 0.0001947692861533232 + | > current_lr_1: 0.0001947692861533232 + | > step_time: 0.4528 (0.38737793963590106) + | > loader_time: 0.0078 (0.006349609910155372) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.488973617553711 (2.488973617553711) + | > loss_disc_real_0: 0.27477651834487915 (0.27477651834487915) + | > loss_disc_real_1: 0.17603206634521484 (0.17603206634521484) + | > loss_disc_real_2: 0.236162468791008 (0.236162468791008) + | > loss_disc_real_3: 0.234458327293396 (0.234458327293396) + | > loss_disc_real_4: 0.23345614969730377 (0.23345614969730377) + | > loss_disc_real_5: 0.2510358393192291 (0.2510358393192291) + | > loss_0: 2.488973617553711 (2.488973617553711) + | > loss_gen: 2.5817577838897705 (2.5817577838897705) + | > loss_kl: 2.4060394763946533 (2.4060394763946533) + | > loss_feat: 6.565340518951416 (6.565340518951416) + | > loss_mel: 16.72638511657715 (16.72638511657715) + | > loss_duration: 1.6916241645812988 (1.6916241645812988) + | > loss_1: 29.971147537231445 (29.971147537231445) + + --> STEP: 1 + | > loss_disc: 2.4915239810943604 (2.4915239810943604) + | > loss_disc_real_0: 0.22210650146007538 (0.22210650146007538) + | > loss_disc_real_1: 0.18707460165023804 (0.18707460165023804) + | > loss_disc_real_2: 0.2168768048286438 (0.2168768048286438) + | > loss_disc_real_3: 0.2458135038614273 (0.2458135038614273) + | > loss_disc_real_4: 0.2643713355064392 (0.2643713355064392) + | > loss_disc_real_5: 0.2686743140220642 (0.2686743140220642) + | > loss_0: 2.4915239810943604 (2.4915239810943604) + | > loss_gen: 2.6690151691436768 (2.6690151691436768) + | > loss_kl: 2.718801975250244 (2.718801975250244) + | > loss_feat: 7.152870178222656 (7.152870178222656) + | > loss_mel: 19.68592643737793 (19.68592643737793) + | > loss_duration: 1.696659803390503 (1.696659803390503) + | > loss_1: 33.92327117919922 (33.92327117919922) + + --> STEP: 2 + | > loss_disc: 2.4231295585632324 (2.4573267698287964) + | > loss_disc_real_0: 0.2270667850971222 (0.22458664327859879) + | > loss_disc_real_1: 0.16160239279270172 (0.17433849722146988) + | > loss_disc_real_2: 0.21902821958065033 (0.21795251220464706) + | > loss_disc_real_3: 0.2565130889415741 (0.2511632964015007) + | > loss_disc_real_4: 0.22220024466514587 (0.24328579008579254) + | > loss_disc_real_5: 0.21502640843391418 (0.2418503612279892) + | > loss_0: 2.4231295585632324 (2.4573267698287964) + | > loss_gen: 2.4436864852905273 (2.556350827217102) + | > loss_kl: 2.1790847778320312 (2.4489433765411377) + | > loss_feat: 6.496166706085205 (6.824518442153931) + | > loss_mel: 17.39714241027832 (18.541534423828125) + | > loss_duration: 1.6639106273651123 (1.6802852153778076) + | > loss_1: 30.179988861083984 (32.0516300201416) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037511587142944336 (+0.00029027462005615234) + | > avg_loss_disc: 2.4573267698287964 (-0.04687178134918213) + | > avg_loss_disc_real_0: 0.22458664327859879 (+0.03251127153635025) + | > avg_loss_disc_real_1: 0.17433849722146988 (-0.011445440351963043) + | > avg_loss_disc_real_2: 0.21795251220464706 (-0.017048567533493042) + | > avg_loss_disc_real_3: 0.2511632964015007 (-0.007347807288169861) + | > avg_loss_disc_real_4: 0.24328579008579254 (+0.007781893014907837) + | > avg_loss_disc_real_5: 0.2418503612279892 (+0.02563413977622986) + | > avg_loss_0: 2.4573267698287964 (-0.04687178134918213) + | > avg_loss_gen: 2.556350827217102 (+0.16070187091827393) + | > avg_loss_kl: 2.4489433765411377 (+0.09118187427520752) + | > avg_loss_feat: 6.824518442153931 (-0.0023458003997802734) + | > avg_loss_mel: 18.541534423828125 (+0.11358356475830078) + | > avg_loss_duration: 1.6802852153778076 (-0.008520781993865967) + | > avg_loss_1: 32.0516300201416 (+0.3545989990234375) + + + > EPOCH: 213/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:29:55)  + + --> TIME: 2025-04-19 02:30:03 -- STEP: 22/142 -- GLOBAL_STEP: 409550 + | > loss_disc: 2.5030999183654785 (2.4816665757786143) + | > loss_disc_real_0: 0.1812422275543213 (0.1289697607809847) + | > loss_disc_real_1: 0.19866372644901276 (0.2108187424865636) + | > loss_disc_real_2: 0.24369986355304718 (0.22220189395275983) + | > loss_disc_real_3: 0.26657217741012573 (0.23536897789348254) + | > loss_disc_real_4: 0.2462577521800995 (0.23220469125292517) + | > loss_disc_real_5: 0.29027751088142395 (0.2290339402177117) + | > loss_0: 2.5030999183654785 (2.4816665757786143) + | > grad_norm_0: tensor(14.9242, device='cuda:0') (tensor(8.6488, device='cuda:0')) + | > loss_gen: 2.411970376968384 (2.352394938468933) + | > loss_kl: 1.8975635766983032 (1.817912226373499) + | > loss_feat: 7.670938014984131 (7.181485262784091) + | > loss_mel: 17.889373779296875 (17.90056948228316) + | > loss_duration: 1.4139270782470703 (1.4303559931841763) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.283771514892578 (30.682717843489215) + | > grad_norm_1: tensor(344.3430, device='cuda:0') (tensor(185.4641, device='cuda:0')) + | > current_lr_0: 0.000194744939992554 + | > current_lr_1: 0.000194744939992554 + | > step_time: 0.3463 (0.3322933262044734) + | > loader_time: 0.005 (0.004888426173817027) + + + --> TIME: 2025-04-19 02:30:12 -- STEP: 47/142 -- GLOBAL_STEP: 409575 + | > loss_disc: 2.391667366027832 (2.4649931927944753) + | > loss_disc_real_0: 0.17331989109516144 (0.12506407673688644) + | > loss_disc_real_1: 0.19794981181621552 (0.20805304323105103) + | > loss_disc_real_2: 0.21095789968967438 (0.22495324370708872) + | > loss_disc_real_3: 0.2078542709350586 (0.23078350152106994) + | > loss_disc_real_4: 0.25620564818382263 (0.22947436063847643) + | > loss_disc_real_5: 0.17987093329429626 (0.22906593090676247) + | > loss_0: 2.391667366027832 (2.4649931927944753) + | > grad_norm_0: tensor(8.7991, device='cuda:0') (tensor(11.3472, device='cuda:0')) + | > loss_gen: 2.6488723754882812 (2.3721490413584605) + | > loss_kl: 1.886710286140442 (1.8454840994895774) + | > loss_feat: 7.2774977684021 (7.130669989484422) + | > loss_mel: 17.854938507080078 (17.96054730516799) + | > loss_duration: 1.4273724555969238 (1.442241369409764) + | > amp_scaler: 256.0 (468.4255319148936) + | > loss_1: 31.09539031982422 (30.7510917744738) + | > grad_norm_1: tensor(132.9440, device='cuda:0') (tensor(216.1565, device='cuda:0')) + | > current_lr_0: 0.000194744939992554 + | > current_lr_1: 0.000194744939992554 + | > step_time: 0.3642 (0.34476521167349305) + | > loader_time: 0.0063 (0.005329066134513693) + + + --> TIME: 2025-04-19 02:30:22 -- STEP: 72/142 -- GLOBAL_STEP: 409600 + | > loss_disc: 2.488593339920044 (2.4707798626687794) + | > loss_disc_real_0: 0.15183933079242706 (0.12919210202784998) + | > loss_disc_real_1: 0.22428473830223083 (0.20770245687001282) + | > loss_disc_real_2: 0.22056132555007935 (0.22508466181655729) + | > loss_disc_real_3: 0.23391827940940857 (0.2308974644790093) + | > loss_disc_real_4: 0.23035173118114471 (0.2298850706881947) + | > loss_disc_real_5: 0.18709513545036316 (0.22770533483061525) + | > loss_0: 2.488593339920044 (2.4707798626687794) + | > grad_norm_0: tensor(18.1460, device='cuda:0') (tensor(11.8230, device='cuda:0')) + | > loss_gen: 2.194333076477051 (2.3474983837869425) + | > loss_kl: 1.9314454793930054 (1.8589906477265887) + | > loss_feat: 7.151262283325195 (7.048332975970374) + | > loss_mel: 18.216976165771484 (17.86242871814304) + | > loss_duration: 1.4570152759552002 (1.455225086874432) + | > amp_scaler: 256.0 (394.6666666666667) + | > loss_1: 30.951030731201172 (30.57247567176819) + | > grad_norm_1: tensor(259.4131, device='cuda:0') (tensor(228.8478, device='cuda:0')) + | > current_lr_0: 0.000194744939992554 + | > current_lr_1: 0.000194744939992554 + | > step_time: 0.392 (0.356662329700258) + | > loader_time: 0.0064 (0.005610651440090602) + + + --> TIME: 2025-04-19 02:30:32 -- STEP: 97/142 -- GLOBAL_STEP: 409625 + | > loss_disc: 2.366028308868408 (2.471413420647689) + | > loss_disc_real_0: 0.1390514224767685 (0.1287300691546237) + | > loss_disc_real_1: 0.2220386415719986 (0.20757848116540417) + | > loss_disc_real_2: 0.181098073720932 (0.22498690989828604) + | > loss_disc_real_3: 0.23030738532543182 (0.23018661218205677) + | > loss_disc_real_4: 0.20854853093624115 (0.22973871092820905) + | > loss_disc_real_5: 0.20950590074062347 (0.23027355415919393) + | > loss_0: 2.366028308868408 (2.471413420647689) + | > grad_norm_0: tensor(16.9935, device='cuda:0') (tensor(11.6082, device='cuda:0')) + | > loss_gen: 2.448991298675537 (2.347860658291689) + | > loss_kl: 1.7672507762908936 (1.872266265534863) + | > loss_feat: 7.086747169494629 (7.0489723461190446) + | > loss_mel: 18.42631721496582 (17.85228963242364) + | > loss_duration: 1.4861866235733032 (1.466724114319713) + | > amp_scaler: 256.0 (358.92783505154637) + | > loss_1: 31.21549415588379 (30.58811289993758) + | > grad_norm_1: tensor(343.8297, device='cuda:0') (tensor(242.1628, device='cuda:0')) + | > current_lr_0: 0.000194744939992554 + | > current_lr_1: 0.000194744939992554 + | > step_time: 0.4105 (0.3673992034086248) + | > loader_time: 0.0066 (0.0058565360983622436) + + + --> TIME: 2025-04-19 02:30:43 -- STEP: 122/142 -- GLOBAL_STEP: 409650 + | > loss_disc: 2.5908544063568115 (2.4714540594913914) + | > loss_disc_real_0: 0.10645132511854172 (0.12883583789111142) + | > loss_disc_real_1: 0.1831173300743103 (0.20616086975472872) + | > loss_disc_real_2: 0.2382066547870636 (0.22571379410438852) + | > loss_disc_real_3: 0.21258136630058289 (0.23025739962448838) + | > loss_disc_real_4: 0.21863524615764618 (0.22947275296586458) + | > loss_disc_real_5: 0.22320649027824402 (0.23044536311606892) + | > loss_0: 2.5908544063568115 (2.4714540594913914) + | > grad_norm_0: tensor(16.2335, device='cuda:0') (tensor(11.6615, device='cuda:0')) + | > loss_gen: 2.2091710567474365 (2.3486122225151687) + | > loss_kl: 1.8771635293960571 (1.8897425014464582) + | > loss_feat: 6.23253059387207 (7.0525496162352015) + | > loss_mel: 16.661823272705078 (17.868499466630286) + | > loss_duration: 1.4790329933166504 (1.4736995433197646) + | > amp_scaler: 256.0 (337.8360655737705) + | > loss_1: 28.4597225189209 (30.633103323764487) + | > grad_norm_1: tensor(254.0714, device='cuda:0') (tensor(253.7423, device='cuda:0')) + | > current_lr_0: 0.000194744939992554 + | > current_lr_1: 0.000194744939992554 + | > step_time: 0.4512 (0.37868959200186814) + | > loader_time: 0.007 (0.006102427107388855) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.533475399017334 (2.533475399017334) + | > loss_disc_real_0: 0.10982923954725266 (0.10982923954725266) + | > loss_disc_real_1: 0.1867513656616211 (0.1867513656616211) + | > loss_disc_real_2: 0.21986904740333557 (0.21986904740333557) + | > loss_disc_real_3: 0.2438235878944397 (0.2438235878944397) + | > loss_disc_real_4: 0.20897656679153442 (0.20897656679153442) + | > loss_disc_real_5: 0.2691287100315094 (0.2691287100315094) + | > loss_0: 2.533475399017334 (2.533475399017334) + | > loss_gen: 2.086622953414917 (2.086622953414917) + | > loss_kl: 2.530728340148926 (2.530728340148926) + | > loss_feat: 5.789061546325684 (5.789061546325684) + | > loss_mel: 18.014066696166992 (18.014066696166992) + | > loss_duration: 1.6929917335510254 (1.6929917335510254) + | > loss_1: 30.11347007751465 (30.11347007751465) + + --> STEP: 1 + | > loss_disc: 2.6869094371795654 (2.6869094371795654) + | > loss_disc_real_0: 0.10665621608495712 (0.10665621608495712) + | > loss_disc_real_1: 0.20440231263637543 (0.20440231263637543) + | > loss_disc_real_2: 0.2308339774608612 (0.2308339774608612) + | > loss_disc_real_3: 0.23825794458389282 (0.23825794458389282) + | > loss_disc_real_4: 0.26185473799705505 (0.26185473799705505) + | > loss_disc_real_5: 0.283377468585968 (0.283377468585968) + | > loss_0: 2.6869094371795654 (2.6869094371795654) + | > loss_gen: 1.8851218223571777 (1.8851218223571777) + | > loss_kl: 2.4810428619384766 (2.4810428619384766) + | > loss_feat: 4.909761428833008 (4.909761428833008) + | > loss_mel: 15.520136833190918 (15.520136833190918) + | > loss_duration: 1.7250927686691284 (1.7250927686691284) + | > loss_1: 26.521156311035156 (26.521156311035156) + + --> STEP: 2 + | > loss_disc: 2.2684311866760254 (2.4776703119277954) + | > loss_disc_real_0: 0.0953742265701294 (0.10101522132754326) + | > loss_disc_real_1: 0.205403134226799 (0.20490272343158722) + | > loss_disc_real_2: 0.2220730185508728 (0.226453498005867) + | > loss_disc_real_3: 0.21955665946006775 (0.22890730202198029) + | > loss_disc_real_4: 0.22800900042057037 (0.2449318692088127) + | > loss_disc_real_5: 0.21335941553115845 (0.24836844205856323) + | > loss_0: 2.2684311866760254 (2.4776703119277954) + | > loss_gen: 2.349821090698242 (2.11747145652771) + | > loss_kl: 2.0222580432891846 (2.2516504526138306) + | > loss_feat: 7.091905117034912 (6.00083327293396) + | > loss_mel: 18.926074981689453 (17.223105907440186) + | > loss_duration: 1.6783591508865356 (1.701725959777832) + | > loss_1: 32.068416595458984 (29.29478645324707) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037386417388916016 (-1.2516975402832031e-05) + | > avg_loss_disc: 2.4776703119277954 (+0.020343542098999023) + | > avg_loss_disc_real_0: 0.10101522132754326 (-0.12357142195105553) + | > avg_loss_disc_real_1: 0.20490272343158722 (+0.03056422621011734) + | > avg_loss_disc_real_2: 0.226453498005867 (+0.00850098580121994) + | > avg_loss_disc_real_3: 0.22890730202198029 (-0.022255994379520416) + | > avg_loss_disc_real_4: 0.2449318692088127 (+0.0016460791230201721) + | > avg_loss_disc_real_5: 0.24836844205856323 (+0.006518080830574036) + | > avg_loss_0: 2.4776703119277954 (+0.020343542098999023) + | > avg_loss_gen: 2.11747145652771 (-0.4388793706893921) + | > avg_loss_kl: 2.2516504526138306 (-0.19729292392730713) + | > avg_loss_feat: 6.00083327293396 (-0.8236851692199707) + | > avg_loss_mel: 17.223105907440186 (-1.3184285163879395) + | > avg_loss_duration: 1.701725959777832 (+0.021440744400024414) + | > avg_loss_1: 29.29478645324707 (-2.7568435668945312) + + + > EPOCH: 214/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:30:53)  + + --> TIME: 2025-04-19 02:30:56 -- STEP: 5/142 -- GLOBAL_STEP: 409675 + | > loss_disc: 2.4176535606384277 (2.4626036167144774) + | > loss_disc_real_0: 0.09633380174636841 (0.1212809920310974) + | > loss_disc_real_1: 0.18616575002670288 (0.2099580556154251) + | > loss_disc_real_2: 0.21371325850486755 (0.22208209037780763) + | > loss_disc_real_3: 0.2114000767469406 (0.22690486907958984) + | > loss_disc_real_4: 0.20225536823272705 (0.23685851991176604) + | > loss_disc_real_5: 0.21567171812057495 (0.23022693693637847) + | > loss_0: 2.4176535606384277 (2.4626036167144774) + | > grad_norm_0: tensor(16.0395, device='cuda:0') (tensor(11.2804, device='cuda:0')) + | > loss_gen: 2.2743780612945557 (2.311088037490845) + | > loss_kl: 1.8626830577850342 (1.8303479433059693) + | > loss_feat: 6.87839937210083 (6.738849830627442) + | > loss_mel: 17.579885482788086 (17.381547737121583) + | > loss_duration: 1.4404383897781372 (1.4335525035858154) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.035785675048828 (29.695386505126955) + | > grad_norm_1: tensor(309.0533, device='cuda:0') (tensor(321.9120, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.3314 (0.32698793411254884) + | > loader_time: 0.0046 (0.004826068878173828) + + + --> TIME: 2025-04-19 02:31:04 -- STEP: 30/142 -- GLOBAL_STEP: 409700 + | > loss_disc: 2.3498616218566895 (2.432861264546712) + | > loss_disc_real_0: 0.1183498427271843 (0.12289377177755038) + | > loss_disc_real_1: 0.21842248737812042 (0.2071110705534617) + | > loss_disc_real_2: 0.22297795116901398 (0.2188148443897565) + | > loss_disc_real_3: 0.2259928435087204 (0.22439694901307425) + | > loss_disc_real_4: 0.23745134472846985 (0.22908980995416642) + | > loss_disc_real_5: 0.20170502364635468 (0.22655312418937684) + | > loss_0: 2.3498616218566895 (2.432861264546712) + | > grad_norm_0: tensor(14.0222, device='cuda:0') (tensor(10.1211, device='cuda:0')) + | > loss_gen: 2.4482362270355225 (2.3576135873794555) + | > loss_kl: 1.931470274925232 (1.8741566061973571) + | > loss_feat: 7.005573272705078 (7.082661962509155) + | > loss_mel: 16.942216873168945 (17.554695415496827) + | > loss_duration: 1.4402953386306763 (1.439793050289154) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.767793655395508 (30.308920923868815) + | > grad_norm_1: tensor(43.5090, device='cuda:0') (tensor(220.3488, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.3504 (0.3377563873926798) + | > loader_time: 0.0056 (0.00513311227162679) + + + --> TIME: 2025-04-19 02:31:14 -- STEP: 55/142 -- GLOBAL_STEP: 409725 + | > loss_disc: 2.5118978023529053 (2.4346914638172494) + | > loss_disc_real_0: 0.1290895938873291 (0.11915986348282206) + | > loss_disc_real_1: 0.20906412601470947 (0.20537513927979903) + | > loss_disc_real_2: 0.2567143738269806 (0.22240850627422332) + | > loss_disc_real_3: 0.22524045407772064 (0.22681931988759474) + | > loss_disc_real_4: 0.24782010912895203 (0.22960941439325158) + | > loss_disc_real_5: 0.25554847717285156 (0.22875784174962477) + | > loss_0: 2.5118978023529053 (2.4346914638172494) + | > grad_norm_0: tensor(7.5518, device='cuda:0') (tensor(10.2766, device='cuda:0')) + | > loss_gen: 2.1743698120117188 (2.36642801544883) + | > loss_kl: 1.9864603281021118 (1.8609923644499344) + | > loss_feat: 5.969503879547119 (7.166179240833629) + | > loss_mel: 16.860031127929688 (17.651333982294254) + | > loss_duration: 1.5006834268569946 (1.44869404706088) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.491046905517578 (30.493627895008434) + | > grad_norm_1: tensor(87.6051, device='cuda:0') (tensor(244.1129, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.3737 (0.3530362302606756) + | > loader_time: 0.006 (0.005402599681507457) + + + --> TIME: 2025-04-19 02:31:24 -- STEP: 80/142 -- GLOBAL_STEP: 409750 + | > loss_disc: 2.421321392059326 (2.4537922382354727) + | > loss_disc_real_0: 0.17674028873443604 (0.12217637477442622) + | > loss_disc_real_1: 0.22874128818511963 (0.20630270466208459) + | > loss_disc_real_2: 0.21381305158138275 (0.22503616698086262) + | > loss_disc_real_3: 0.22351373732089996 (0.22948015704751015) + | > loss_disc_real_4: 0.2140904664993286 (0.22974575031548738) + | > loss_disc_real_5: 0.22486206889152527 (0.22992777395993472) + | > loss_0: 2.421321392059326 (2.4537922382354727) + | > grad_norm_0: tensor(20.8155, device='cuda:0') (tensor(12.1353, device='cuda:0')) + | > loss_gen: 2.363046169281006 (2.3605441629886625) + | > loss_kl: 1.954229712486267 (1.8745043888688087) + | > loss_feat: 6.333109378814697 (7.052784746885299) + | > loss_mel: 16.574066162109375 (17.552285623550414) + | > loss_duration: 1.5015333890914917 (1.4607580527663226) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.725984573364258 (30.300877189636232) + | > grad_norm_1: tensor(208.8321, device='cuda:0') (tensor(268.8209, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.4069 (0.3652284502983093) + | > loader_time: 0.007 (0.005838590860366821) + + + --> TIME: 2025-04-19 02:31:34 -- STEP: 105/142 -- GLOBAL_STEP: 409775 + | > loss_disc: 2.6033473014831543 (2.4654670056842614) + | > loss_disc_real_0: 0.1553642600774765 (0.12430643985668818) + | > loss_disc_real_1: 0.2393646389245987 (0.20755599126929328) + | > loss_disc_real_2: 0.21537533402442932 (0.22527856159777868) + | > loss_disc_real_3: 0.23653621971607208 (0.229864374370802) + | > loss_disc_real_4: 0.2206806093454361 (0.23044798260643368) + | > loss_disc_real_5: 0.2095038741827011 (0.23080278493109205) + | > loss_0: 2.6033473014831543 (2.4654670056842614) + | > grad_norm_0: tensor(13.2952, device='cuda:0') (tensor(12.2999, device='cuda:0')) + | > loss_gen: 2.235481023788452 (2.3476472854614254) + | > loss_kl: 2.0367491245269775 (1.891166457675752) + | > loss_feat: 6.855247497558594 (6.995795944758824) + | > loss_mel: 17.977306365966797 (17.591354315621516) + | > loss_duration: 1.5174847841262817 (1.4712295986357184) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.622268676757812 (30.297193745204382) + | > grad_norm_1: tensor(81.3891, device='cuda:0') (tensor(275.3816, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.4167 (0.374623187383016) + | > loader_time: 0.0069 (0.0061199438004266625) + + + --> TIME: 2025-04-19 02:31:45 -- STEP: 130/142 -- GLOBAL_STEP: 409800 + | > loss_disc: 2.587263822555542 (2.471027189034681) + | > loss_disc_real_0: 0.1350710093975067 (0.12636067792773248) + | > loss_disc_real_1: 0.19995208084583282 (0.2079381580536182) + | > loss_disc_real_2: 0.22555451095104218 (0.22587819970571077) + | > loss_disc_real_3: 0.23137079179286957 (0.22962470409961847) + | > loss_disc_real_4: 0.2417605221271515 (0.2301957049048864) + | > loss_disc_real_5: 0.22457832098007202 (0.2312922125825515) + | > loss_0: 2.587263822555542 (2.471027189034681) + | > grad_norm_0: tensor(23.4933, device='cuda:0') (tensor(12.6872, device='cuda:0')) + | > loss_gen: 2.2694451808929443 (2.3395769155942476) + | > loss_kl: 1.9860491752624512 (1.894442343711853) + | > loss_feat: 5.3308939933776855 (6.948212806995098) + | > loss_mel: 14.786924362182617 (17.544489787175113) + | > loss_duration: 1.4806060791015625 (1.4771873602500325) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.853918075561523 (30.203909331101638) + | > grad_norm_1: tensor(238.6351, device='cuda:0') (tensor(279.6476, device='cuda:0')) + | > current_lr_0: 0.00019472059687505495 + | > current_lr_1: 0.00019472059687505495 + | > step_time: 0.4505 (0.38576669142796444) + | > loader_time: 0.0076 (0.006363824697641224) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.368161916732788 (2.368161916732788) + | > loss_disc_real_0: 0.23898954689502716 (0.23898954689502716) + | > loss_disc_real_1: 0.25086331367492676 (0.25086331367492676) + | > loss_disc_real_2: 0.2157905101776123 (0.2157905101776123) + | > loss_disc_real_3: 0.20815016329288483 (0.20815016329288483) + | > loss_disc_real_4: 0.2426658719778061 (0.2426658719778061) + | > loss_disc_real_5: 0.2184673547744751 (0.2184673547744751) + | > loss_0: 2.368161916732788 (2.368161916732788) + | > loss_gen: 2.7039713859558105 (2.7039713859558105) + | > loss_kl: 2.6356751918792725 (2.6356751918792725) + | > loss_feat: 6.665838718414307 (6.665838718414307) + | > loss_mel: 17.343870162963867 (17.343870162963867) + | > loss_duration: 1.694384217262268 (1.694384217262268) + | > loss_1: 31.043739318847656 (31.043739318847656) + + --> STEP: 1 + | > loss_disc: 2.54123854637146 (2.54123854637146) + | > loss_disc_real_0: 0.27990326285362244 (0.27990326285362244) + | > loss_disc_real_1: 0.2420407384634018 (0.2420407384634018) + | > loss_disc_real_2: 0.2026231437921524 (0.2026231437921524) + | > loss_disc_real_3: 0.23239362239837646 (0.23239362239837646) + | > loss_disc_real_4: 0.2527669072151184 (0.2527669072151184) + | > loss_disc_real_5: 0.26308372616767883 (0.26308372616767883) + | > loss_0: 2.54123854637146 (2.54123854637146) + | > loss_gen: 2.878112554550171 (2.878112554550171) + | > loss_kl: 2.495199203491211 (2.495199203491211) + | > loss_feat: 7.2990803718566895 (7.2990803718566895) + | > loss_mel: 19.423952102661133 (19.423952102661133) + | > loss_duration: 1.7079057693481445 (1.7079057693481445) + | > loss_1: 33.80424880981445 (33.80424880981445) + + --> STEP: 2 + | > loss_disc: 2.514388084411621 (2.5278133153915405) + | > loss_disc_real_0: 0.29020023345947266 (0.28505174815654755) + | > loss_disc_real_1: 0.2098063975572586 (0.2259235680103302) + | > loss_disc_real_2: 0.21941694617271423 (0.21102004498243332) + | > loss_disc_real_3: 0.2536346912384033 (0.2430141568183899) + | > loss_disc_real_4: 0.26397189497947693 (0.25836940109729767) + | > loss_disc_real_5: 0.22385260462760925 (0.24346816539764404) + | > loss_0: 2.514388084411621 (2.5278133153915405) + | > loss_gen: 2.7484309673309326 (2.8132717609405518) + | > loss_kl: 2.4778192043304443 (2.4865092039108276) + | > loss_feat: 7.641537189483643 (7.470308780670166) + | > loss_mel: 18.32709503173828 (18.875523567199707) + | > loss_duration: 1.6773350238800049 (1.6926203966140747) + | > loss_1: 32.872215270996094 (33.33823204040527) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003370523452758789 (-0.0003681182861328125) + | > avg_loss_disc: 2.5278133153915405 (+0.05014300346374512) + | > avg_loss_disc_real_0: 0.28505174815654755 (+0.1840365268290043) + | > avg_loss_disc_real_1: 0.2259235680103302 (+0.02102084457874298) + | > avg_loss_disc_real_2: 0.21102004498243332 (-0.015433453023433685) + | > avg_loss_disc_real_3: 0.2430141568183899 (+0.014106854796409607) + | > avg_loss_disc_real_4: 0.25836940109729767 (+0.013437531888484955) + | > avg_loss_disc_real_5: 0.24346816539764404 (-0.0049002766609191895) + | > avg_loss_0: 2.5278133153915405 (+0.05014300346374512) + | > avg_loss_gen: 2.8132717609405518 (+0.6958003044128418) + | > avg_loss_kl: 2.4865092039108276 (+0.23485875129699707) + | > avg_loss_feat: 7.470308780670166 (+1.469475507736206) + | > avg_loss_mel: 18.875523567199707 (+1.6524176597595215) + | > avg_loss_duration: 1.6926203966140747 (-0.009105563163757324) + | > avg_loss_1: 33.33823204040527 (+4.043445587158203) + + + > EPOCH: 215/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:31:52)  + + --> TIME: 2025-04-19 02:31:57 -- STEP: 13/142 -- GLOBAL_STEP: 409825 + | > loss_disc: 2.5184988975524902 (2.44856282380911) + | > loss_disc_real_0: 0.1041177287697792 (0.12271936180499882) + | > loss_disc_real_1: 0.21907749772071838 (0.2038596742428266) + | > loss_disc_real_2: 0.2299240231513977 (0.23286108787243182) + | > loss_disc_real_3: 0.24739274382591248 (0.23530918589005104) + | > loss_disc_real_4: 0.28403225541114807 (0.23114865674422339) + | > loss_disc_real_5: 0.2497500330209732 (0.2328311032973803) + | > loss_0: 2.5184988975524902 (2.44856282380911) + | > grad_norm_0: tensor(8.0550, device='cuda:0') (tensor(12.0814, device='cuda:0')) + | > loss_gen: 2.2816739082336426 (2.3958974251380334) + | > loss_kl: 1.9734305143356323 (1.806885315821721) + | > loss_feat: 7.335697650909424 (7.400023937225342) + | > loss_mel: 18.635732650756836 (18.125680703383225) + | > loss_duration: 1.4379684925079346 (1.4258094384120061) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.66450309753418 (31.154297021719124) + | > grad_norm_1: tensor(51.2468, device='cuda:0') (tensor(282.5679, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.333 (0.3288882879110483) + | > loader_time: 0.0051 (0.004651436438927284) + + + --> TIME: 2025-04-19 02:32:06 -- STEP: 38/142 -- GLOBAL_STEP: 409850 + | > loss_disc: 2.466337203979492 (2.45555359438846) + | > loss_disc_real_0: 0.10956782102584839 (0.12288027217513636) + | > loss_disc_real_1: 0.24048374593257904 (0.20625904124034078) + | > loss_disc_real_2: 0.21875183284282684 (0.22810862370227514) + | > loss_disc_real_3: 0.19682146608829498 (0.22806760668754578) + | > loss_disc_real_4: 0.2095777541399002 (0.22954800842623962) + | > loss_disc_real_5: 0.230494886636734 (0.23264724918101964) + | > loss_0: 2.466337203979492 (2.45555359438846) + | > grad_norm_0: tensor(17.7592, device='cuda:0') (tensor(11.2643, device='cuda:0')) + | > loss_gen: 2.2757318019866943 (2.3606494163212024) + | > loss_kl: 1.9698591232299805 (1.8491987491908826) + | > loss_feat: 7.194344997406006 (7.158792483179193) + | > loss_mel: 17.832136154174805 (17.781535650554456) + | > loss_duration: 1.4504042863845825 (1.4373571841340316) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.722476959228516 (30.587533800225508) + | > grad_norm_1: tensor(354.4929, device='cuda:0') (tensor(281.1553, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.3529 (0.3414073868801719) + | > loader_time: 0.0058 (0.005133904908832751) + + + --> TIME: 2025-04-19 02:32:16 -- STEP: 63/142 -- GLOBAL_STEP: 409875 + | > loss_disc: 2.4963698387145996 (2.4648010465833874) + | > loss_disc_real_0: 0.18002818524837494 (0.12494557894884595) + | > loss_disc_real_1: 0.23599602282047272 (0.20894722522251188) + | > loss_disc_real_2: 0.2544715106487274 (0.2292889245445766) + | > loss_disc_real_3: 0.22988417744636536 (0.22786148081696223) + | > loss_disc_real_4: 0.2636265754699707 (0.22785099205516635) + | > loss_disc_real_5: 0.23730002343654633 (0.2316122752806497) + | > loss_0: 2.4963698387145996 (2.4648010465833874) + | > grad_norm_0: tensor(11.6637, device='cuda:0') (tensor(11.2202, device='cuda:0')) + | > loss_gen: 2.4892115592956543 (2.3504722118377686) + | > loss_kl: 1.9307278394699097 (1.8613292395122467) + | > loss_feat: 6.820747375488281 (7.063789397951156) + | > loss_mel: 18.412668228149414 (17.721014507233143) + | > loss_duration: 1.514906883239746 (1.4514080172493347) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.168262481689453 (30.448013547867063) + | > grad_norm_1: tensor(279.9134, device='cuda:0') (tensor(271.6568, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.3875 (0.353358094654386) + | > loader_time: 0.0062 (0.005560780328417581) + + + --> TIME: 2025-04-19 02:32:26 -- STEP: 88/142 -- GLOBAL_STEP: 409900 + | > loss_disc: 2.482466697692871 (2.4740523885596883) + | > loss_disc_real_0: 0.14007361233234406 (0.1278653648597273) + | > loss_disc_real_1: 0.20115159451961517 (0.20838721892373124) + | > loss_disc_real_2: 0.2470390349626541 (0.22903144139457832) + | > loss_disc_real_3: 0.20668090879917145 (0.22965458323332397) + | > loss_disc_real_4: 0.275810182094574 (0.22853576916862617) + | > loss_disc_real_5: 0.2096811830997467 (0.2329305632209236) + | > loss_0: 2.482466697692871 (2.4740523885596883) + | > grad_norm_0: tensor(7.9931, device='cuda:0') (tensor(11.6642, device='cuda:0')) + | > loss_gen: 2.3814778327941895 (2.3474327109076754) + | > loss_kl: 1.9330552816390991 (1.875617359172214) + | > loss_feat: 6.6426496505737305 (6.999289534308693) + | > loss_mel: 17.197147369384766 (17.645818439396937) + | > loss_duration: 1.5473318099975586 (1.4638090377504178) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70166015625 (30.33196713707664) + | > grad_norm_1: tensor(92.5604, device='cuda:0') (tensor(263.5867, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.3969 (0.364448688246987) + | > loader_time: 0.0069 (0.005826248364015059) + + + --> TIME: 2025-04-19 02:32:36 -- STEP: 113/142 -- GLOBAL_STEP: 409925 + | > loss_disc: 2.415914535522461 (2.4724195805271116) + | > loss_disc_real_0: 0.13033612072467804 (0.1286548101796513) + | > loss_disc_real_1: 0.21220557391643524 (0.20777564304592333) + | > loss_disc_real_2: 0.21753394603729248 (0.22857894398997317) + | > loss_disc_real_3: 0.19919157028198242 (0.2292742750286001) + | > loss_disc_real_4: 0.17977510392665863 (0.22756199617828943) + | > loss_disc_real_5: 0.21405985951423645 (0.23144723857398583) + | > loss_0: 2.415914535522461 (2.4724195805271116) + | > grad_norm_0: tensor(13.0811, device='cuda:0') (tensor(10.7599, device='cuda:0')) + | > loss_gen: 2.2734122276306152 (2.3291470898991133) + | > loss_kl: 1.8571383953094482 (1.8838979512189342) + | > loss_feat: 6.819164276123047 (6.911399583900924) + | > loss_mel: 16.922874450683594 (17.508761068360982) + | > loss_duration: 1.506184697151184 (1.4712534035201623) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.37877655029297 (30.104459138043158) + | > grad_norm_1: tensor(190.0433, device='cuda:0') (tensor(244.7262, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.4257 (0.37513988207926785) + | > loader_time: 0.0076 (0.006109733497147012) + + + --> TIME: 2025-04-19 02:32:48 -- STEP: 138/142 -- GLOBAL_STEP: 409950 + | > loss_disc: 2.526099681854248 (2.4758936218593437) + | > loss_disc_real_0: 0.13107037544250488 (0.13067460308472315) + | > loss_disc_real_1: 0.1763143539428711 (0.2076184122242789) + | > loss_disc_real_2: 0.22245706617832184 (0.22839543536521387) + | > loss_disc_real_3: 0.21560929715633392 (0.22880821167558862) + | > loss_disc_real_4: 0.22990310192108154 (0.22878968694071836) + | > loss_disc_real_5: 0.22411474585533142 (0.23218351138242777) + | > loss_0: 2.526099681854248 (2.4758936218593437) + | > grad_norm_0: tensor(13.7176, device='cuda:0') (tensor(11.2538, device='cuda:0')) + | > loss_gen: 2.2408010959625244 (2.329317937726559) + | > loss_kl: 1.971704125404358 (1.8927132495935413) + | > loss_feat: 6.184772968292236 (6.911470838215039) + | > loss_mel: 17.693302154541016 (17.547685685365085) + | > loss_duration: 1.555983066558838 (1.4781279373860015) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.646562576293945 (30.159315662107606) + | > grad_norm_1: tensor(205.0663, device='cuda:0') (tensor(246.9361, device='cuda:0')) + | > current_lr_0: 0.00019469625680044555 + | > current_lr_1: 0.00019469625680044555 + | > step_time: 0.4525 (0.3872935892879099) + | > loader_time: 0.0077 (0.0063542093055835676) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4887115955352783 (2.4887115955352783) + | > loss_disc_real_0: 0.15918612480163574 (0.15918612480163574) + | > loss_disc_real_1: 0.2490796446800232 (0.2490796446800232) + | > loss_disc_real_2: 0.288017213344574 (0.288017213344574) + | > loss_disc_real_3: 0.258716344833374 (0.258716344833374) + | > loss_disc_real_4: 0.21219603717327118 (0.21219603717327118) + | > loss_disc_real_5: 0.2378978431224823 (0.2378978431224823) + | > loss_0: 2.4887115955352783 (2.4887115955352783) + | > loss_gen: 2.409510850906372 (2.409510850906372) + | > loss_kl: 2.4335885047912598 (2.4335885047912598) + | > loss_feat: 6.462751388549805 (6.462751388549805) + | > loss_mel: 16.94366455078125 (16.94366455078125) + | > loss_duration: 1.677869200706482 (1.677869200706482) + | > loss_1: 29.927385330200195 (29.927385330200195) + + --> STEP: 1 + | > loss_disc: 2.5204451084136963 (2.5204451084136963) + | > loss_disc_real_0: 0.08792425692081451 (0.08792425692081451) + | > loss_disc_real_1: 0.2644912302494049 (0.2644912302494049) + | > loss_disc_real_2: 0.316226989030838 (0.316226989030838) + | > loss_disc_real_3: 0.2821929454803467 (0.2821929454803467) + | > loss_disc_real_4: 0.25459200143814087 (0.25459200143814087) + | > loss_disc_real_5: 0.20577780902385712 (0.20577780902385712) + | > loss_0: 2.5204451084136963 (2.5204451084136963) + | > loss_gen: 2.3600144386291504 (2.3600144386291504) + | > loss_kl: 2.662287473678589 (2.662287473678589) + | > loss_feat: 7.209917068481445 (7.209917068481445) + | > loss_mel: 18.83926010131836 (18.83926010131836) + | > loss_duration: 1.7197743654251099 (1.7197743654251099) + | > loss_1: 32.79125213623047 (32.79125213623047) + + --> STEP: 2 + | > loss_disc: 2.4619638919830322 (2.4912045001983643) + | > loss_disc_real_0: 0.11036708950996399 (0.09914567321538925) + | > loss_disc_real_1: 0.19475558400154114 (0.22962340712547302) + | > loss_disc_real_2: 0.24973717331886292 (0.28298208117485046) + | > loss_disc_real_3: 0.2706620693206787 (0.2764275074005127) + | > loss_disc_real_4: 0.23897798359394073 (0.2467849925160408) + | > loss_disc_real_5: 0.19750189781188965 (0.20163985341787338) + | > loss_0: 2.4619638919830322 (2.4912045001983643) + | > loss_gen: 2.3224520683288574 (2.341233253479004) + | > loss_kl: 2.253552198410034 (2.4579198360443115) + | > loss_feat: 7.64772367477417 (7.428820371627808) + | > loss_mel: 19.050439834594727 (18.944849967956543) + | > loss_duration: 1.6636204719543457 (1.6916974186897278) + | > loss_1: 32.93778991699219 (32.86452102661133) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034263134002685547 (+5.5789947509765625e-05) + | > avg_loss_disc: 2.4912045001983643 (-0.03660881519317627) + | > avg_loss_disc_real_0: 0.09914567321538925 (-0.1859060749411583) + | > avg_loss_disc_real_1: 0.22962340712547302 (+0.0036998391151428223) + | > avg_loss_disc_real_2: 0.28298208117485046 (+0.07196203619241714) + | > avg_loss_disc_real_3: 0.2764275074005127 (+0.0334133505821228) + | > avg_loss_disc_real_4: 0.2467849925160408 (-0.011584408581256866) + | > avg_loss_disc_real_5: 0.20163985341787338 (-0.04182831197977066) + | > avg_loss_0: 2.4912045001983643 (-0.03660881519317627) + | > avg_loss_gen: 2.341233253479004 (-0.47203850746154785) + | > avg_loss_kl: 2.4579198360443115 (-0.028589367866516113) + | > avg_loss_feat: 7.428820371627808 (-0.0414884090423584) + | > avg_loss_mel: 18.944849967956543 (+0.06932640075683594) + | > avg_loss_duration: 1.6916974186897278 (-0.0009229779243469238) + | > avg_loss_1: 32.86452102661133 (-0.4737110137939453) + + + > EPOCH: 216/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:32:51)  + + --> TIME: 2025-04-19 02:32:59 -- STEP: 21/142 -- GLOBAL_STEP: 409975 + | > loss_disc: 2.6082913875579834 (2.4103567600250244) + | > loss_disc_real_0: 0.09227945655584335 (0.11373759841635114) + | > loss_disc_real_1: 0.20198607444763184 (0.20569418228807904) + | > loss_disc_real_2: 0.23619551956653595 (0.2293932161160878) + | > loss_disc_real_3: 0.2223750799894333 (0.22310984489463626) + | > loss_disc_real_4: 0.24149318039417267 (0.22788265560354506) + | > loss_disc_real_5: 0.2625225782394409 (0.2294439517316364) + | > loss_0: 2.6082913875579834 (2.4103567600250244) + | > grad_norm_0: tensor(7.5715, device='cuda:0') (tensor(9.1321, device='cuda:0')) + | > loss_gen: 2.243168830871582 (2.4088142599378313) + | > loss_kl: 1.9242857694625854 (1.834202181725275) + | > loss_feat: 6.832880973815918 (7.3246307373046875) + | > loss_mel: 17.800064086914062 (17.890804835728236) + | > loss_duration: 1.433197259902954 (1.432768430028643) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.233596801757812 (30.89122045607794) + | > grad_norm_1: tensor(74.7667, device='cuda:0') (tensor(298.0114, device='cuda:0')) + | > current_lr_0: 0.0001946719197683455 + | > current_lr_1: 0.0001946719197683455 + | > step_time: 0.3413 (0.33169115157354445) + | > loader_time: 0.0055 (0.004967632747831799) + + + --> TIME: 2025-04-19 02:33:08 -- STEP: 46/142 -- GLOBAL_STEP: 410000 + | > loss_disc: 2.3594343662261963 (2.434676258460334) + | > loss_disc_real_0: 0.10719563066959381 (0.11775656433209129) + | > loss_disc_real_1: 0.15433937311172485 (0.20481251115384308) + | > loss_disc_real_2: 0.2268223911523819 (0.22810352820417154) + | > loss_disc_real_3: 0.234731063246727 (0.2269934730037399) + | > loss_disc_real_4: 0.2181791216135025 (0.22895408033028894) + | > loss_disc_real_5: 0.22288204729557037 (0.2314872340015743) + | > loss_0: 2.3594343662261963 (2.434676258460334) + | > grad_norm_0: tensor(11.0697, device='cuda:0') (tensor(9.2269, device='cuda:0')) + | > loss_gen: 2.5565853118896484 (2.3845284399778945) + | > loss_kl: 1.8821433782577515 (1.858838571154553) + | > loss_feat: 7.989670753479004 (7.1742571644161055) + | > loss_mel: 18.42737579345703 (17.876809617747426) + | > loss_duration: 1.4345287084579468 (1.444699041221453) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.290306091308594 (30.73913288116455) + | > grad_norm_1: tensor(302.0079, device='cuda:0') (tensor(252.6017, device='cuda:0')) + | > current_lr_0: 0.0001946719197683455 + | > current_lr_1: 0.0001946719197683455 + | > step_time: 0.367 (0.3443162700404291) + | > loader_time: 0.0059 (0.005303211834119715) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_410000.pth + + --> TIME: 2025-04-19 02:33:19 -- STEP: 71/142 -- GLOBAL_STEP: 410025 + | > loss_disc: 2.509343147277832 (2.4460681223533514) + | > loss_disc_real_0: 0.11583937704563141 (0.12149000598091475) + | > loss_disc_real_1: 0.2230815291404724 (0.2069652712261173) + | > loss_disc_real_2: 0.26223132014274597 (0.22888172680223492) + | > loss_disc_real_3: 0.2443753480911255 (0.2275074063472345) + | > loss_disc_real_4: 0.28264033794403076 (0.22809740942968448) + | > loss_disc_real_5: 0.23747968673706055 (0.23210295090373134) + | > loss_0: 2.509343147277832 (2.4460681223533514) + | > grad_norm_0: tensor(18.8291, device='cuda:0') (tensor(9.8264, device='cuda:0')) + | > loss_gen: 2.382483959197998 (2.3827034352530903) + | > loss_kl: 1.8256617784500122 (1.8655924612367656) + | > loss_feat: 6.734623908996582 (7.184850054727474) + | > loss_mel: 17.960575103759766 (18.00345243534571) + | > loss_duration: 1.488119125366211 (1.4548310108587779) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.391464233398438 (30.89142949816207) + | > grad_norm_1: tensor(320.4572, device='cuda:0') (tensor(256.2108, device='cuda:0')) + | > current_lr_0: 0.0001946719197683455 + | > current_lr_1: 0.0001946719197683455 + | > step_time: 0.3877 (0.356098732478182) + | > loader_time: 0.0063 (0.0055935349262936016) + + + --> TIME: 2025-04-19 02:33:29 -- STEP: 96/142 -- GLOBAL_STEP: 410050 + | > loss_disc: 2.6294314861297607 (2.470764063298702) + | > loss_disc_real_0: 0.3365609347820282 (0.12882697178671756) + | > loss_disc_real_1: 0.23051048815250397 (0.2088279826566577) + | > loss_disc_real_2: 0.2043173760175705 (0.22933543333783746) + | > loss_disc_real_3: 0.18659357726573944 (0.22802630749841532) + | > loss_disc_real_4: 0.21880023181438446 (0.22929187258705497) + | > loss_disc_real_5: 0.2605881094932556 (0.2326869696068267) + | > loss_0: 2.6294314861297607 (2.470764063298702) + | > grad_norm_0: tensor(26.2845, device='cuda:0') (tensor(12.7403, device='cuda:0')) + | > loss_gen: 2.105616569519043 (2.375869390865168) + | > loss_kl: 1.9285024404525757 (1.8717381867269676) + | > loss_feat: 5.3646769523620605 (7.140764688452085) + | > loss_mel: 14.800355911254883 (18.032021145025887) + | > loss_duration: 1.490361213684082 (1.4664652161300182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.68951416015625 (30.886858701705933) + | > grad_norm_1: tensor(187.5746, device='cuda:0') (tensor(277.0666, device='cuda:0')) + | > current_lr_0: 0.0001946719197683455 + | > current_lr_1: 0.0001946719197683455 + | > step_time: 0.4055 (0.3679232622186343) + | > loader_time: 0.0068 (0.006009422242641449) + + + --> TIME: 2025-04-19 02:33:40 -- STEP: 121/142 -- GLOBAL_STEP: 410075 + | > loss_disc: 2.504845142364502 (2.466193175512897) + | > loss_disc_real_0: 0.15057595074176788 (0.12735741640910628) + | > loss_disc_real_1: 0.2451099306344986 (0.20918007626021204) + | > loss_disc_real_2: 0.22855333983898163 (0.22785980630019478) + | > loss_disc_real_3: 0.25992029905319214 (0.22813784115570634) + | > loss_disc_real_4: 0.27997392416000366 (0.22960213577944386) + | > loss_disc_real_5: 0.27881044149398804 (0.23143687671866298) + | > loss_0: 2.504845142364502 (2.466193175512897) + | > grad_norm_0: tensor(7.6958, device='cuda:0') (tensor(13.0392, device='cuda:0')) + | > loss_gen: 2.381566286087036 (2.3744752525298067) + | > loss_kl: 2.1855881214141846 (1.8795285481066744) + | > loss_feat: 6.657866954803467 (7.090280855982757) + | > loss_mel: 18.399044036865234 (17.930108677257188) + | > loss_duration: 1.5364848375320435 (1.4735167528972153) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.160552978515625 (30.747910144900487) + | > grad_norm_1: tensor(235.4446, device='cuda:0') (tensor(278.5031, device='cuda:0')) + | > current_lr_0: 0.0001946719197683455 + | > current_lr_1: 0.0001946719197683455 + | > step_time: 0.4409 (0.378970018103103) + | > loader_time: 0.0075 (0.006248424861056746) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6018946170806885 (2.6018946170806885) + | > loss_disc_real_0: 0.13464964926242828 (0.13464964926242828) + | > loss_disc_real_1: 0.23323658108711243 (0.23323658108711243) + | > loss_disc_real_2: 0.2439572662115097 (0.2439572662115097) + | > loss_disc_real_3: 0.24724702537059784 (0.24724702537059784) + | > loss_disc_real_4: 0.2539263069629669 (0.2539263069629669) + | > loss_disc_real_5: 0.24034607410430908 (0.24034607410430908) + | > loss_0: 2.6018946170806885 (2.6018946170806885) + | > loss_gen: 2.113621711730957 (2.113621711730957) + | > loss_kl: 2.234266996383667 (2.234266996383667) + | > loss_feat: 5.568511962890625 (5.568511962890625) + | > loss_mel: 14.776054382324219 (14.776054382324219) + | > loss_duration: 1.6837124824523926 (1.6837124824523926) + | > loss_1: 26.37616729736328 (26.37616729736328) + + --> STEP: 1 + | > loss_disc: 2.5718019008636475 (2.5718019008636475) + | > loss_disc_real_0: 0.17053821682929993 (0.17053821682929993) + | > loss_disc_real_1: 0.1941945105791092 (0.1941945105791092) + | > loss_disc_real_2: 0.25328686833381653 (0.25328686833381653) + | > loss_disc_real_3: 0.2633092403411865 (0.2633092403411865) + | > loss_disc_real_4: 0.26591646671295166 (0.26591646671295166) + | > loss_disc_real_5: 0.23820920288562775 (0.23820920288562775) + | > loss_0: 2.5718019008636475 (2.5718019008636475) + | > loss_gen: 2.2106215953826904 (2.2106215953826904) + | > loss_kl: 2.5930025577545166 (2.5930025577545166) + | > loss_feat: 6.7871575355529785 (6.7871575355529785) + | > loss_mel: 18.294021606445312 (18.294021606445312) + | > loss_duration: 1.6992628574371338 (1.6992628574371338) + | > loss_1: 31.58406639099121 (31.58406639099121) + + --> STEP: 2 + | > loss_disc: 2.5941317081451416 (2.5829668045043945) + | > loss_disc_real_0: 0.1326868087053299 (0.1516125127673149) + | > loss_disc_real_1: 0.18725605309009552 (0.19072528183460236) + | > loss_disc_real_2: 0.2560480535030365 (0.2546674609184265) + | > loss_disc_real_3: 0.24187006056308746 (0.252589650452137) + | > loss_disc_real_4: 0.23825792968273163 (0.25208719819784164) + | > loss_disc_real_5: 0.2981509864330292 (0.26818009465932846) + | > loss_0: 2.5941317081451416 (2.5829668045043945) + | > loss_gen: 2.172773599624634 (2.191697597503662) + | > loss_kl: 2.390066385269165 (2.491534471511841) + | > loss_feat: 5.970633029937744 (6.378895282745361) + | > loss_mel: 19.655752182006836 (18.974886894226074) + | > loss_duration: 1.6520118713378906 (1.6756373643875122) + | > loss_1: 31.841238021850586 (31.7126522064209) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032483339309692383 (-0.0001779794692993164) + | > avg_loss_disc: 2.5829668045043945 (+0.09176230430603027) + | > avg_loss_disc_real_0: 0.1516125127673149 (+0.05246683955192566) + | > avg_loss_disc_real_1: 0.19072528183460236 (-0.038898125290870667) + | > avg_loss_disc_real_2: 0.2546674609184265 (-0.02831462025642395) + | > avg_loss_disc_real_3: 0.252589650452137 (-0.023837856948375702) + | > avg_loss_disc_real_4: 0.25208719819784164 (+0.005302205681800842) + | > avg_loss_disc_real_5: 0.26818009465932846 (+0.06654024124145508) + | > avg_loss_0: 2.5829668045043945 (+0.09176230430603027) + | > avg_loss_gen: 2.191697597503662 (-0.1495356559753418) + | > avg_loss_kl: 2.491534471511841 (+0.0336146354675293) + | > avg_loss_feat: 6.378895282745361 (-1.0499250888824463) + | > avg_loss_mel: 18.974886894226074 (+0.03003692626953125) + | > avg_loss_duration: 1.6756373643875122 (-0.016060054302215576) + | > avg_loss_1: 31.7126522064209 (-1.1518688201904297) + + + > EPOCH: 217/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:33:51)  + + --> TIME: 2025-04-19 02:33:53 -- STEP: 4/142 -- GLOBAL_STEP: 410100 + | > loss_disc: 2.4391002655029297 (2.4375827312469482) + | > loss_disc_real_0: 0.11536610126495361 (0.11291799135506153) + | > loss_disc_real_1: 0.22673316299915314 (0.19837025552988052) + | > loss_disc_real_2: 0.22867338359355927 (0.22223537042737007) + | > loss_disc_real_3: 0.2558484971523285 (0.2482111193239689) + | > loss_disc_real_4: 0.23627708852291107 (0.23190096393227577) + | > loss_disc_real_5: 0.2384248524904251 (0.25057240575551987) + | > loss_0: 2.4391002655029297 (2.4375827312469482) + | > grad_norm_0: tensor(11.2399, device='cuda:0') (tensor(10.4225, device='cuda:0')) + | > loss_gen: 2.3726694583892822 (2.412177562713623) + | > loss_kl: 1.9674263000488281 (1.8857066929340363) + | > loss_feat: 7.075225830078125 (7.250134229660034) + | > loss_mel: 17.745004653930664 (17.895233154296875) + | > loss_duration: 1.4224334955215454 (1.4309057593345642) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.582759857177734 (30.874157428741455) + | > grad_norm_1: tensor(295.2017, device='cuda:0') (tensor(345.9798, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.3248 (0.3247462511062622) + | > loader_time: 0.006 (0.004517257213592529) + + + --> TIME: 2025-04-19 02:34:02 -- STEP: 29/142 -- GLOBAL_STEP: 410125 + | > loss_disc: 2.5556490421295166 (2.460941783313093) + | > loss_disc_real_0: 0.186646968126297 (0.12889480924811855) + | > loss_disc_real_1: 0.23377291858196259 (0.20684758393928923) + | > loss_disc_real_2: 0.24106675386428833 (0.22772535065124774) + | > loss_disc_real_3: 0.2387017011642456 (0.22702924290607715) + | > loss_disc_real_4: 0.22728468477725983 (0.22895219819299106) + | > loss_disc_real_5: 0.30104613304138184 (0.23757570262612968) + | > loss_0: 2.5556490421295166 (2.460941783313093) + | > grad_norm_0: tensor(11.2573, device='cuda:0') (tensor(11.5126, device='cuda:0')) + | > loss_gen: 2.2494027614593506 (2.365431514279596) + | > loss_kl: 1.7425519227981567 (1.8438610208445583) + | > loss_feat: 7.107931137084961 (7.108133759991876) + | > loss_mel: 15.748825073242188 (17.51114687426337) + | > loss_duration: 1.4348962306976318 (1.4376742922026535) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.283607482910156 (30.266247124507508) + | > grad_norm_1: tensor(457.7295, device='cuda:0') (tensor(265.1264, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.3457 (0.3356037797599003) + | > loader_time: 0.0054 (0.004768568893958782) + + + --> TIME: 2025-04-19 02:34:11 -- STEP: 54/142 -- GLOBAL_STEP: 410150 + | > loss_disc: 2.4877524375915527 (2.4612842356717133) + | > loss_disc_real_0: 0.14858309924602509 (0.12560118251928576) + | > loss_disc_real_1: 0.2006659358739853 (0.20445964540596362) + | > loss_disc_real_2: 0.20846876502037048 (0.22751495959582152) + | > loss_disc_real_3: 0.21224379539489746 (0.2278867151450228) + | > loss_disc_real_4: 0.22902986407279968 (0.22770610562077276) + | > loss_disc_real_5: 0.24814966320991516 (0.23694625386485346) + | > loss_0: 2.4877524375915527 (2.4612842356717133) + | > grad_norm_0: tensor(5.1109, device='cuda:0') (tensor(9.3447, device='cuda:0')) + | > loss_gen: 2.335730791091919 (2.3559563071639458) + | > loss_kl: 1.9202250242233276 (1.863152269963865) + | > loss_feat: 7.270750999450684 (7.063257685414067) + | > loss_mel: 17.781482696533203 (17.546354064234986) + | > loss_duration: 1.4783527851104736 (1.4464284159519054) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.786542892456055 (30.27514860365126) + | > grad_norm_1: tensor(51.3901, device='cuda:0') (tensor(194.4876, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.3709 (0.3516381405017994) + | > loader_time: 0.0057 (0.005159051330001266) + + + --> TIME: 2025-04-19 02:34:21 -- STEP: 79/142 -- GLOBAL_STEP: 410175 + | > loss_disc: 2.378838300704956 (2.476690008670468) + | > loss_disc_real_0: 0.13706709444522858 (0.12928205332424061) + | > loss_disc_real_1: 0.20642900466918945 (0.20702135015892073) + | > loss_disc_real_2: 0.2622644901275635 (0.22800218719470355) + | > loss_disc_real_3: 0.25837960839271545 (0.23015221406387376) + | > loss_disc_real_4: 0.26892945170402527 (0.22895875007291383) + | > loss_disc_real_5: 0.19325891137123108 (0.23771175440353684) + | > loss_0: 2.378838300704956 (2.476690008670468) + | > grad_norm_0: tensor(6.2620, device='cuda:0') (tensor(8.5081, device='cuda:0')) + | > loss_gen: 2.4290809631347656 (2.3374744095379802) + | > loss_kl: 1.830722689628601 (1.8826849536050725) + | > loss_feat: 7.554801940917969 (7.0357820414289645) + | > loss_mel: 19.219667434692383 (17.70148372650146) + | > loss_duration: 1.5315067768096924 (1.4591465750827066) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.56578063964844 (30.41657174991656) + | > grad_norm_1: tensor(150.1777, device='cuda:0') (tensor(167.2744, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.3845 (0.36244696303258966) + | > loader_time: 0.0062 (0.005441457410401937) + + + --> TIME: 2025-04-19 02:34:31 -- STEP: 104/142 -- GLOBAL_STEP: 410200 + | > loss_disc: 2.4668660163879395 (2.4764642623754636) + | > loss_disc_real_0: 0.13793882727622986 (0.1288981189091618) + | > loss_disc_real_1: 0.21960103511810303 (0.2074923695853123) + | > loss_disc_real_2: 0.2385532557964325 (0.22841310443786478) + | > loss_disc_real_3: 0.23546048998832703 (0.2299138823380837) + | > loss_disc_real_4: 0.2787066400051117 (0.22952268458902836) + | > loss_disc_real_5: 0.22605490684509277 (0.2353060944722249) + | > loss_0: 2.4668660163879395 (2.4764642623754636) + | > grad_norm_0: tensor(12.4476, device='cuda:0') (tensor(7.9942, device='cuda:0')) + | > loss_gen: 2.4066874980926514 (2.32827230829459) + | > loss_kl: 1.9357097148895264 (1.8880496609669466) + | > loss_feat: 7.052367210388184 (6.979596935785734) + | > loss_mel: 17.919483184814453 (17.674571908437287) + | > loss_duration: 1.5042049884796143 (1.469591381458136) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.818452835083008 (30.340082241938664) + | > grad_norm_1: tensor(213.2205, device='cuda:0') (tensor(161.0386, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.409 (0.37262447751485384) + | > loader_time: 0.0066 (0.005707461100358229) + + + --> TIME: 2025-04-19 02:34:42 -- STEP: 129/142 -- GLOBAL_STEP: 410225 + | > loss_disc: 2.469287872314453 (2.4794011171474004) + | > loss_disc_real_0: 0.13589496910572052 (0.1298471392933712) + | > loss_disc_real_1: 0.2458820939064026 (0.20793106285638585) + | > loss_disc_real_2: 0.21456770598888397 (0.22826324796029768) + | > loss_disc_real_3: 0.2093314230442047 (0.22962943424088086) + | > loss_disc_real_4: 0.19341814517974854 (0.22907448022864585) + | > loss_disc_real_5: 0.20483002066612244 (0.2355617640554443) + | > loss_0: 2.469287872314453 (2.4794011171474004) + | > grad_norm_0: tensor(8.9801, device='cuda:0') (tensor(7.9919, device='cuda:0')) + | > loss_gen: 2.3316142559051514 (2.321237240650857) + | > loss_kl: 1.8430041074752808 (1.8884733268456866) + | > loss_feat: 6.988562107086182 (6.959919545077538) + | > loss_mel: 18.154380798339844 (17.697568996932155) + | > loss_duration: 1.4894505739212036 (1.475836981174558) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.807010650634766 (30.34303608975669) + | > grad_norm_1: tensor(213.1227, device='cuda:0') (tensor(164.2321, device='cuda:0')) + | > current_lr_0: 0.00019464758577837445 + | > current_lr_1: 0.00019464758577837445 + | > step_time: 0.4457 (0.3839152524637621) + | > loader_time: 0.0074 (0.005963131438854129) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.592487335205078 (2.592487335205078) + | > loss_disc_real_0: 0.1838187724351883 (0.1838187724351883) + | > loss_disc_real_1: 0.26954808831214905 (0.26954808831214905) + | > loss_disc_real_2: 0.245696559548378 (0.245696559548378) + | > loss_disc_real_3: 0.2167426347732544 (0.2167426347732544) + | > loss_disc_real_4: 0.238589346408844 (0.238589346408844) + | > loss_disc_real_5: 0.23256328701972961 (0.23256328701972961) + | > loss_0: 2.592487335205078 (2.592487335205078) + | > loss_gen: 2.176481246948242 (2.176481246948242) + | > loss_kl: 2.415705919265747 (2.415705919265747) + | > loss_feat: 5.890097618103027 (5.890097618103027) + | > loss_mel: 16.743417739868164 (16.743417739868164) + | > loss_duration: 1.6681301593780518 (1.6681301593780518) + | > loss_1: 28.89383316040039 (28.89383316040039) + + --> STEP: 1 + | > loss_disc: 2.5646345615386963 (2.5646345615386963) + | > loss_disc_real_0: 0.12614575028419495 (0.12614575028419495) + | > loss_disc_real_1: 0.23868581652641296 (0.23868581652641296) + | > loss_disc_real_2: 0.21097762882709503 (0.21097762882709503) + | > loss_disc_real_3: 0.21114028990268707 (0.21114028990268707) + | > loss_disc_real_4: 0.2382219433784485 (0.2382219433784485) + | > loss_disc_real_5: 0.21212321519851685 (0.21212321519851685) + | > loss_0: 2.5646345615386963 (2.5646345615386963) + | > loss_gen: 2.1043553352355957 (2.1043553352355957) + | > loss_kl: 2.5953829288482666 (2.5953829288482666) + | > loss_feat: 6.973047256469727 (6.973047256469727) + | > loss_mel: 17.76988410949707 (17.76988410949707) + | > loss_duration: 1.7381128072738647 (1.7381128072738647) + | > loss_1: 31.180782318115234 (31.180782318115234) + + --> STEP: 2 + | > loss_disc: 2.394827127456665 (2.4797308444976807) + | > loss_disc_real_0: 0.07938559353351593 (0.10276567190885544) + | > loss_disc_real_1: 0.2071717083454132 (0.22292876243591309) + | > loss_disc_real_2: 0.20385318994522095 (0.207415409386158) + | > loss_disc_real_3: 0.22613544762134552 (0.2186378687620163) + | > loss_disc_real_4: 0.23450413346290588 (0.23636303842067719) + | > loss_disc_real_5: 0.26590028405189514 (0.239011749625206) + | > loss_0: 2.394827127456665 (2.4797308444976807) + | > loss_gen: 2.311115264892578 (2.207735300064087) + | > loss_kl: 2.2771458625793457 (2.436264395713806) + | > loss_feat: 7.441821098327637 (7.207434177398682) + | > loss_mel: 19.04422950744629 (18.40705680847168) + | > loss_duration: 1.6650135517120361 (1.7015631794929504) + | > loss_1: 32.73932647705078 (31.960054397583008) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036171674728393555 (+0.0003688335418701172) + | > avg_loss_disc: 2.4797308444976807 (-0.10323596000671387) + | > avg_loss_disc_real_0: 0.10276567190885544 (-0.04884684085845947) + | > avg_loss_disc_real_1: 0.22292876243591309 (+0.03220348060131073) + | > avg_loss_disc_real_2: 0.207415409386158 (-0.047252051532268524) + | > avg_loss_disc_real_3: 0.2186378687620163 (-0.0339517816901207) + | > avg_loss_disc_real_4: 0.23636303842067719 (-0.01572415977716446) + | > avg_loss_disc_real_5: 0.239011749625206 (-0.029168345034122467) + | > avg_loss_0: 2.4797308444976807 (-0.10323596000671387) + | > avg_loss_gen: 2.207735300064087 (+0.016037702560424805) + | > avg_loss_kl: 2.436264395713806 (-0.05527007579803467) + | > avg_loss_feat: 7.207434177398682 (+0.8285388946533203) + | > avg_loss_mel: 18.40705680847168 (-0.5678300857543945) + | > avg_loss_duration: 1.7015631794929504 (+0.025925815105438232) + | > avg_loss_1: 31.960054397583008 (+0.24740219116210938) + + + > EPOCH: 218/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:34:50)  + + --> TIME: 2025-04-19 02:34:54 -- STEP: 12/142 -- GLOBAL_STEP: 410250 + | > loss_disc: 2.3370327949523926 (2.4033917784690857) + | > loss_disc_real_0: 0.1616218537092209 (0.13053558145960173) + | > loss_disc_real_1: 0.16633488237857819 (0.20038955037792525) + | > loss_disc_real_2: 0.22154293954372406 (0.22104395305116972) + | > loss_disc_real_3: 0.21278482675552368 (0.22086058308680853) + | > loss_disc_real_4: 0.22287119925022125 (0.21847106019655863) + | > loss_disc_real_5: 0.26326650381088257 (0.2247204432884852) + | > loss_0: 2.3370327949523926 (2.4033917784690857) + | > grad_norm_0: tensor(24.6129, device='cuda:0') (tensor(13.5250, device='cuda:0')) + | > loss_gen: 2.5505290031433105 (2.4180060227711997) + | > loss_kl: 1.8653680086135864 (1.867470512787501) + | > loss_feat: 7.648806095123291 (7.44810152053833) + | > loss_mel: 18.710437774658203 (18.066335837046307) + | > loss_duration: 1.4480319023132324 (1.4259177943070729) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.223175048828125 (31.22583230336507) + | > grad_norm_1: tensor(367.9529, device='cuda:0') (tensor(287.3633, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.332 (0.32615047693252563) + | > loader_time: 0.005 (0.004628300666809082) + + + --> TIME: 2025-04-19 02:35:03 -- STEP: 37/142 -- GLOBAL_STEP: 410275 + | > loss_disc: 2.4484262466430664 (2.438708614658665) + | > loss_disc_real_0: 0.1106683760881424 (0.12147464452160371) + | > loss_disc_real_1: 0.179004967212677 (0.20188960392732877) + | > loss_disc_real_2: 0.2310609221458435 (0.22524224141159574) + | > loss_disc_real_3: 0.2014036476612091 (0.22398524791807742) + | > loss_disc_real_4: 0.21115577220916748 (0.22250960243714824) + | > loss_disc_real_5: 0.2281787097454071 (0.23232214273633184) + | > loss_0: 2.4484262466430664 (2.438708614658665) + | > grad_norm_0: tensor(20.7325, device='cuda:0') (tensor(15.5196, device='cuda:0')) + | > loss_gen: 2.317816734313965 (2.4185053786715947) + | > loss_kl: 1.8878785371780396 (1.8876205425004702) + | > loss_feat: 6.54681396484375 (7.340966327770336) + | > loss_mel: 18.51564598083496 (18.007707286525417) + | > loss_duration: 1.4465548992156982 (1.4376978745331634) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.71470832824707 (31.092497438997835) + | > grad_norm_1: tensor(393.7217, device='cuda:0') (tensor(304.8571, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.3531 (0.33965910447610387) + | > loader_time: 0.0052 (0.005038119651175834) + + + --> TIME: 2025-04-19 02:35:12 -- STEP: 62/142 -- GLOBAL_STEP: 410300 + | > loss_disc: 2.448481559753418 (2.4391412388893867) + | > loss_disc_real_0: 0.13641050457954407 (0.12033179612650026) + | > loss_disc_real_1: 0.2132304310798645 (0.20268249295411572) + | > loss_disc_real_2: 0.22590255737304688 (0.22407550701210577) + | > loss_disc_real_3: 0.21212588250637054 (0.22587332345785632) + | > loss_disc_real_4: 0.20518018305301666 (0.22462704104761924) + | > loss_disc_real_5: 0.24005481600761414 (0.23265025692601357) + | > loss_0: 2.448481559753418 (2.4391412388893867) + | > grad_norm_0: tensor(10.1162, device='cuda:0') (tensor(14.7790, device='cuda:0')) + | > loss_gen: 2.3886899948120117 (2.397271906175921) + | > loss_kl: 1.8480173349380493 (1.8793747982671183) + | > loss_feat: 6.867773056030273 (7.270683042464718) + | > loss_mel: 17.119592666625977 (18.00565267378284) + | > loss_duration: 1.5243059396743774 (1.4511520401124032) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.748380661010742 (31.00413470114431) + | > grad_norm_1: tensor(223.4004, device='cuda:0') (tensor(317.3661, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.382 (0.35139904868218197) + | > loader_time: 0.0062 (0.005319749155352193) + + + --> TIME: 2025-04-19 02:35:23 -- STEP: 87/142 -- GLOBAL_STEP: 410325 + | > loss_disc: 2.501634359359741 (2.4505437050742667) + | > loss_disc_real_0: 0.12946410477161407 (0.12368429413643377) + | > loss_disc_real_1: 0.19298526644706726 (0.20422823538725404) + | > loss_disc_real_2: 0.2505401074886322 (0.22409689340783262) + | > loss_disc_real_3: 0.27084633708000183 (0.22743801327272392) + | > loss_disc_real_4: 0.24501772224903107 (0.2247814056517064) + | > loss_disc_real_5: 0.2515098452568054 (0.23385999428814855) + | > loss_0: 2.501634359359741 (2.4505437050742667) + | > grad_norm_0: tensor(8.5203, device='cuda:0') (tensor(13.3015, device='cuda:0')) + | > loss_gen: 2.351116418838501 (2.375119505257442) + | > loss_kl: 2.0351006984710693 (1.8911498812423355) + | > loss_feat: 6.176297664642334 (7.1518188619065555) + | > loss_mel: 17.693279266357422 (17.89454308871565) + | > loss_duration: 1.5130136013031006 (1.4624364883050154) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.768808364868164 (30.775068019998486) + | > grad_norm_1: tensor(226.2926, device='cuda:0') (tensor(287.3096, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.4003 (0.36303546236849366) + | > loader_time: 0.0063 (0.00559224205455561) + + + --> TIME: 2025-04-19 02:35:33 -- STEP: 112/142 -- GLOBAL_STEP: 410350 + | > loss_disc: 2.541059970855713 (2.460051134228707) + | > loss_disc_real_0: 0.10903209447860718 (0.12567383047592423) + | > loss_disc_real_1: 0.22731608152389526 (0.20568151266447135) + | > loss_disc_real_2: 0.24023261666297913 (0.22567029523530177) + | > loss_disc_real_3: 0.2400256097316742 (0.2277364017707961) + | > loss_disc_real_4: 0.24880725145339966 (0.22633601738406078) + | > loss_disc_real_5: 0.23455552756786346 (0.23402032775006124) + | > loss_0: 2.541059970855713 (2.460051134228707) + | > grad_norm_0: tensor(5.9494, device='cuda:0') (tensor(12.1440, device='cuda:0')) + | > loss_gen: 2.328645944595337 (2.3546122163534178) + | > loss_kl: 1.783146858215332 (1.9002919069358282) + | > loss_feat: 6.526373386383057 (7.062601170369557) + | > loss_mel: 17.14214324951172 (17.826745101383757) + | > loss_duration: 1.4740402698516846 (1.4705501507435528) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.254350662231445 (30.614800657544816) + | > grad_norm_1: tensor(75.3540, device='cuda:0') (tensor(263.0135, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.4272 (0.37494656656469605) + | > loader_time: 0.0067 (0.005884566477366858) + + + --> TIME: 2025-04-19 02:35:44 -- STEP: 137/142 -- GLOBAL_STEP: 410375 + | > loss_disc: 2.5203843116760254 (2.4606615626899004) + | > loss_disc_real_0: 0.1153358742594719 (0.12635552113617424) + | > loss_disc_real_1: 0.21065109968185425 (0.20510950956466426) + | > loss_disc_real_2: 0.273169606924057 (0.22649000084748233) + | > loss_disc_real_3: 0.25104281306266785 (0.22768137881355563) + | > loss_disc_real_4: 0.23742635548114777 (0.22635327714638118) + | > loss_disc_real_5: 0.2493499517440796 (0.233587385311614) + | > loss_0: 2.5203843116760254 (2.4606615626899004) + | > grad_norm_0: tensor(11.3944, device='cuda:0') (tensor(11.6522, device='cuda:0')) + | > loss_gen: 2.4265971183776855 (2.351381757833663) + | > loss_kl: 1.9095629453659058 (1.904771315790441) + | > loss_feat: 6.731918811798096 (7.040837597672957) + | > loss_mel: 18.59233856201172 (17.80329128599515) + | > loss_duration: 1.4892513751983643 (1.4760346482269957) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.149669647216797 (30.57631670819582) + | > grad_norm_1: tensor(302.0987, device='cuda:0') (tensor(256.9201, device='cuda:0')) + | > current_lr_0: 0.00019462325483015215 + | > current_lr_1: 0.00019462325483015215 + | > step_time: 0.457 (0.3867570083506785) + | > loader_time: 0.0075 (0.006162612107548402) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5221455097198486 (2.5221455097198486) + | > loss_disc_real_0: 0.16575227677822113 (0.16575227677822113) + | > loss_disc_real_1: 0.2510914206504822 (0.2510914206504822) + | > loss_disc_real_2: 0.24264001846313477 (0.24264001846313477) + | > loss_disc_real_3: 0.23120594024658203 (0.23120594024658203) + | > loss_disc_real_4: 0.2419695407152176 (0.2419695407152176) + | > loss_disc_real_5: 0.24449434876441956 (0.24449434876441956) + | > loss_0: 2.5221455097198486 (2.5221455097198486) + | > loss_gen: 2.464341163635254 (2.464341163635254) + | > loss_kl: 2.4031779766082764 (2.4031779766082764) + | > loss_feat: 6.581645488739014 (6.581645488739014) + | > loss_mel: 17.56856918334961 (17.56856918334961) + | > loss_duration: 1.6764304637908936 (1.6764304637908936) + | > loss_1: 30.694164276123047 (30.694164276123047) + + --> STEP: 1 + | > loss_disc: 2.527118682861328 (2.527118682861328) + | > loss_disc_real_0: 0.15944933891296387 (0.15944933891296387) + | > loss_disc_real_1: 0.2440745085477829 (0.2440745085477829) + | > loss_disc_real_2: 0.22879190742969513 (0.22879190742969513) + | > loss_disc_real_3: 0.2284107804298401 (0.2284107804298401) + | > loss_disc_real_4: 0.2636623978614807 (0.2636623978614807) + | > loss_disc_real_5: 0.26725783944129944 (0.26725783944129944) + | > loss_0: 2.527118682861328 (2.527118682861328) + | > loss_gen: 2.4010772705078125 (2.4010772705078125) + | > loss_kl: 2.6355807781219482 (2.6355807781219482) + | > loss_feat: 6.6183648109436035 (6.6183648109436035) + | > loss_mel: 18.48004150390625 (18.48004150390625) + | > loss_duration: 1.7010201215744019 (1.7010201215744019) + | > loss_1: 31.836082458496094 (31.836082458496094) + + --> STEP: 2 + | > loss_disc: 2.4865598678588867 (2.5068392753601074) + | > loss_disc_real_0: 0.24912306666374207 (0.20428620278835297) + | > loss_disc_real_1: 0.23324139416217804 (0.23865795135498047) + | > loss_disc_real_2: 0.21175651252269745 (0.2202742099761963) + | > loss_disc_real_3: 0.22166083753108978 (0.22503580898046494) + | > loss_disc_real_4: 0.2451104372739792 (0.25438641756772995) + | > loss_disc_real_5: 0.23562198877334595 (0.2514399141073227) + | > loss_0: 2.4865598678588867 (2.5068392753601074) + | > loss_gen: 2.5890164375305176 (2.495046854019165) + | > loss_kl: 2.3770809173583984 (2.5063308477401733) + | > loss_feat: 7.3325300216674805 (6.975447416305542) + | > loss_mel: 17.466487884521484 (17.973264694213867) + | > loss_duration: 1.662294626235962 (1.6816573739051819) + | > loss_1: 31.42740821838379 (31.63174533843994) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003517627716064453 (-9.953975677490234e-05) + | > avg_loss_disc: 2.5068392753601074 (+0.027108430862426758) + | > avg_loss_disc_real_0: 0.20428620278835297 (+0.10152053087949753) + | > avg_loss_disc_real_1: 0.23865795135498047 (+0.015729188919067383) + | > avg_loss_disc_real_2: 0.2202742099761963 (+0.0128588005900383) + | > avg_loss_disc_real_3: 0.22503580898046494 (+0.006397940218448639) + | > avg_loss_disc_real_4: 0.25438641756772995 (+0.018023379147052765) + | > avg_loss_disc_real_5: 0.2514399141073227 (+0.0124281644821167) + | > avg_loss_0: 2.5068392753601074 (+0.027108430862426758) + | > avg_loss_gen: 2.495046854019165 (+0.2873115539550781) + | > avg_loss_kl: 2.5063308477401733 (+0.07006645202636719) + | > avg_loss_feat: 6.975447416305542 (-0.23198676109313965) + | > avg_loss_mel: 17.973264694213867 (-0.4337921142578125) + | > avg_loss_duration: 1.6816573739051819 (-0.019905805587768555) + | > avg_loss_1: 31.63174533843994 (-0.3283090591430664) + + + > EPOCH: 219/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:35:48)  + + --> TIME: 2025-04-19 02:35:56 -- STEP: 20/142 -- GLOBAL_STEP: 410400 + | > loss_disc: 2.3750054836273193 (2.4203128576278683) + | > loss_disc_real_0: 0.10650495439767838 (0.11983391977846622) + | > loss_disc_real_1: 0.2078513205051422 (0.20272488370537758) + | > loss_disc_real_2: 0.25030431151390076 (0.22347289994359015) + | > loss_disc_real_3: 0.25059667229652405 (0.22806113064289094) + | > loss_disc_real_4: 0.2235061526298523 (0.22579051032662392) + | > loss_disc_real_5: 0.23484621942043304 (0.2269828975200653) + | > loss_0: 2.3750054836273193 (2.4203128576278683) + | > grad_norm_0: tensor(8.2951, device='cuda:0') (tensor(8.0653, device='cuda:0')) + | > loss_gen: 2.3994300365448 (2.3568177938461305) + | > loss_kl: 1.8904742002487183 (1.8572184085845946) + | > loss_feat: 6.419857978820801 (7.118481826782227) + | > loss_mel: 16.042142868041992 (17.355669260025024) + | > loss_duration: 1.4848015308380127 (1.4327083289623261) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.236705780029297 (30.120895767211913) + | > grad_norm_1: tensor(281.0264, device='cuda:0') (tensor(209.3033, device='cuda:0')) + | > current_lr_0: 0.00019459892692329838 + | > current_lr_1: 0.00019459892692329838 + | > step_time: 0.3456 (0.3422015905380249) + | > loader_time: 0.0056 (0.00474081039428711) + + + --> TIME: 2025-04-19 02:36:05 -- STEP: 45/142 -- GLOBAL_STEP: 410425 + | > loss_disc: 2.4131240844726562 (2.4478948593139647) + | > loss_disc_real_0: 0.09306921809911728 (0.12344181703196631) + | > loss_disc_real_1: 0.20699520409107208 (0.20693508320384554) + | > loss_disc_real_2: 0.22109383344650269 (0.22623499002721575) + | > loss_disc_real_3: 0.22743818163871765 (0.22884530656867558) + | > loss_disc_real_4: 0.2237139493227005 (0.22810282210508984) + | > loss_disc_real_5: 0.20086175203323364 (0.22876435617605845) + | > loss_0: 2.4131240844726562 (2.4478948593139647) + | > grad_norm_0: tensor(13.5501, device='cuda:0') (tensor(7.9172, device='cuda:0')) + | > loss_gen: 2.2367801666259766 (2.3656409740448003) + | > loss_kl: 1.8915824890136719 (1.8667244593302408) + | > loss_feat: 7.063828945159912 (7.2098836792839895) + | > loss_mel: 18.137741088867188 (17.770443322923448) + | > loss_duration: 1.4639692306518555 (1.4426562574174668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.793903350830078 (30.655348841349284) + | > grad_norm_1: tensor(357.3956, device='cuda:0') (tensor(176.7084, device='cuda:0')) + | > current_lr_0: 0.00019459892692329838 + | > current_lr_1: 0.00019459892692329838 + | > step_time: 0.3685 (0.35032288763258196) + | > loader_time: 0.0058 (0.005228148566351997) + + + --> TIME: 2025-04-19 02:36:15 -- STEP: 70/142 -- GLOBAL_STEP: 410450 + | > loss_disc: 2.5196495056152344 (2.4535486016954686) + | > loss_disc_real_0: 0.1542925387620926 (0.1269268420125757) + | > loss_disc_real_1: 0.21174760162830353 (0.20819353312253952) + | > loss_disc_real_2: 0.26643720269203186 (0.22575415372848512) + | > loss_disc_real_3: 0.25812476873397827 (0.22883593014308384) + | > loss_disc_real_4: 0.2824106216430664 (0.2287089969430651) + | > loss_disc_real_5: 0.2941100597381592 (0.22866559645959308) + | > loss_0: 2.5196495056152344 (2.4535486016954686) + | > grad_norm_0: tensor(9.4386, device='cuda:0') (tensor(9.4716, device='cuda:0')) + | > loss_gen: 2.3852627277374268 (2.3650312628064847) + | > loss_kl: 1.8562088012695312 (1.8718819260597228) + | > loss_feat: 7.364962100982666 (7.195433698381696) + | > loss_mel: 17.7852840423584 (17.862955052512024) + | > loss_duration: 1.4756970405578613 (1.4529243843896047) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.867414474487305 (30.748226383754186) + | > grad_norm_1: tensor(249.8627, device='cuda:0') (tensor(208.3427, device='cuda:0')) + | > current_lr_0: 0.00019459892692329838 + | > current_lr_1: 0.00019459892692329838 + | > step_time: 0.3879 (0.3599420649664742) + | > loader_time: 0.0061 (0.005541208812168666) + + + --> TIME: 2025-04-19 02:36:25 -- STEP: 95/142 -- GLOBAL_STEP: 410475 + | > loss_disc: 2.3517627716064453 (2.4535596195020166) + | > loss_disc_real_0: 0.09914302080869675 (0.12618517577648158) + | > loss_disc_real_1: 0.183040589094162 (0.20804183279213154) + | > loss_disc_real_2: 0.20529747009277344 (0.22507301882693642) + | > loss_disc_real_3: 0.21766696870326996 (0.2269162074515694) + | > loss_disc_real_4: 0.2193775773048401 (0.22836497049582632) + | > loss_disc_real_5: 0.23669295012950897 (0.22872454806378012) + | > loss_0: 2.3517627716064453 (2.4535596195020166) + | > grad_norm_0: tensor(4.6266, device='cuda:0') (tensor(9.4592, device='cuda:0')) + | > loss_gen: 2.2798523902893066 (2.3524797665445427) + | > loss_kl: 1.898439884185791 (1.881245615607814) + | > loss_feat: 7.245157241821289 (7.114427907843339) + | > loss_mel: 18.856191635131836 (17.846401134290183) + | > loss_duration: 1.4986956119537354 (1.4649003217094823) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.778335571289062 (30.659454807482266) + | > grad_norm_1: tensor(33.5856, device='cuda:0') (tensor(204.7219, device='cuda:0')) + | > current_lr_0: 0.00019459892692329838 + | > current_lr_1: 0.00019459892692329838 + | > step_time: 0.4046 (0.3699805284801281) + | > loader_time: 0.007 (0.005821132659912109) + + + --> TIME: 2025-04-19 02:36:35 -- STEP: 120/142 -- GLOBAL_STEP: 410500 + | > loss_disc: 2.5257842540740967 (2.452969801425934) + | > loss_disc_real_0: 0.14967773854732513 (0.12707482681920126) + | > loss_disc_real_1: 0.21224914491176605 (0.2073257241398096) + | > loss_disc_real_2: 0.19582469761371613 (0.22450201051930588) + | > loss_disc_real_3: 0.19638200104236603 (0.22686130131284396) + | > loss_disc_real_4: 0.20218133926391602 (0.22777030418316524) + | > loss_disc_real_5: 0.24105222523212433 (0.22972889207303523) + | > loss_0: 2.5257842540740967 (2.452969801425934) + | > grad_norm_0: tensor(21.2029, device='cuda:0') (tensor(9.3340, device='cuda:0')) + | > loss_gen: 2.145582914352417 (2.3469826380411787) + | > loss_kl: 1.9854804277420044 (1.8902894308169682) + | > loss_feat: 6.180778503417969 (7.066481717427572) + | > loss_mel: 16.789772033691406 (17.762153132756545) + | > loss_duration: 1.5074902772903442 (1.4723622093598048) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.60910415649414 (30.538269170125325) + | > grad_norm_1: tensor(307.1460, device='cuda:0') (tensor(200.1789, device='cuda:0')) + | > current_lr_0: 0.00019459892692329838 + | > current_lr_1: 0.00019459892692329838 + | > step_time: 0.4361 (0.380670791864395) + | > loader_time: 0.0086 (0.006114387512207031) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3870081901550293 (2.3870081901550293) + | > loss_disc_real_0: 0.18599967658519745 (0.18599967658519745) + | > loss_disc_real_1: 0.18384593725204468 (0.18384593725204468) + | > loss_disc_real_2: 0.25839152932167053 (0.25839152932167053) + | > loss_disc_real_3: 0.24493013322353363 (0.24493013322353363) + | > loss_disc_real_4: 0.21152032911777496 (0.21152032911777496) + | > loss_disc_real_5: 0.2412024289369583 (0.2412024289369583) + | > loss_0: 2.3870081901550293 (2.3870081901550293) + | > loss_gen: 2.553636312484741 (2.553636312484741) + | > loss_kl: 2.445521354675293 (2.445521354675293) + | > loss_feat: 7.561342239379883 (7.561342239379883) + | > loss_mel: 19.64495849609375 (19.64495849609375) + | > loss_duration: 1.6811860799789429 (1.6811860799789429) + | > loss_1: 33.88664245605469 (33.88664245605469) + + --> STEP: 1 + | > loss_disc: 2.697500467300415 (2.697500467300415) + | > loss_disc_real_0: 0.26621025800704956 (0.26621025800704956) + | > loss_disc_real_1: 0.21945108473300934 (0.21945108473300934) + | > loss_disc_real_2: 0.29946261644363403 (0.29946261644363403) + | > loss_disc_real_3: 0.26880529522895813 (0.26880529522895813) + | > loss_disc_real_4: 0.26992303133010864 (0.26992303133010864) + | > loss_disc_real_5: 0.2536059617996216 (0.2536059617996216) + | > loss_0: 2.697500467300415 (2.697500467300415) + | > loss_gen: 2.522702932357788 (2.522702932357788) + | > loss_kl: 2.5752577781677246 (2.5752577781677246) + | > loss_feat: 7.311907768249512 (7.311907768249512) + | > loss_mel: 18.933679580688477 (18.933679580688477) + | > loss_duration: 1.7096939086914062 (1.7096939086914062) + | > loss_1: 33.05324172973633 (33.05324172973633) + + --> STEP: 2 + | > loss_disc: 2.470297336578369 (2.583898901939392) + | > loss_disc_real_0: 0.16147035360336304 (0.2138403058052063) + | > loss_disc_real_1: 0.23165862262248993 (0.22555485367774963) + | > loss_disc_real_2: 0.26805078983306885 (0.28375670313835144) + | > loss_disc_real_3: 0.2557600438594818 (0.26228266954421997) + | > loss_disc_real_4: 0.22726057469844818 (0.2485918030142784) + | > loss_disc_real_5: 0.26587557792663574 (0.25974076986312866) + | > loss_0: 2.470297336578369 (2.583898901939392) + | > loss_gen: 2.5077784061431885 (2.5152406692504883) + | > loss_kl: 2.4470672607421875 (2.511162519454956) + | > loss_feat: 7.191281795501709 (7.25159478187561) + | > loss_mel: 18.5129451751709 (18.723312377929688) + | > loss_duration: 1.6757229566574097 (1.692708432674408) + | > loss_1: 32.33479690551758 (32.69401931762695) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004036307334899902 (+0.0005186796188354492) + | > avg_loss_disc: 2.583898901939392 (+0.07705962657928467) + | > avg_loss_disc_real_0: 0.2138403058052063 (+0.009554103016853333) + | > avg_loss_disc_real_1: 0.22555485367774963 (-0.013103097677230835) + | > avg_loss_disc_real_2: 0.28375670313835144 (+0.06348249316215515) + | > avg_loss_disc_real_3: 0.26228266954421997 (+0.037246860563755035) + | > avg_loss_disc_real_4: 0.2485918030142784 (-0.005794614553451538) + | > avg_loss_disc_real_5: 0.25974076986312866 (+0.00830085575580597) + | > avg_loss_0: 2.583898901939392 (+0.07705962657928467) + | > avg_loss_gen: 2.5152406692504883 (+0.020193815231323242) + | > avg_loss_kl: 2.511162519454956 (+0.004831671714782715) + | > avg_loss_feat: 7.25159478187561 (+0.27614736557006836) + | > avg_loss_mel: 18.723312377929688 (+0.7500476837158203) + | > avg_loss_duration: 1.692708432674408 (+0.011051058769226074) + | > avg_loss_1: 32.69401931762695 (+1.0622739791870117) + + + > EPOCH: 220/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:36:47)  + + --> TIME: 2025-04-19 02:36:49 -- STEP: 3/142 -- GLOBAL_STEP: 410525 + | > loss_disc: 2.4308440685272217 (2.3901309967041016) + | > loss_disc_real_0: 0.1002541333436966 (0.10764565815528233) + | > loss_disc_real_1: 0.19910019636154175 (0.20510505139827728) + | > loss_disc_real_2: 0.21636036038398743 (0.23304385443528494) + | > loss_disc_real_3: 0.23421259224414825 (0.2283256302277247) + | > loss_disc_real_4: 0.22345198690891266 (0.20591236154238382) + | > loss_disc_real_5: 0.25540071725845337 (0.22711492081483206) + | > loss_0: 2.4308440685272217 (2.3901309967041016) + | > grad_norm_0: tensor(8.8409, device='cuda:0') (tensor(10.9769, device='cuda:0')) + | > loss_gen: 2.431499719619751 (2.3479129473368325) + | > loss_kl: 1.7874536514282227 (1.8727615276972454) + | > loss_feat: 7.3297858238220215 (7.480002403259277) + | > loss_mel: 18.527841567993164 (18.002273559570312) + | > loss_duration: 1.4370980262756348 (1.4387411673863728) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.5136775970459 (31.14169184366862) + | > grad_norm_1: tensor(215.0466, device='cuda:0') (tensor(222.3348, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.3335 (0.33124740918477374) + | > loader_time: 0.0048 (0.004196723302205403) + + + --> TIME: 2025-04-19 02:36:57 -- STEP: 28/142 -- GLOBAL_STEP: 410550 + | > loss_disc: 2.3688859939575195 (2.4312420402254378) + | > loss_disc_real_0: 0.14736732840538025 (0.11425591366631645) + | > loss_disc_real_1: 0.17846617102622986 (0.2031229989869254) + | > loss_disc_real_2: 0.24122314155101776 (0.22224383375474385) + | > loss_disc_real_3: 0.22695213556289673 (0.22705908492207527) + | > loss_disc_real_4: 0.24300070106983185 (0.2305977897984641) + | > loss_disc_real_5: 0.2352007031440735 (0.22992292046546936) + | > loss_0: 2.3688859939575195 (2.4312420402254378) + | > grad_norm_0: tensor(9.0006, device='cuda:0') (tensor(9.9454, device='cuda:0')) + | > loss_gen: 2.5323269367218018 (2.3878680723054067) + | > loss_kl: 1.955505132675171 (1.8751991731779916) + | > loss_feat: 6.524513244628906 (7.284716520990644) + | > loss_mel: 17.336170196533203 (17.83358478546143) + | > loss_duration: 1.488895058631897 (1.4378309420176916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.837411880493164 (30.81919949395316) + | > grad_norm_1: tensor(193.1621, device='cuda:0') (tensor(264.0468, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.3458 (0.3366234898567201) + | > loader_time: 0.0054 (0.004959149020058768) + + + --> TIME: 2025-04-19 02:37:06 -- STEP: 53/142 -- GLOBAL_STEP: 410575 + | > loss_disc: 2.5131492614746094 (2.4347946868752532) + | > loss_disc_real_0: 0.160828098654747 (0.11886744144952523) + | > loss_disc_real_1: 0.20146213471889496 (0.20370091469782703) + | > loss_disc_real_2: 0.24056415259838104 (0.22426754095644322) + | > loss_disc_real_3: 0.22731395065784454 (0.22773492083234606) + | > loss_disc_real_4: 0.24031507968902588 (0.23059408338564746) + | > loss_disc_real_5: 0.2064533829689026 (0.2287537635497327) + | > loss_0: 2.5131492614746094 (2.4347946868752532) + | > grad_norm_0: tensor(7.4758, device='cuda:0') (tensor(8.6995, device='cuda:0')) + | > loss_gen: 2.2338855266571045 (2.3647875740843) + | > loss_kl: 1.8898797035217285 (1.8717236788767688) + | > loss_feat: 7.760616779327393 (7.184648486803162) + | > loss_mel: 17.45284652709961 (17.74598621872236) + | > loss_duration: 1.4353364706039429 (1.4459361917567704) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.772565841674805 (30.613082058024855) + | > grad_norm_1: tensor(115.0533, device='cuda:0') (tensor(210.4646, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.368 (0.3476769204409617) + | > loader_time: 0.0058 (0.0053099371352285695) + + + --> TIME: 2025-04-19 02:37:16 -- STEP: 78/142 -- GLOBAL_STEP: 410600 + | > loss_disc: 2.4789350032806396 (2.4434487391740847) + | > loss_disc_real_0: 0.10796783119440079 (0.12170679800403424) + | > loss_disc_real_1: 0.20130325853824615 (0.2041538090277941) + | > loss_disc_real_2: 0.20808018743991852 (0.22539429041819695) + | > loss_disc_real_3: 0.2472119927406311 (0.22823551755685073) + | > loss_disc_real_4: 0.21464885771274567 (0.229529123657789) + | > loss_disc_real_5: 0.2542705535888672 (0.22896302090241358) + | > loss_0: 2.4789350032806396 (2.4434487391740847) + | > grad_norm_0: tensor(5.9451, device='cuda:0') (tensor(8.7571, device='cuda:0')) + | > loss_gen: 2.339759349822998 (2.3714802448566146) + | > loss_kl: 2.0391016006469727 (1.8765944273043902) + | > loss_feat: 7.138174533843994 (7.175071465663421) + | > loss_mel: 18.01473617553711 (17.854038935441242) + | > loss_duration: 1.5026075839996338 (1.457611791598491) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.034379959106445 (30.734796817486103) + | > grad_norm_1: tensor(170.8361, device='cuda:0') (tensor(191.8830, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.3922 (0.35967167524191046) + | > loader_time: 0.0063 (0.00560773335970365) + + + --> TIME: 2025-04-19 02:37:27 -- STEP: 103/142 -- GLOBAL_STEP: 410625 + | > loss_disc: 2.4222307205200195 (2.462627510422642) + | > loss_disc_real_0: 0.09400245547294617 (0.126544595705074) + | > loss_disc_real_1: 0.18514776229858398 (0.2060172980271497) + | > loss_disc_real_2: 0.22779737412929535 (0.22649018104793955) + | > loss_disc_real_3: 0.2183007448911667 (0.22965047920791848) + | > loss_disc_real_4: 0.23080214858055115 (0.23018540239449845) + | > loss_disc_real_5: 0.22463683784008026 (0.22995071662861166) + | > loss_0: 2.4222307205200195 (2.462627510422642) + | > grad_norm_0: tensor(9.3849, device='cuda:0') (tensor(8.7073, device='cuda:0')) + | > loss_gen: 2.1726481914520264 (2.3474209609540924) + | > loss_kl: 1.9659123420715332 (1.8911601617498306) + | > loss_feat: 7.046856880187988 (7.055464480687114) + | > loss_mel: 17.991640090942383 (17.758017965890836) + | > loss_duration: 1.5254671573638916 (1.4685404809933265) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.702524185180664 (30.52060388361366) + | > grad_norm_1: tensor(190.0983, device='cuda:0') (tensor(190.7839, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.4108 (0.3700920216088156) + | > loader_time: 0.0069 (0.005861532340929346) + + + --> TIME: 2025-04-19 02:37:38 -- STEP: 128/142 -- GLOBAL_STEP: 410650 + | > loss_disc: 2.600205898284912 (2.468581721186638) + | > loss_disc_real_0: 0.17868398129940033 (0.12701589317293832) + | > loss_disc_real_1: 0.24889829754829407 (0.2066884197993204) + | > loss_disc_real_2: 0.26301342248916626 (0.22738247783854604) + | > loss_disc_real_3: 0.27210110425949097 (0.22996371856424958) + | > loss_disc_real_4: 0.2176891416311264 (0.23101592750754207) + | > loss_disc_real_5: 0.25161871314048767 (0.23046494962181896) + | > loss_0: 2.600205898284912 (2.468581721186638) + | > grad_norm_0: tensor(15.1495, device='cuda:0') (tensor(9.4899, device='cuda:0')) + | > loss_gen: 2.309910774230957 (2.347913989797235) + | > loss_kl: 1.8358030319213867 (1.8950943099334836) + | > loss_feat: 6.1684699058532715 (7.040210630744696) + | > loss_mel: 17.521520614624023 (17.711115397512916) + | > loss_duration: 1.4966108798980713 (1.4746668254956603) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.33231544494629 (30.46900101006031) + | > grad_norm_1: tensor(225.3855, device='cuda:0') (tensor(208.4276, device='cuda:0')) + | > current_lr_0: 0.00019457460205743297 + | > current_lr_1: 0.00019457460205743297 + | > step_time: 0.4363 (0.3827276583760977) + | > loader_time: 0.0076 (0.006167922168970108) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.509352207183838 (2.509352207183838) + | > loss_disc_real_0: 0.09569523483514786 (0.09569523483514786) + | > loss_disc_real_1: 0.3061310648918152 (0.3061310648918152) + | > loss_disc_real_2: 0.2703113257884979 (0.2703113257884979) + | > loss_disc_real_3: 0.2555268704891205 (0.2555268704891205) + | > loss_disc_real_4: 0.2598305344581604 (0.2598305344581604) + | > loss_disc_real_5: 0.21209116280078888 (0.21209116280078888) + | > loss_0: 2.509352207183838 (2.509352207183838) + | > loss_gen: 2.3417723178863525 (2.3417723178863525) + | > loss_kl: 2.266204833984375 (2.266204833984375) + | > loss_feat: 6.087337493896484 (6.087337493896484) + | > loss_mel: 16.631206512451172 (16.631206512451172) + | > loss_duration: 1.6953468322753906 (1.6953468322753906) + | > loss_1: 29.021867752075195 (29.021867752075195) + + --> STEP: 1 + | > loss_disc: 2.5358316898345947 (2.5358316898345947) + | > loss_disc_real_0: 0.09028464555740356 (0.09028464555740356) + | > loss_disc_real_1: 0.2678530514240265 (0.2678530514240265) + | > loss_disc_real_2: 0.2735649645328522 (0.2735649645328522) + | > loss_disc_real_3: 0.23978452384471893 (0.23978452384471893) + | > loss_disc_real_4: 0.2805890738964081 (0.2805890738964081) + | > loss_disc_real_5: 0.24002686142921448 (0.24002686142921448) + | > loss_0: 2.5358316898345947 (2.5358316898345947) + | > loss_gen: 2.3157474994659424 (2.3157474994659424) + | > loss_kl: 2.6689565181732178 (2.6689565181732178) + | > loss_feat: 7.106986999511719 (7.106986999511719) + | > loss_mel: 18.800153732299805 (18.800153732299805) + | > loss_duration: 1.7110278606414795 (1.7110278606414795) + | > loss_1: 32.602874755859375 (32.602874755859375) + + --> STEP: 2 + | > loss_disc: 2.408601760864258 (2.4722167253494263) + | > loss_disc_real_0: 0.09141784906387329 (0.09085124731063843) + | > loss_disc_real_1: 0.2507825493812561 (0.2593178004026413) + | > loss_disc_real_2: 0.2536230683326721 (0.26359401643276215) + | > loss_disc_real_3: 0.2171514481306076 (0.22846798598766327) + | > loss_disc_real_4: 0.22473040223121643 (0.25265973806381226) + | > loss_disc_real_5: 0.21202336251735687 (0.22602511197328568) + | > loss_0: 2.408601760864258 (2.4722167253494263) + | > loss_gen: 2.354091167449951 (2.3349193334579468) + | > loss_kl: 2.1305603981018066 (2.399758458137512) + | > loss_feat: 7.985709190368652 (7.5463480949401855) + | > loss_mel: 18.198984146118164 (18.499568939208984) + | > loss_duration: 1.6717915534973145 (1.691409707069397) + | > loss_1: 32.34113693237305 (32.47200584411621) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035265684127807617 (-0.0005097389221191406) + | > avg_loss_disc: 2.4722167253494263 (-0.11168217658996582) + | > avg_loss_disc_real_0: 0.09085124731063843 (-0.12298905849456787) + | > avg_loss_disc_real_1: 0.2593178004026413 (+0.03376294672489166) + | > avg_loss_disc_real_2: 0.26359401643276215 (-0.020162686705589294) + | > avg_loss_disc_real_3: 0.22846798598766327 (-0.0338146835565567) + | > avg_loss_disc_real_4: 0.25265973806381226 (+0.004067935049533844) + | > avg_loss_disc_real_5: 0.22602511197328568 (-0.03371565788984299) + | > avg_loss_0: 2.4722167253494263 (-0.11168217658996582) + | > avg_loss_gen: 2.3349193334579468 (-0.1803213357925415) + | > avg_loss_kl: 2.399758458137512 (-0.11140406131744385) + | > avg_loss_feat: 7.5463480949401855 (+0.2947533130645752) + | > avg_loss_mel: 18.499568939208984 (-0.22374343872070312) + | > avg_loss_duration: 1.691409707069397 (-0.0012987256050109863) + | > avg_loss_1: 32.47200584411621 (-0.2220134735107422) + + + > EPOCH: 221/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:37:46)  + + --> TIME: 2025-04-19 02:37:50 -- STEP: 11/142 -- GLOBAL_STEP: 410675 + | > loss_disc: 2.386430025100708 (2.4881053187630395) + | > loss_disc_real_0: 0.13429491221904755 (0.1299294585531408) + | > loss_disc_real_1: 0.20401905477046967 (0.21624786474487998) + | > loss_disc_real_2: 0.23037727177143097 (0.23050537028095938) + | > loss_disc_real_3: 0.239949569106102 (0.2313309068029577) + | > loss_disc_real_4: 0.25456735491752625 (0.23522719334472308) + | > loss_disc_real_5: 0.23906539380550385 (0.23480614748868076) + | > loss_0: 2.386430025100708 (2.4881053187630395) + | > grad_norm_0: tensor(17.8990, device='cuda:0') (tensor(17.9787, device='cuda:0')) + | > loss_gen: 2.3759002685546875 (2.3995100368152964) + | > loss_kl: 1.8416849374771118 (1.8091662471944636) + | > loss_feat: 7.326299667358398 (7.2144832177595655) + | > loss_mel: 17.919132232666016 (17.851910851218484) + | > loss_duration: 1.4182896614074707 (1.4230536330829968) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.881305694580078 (30.698124278675426) + | > grad_norm_1: tensor(430.3949, device='cuda:0') (tensor(323.3196, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.3291 (0.3273892185904763) + | > loader_time: 0.005 (0.004625775597312234) + + + --> TIME: 2025-04-19 02:37:59 -- STEP: 36/142 -- GLOBAL_STEP: 410700 + | > loss_disc: 2.4007976055145264 (2.440210097365909) + | > loss_disc_real_0: 0.094479501247406 (0.11713123155964746) + | > loss_disc_real_1: 0.20385682582855225 (0.2070246727930175) + | > loss_disc_real_2: 0.23426295816898346 (0.22566007119086054) + | > loss_disc_real_3: 0.2625437080860138 (0.22987060704165035) + | > loss_disc_real_4: 0.20794907212257385 (0.22909239307045937) + | > loss_disc_real_5: 0.2381429523229599 (0.22809719501270187) + | > loss_0: 2.4007976055145264 (2.440210097365909) + | > grad_norm_0: tensor(5.1014, device='cuda:0') (tensor(12.4140, device='cuda:0')) + | > loss_gen: 2.408965587615967 (2.39547500345442) + | > loss_kl: 1.934093713760376 (1.8413620856073167) + | > loss_feat: 7.634005546569824 (7.28518173429701) + | > loss_mel: 17.866920471191406 (17.957802878485783) + | > loss_duration: 1.4443044662475586 (1.4390565620528326) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.288291931152344 (30.918878343370224) + | > grad_norm_1: tensor(54.4837, device='cuda:0') (tensor(260.9998, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.3538 (0.3398231400383843) + | > loader_time: 0.0056 (0.005167808797624376) + + + --> TIME: 2025-04-19 02:38:08 -- STEP: 61/142 -- GLOBAL_STEP: 410725 + | > loss_disc: 2.5358169078826904 (2.4460494283769947) + | > loss_disc_real_0: 0.12904243171215057 (0.11950958410247428) + | > loss_disc_real_1: 0.27855220437049866 (0.20910339057445526) + | > loss_disc_real_2: 0.22765348851680756 (0.2283442289125724) + | > loss_disc_real_3: 0.25509461760520935 (0.22989565402757925) + | > loss_disc_real_4: 0.23521359264850616 (0.2295157740839192) + | > loss_disc_real_5: 0.24146492779254913 (0.2293896648238917) + | > loss_0: 2.5358169078826904 (2.4460494283769947) + | > grad_norm_0: tensor(7.3585, device='cuda:0') (tensor(11.1847, device='cuda:0')) + | > loss_gen: 2.217271566390991 (2.382443471033065) + | > loss_kl: 1.9098197221755981 (1.8481208359608885) + | > loss_feat: 6.196746826171875 (7.226383615712651) + | > loss_mel: 17.031787872314453 (17.939833156398077) + | > loss_duration: 1.4588937759399414 (1.4506632304582439) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.81452178955078 (30.847444409229716) + | > grad_norm_1: tensor(226.4017, device='cuda:0') (tensor(249.3342, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.3813 (0.3512609473994521) + | > loader_time: 0.0063 (0.005397077466620773) + + + --> TIME: 2025-04-19 02:38:18 -- STEP: 86/142 -- GLOBAL_STEP: 410750 + | > loss_disc: 2.3962202072143555 (2.4451445756956587) + | > loss_disc_real_0: 0.09294616430997849 (0.12081324534360753) + | > loss_disc_real_1: 0.18337911367416382 (0.20692443934290908) + | > loss_disc_real_2: 0.23371924459934235 (0.22778413808623024) + | > loss_disc_real_3: 0.22775931656360626 (0.22894322525623234) + | > loss_disc_real_4: 0.2514534592628479 (0.22872180214454962) + | > loss_disc_real_5: 0.18147681653499603 (0.2273757903035297) + | > loss_0: 2.3962202072143555 (2.4451445756956587) + | > grad_norm_0: tensor(5.6212, device='cuda:0') (tensor(11.0450, device='cuda:0')) + | > loss_gen: 2.463719606399536 (2.3688821459925444) + | > loss_kl: 1.9414139986038208 (1.8646259751430778) + | > loss_feat: 7.9499053955078125 (7.15779924947162) + | > loss_mel: 18.4691219329834 (17.775156231813654) + | > loss_duration: 1.5194036960601807 (1.462532979111339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.34356689453125 (30.628996627275335) + | > grad_norm_1: tensor(173.8273, device='cuda:0') (tensor(252.0119, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.3974 (0.362987809402998) + | > loader_time: 0.0067 (0.0056971893754116315) + + + --> TIME: 2025-04-19 02:38:29 -- STEP: 111/142 -- GLOBAL_STEP: 410775 + | > loss_disc: 2.3974223136901855 (2.4508913315094265) + | > loss_disc_real_0: 0.17723165452480316 (0.12410858052002417) + | > loss_disc_real_1: 0.18088790774345398 (0.2083303398108697) + | > loss_disc_real_2: 0.22056159377098083 (0.22708734628316518) + | > loss_disc_real_3: 0.23773866891860962 (0.22896494065319095) + | > loss_disc_real_4: 0.2776522934436798 (0.22965035567412506) + | > loss_disc_real_5: 0.24219682812690735 (0.2279307922801456) + | > loss_0: 2.3974223136901855 (2.4508913315094265) + | > grad_norm_0: tensor(17.0577, device='cuda:0') (tensor(11.3383, device='cuda:0')) + | > loss_gen: 2.415585517883301 (2.3618780471183163) + | > loss_kl: 1.8987895250320435 (1.8772314496942468) + | > loss_feat: 7.095417022705078 (7.101552976144327) + | > loss_mel: 18.35042953491211 (17.786543932046975) + | > loss_duration: 1.5154508352279663 (1.4709173193922993) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.275671005249023 (30.598123670698286) + | > grad_norm_1: tensor(425.8926, device='cuda:0') (tensor(251.5859, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.4248 (0.3735549922461982) + | > loader_time: 0.008 (0.005949778599782033) + + + --> TIME: 2025-04-19 02:38:40 -- STEP: 136/142 -- GLOBAL_STEP: 410800 + | > loss_disc: 2.459022045135498 (2.451175212860107) + | > loss_disc_real_0: 0.1290816366672516 (0.12432539265821962) + | > loss_disc_real_1: 0.21184849739074707 (0.20762570673490272) + | > loss_disc_real_2: 0.21804559230804443 (0.22624900553594618) + | > loss_disc_real_3: 0.21787719428539276 (0.22856525353649082) + | > loss_disc_real_4: 0.2258899211883545 (0.23084278492366567) + | > loss_disc_real_5: 0.22673311829566956 (0.22785351633587303) + | > loss_0: 2.459022045135498 (2.451175212860107) + | > grad_norm_0: tensor(8.6590, device='cuda:0') (tensor(11.1560, device='cuda:0')) + | > loss_gen: 2.3055834770202637 (2.3612193465232854) + | > loss_kl: 1.9647862911224365 (1.8833533902378643) + | > loss_feat: 7.009365081787109 (7.098438809899723) + | > loss_mel: 17.441171646118164 (17.775878527585206) + | > loss_duration: 1.5103683471679688 (1.4766896849169455) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.23127555847168 (30.595579708323758) + | > grad_norm_1: tensor(106.1684, device='cuda:0') (tensor(250.8565, device='cuda:0')) + | > current_lr_0: 0.00019455028023217577 + | > current_lr_1: 0.00019455028023217577 + | > step_time: 0.4511 (0.3854203732574687) + | > loader_time: 0.0078 (0.006204883841907277) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5188512802124023 (2.5188512802124023) + | > loss_disc_real_0: 0.17397074401378632 (0.17397074401378632) + | > loss_disc_real_1: 0.22933007776737213 (0.22933007776737213) + | > loss_disc_real_2: 0.20982739329338074 (0.20982739329338074) + | > loss_disc_real_3: 0.20727230608463287 (0.20727230608463287) + | > loss_disc_real_4: 0.24448806047439575 (0.24448806047439575) + | > loss_disc_real_5: 0.2290363609790802 (0.2290363609790802) + | > loss_0: 2.5188512802124023 (2.5188512802124023) + | > loss_gen: 2.164100408554077 (2.164100408554077) + | > loss_kl: 2.4796056747436523 (2.4796056747436523) + | > loss_feat: 6.346262454986572 (6.346262454986572) + | > loss_mel: 18.72978973388672 (18.72978973388672) + | > loss_duration: 1.6749956607818604 (1.6749956607818604) + | > loss_1: 31.39475440979004 (31.39475440979004) + + --> STEP: 1 + | > loss_disc: 2.534224271774292 (2.534224271774292) + | > loss_disc_real_0: 0.16460250318050385 (0.16460250318050385) + | > loss_disc_real_1: 0.22809134423732758 (0.22809134423732758) + | > loss_disc_real_2: 0.22306868433952332 (0.22306868433952332) + | > loss_disc_real_3: 0.25063037872314453 (0.25063037872314453) + | > loss_disc_real_4: 0.23994240164756775 (0.23994240164756775) + | > loss_disc_real_5: 0.250997394323349 (0.250997394323349) + | > loss_0: 2.534224271774292 (2.534224271774292) + | > loss_gen: 2.2634754180908203 (2.2634754180908203) + | > loss_kl: 2.6139261722564697 (2.6139261722564697) + | > loss_feat: 5.4268999099731445 (5.4268999099731445) + | > loss_mel: 16.74974250793457 (16.74974250793457) + | > loss_duration: 1.6936545372009277 (1.6936545372009277) + | > loss_1: 28.747699737548828 (28.747699737548828) + + --> STEP: 2 + | > loss_disc: 2.2940516471862793 (2.4141379594802856) + | > loss_disc_real_0: 0.12149270623922348 (0.14304760470986366) + | > loss_disc_real_1: 0.21670043468475342 (0.2223958894610405) + | > loss_disc_real_2: 0.2034311294555664 (0.21324990689754486) + | > loss_disc_real_3: 0.2045060694217682 (0.22756822407245636) + | > loss_disc_real_4: 0.20354866981506348 (0.2217455357313156) + | > loss_disc_real_5: 0.23777413368225098 (0.2443857640028) + | > loss_0: 2.2940516471862793 (2.4141379594802856) + | > loss_gen: 2.519674777984619 (2.3915750980377197) + | > loss_kl: 2.0593457221984863 (2.336635947227478) + | > loss_feat: 8.003422737121582 (6.715161323547363) + | > loss_mel: 19.645305633544922 (18.197524070739746) + | > loss_duration: 1.675248622894287 (1.6844515800476074) + | > loss_1: 33.90299606323242 (31.325347900390625) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036504268646240234 (+0.00012385845184326172) + | > avg_loss_disc: 2.4141379594802856 (-0.058078765869140625) + | > avg_loss_disc_real_0: 0.14304760470986366 (+0.052196357399225235) + | > avg_loss_disc_real_1: 0.2223958894610405 (-0.0369219109416008) + | > avg_loss_disc_real_2: 0.21324990689754486 (-0.050344109535217285) + | > avg_loss_disc_real_3: 0.22756822407245636 (-0.0008997619152069092) + | > avg_loss_disc_real_4: 0.2217455357313156 (-0.030914202332496643) + | > avg_loss_disc_real_5: 0.2443857640028 (+0.018360652029514313) + | > avg_loss_0: 2.4141379594802856 (-0.058078765869140625) + | > avg_loss_gen: 2.3915750980377197 (+0.05665576457977295) + | > avg_loss_kl: 2.336635947227478 (-0.06312251091003418) + | > avg_loss_feat: 6.715161323547363 (-0.8311867713928223) + | > avg_loss_mel: 18.197524070739746 (-0.3020448684692383) + | > avg_loss_duration: 1.6844515800476074 (-0.006958127021789551) + | > avg_loss_1: 31.325347900390625 (-1.146657943725586) + + + > EPOCH: 222/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:38:44)  + + --> TIME: 2025-04-19 02:38:51 -- STEP: 19/142 -- GLOBAL_STEP: 410825 + | > loss_disc: 2.509788751602173 (2.4485494463067305) + | > loss_disc_real_0: 0.17054413259029388 (0.12693729212409574) + | > loss_disc_real_1: 0.2078186422586441 (0.2083781338051746) + | > loss_disc_real_2: 0.21168406307697296 (0.22955910077220515) + | > loss_disc_real_3: 0.22271224856376648 (0.22473711716501335) + | > loss_disc_real_4: 0.21842093765735626 (0.22886422747059873) + | > loss_disc_real_5: 0.19743405282497406 (0.22392797313238444) + | > loss_0: 2.509788751602173 (2.4485494463067305) + | > grad_norm_0: tensor(12.8616, device='cuda:0') (tensor(10.4871, device='cuda:0')) + | > loss_gen: 2.2092232704162598 (2.3627953905808297) + | > loss_kl: 1.8949289321899414 (1.8011606304269088) + | > loss_feat: 7.3272929191589355 (7.1191837160210865) + | > loss_mel: 16.949630737304688 (17.85450774744937) + | > loss_duration: 1.4352368116378784 (1.424166020594145) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.81631088256836 (30.561813655652497) + | > grad_norm_1: tensor(341.4756, device='cuda:0') (tensor(221.2506, device='cuda:0')) + | > current_lr_0: 0.00019452596144714675 + | > current_lr_1: 0.00019452596144714675 + | > step_time: 0.3361 (0.3303401846634714) + | > loader_time: 0.0053 (0.005016427291067023) + + + --> TIME: 2025-04-19 02:39:01 -- STEP: 44/142 -- GLOBAL_STEP: 410850 + | > loss_disc: 2.4954452514648438 (2.443115494467995) + | > loss_disc_real_0: 0.11062786728143692 (0.12207843972878023) + | > loss_disc_real_1: 0.20206111669540405 (0.20791194554079662) + | > loss_disc_real_2: 0.2260759472846985 (0.22737678343599493) + | > loss_disc_real_3: 0.2472745180130005 (0.22588609328324144) + | > loss_disc_real_4: 0.2338574379682541 (0.2287544391371987) + | > loss_disc_real_5: 0.21850647032260895 (0.22750700028105217) + | > loss_0: 2.4954452514648438 (2.443115494467995) + | > grad_norm_0: tensor(6.9490, device='cuda:0') (tensor(9.2409, device='cuda:0')) + | > loss_gen: 2.3147096633911133 (2.3606330102140247) + | > loss_kl: 1.9351096153259277 (1.8186271461573513) + | > loss_feat: 6.722558975219727 (7.1403046954761855) + | > loss_mel: 18.10428237915039 (17.9041693427346) + | > loss_duration: 1.475337028503418 (1.4411200366236947) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.551998138427734 (30.664854179729115) + | > grad_norm_1: tensor(183.9704, device='cuda:0') (tensor(227.0119, device='cuda:0')) + | > current_lr_0: 0.00019452596144714675 + | > current_lr_1: 0.00019452596144714675 + | > step_time: 0.3774 (0.35237635265697126) + | > loader_time: 0.0059 (0.0053615732626481494) + + + --> TIME: 2025-04-19 02:39:10 -- STEP: 69/142 -- GLOBAL_STEP: 410875 + | > loss_disc: 2.36864972114563 (2.4462209508038946) + | > loss_disc_real_0: 0.10296058654785156 (0.12432449937298679) + | > loss_disc_real_1: 0.16247960925102234 (0.20683456179888352) + | > loss_disc_real_2: 0.2502251863479614 (0.22632271548112234) + | > loss_disc_real_3: 0.2489553689956665 (0.22747666269972705) + | > loss_disc_real_4: 0.2369871288537979 (0.22825355007164722) + | > loss_disc_real_5: 0.20820115506649017 (0.2274269409801649) + | > loss_0: 2.36864972114563 (2.4462209508038946) + | > grad_norm_0: tensor(6.7424, device='cuda:0') (tensor(9.9275, device='cuda:0')) + | > loss_gen: 2.4882571697235107 (2.364350007927936) + | > loss_kl: 1.887048602104187 (1.8471724054087764) + | > loss_feat: 6.860759258270264 (7.123830712359885) + | > loss_mel: 17.93602180480957 (17.91443114349808) + | > loss_duration: 1.5278828144073486 (1.4522858892661938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.699970245361328 (30.702070070349652) + | > grad_norm_1: tensor(48.0225, device='cuda:0') (tensor(227.4622, device='cuda:0')) + | > current_lr_0: 0.00019452596144714675 + | > current_lr_1: 0.00019452596144714675 + | > step_time: 0.3986 (0.3614212568255437) + | > loader_time: 0.0061 (0.005642103112262228) + + + --> TIME: 2025-04-19 02:39:20 -- STEP: 94/142 -- GLOBAL_STEP: 410900 + | > loss_disc: 2.331737518310547 (2.4512847180062147) + | > loss_disc_real_0: 0.11878234148025513 (0.12752686278458608) + | > loss_disc_real_1: 0.17791910469532013 (0.20607666037184127) + | > loss_disc_real_2: 0.2234463393688202 (0.22576057054895035) + | > loss_disc_real_3: 0.21756617724895477 (0.22775289876029847) + | > loss_disc_real_4: 0.20242588222026825 (0.22807625221445205) + | > loss_disc_real_5: 0.19623610377311707 (0.22868227847713105) + | > loss_0: 2.331737518310547 (2.4512847180062147) + | > grad_norm_0: tensor(13.5126, device='cuda:0') (tensor(10.8322, device='cuda:0')) + | > loss_gen: 2.456925392150879 (2.3615763162044767) + | > loss_kl: 1.8508012294769287 (1.8647003744510895) + | > loss_feat: 7.095354080200195 (7.060907622601124) + | > loss_mel: 18.539012908935547 (17.84985939999844) + | > loss_duration: 1.5013680458068848 (1.4637513376296836) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.44346046447754 (30.600794954502835) + | > grad_norm_1: tensor(340.1069, device='cuda:0') (tensor(238.8604, device='cuda:0')) + | > current_lr_0: 0.00019452596144714675 + | > current_lr_1: 0.00019452596144714675 + | > step_time: 0.3971 (0.3709590232118647) + | > loader_time: 0.0069 (0.005844532175266997) + + + --> TIME: 2025-04-19 02:39:31 -- STEP: 119/142 -- GLOBAL_STEP: 410925 + | > loss_disc: 2.4089441299438477 (2.4569704833150903) + | > loss_disc_real_0: 0.1403447687625885 (0.1280994019901552) + | > loss_disc_real_1: 0.1951235681772232 (0.206926152736199) + | > loss_disc_real_2: 0.2194082885980606 (0.22622225790464578) + | > loss_disc_real_3: 0.21868452429771423 (0.22785103596559092) + | > loss_disc_real_4: 0.22260314226150513 (0.22902833351067134) + | > loss_disc_real_5: 0.21750257909297943 (0.2287231599833785) + | > loss_0: 2.4089441299438477 (2.4569704833150903) + | > grad_norm_0: tensor(5.4385, device='cuda:0') (tensor(10.9076, device='cuda:0')) + | > loss_gen: 2.3160550594329834 (2.3582317969378304) + | > loss_kl: 1.9163587093353271 (1.8765533050569165) + | > loss_feat: 6.486783504486084 (7.036969152819209) + | > loss_mel: 17.522930145263672 (17.809865053962255) + | > loss_duration: 1.5606505870819092 (1.470880703765805) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.802776336669922 (30.55249991537142) + | > grad_norm_1: tensor(159.9352, device='cuda:0') (tensor(242.5776, device='cuda:0')) + | > current_lr_0: 0.00019452596144714675 + | > current_lr_1: 0.00019452596144714675 + | > step_time: 0.4258 (0.38114458773316456) + | > loader_time: 0.0068 (0.006065807422669996) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.552687644958496 (2.552687644958496) + | > loss_disc_real_0: 0.1258264034986496 (0.1258264034986496) + | > loss_disc_real_1: 0.20085448026657104 (0.20085448026657104) + | > loss_disc_real_2: 0.2522105574607849 (0.2522105574607849) + | > loss_disc_real_3: 0.224677175283432 (0.224677175283432) + | > loss_disc_real_4: 0.2605230510234833 (0.2605230510234833) + | > loss_disc_real_5: 0.26542049646377563 (0.26542049646377563) + | > loss_0: 2.552687644958496 (2.552687644958496) + | > loss_gen: 2.305302619934082 (2.305302619934082) + | > loss_kl: 2.446013927459717 (2.446013927459717) + | > loss_feat: 7.40711784362793 (7.40711784362793) + | > loss_mel: 19.24127769470215 (19.24127769470215) + | > loss_duration: 1.6646170616149902 (1.6646170616149902) + | > loss_1: 33.064327239990234 (33.064327239990234) + + --> STEP: 1 + | > loss_disc: 2.5587351322174072 (2.5587351322174072) + | > loss_disc_real_0: 0.11367616802453995 (0.11367616802453995) + | > loss_disc_real_1: 0.2349371314048767 (0.2349371314048767) + | > loss_disc_real_2: 0.2523438632488251 (0.2523438632488251) + | > loss_disc_real_3: 0.2164226472377777 (0.2164226472377777) + | > loss_disc_real_4: 0.21502193808555603 (0.21502193808555603) + | > loss_disc_real_5: 0.20612183213233948 (0.20612183213233948) + | > loss_0: 2.5587351322174072 (2.5587351322174072) + | > loss_gen: 2.193553924560547 (2.193553924560547) + | > loss_kl: 2.6367104053497314 (2.6367104053497314) + | > loss_feat: 6.788750171661377 (6.788750171661377) + | > loss_mel: 18.240825653076172 (18.240825653076172) + | > loss_duration: 1.7140260934829712 (1.7140260934829712) + | > loss_1: 31.57386589050293 (31.57386589050293) + + --> STEP: 2 + | > loss_disc: 2.6216700077056885 (2.590202569961548) + | > loss_disc_real_0: 0.15154775977134705 (0.1326119638979435) + | > loss_disc_real_1: 0.22360405325889587 (0.2292705923318863) + | > loss_disc_real_2: 0.27851155400276184 (0.26542770862579346) + | > loss_disc_real_3: 0.22025737166404724 (0.21834000945091248) + | > loss_disc_real_4: 0.22736817598342896 (0.2211950570344925) + | > loss_disc_real_5: 0.20987871289253235 (0.2080002725124359) + | > loss_0: 2.6216700077056885 (2.590202569961548) + | > loss_gen: 2.210094451904297 (2.201824188232422) + | > loss_kl: 2.116065502166748 (2.3763879537582397) + | > loss_feat: 7.670219898223877 (7.229485034942627) + | > loss_mel: 19.39368438720703 (18.8172550201416) + | > loss_duration: 1.6569275856018066 (1.685476839542389) + | > loss_1: 33.046993255615234 (32.31042957305908) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037376880645751953 (+8.726119995117188e-05) + | > avg_loss_disc: 2.590202569961548 (+0.1760646104812622) + | > avg_loss_disc_real_0: 0.1326119638979435 (-0.010435640811920166) + | > avg_loss_disc_real_1: 0.2292705923318863 (+0.006874702870845795) + | > avg_loss_disc_real_2: 0.26542770862579346 (+0.052177801728248596) + | > avg_loss_disc_real_3: 0.21834000945091248 (-0.009228214621543884) + | > avg_loss_disc_real_4: 0.2211950570344925 (-0.0005504786968231201) + | > avg_loss_disc_real_5: 0.2080002725124359 (-0.036385491490364075) + | > avg_loss_0: 2.590202569961548 (+0.1760646104812622) + | > avg_loss_gen: 2.201824188232422 (-0.18975090980529785) + | > avg_loss_kl: 2.3763879537582397 (+0.03975200653076172) + | > avg_loss_feat: 7.229485034942627 (+0.5143237113952637) + | > avg_loss_mel: 18.8172550201416 (+0.6197309494018555) + | > avg_loss_duration: 1.685476839542389 (+0.0010252594947814941) + | > avg_loss_1: 32.31042957305908 (+0.985081672668457) + + + > EPOCH: 223/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:39:43)  + + --> TIME: 2025-04-19 02:39:44 -- STEP: 2/142 -- GLOBAL_STEP: 410950 + | > loss_disc: 2.596602439880371 (2.5404659509658813) + | > loss_disc_real_0: 0.13414707779884338 (0.1238761804997921) + | > loss_disc_real_1: 0.2136765718460083 (0.2128024622797966) + | > loss_disc_real_2: 0.22762078046798706 (0.23582202196121216) + | > loss_disc_real_3: 0.2237357497215271 (0.21281369775533676) + | > loss_disc_real_4: 0.23163117468357086 (0.2244030460715294) + | > loss_disc_real_5: 0.2103913426399231 (0.20787762105464935) + | > loss_0: 2.596602439880371 (2.5404659509658813) + | > grad_norm_0: tensor(4.0643, device='cuda:0') (tensor(8.8614, device='cuda:0')) + | > loss_gen: 2.2139086723327637 (2.2392265796661377) + | > loss_kl: 1.7297492027282715 (1.7208806276321411) + | > loss_feat: 6.963409900665283 (6.866655588150024) + | > loss_mel: 17.555255889892578 (17.274066925048828) + | > loss_duration: 1.440500020980835 (1.4245383143424988) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.90282440185547 (29.52536964416504) + | > grad_norm_1: tensor(125.9616, device='cuda:0') (tensor(268.1993, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.3261 (0.32726478576660156) + | > loader_time: 0.004 (0.004118919372558594) + + + --> TIME: 2025-04-19 02:39:53 -- STEP: 27/142 -- GLOBAL_STEP: 410975 + | > loss_disc: 2.4122626781463623 (2.4393040515758373) + | > loss_disc_real_0: 0.14978136122226715 (0.11730408530544352) + | > loss_disc_real_1: 0.19980520009994507 (0.20064882161440672) + | > loss_disc_real_2: 0.2359275221824646 (0.22749317796142013) + | > loss_disc_real_3: 0.21607054769992828 (0.22778435824093995) + | > loss_disc_real_4: 0.20825502276420593 (0.23029212046552588) + | > loss_disc_real_5: 0.209145650267601 (0.23231363186129816) + | > loss_0: 2.4122626781463623 (2.4393040515758373) + | > grad_norm_0: tensor(17.6347, device='cuda:0') (tensor(10.7907, device='cuda:0')) + | > loss_gen: 2.457317352294922 (2.3830707603030734) + | > loss_kl: 2.0898826122283936 (1.8333140523345381) + | > loss_feat: 8.096632957458496 (7.279149991494638) + | > loss_mel: 18.131256103515625 (17.80168582774975) + | > loss_duration: 1.44368577003479 (1.4346659492563318) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.21877670288086 (30.731886828387225) + | > grad_norm_1: tensor(516.4981, device='cuda:0') (tensor(244.6674, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.3581 (0.33906616104973686) + | > loader_time: 0.0053 (0.004976466849998194) + + + --> TIME: 2025-04-19 02:40:02 -- STEP: 52/142 -- GLOBAL_STEP: 411000 + | > loss_disc: 2.570760726928711 (2.452461591133705) + | > loss_disc_real_0: 0.1058812364935875 (0.11995811531176934) + | > loss_disc_real_1: 0.2220095545053482 (0.20404942648915145) + | > loss_disc_real_2: 0.24340176582336426 (0.22682917232696825) + | > loss_disc_real_3: 0.23714271187782288 (0.2269109719647811) + | > loss_disc_real_4: 0.23582330346107483 (0.2288491390645504) + | > loss_disc_real_5: 0.26575422286987305 (0.23345620099168557) + | > loss_0: 2.570760726928711 (2.452461591133705) + | > grad_norm_0: tensor(12.4484, device='cuda:0') (tensor(10.4261, device='cuda:0')) + | > loss_gen: 2.385976552963257 (2.3708461339657116) + | > loss_kl: 1.945945382118225 (1.861750141932414) + | > loss_feat: 7.081234455108643 (7.153907133982732) + | > loss_mel: 18.947704315185547 (17.648817722613998) + | > loss_duration: 1.5103366374969482 (1.443656754035216) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.871196746826172 (30.47897819372324) + | > grad_norm_1: tensor(260.9417, device='cuda:0') (tensor(242.0843, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.3661 (0.3488143040583684) + | > loader_time: 0.0055 (0.0053514563120328484) + + + --> TIME: 2025-04-19 02:40:12 -- STEP: 77/142 -- GLOBAL_STEP: 411025 + | > loss_disc: 2.489771604537964 (2.458831601328664) + | > loss_disc_real_0: 0.11721990257501602 (0.12260276891968468) + | > loss_disc_real_1: 0.20627202093601227 (0.20463902416167323) + | > loss_disc_real_2: 0.2175230234861374 (0.22629669134492997) + | > loss_disc_real_3: 0.23754246532917023 (0.22740305611839542) + | > loss_disc_real_4: 0.22326694428920746 (0.22849534568074462) + | > loss_disc_real_5: 0.237664595246315 (0.2342084479796422) + | > loss_0: 2.489771604537964 (2.458831601328664) + | > grad_norm_0: tensor(5.4583, device='cuda:0') (tensor(9.8369, device='cuda:0')) + | > loss_gen: 2.234574794769287 (2.350693213475215) + | > loss_kl: 1.964594841003418 (1.8631702800849816) + | > loss_feat: 6.77169942855835 (7.102563839454154) + | > loss_mel: 17.267223358154297 (17.687963634342342) + | > loss_duration: 1.4862074851989746 (1.4554765116084707) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.724300384521484 (30.4598676012708) + | > grad_norm_1: tensor(83.5507, device='cuda:0') (tensor(229.5620, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.3888 (0.36079177299103177) + | > loader_time: 0.0065 (0.005669928216314937) + + + --> TIME: 2025-04-19 02:40:23 -- STEP: 102/142 -- GLOBAL_STEP: 411050 + | > loss_disc: 2.5744898319244385 (2.456982647671421) + | > loss_disc_real_0: 0.12380129098892212 (0.12253283986858293) + | > loss_disc_real_1: 0.21885986626148224 (0.2041536597352402) + | > loss_disc_real_2: 0.2925717830657959 (0.22690893417479946) + | > loss_disc_real_3: 0.229679137468338 (0.22717020441504085) + | > loss_disc_real_4: 0.26039573550224304 (0.2289786860346794) + | > loss_disc_real_5: 0.25184115767478943 (0.23406034021400937) + | > loss_0: 2.5744898319244385 (2.456982647671421) + | > grad_norm_0: tensor(6.2178, device='cuda:0') (tensor(9.1287, device='cuda:0')) + | > loss_gen: 2.230289936065674 (2.3491729053796506) + | > loss_kl: 1.9140570163726807 (1.8756837634479298) + | > loss_feat: 7.399343013763428 (7.112922701181149) + | > loss_mel: 19.058696746826172 (17.784568599626144) + | > loss_duration: 1.4760963916778564 (1.4663470235525393) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.07848358154297 (30.588695002537147) + | > grad_norm_1: tensor(100.8103, device='cuda:0') (tensor(216.0575, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.4178 (0.37126950656666474) + | > loader_time: 0.0068 (0.005932010856329227) + + + --> TIME: 2025-04-19 02:40:34 -- STEP: 127/142 -- GLOBAL_STEP: 411075 + | > loss_disc: 2.430530071258545 (2.472595909449061) + | > loss_disc_real_0: 0.07526566833257675 (0.1254074325946372) + | > loss_disc_real_1: 0.1820129156112671 (0.20559851625772912) + | > loss_disc_real_2: 0.23622138798236847 (0.2280064265089711) + | > loss_disc_real_3: 0.2281217724084854 (0.22889296006499313) + | > loss_disc_real_4: 0.2576636075973511 (0.23020089435295796) + | > loss_disc_real_5: 0.22327899932861328 (0.23443907933441674) + | > loss_0: 2.430530071258545 (2.472595909449061) + | > grad_norm_0: tensor(14.9817, device='cuda:0') (tensor(10.3806, device='cuda:0')) + | > loss_gen: 2.505535840988159 (2.342609157712441) + | > loss_kl: 1.9060966968536377 (1.8821809920739) + | > loss_feat: 7.329451084136963 (7.044134928485539) + | > loss_mel: 18.344148635864258 (17.71001966731756) + | > loss_duration: 1.4940524101257324 (1.4734524235012025) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.57928466796875 (30.452397143746925) + | > grad_norm_1: tensor(483.2680, device='cuda:0') (tensor(231.8929, device='cuda:0')) + | > current_lr_0: 0.00019450164570196585 + | > current_lr_1: 0.00019450164570196585 + | > step_time: 0.5986 (0.38377932863911307) + | > loader_time: 0.0073 (0.006190161066731131) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.507126569747925 (2.507126569747925) + | > loss_disc_real_0: 0.17073093354701996 (0.17073093354701996) + | > loss_disc_real_1: 0.212490051984787 (0.212490051984787) + | > loss_disc_real_2: 0.2649891972541809 (0.2649891972541809) + | > loss_disc_real_3: 0.26978328824043274 (0.26978328824043274) + | > loss_disc_real_4: 0.2595888078212738 (0.2595888078212738) + | > loss_disc_real_5: 0.2436225265264511 (0.2436225265264511) + | > loss_0: 2.507126569747925 (2.507126569747925) + | > loss_gen: 2.5044069290161133 (2.5044069290161133) + | > loss_kl: 2.4272751808166504 (2.4272751808166504) + | > loss_feat: 6.579438209533691 (6.579438209533691) + | > loss_mel: 18.043365478515625 (18.043365478515625) + | > loss_duration: 1.6928232908248901 (1.6928232908248901) + | > loss_1: 31.2473087310791 (31.2473087310791) + + --> STEP: 1 + | > loss_disc: 2.484560012817383 (2.484560012817383) + | > loss_disc_real_0: 0.14882519841194153 (0.14882519841194153) + | > loss_disc_real_1: 0.2400549352169037 (0.2400549352169037) + | > loss_disc_real_2: 0.23389026522636414 (0.23389026522636414) + | > loss_disc_real_3: 0.24220679700374603 (0.24220679700374603) + | > loss_disc_real_4: 0.23600001633167267 (0.23600001633167267) + | > loss_disc_real_5: 0.1969512701034546 (0.1969512701034546) + | > loss_0: 2.484560012817383 (2.484560012817383) + | > loss_gen: 2.3104822635650635 (2.3104822635650635) + | > loss_kl: 2.7536466121673584 (2.7536466121673584) + | > loss_feat: 6.531574249267578 (6.531574249267578) + | > loss_mel: 18.12137222290039 (18.12137222290039) + | > loss_duration: 1.729050874710083 (1.729050874710083) + | > loss_1: 31.44612693786621 (31.44612693786621) + + --> STEP: 2 + | > loss_disc: 2.4418349266052246 (2.4631974697113037) + | > loss_disc_real_0: 0.2221614420413971 (0.1854933202266693) + | > loss_disc_real_1: 0.24885915219783783 (0.24445704370737076) + | > loss_disc_real_2: 0.23234084248542786 (0.233115553855896) + | > loss_disc_real_3: 0.23693415522575378 (0.2395704761147499) + | > loss_disc_real_4: 0.23280136287212372 (0.2344006896018982) + | > loss_disc_real_5: 0.23362849652767181 (0.2152898833155632) + | > loss_0: 2.4418349266052246 (2.4631974697113037) + | > loss_gen: 2.633404493331909 (2.4719433784484863) + | > loss_kl: 2.1368885040283203 (2.4452675580978394) + | > loss_feat: 6.881626605987549 (6.7066004276275635) + | > loss_mel: 17.169349670410156 (17.645360946655273) + | > loss_duration: 1.6648632287979126 (1.6969570517539978) + | > loss_1: 30.48613166809082 (30.966129302978516) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033315420150756836 (-0.0004061460494995117) + | > avg_loss_disc: 2.4631974697113037 (-0.12700510025024414) + | > avg_loss_disc_real_0: 0.1854933202266693 (+0.052881356328725815) + | > avg_loss_disc_real_1: 0.24445704370737076 (+0.015186451375484467) + | > avg_loss_disc_real_2: 0.233115553855896 (-0.03231215476989746) + | > avg_loss_disc_real_3: 0.2395704761147499 (+0.021230466663837433) + | > avg_loss_disc_real_4: 0.2344006896018982 (+0.0132056325674057) + | > avg_loss_disc_real_5: 0.2152898833155632 (+0.007289610803127289) + | > avg_loss_0: 2.4631974697113037 (-0.12700510025024414) + | > avg_loss_gen: 2.4719433784484863 (+0.27011919021606445) + | > avg_loss_kl: 2.4452675580978394 (+0.06887960433959961) + | > avg_loss_feat: 6.7066004276275635 (-0.5228846073150635) + | > avg_loss_mel: 17.645360946655273 (-1.1718940734863281) + | > avg_loss_duration: 1.6969570517539978 (+0.011480212211608887) + | > avg_loss_1: 30.966129302978516 (-1.3443002700805664) + + + > EPOCH: 224/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:40:42)  + + --> TIME: 2025-04-19 02:40:46 -- STEP: 10/142 -- GLOBAL_STEP: 411100 + | > loss_disc: 2.3484530448913574 (2.4241819858551024) + | > loss_disc_real_0: 0.12389188259840012 (0.1123325727880001) + | > loss_disc_real_1: 0.20522862672805786 (0.20307753533124923) + | > loss_disc_real_2: 0.22429318726062775 (0.2324380099773407) + | > loss_disc_real_3: 0.23062622547149658 (0.22385244220495223) + | > loss_disc_real_4: 0.23507113754749298 (0.22857167869806289) + | > loss_disc_real_5: 0.22774043679237366 (0.23184597492218018) + | > loss_0: 2.3484530448913574 (2.4241819858551024) + | > grad_norm_0: tensor(25.0350, device='cuda:0') (tensor(16.9004, device='cuda:0')) + | > loss_gen: 2.4092695713043213 (2.4298530340194704) + | > loss_kl: 2.0104267597198486 (1.8783828496932984) + | > loss_feat: 7.575921058654785 (7.394171571731567) + | > loss_mel: 18.966190338134766 (18.11826686859131) + | > loss_duration: 1.403531551361084 (1.4272940516471864) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.36533737182617 (31.247968482971192) + | > grad_norm_1: tensor(341.0111, device='cuda:0') (tensor(334.9010, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.3374 (0.3262324333190918) + | > loader_time: 0.0047 (0.004399394989013672) + + + --> TIME: 2025-04-19 02:40:55 -- STEP: 35/142 -- GLOBAL_STEP: 411125 + | > loss_disc: 2.5282111167907715 (2.4512250082833424) + | > loss_disc_real_0: 0.19914673268795013 (0.1179672366806439) + | > loss_disc_real_1: 0.2263045608997345 (0.20522221411977495) + | > loss_disc_real_2: 0.18288470804691315 (0.2271741624389376) + | > loss_disc_real_3: 0.19784225523471832 (0.22560766211577823) + | > loss_disc_real_4: 0.17074714601039886 (0.22960773067814963) + | > loss_disc_real_5: 0.1924247443675995 (0.23173470156533377) + | > loss_0: 2.5282111167907715 (2.4512250082833424) + | > grad_norm_0: tensor(6.9623, device='cuda:0') (tensor(12.6981, device='cuda:0')) + | > loss_gen: 2.274376392364502 (2.3883790833609444) + | > loss_kl: 1.745754599571228 (1.8770957367760794) + | > loss_feat: 7.4454755783081055 (7.325463485717774) + | > loss_mel: 17.484989166259766 (18.056754193987164) + | > loss_duration: 1.4612537622451782 (1.438672069140843) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.411849975585938 (31.086364473615372) + | > grad_norm_1: tensor(244.5309, device='cuda:0') (tensor(269.3407, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.3583 (0.3406017303466797) + | > loader_time: 0.0056 (0.00503568649291992) + + + --> TIME: 2025-04-19 02:41:04 -- STEP: 60/142 -- GLOBAL_STEP: 411150 + | > loss_disc: 2.3733363151550293 (2.470180332660675) + | > loss_disc_real_0: 0.13244086503982544 (0.12063913494348526) + | > loss_disc_real_1: 0.19764667749404907 (0.20803245529532433) + | > loss_disc_real_2: 0.22262737154960632 (0.22926617662111917) + | > loss_disc_real_3: 0.20906682312488556 (0.22829109877347947) + | > loss_disc_real_4: 0.2754463851451874 (0.232119316359361) + | > loss_disc_real_5: 0.20361025631427765 (0.23490902508298556) + | > loss_0: 2.3733363151550293 (2.470180332660675) + | > grad_norm_0: tensor(12.9065, device='cuda:0') (tensor(11.7961, device='cuda:0')) + | > loss_gen: 2.505462408065796 (2.382897869745891) + | > loss_kl: 1.9373966455459595 (1.8843518575032552) + | > loss_feat: 7.915652751922607 (7.211863787968953) + | > loss_mel: 19.24600601196289 (18.00968030293783) + | > loss_duration: 1.491603136062622 (1.4501687268416086) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.09612274169922 (30.938962586720784) + | > grad_norm_1: tensor(326.8511, device='cuda:0') (tensor(274.9701, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.3752 (0.3517341732978821) + | > loader_time: 0.0063 (0.005354932943979899) + + + --> TIME: 2025-04-19 02:41:14 -- STEP: 85/142 -- GLOBAL_STEP: 411175 + | > loss_disc: 2.468991279602051 (2.4796871830435365) + | > loss_disc_real_0: 0.12382467091083527 (0.12578994964852055) + | > loss_disc_real_1: 0.24259395897388458 (0.20987347855287436) + | > loss_disc_real_2: 0.232636496424675 (0.2283353600431891) + | > loss_disc_real_3: 0.2351124882698059 (0.22762204952099743) + | > loss_disc_real_4: 0.23553408682346344 (0.2321590171140783) + | > loss_disc_real_5: 0.2296038568019867 (0.23438087771920596) + | > loss_0: 2.468991279602051 (2.4796871830435365) + | > grad_norm_0: tensor(21.0230, device='cuda:0') (tensor(11.4855, device='cuda:0')) + | > loss_gen: 2.228516101837158 (2.357590035831228) + | > loss_kl: 1.8509042263031006 (1.883665511187385) + | > loss_feat: 7.040395736694336 (7.140964182685403) + | > loss_mel: 17.925519943237305 (17.964321158914004) + | > loss_duration: 1.4904701709747314 (1.4621324609307684) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.53580665588379 (30.808673297657688) + | > grad_norm_1: tensor(525.4625, device='cuda:0') (tensor(262.6277, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.3971 (0.3632756008821376) + | > loader_time: 0.0071 (0.0056731756995706) + + + --> TIME: 2025-04-19 02:41:25 -- STEP: 110/142 -- GLOBAL_STEP: 411200 + | > loss_disc: 2.394026756286621 (2.476318157802928) + | > loss_disc_real_0: 0.11936159431934357 (0.1251054229384119) + | > loss_disc_real_1: 0.20704226195812225 (0.2097496854988011) + | > loss_disc_real_2: 0.21676452457904816 (0.227902251617475) + | > loss_disc_real_3: 0.23129315674304962 (0.22779822579839012) + | > loss_disc_real_4: 0.21993176639080048 (0.23077004646713084) + | > loss_disc_real_5: 0.2122013121843338 (0.23395688086748123) + | > loss_0: 2.394026756286621 (2.476318157802928) + | > grad_norm_0: tensor(15.0907, device='cuda:0') (tensor(11.5556, device='cuda:0')) + | > loss_gen: 2.454859495162964 (2.3564976237036976) + | > loss_kl: 2.015780210494995 (1.8908618265932255) + | > loss_feat: 7.301165580749512 (7.11997899575667) + | > loss_mel: 19.470985412597656 (17.943102368441494) + | > loss_duration: 1.4906716346740723 (1.4705275784839282) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.733463287353516 (30.7809684233232) + | > grad_norm_1: tensor(431.8018, device='cuda:0') (tensor(273.4961, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.4323 (0.3735085617412221) + | > loader_time: 0.0071 (0.005911467292092063) + + + --> TIME: 2025-04-19 02:41:36 -- STEP: 135/142 -- GLOBAL_STEP: 411225 + | > loss_disc: 2.5183169841766357 (2.4838889316276265) + | > loss_disc_real_0: 0.10062427818775177 (0.1270054242677159) + | > loss_disc_real_1: 0.2187674641609192 (0.21033705274264014) + | > loss_disc_real_2: 0.2325029820203781 (0.2280999203523) + | > loss_disc_real_3: 0.23712873458862305 (0.2284654258577912) + | > loss_disc_real_4: 0.2604139447212219 (0.23190095325311025) + | > loss_disc_real_5: 0.2731289863586426 (0.23468382623460557) + | > loss_0: 2.5183169841766357 (2.4838889316276265) + | > grad_norm_0: tensor(10.8595, device='cuda:0') (tensor(11.8578, device='cuda:0')) + | > loss_gen: 2.3162200450897217 (2.34668041865031) + | > loss_kl: 2.041306734085083 (1.8929930554495917) + | > loss_feat: 7.362415313720703 (7.077622975243463) + | > loss_mel: 18.29562759399414 (17.90157922108968) + | > loss_duration: 1.482541561126709 (1.4772082911597357) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.498109817504883 (30.696083958943685) + | > grad_norm_1: tensor(149.4249, device='cuda:0') (tensor(272.3187, device='cuda:0')) + | > current_lr_0: 0.0001944773329962531 + | > current_lr_1: 0.0001944773329962531 + | > step_time: 0.4567 (0.3853627381501375) + | > loader_time: 0.0076 (0.006170513011791088) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6763765811920166 (2.6763765811920166) + | > loss_disc_real_0: 0.14037583768367767 (0.14037583768367767) + | > loss_disc_real_1: 0.28425538539886475 (0.28425538539886475) + | > loss_disc_real_2: 0.2125631719827652 (0.2125631719827652) + | > loss_disc_real_3: 0.2545841932296753 (0.2545841932296753) + | > loss_disc_real_4: 0.19804823398590088 (0.19804823398590088) + | > loss_disc_real_5: 0.2652870714664459 (0.2652870714664459) + | > loss_0: 2.6763765811920166 (2.6763765811920166) + | > loss_gen: 1.9953166246414185 (1.9953166246414185) + | > loss_kl: 2.1926028728485107 (2.1926028728485107) + | > loss_feat: 4.99729585647583 (4.99729585647583) + | > loss_mel: 13.879694938659668 (13.879694938659668) + | > loss_duration: 1.708071231842041 (1.708071231842041) + | > loss_1: 24.772981643676758 (24.772981643676758) + + --> STEP: 1 + | > loss_disc: 2.4365668296813965 (2.4365668296813965) + | > loss_disc_real_0: 0.1338641494512558 (0.1338641494512558) + | > loss_disc_real_1: 0.26546937227249146 (0.26546937227249146) + | > loss_disc_real_2: 0.22485171258449554 (0.22485171258449554) + | > loss_disc_real_3: 0.22131036221981049 (0.22131036221981049) + | > loss_disc_real_4: 0.23398323357105255 (0.23398323357105255) + | > loss_disc_real_5: 0.24175415933132172 (0.24175415933132172) + | > loss_0: 2.4365668296813965 (2.4365668296813965) + | > loss_gen: 2.3806803226470947 (2.3806803226470947) + | > loss_kl: 2.8194727897644043 (2.8194727897644043) + | > loss_feat: 6.866987228393555 (6.866987228393555) + | > loss_mel: 16.766433715820312 (16.766433715820312) + | > loss_duration: 1.729053020477295 (1.729053020477295) + | > loss_1: 30.5626277923584 (30.5626277923584) + + --> STEP: 2 + | > loss_disc: 2.343986749649048 (2.390276789665222) + | > loss_disc_real_0: 0.1339477151632309 (0.13390593230724335) + | > loss_disc_real_1: 0.2483881115913391 (0.2569287419319153) + | > loss_disc_real_2: 0.20907515287399292 (0.21696343272924423) + | > loss_disc_real_3: 0.2112223356962204 (0.21626634895801544) + | > loss_disc_real_4: 0.22572439908981323 (0.2298538163304329) + | > loss_disc_real_5: 0.23374077677726746 (0.2377474680542946) + | > loss_0: 2.343986749649048 (2.390276789665222) + | > loss_gen: 2.5565595626831055 (2.4686199426651) + | > loss_kl: 2.172304153442383 (2.4958884716033936) + | > loss_feat: 7.66238260269165 (7.2646849155426025) + | > loss_mel: 17.383594512939453 (17.075014114379883) + | > loss_duration: 1.6775941848754883 (1.7033236026763916) + | > loss_1: 31.452434539794922 (31.00753116607666) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034542083740234375 (+0.0001226663589477539) + | > avg_loss_disc: 2.390276789665222 (-0.07292068004608154) + | > avg_loss_disc_real_0: 0.13390593230724335 (-0.051587387919425964) + | > avg_loss_disc_real_1: 0.2569287419319153 (+0.012471698224544525) + | > avg_loss_disc_real_2: 0.21696343272924423 (-0.016152121126651764) + | > avg_loss_disc_real_3: 0.21626634895801544 (-0.023304127156734467) + | > avg_loss_disc_real_4: 0.2298538163304329 (-0.0045468732714653015) + | > avg_loss_disc_real_5: 0.2377474680542946 (+0.022457584738731384) + | > avg_loss_0: 2.390276789665222 (-0.07292068004608154) + | > avg_loss_gen: 2.4686199426651 (-0.0033234357833862305) + | > avg_loss_kl: 2.4958884716033936 (+0.0506209135055542) + | > avg_loss_feat: 7.2646849155426025 (+0.5580844879150391) + | > avg_loss_mel: 17.075014114379883 (-0.5703468322753906) + | > avg_loss_duration: 1.7033236026763916 (+0.006366550922393799) + | > avg_loss_1: 31.00753116607666 (+0.04140186309814453) + + + > EPOCH: 225/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:41:40)  + + --> TIME: 2025-04-19 02:41:47 -- STEP: 18/142 -- GLOBAL_STEP: 411250 + | > loss_disc: 2.4002511501312256 (2.422521869341533) + | > loss_disc_real_0: 0.11512588709592819 (0.11737519668208228) + | > loss_disc_real_1: 0.22017858922481537 (0.20487542202075323) + | > loss_disc_real_2: 0.29135435819625854 (0.22817032784223557) + | > loss_disc_real_3: 0.2640385031700134 (0.22411028378539616) + | > loss_disc_real_4: 0.23353897035121918 (0.2233621883723471) + | > loss_disc_real_5: 0.23570212721824646 (0.23305797659688526) + | > loss_0: 2.4002511501312256 (2.422521869341533) + | > grad_norm_0: tensor(11.1899, device='cuda:0') (tensor(9.1609, device='cuda:0')) + | > loss_gen: 2.4805195331573486 (2.4077574147118463) + | > loss_kl: 1.8100584745407104 (1.8247186409102545) + | > loss_feat: 7.577336311340332 (7.333210733201769) + | > loss_mel: 18.915870666503906 (17.75874423980713) + | > loss_duration: 1.4522013664245605 (1.4265255795584784) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.235984802246094 (30.75095600552029) + | > grad_norm_1: tensor(278.7152, device='cuda:0') (tensor(236.8023, device='cuda:0')) + | > current_lr_0: 0.00019445302332962857 + | > current_lr_1: 0.00019445302332962857 + | > step_time: 0.3465 (0.33467239803738064) + | > loader_time: 0.0052 (0.005020711157057021) + + + --> TIME: 2025-04-19 02:41:57 -- STEP: 43/142 -- GLOBAL_STEP: 411275 + | > loss_disc: 2.4092330932617188 (2.4255663516909576) + | > loss_disc_real_0: 0.1314675658941269 (0.11647016191205313) + | > loss_disc_real_1: 0.24749861657619476 (0.2026237093432005) + | > loss_disc_real_2: 0.20323026180267334 (0.22541978227537732) + | > loss_disc_real_3: 0.22356581687927246 (0.22676878474479498) + | > loss_disc_real_4: 0.19977909326553345 (0.22580361608849014) + | > loss_disc_real_5: 0.23201188445091248 (0.23412769201189973) + | > loss_0: 2.4092330932617188 (2.4255663516909576) + | > grad_norm_0: tensor(7.8124, device='cuda:0') (tensor(8.5462, device='cuda:0')) + | > loss_gen: 2.209465265274048 (2.3874177766400706) + | > loss_kl: 1.844496726989746 (1.8562289088271384) + | > loss_feat: 6.523077964782715 (7.250438346419224) + | > loss_mel: 17.210660934448242 (17.86683836648631) + | > loss_duration: 1.4661732912063599 (1.4420554748801298) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.253873825073242 (30.802978515625) + | > grad_norm_1: tensor(122.4745, device='cuda:0') (tensor(229.6529, device='cuda:0')) + | > current_lr_0: 0.00019445302332962857 + | > current_lr_1: 0.00019445302332962857 + | > step_time: 0.3736 (0.34973754993704864) + | > loader_time: 0.006 (0.005380907724069995) + + + --> TIME: 2025-04-19 02:42:07 -- STEP: 68/142 -- GLOBAL_STEP: 411300 + | > loss_disc: 2.522831916809082 (2.435118962736691) + | > loss_disc_real_0: 0.15778453648090363 (0.11813289683093042) + | > loss_disc_real_1: 0.23069369792938232 (0.20450343520325773) + | > loss_disc_real_2: 0.22084228694438934 (0.22591661026372628) + | > loss_disc_real_3: 0.2105446606874466 (0.2264619515660931) + | > loss_disc_real_4: 0.2568714916706085 (0.22498682952102492) + | > loss_disc_real_5: 0.23952841758728027 (0.23243229397956064) + | > loss_0: 2.522831916809082 (2.435118962736691) + | > grad_norm_0: tensor(4.3010, device='cuda:0') (tensor(7.7398, device='cuda:0')) + | > loss_gen: 2.3179314136505127 (2.3702412563211777) + | > loss_kl: 1.7317668199539185 (1.8677815602106207) + | > loss_feat: 6.646949768066406 (7.1692057847976685) + | > loss_mel: 16.87451171875 (17.714613044963162) + | > loss_duration: 1.4715676307678223 (1.453207692679237) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.042726516723633 (30.57504914788639) + | > grad_norm_1: tensor(114.5695, device='cuda:0') (tensor(202.3016, device='cuda:0')) + | > current_lr_0: 0.00019445302332962857 + | > current_lr_1: 0.00019445302332962857 + | > step_time: 0.3914 (0.3632676320917466) + | > loader_time: 0.0062 (0.005641772466547349) + + + --> TIME: 2025-04-19 02:42:17 -- STEP: 93/142 -- GLOBAL_STEP: 411325 + | > loss_disc: 2.488205909729004 (2.443253273605018) + | > loss_disc_real_0: 0.09718809276819229 (0.12093001115386204) + | > loss_disc_real_1: 0.21701431274414062 (0.20445391776100283) + | > loss_disc_real_2: 0.240173801779747 (0.22643763131351882) + | > loss_disc_real_3: 0.2405545562505722 (0.2273201843102773) + | > loss_disc_real_4: 0.23237264156341553 (0.22569019659872977) + | > loss_disc_real_5: 0.21091918647289276 (0.23255198537021554) + | > loss_0: 2.488205909729004 (2.443253273605018) + | > grad_norm_0: tensor(10.9471, device='cuda:0') (tensor(7.4952, device='cuda:0')) + | > loss_gen: 2.2445156574249268 (2.3619601700895583) + | > loss_kl: 1.8890310525894165 (1.8835073260850803) + | > loss_feat: 6.797016620635986 (7.1226559762031805) + | > loss_mel: 17.39280128479004 (17.690231948770506) + | > loss_duration: 1.4917898178100586 (1.46551864634278) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.815155029296875 (30.523873913672663) + | > grad_norm_1: tensor(111.0388, device='cuda:0') (tensor(170.2397, device='cuda:0')) + | > current_lr_0: 0.00019445302332962857 + | > current_lr_1: 0.00019445302332962857 + | > step_time: 0.3992 (0.3724770545959472) + | > loader_time: 0.0066 (0.005836550907422138) + + + --> TIME: 2025-04-19 02:42:27 -- STEP: 118/142 -- GLOBAL_STEP: 411350 + | > loss_disc: 2.4213643074035645 (2.455754284131325) + | > loss_disc_real_0: 0.12739771604537964 (0.12360062172352257) + | > loss_disc_real_1: 0.18456865847110748 (0.20495308064303155) + | > loss_disc_real_2: 0.23141157627105713 (0.22681198382781723) + | > loss_disc_real_3: 0.21765311062335968 (0.2276131207660093) + | > loss_disc_real_4: 0.23575669527053833 (0.22686442644414256) + | > loss_disc_real_5: 0.2132554054260254 (0.2319771801516161) + | > loss_0: 2.4213643074035645 (2.455754284131325) + | > grad_norm_0: tensor(5.5627, device='cuda:0') (tensor(7.5223, device='cuda:0')) + | > loss_gen: 2.399515151977539 (2.3481649786739047) + | > loss_kl: 1.9841015338897705 (1.8901896325208374) + | > loss_feat: 7.536369800567627 (7.0768433748665505) + | > loss_mel: 18.688358306884766 (17.709499577344474) + | > loss_duration: 1.4684722423553467 (1.4725673774541435) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.07681655883789 (30.497264878224517) + | > grad_norm_1: tensor(115.7594, device='cuda:0') (tensor(170.8191, device='cuda:0')) + | > current_lr_0: 0.00019445302332962857 + | > current_lr_1: 0.00019445302332962857 + | > step_time: 0.4417 (0.3819913177166954) + | > loader_time: 0.0074 (0.0060839935884637346) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5911881923675537 (2.5911881923675537) + | > loss_disc_real_0: 0.2526191174983978 (0.2526191174983978) + | > loss_disc_real_1: 0.2443765550851822 (0.2443765550851822) + | > loss_disc_real_2: 0.2680424451828003 (0.2680424451828003) + | > loss_disc_real_3: 0.25204750895500183 (0.25204750895500183) + | > loss_disc_real_4: 0.26905250549316406 (0.26905250549316406) + | > loss_disc_real_5: 0.2997431457042694 (0.2997431457042694) + | > loss_0: 2.5911881923675537 (2.5911881923675537) + | > loss_gen: 2.7595486640930176 (2.7595486640930176) + | > loss_kl: 2.4770519733428955 (2.4770519733428955) + | > loss_feat: 7.605686187744141 (7.605686187744141) + | > loss_mel: 18.81998062133789 (18.81998062133789) + | > loss_duration: 1.6788036823272705 (1.6788036823272705) + | > loss_1: 33.341068267822266 (33.341068267822266) + + --> STEP: 1 + | > loss_disc: 2.5941250324249268 (2.5941250324249268) + | > loss_disc_real_0: 0.1981893628835678 (0.1981893628835678) + | > loss_disc_real_1: 0.23119619488716125 (0.23119619488716125) + | > loss_disc_real_2: 0.24037687480449677 (0.24037687480449677) + | > loss_disc_real_3: 0.25299978256225586 (0.25299978256225586) + | > loss_disc_real_4: 0.2798693776130676 (0.2798693776130676) + | > loss_disc_real_5: 0.2845887541770935 (0.2845887541770935) + | > loss_0: 2.5941250324249268 (2.5941250324249268) + | > loss_gen: 2.556199789047241 (2.556199789047241) + | > loss_kl: 2.6708383560180664 (2.6708383560180664) + | > loss_feat: 6.051493167877197 (6.051493167877197) + | > loss_mel: 17.2205810546875 (17.2205810546875) + | > loss_duration: 1.763643741607666 (1.763643741607666) + | > loss_1: 30.26275634765625 (30.26275634765625) + + --> STEP: 2 + | > loss_disc: 2.6738266944885254 (2.633975863456726) + | > loss_disc_real_0: 0.20112749934196472 (0.19965843111276627) + | > loss_disc_real_1: 0.2832854390144348 (0.25724081695079803) + | > loss_disc_real_2: 0.24821889400482178 (0.24429788440465927) + | > loss_disc_real_3: 0.2814202904701233 (0.2672100365161896) + | > loss_disc_real_4: 0.24342049658298492 (0.2616449370980263) + | > loss_disc_real_5: 0.31692349910736084 (0.3007561266422272) + | > loss_0: 2.6738266944885254 (2.633975863456726) + | > loss_gen: 2.5105528831481934 (2.5333763360977173) + | > loss_kl: 2.197054147720337 (2.4339462518692017) + | > loss_feat: 6.300365447998047 (6.175929307937622) + | > loss_mel: 17.277236938476562 (17.24890899658203) + | > loss_duration: 1.6784886121749878 (1.721066176891327) + | > loss_1: 29.96369743347168 (30.113226890563965) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035475492477416992 (+9.334087371826172e-05) + | > avg_loss_disc: 2.633975863456726 (+0.2436990737915039) + | > avg_loss_disc_real_0: 0.19965843111276627 (+0.06575249880552292) + | > avg_loss_disc_real_1: 0.25724081695079803 (+0.00031207501888275146) + | > avg_loss_disc_real_2: 0.24429788440465927 (+0.02733445167541504) + | > avg_loss_disc_real_3: 0.2672100365161896 (+0.05094368755817413) + | > avg_loss_disc_real_4: 0.2616449370980263 (+0.031791120767593384) + | > avg_loss_disc_real_5: 0.3007561266422272 (+0.06300865858793259) + | > avg_loss_0: 2.633975863456726 (+0.2436990737915039) + | > avg_loss_gen: 2.5333763360977173 (+0.06475639343261719) + | > avg_loss_kl: 2.4339462518692017 (-0.061942219734191895) + | > avg_loss_feat: 6.175929307937622 (-1.0887556076049805) + | > avg_loss_mel: 17.24890899658203 (+0.17389488220214844) + | > avg_loss_duration: 1.721066176891327 (+0.017742574214935303) + | > avg_loss_1: 30.113226890563965 (-0.8943042755126953) + + + > EPOCH: 226/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:42:40)  + + --> TIME: 2025-04-19 02:42:41 -- STEP: 1/142 -- GLOBAL_STEP: 411375 + | > loss_disc: 2.577150583267212 (2.577150583267212) + | > loss_disc_real_0: 0.2227337807416916 (0.2227337807416916) + | > loss_disc_real_1: 0.2242908924818039 (0.2242908924818039) + | > loss_disc_real_2: 0.2902095317840576 (0.2902095317840576) + | > loss_disc_real_3: 0.23624320328235626 (0.23624320328235626) + | > loss_disc_real_4: 0.29764506220817566 (0.29764506220817566) + | > loss_disc_real_5: 0.2770833671092987 (0.2770833671092987) + | > loss_0: 2.577150583267212 (2.577150583267212) + | > grad_norm_0: tensor(14.0797, device='cuda:0') (tensor(14.0797, device='cuda:0')) + | > loss_gen: 2.436180353164673 (2.436180353164673) + | > loss_kl: 1.7846078872680664 (1.7846078872680664) + | > loss_feat: 6.6717209815979 (6.6717209815979) + | > loss_mel: 16.745403289794922 (16.745403289794922) + | > loss_duration: 1.4109290838241577 (1.4109290838241577) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.04884147644043 (29.04884147644043) + | > grad_norm_1: tensor(400.8186, device='cuda:0') (tensor(400.8186, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.3311 (0.3311026096343994) + | > loader_time: 0.0046 (0.004639387130737305) + + + --> TIME: 2025-04-19 02:42:49 -- STEP: 26/142 -- GLOBAL_STEP: 411400 + | > loss_disc: 2.4835433959960938 (2.4757784605026245) + | > loss_disc_real_0: 0.11759080737829208 (0.12750748774180043) + | > loss_disc_real_1: 0.21144036948680878 (0.20913065969944) + | > loss_disc_real_2: 0.22684817016124725 (0.2270552206497926) + | > loss_disc_real_3: 0.2262086570262909 (0.23030234930606988) + | > loss_disc_real_4: 0.20557411015033722 (0.2305491417646408) + | > loss_disc_real_5: 0.24666418135166168 (0.24103207198473123) + | > loss_0: 2.4835433959960938 (2.4757784605026245) + | > grad_norm_0: tensor(5.4641, device='cuda:0') (tensor(12.4622, device='cuda:0')) + | > loss_gen: 2.215477466583252 (2.384301598255451) + | > loss_kl: 1.8115079402923584 (1.827947029700646) + | > loss_feat: 6.997919082641602 (7.412600462253277) + | > loss_mel: 17.835023880004883 (17.885740060072678) + | > loss_duration: 1.4736841917037964 (1.4360617995262144) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.3336124420166 (30.946651238661545) + | > grad_norm_1: tensor(125.9723, device='cuda:0') (tensor(291.4625, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.3535 (0.3348426176951481) + | > loader_time: 0.0054 (0.004910083917471079) + + + --> TIME: 2025-04-19 02:42:58 -- STEP: 51/142 -- GLOBAL_STEP: 411425 + | > loss_disc: 2.4920897483825684 (2.4479170500063425) + | > loss_disc_real_0: 0.10395147651433945 (0.12218520731902587) + | > loss_disc_real_1: 0.18150927126407623 (0.20370632731447033) + | > loss_disc_real_2: 0.24396531283855438 (0.22559391371175355) + | > loss_disc_real_3: 0.23308314383029938 (0.22721833224390067) + | > loss_disc_real_4: 0.23978838324546814 (0.2287507878214705) + | > loss_disc_real_5: 0.25595927238464355 (0.23510172495654985) + | > loss_0: 2.4920897483825684 (2.4479170500063425) + | > grad_norm_0: tensor(13.7973, device='cuda:0') (tensor(11.5516, device='cuda:0')) + | > loss_gen: 2.3750948905944824 (2.38844942111595) + | > loss_kl: 1.9541252851486206 (1.836894203634823) + | > loss_feat: 7.452569961547852 (7.362578401378557) + | > loss_mel: 18.966428756713867 (17.933967328539087) + | > loss_duration: 1.451348066329956 (1.4419327366585826) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.19956588745117 (30.963822346107634) + | > grad_norm_1: tensor(336.1967, device='cuda:0') (tensor(288.9073, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.3667 (0.3470100702023974) + | > loader_time: 0.0059 (0.005250397850485409) + + + --> TIME: 2025-04-19 02:43:08 -- STEP: 76/142 -- GLOBAL_STEP: 411450 + | > loss_disc: 2.664215087890625 (2.4689337736681893) + | > loss_disc_real_0: 0.05918094143271446 (0.1253287801519036) + | > loss_disc_real_1: 0.21707403659820557 (0.20737377181649205) + | > loss_disc_real_2: 0.2706166207790375 (0.22598893371851822) + | > loss_disc_real_3: 0.26708316802978516 (0.22866970575169512) + | > loss_disc_real_4: 0.2590462565422058 (0.22830156001605487) + | > loss_disc_real_5: 0.2563038766384125 (0.23649435039413602) + | > loss_0: 2.664215087890625 (2.4689337736681893) + | > grad_norm_0: tensor(18.7336, device='cuda:0') (tensor(11.4076, device='cuda:0')) + | > loss_gen: 2.226808547973633 (2.3634775029985526) + | > loss_kl: 2.0152642726898193 (1.856660739371651) + | > loss_feat: 5.732521057128906 (7.263692297433552) + | > loss_mel: 16.206295013427734 (17.85982036590576) + | > loss_duration: 1.5036331415176392 (1.4552360867199148) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.68452262878418 (30.79888710222746) + | > grad_norm_1: tensor(67.1122, device='cuda:0') (tensor(265.3207, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.3938 (0.3592678841791655) + | > loader_time: 0.0069 (0.005557957448457417) + + + --> TIME: 2025-04-19 02:43:19 -- STEP: 101/142 -- GLOBAL_STEP: 411475 + | > loss_disc: 2.516733407974243 (2.4784019843186487) + | > loss_disc_real_0: 0.12325747311115265 (0.12739000237076586) + | > loss_disc_real_1: 0.21022480726242065 (0.2081972488967499) + | > loss_disc_real_2: 0.2531413435935974 (0.22587516653065634) + | > loss_disc_real_3: 0.24186784029006958 (0.22941071326189702) + | > loss_disc_real_4: 0.24691051244735718 (0.2297193381455865) + | > loss_disc_real_5: 0.269889235496521 (0.236016514720303) + | > loss_0: 2.516733407974243 (2.4784019843186487) + | > grad_norm_0: tensor(15.2889, device='cuda:0') (tensor(11.3982, device='cuda:0')) + | > loss_gen: 2.435898542404175 (2.3469861993695242) + | > loss_kl: 1.9865658283233643 (1.8681993059592672) + | > loss_feat: 7.0567851066589355 (7.113052061288664) + | > loss_mel: 17.830032348632812 (17.737518461623996) + | > loss_duration: 1.531522274017334 (1.4660211872346331) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.840803146362305 (30.531777221377535) + | > grad_norm_1: tensor(581.5656, device='cuda:0') (tensor(265.1167, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.4152 (0.37004684457684517) + | > loader_time: 0.0071 (0.005815199106046471) + + + --> TIME: 2025-04-19 02:43:29 -- STEP: 126/142 -- GLOBAL_STEP: 411500 + | > loss_disc: 2.4347281455993652 (2.474243701450409) + | > loss_disc_real_0: 0.160293847322464 (0.1268515912372441) + | > loss_disc_real_1: 0.16728049516677856 (0.20723971558941734) + | > loss_disc_real_2: 0.22171775996685028 (0.22600673612148042) + | > loss_disc_real_3: 0.22910763323307037 (0.2295296797676692) + | > loss_disc_real_4: 0.24682076275348663 (0.23045120979585346) + | > loss_disc_real_5: 0.21690821647644043 (0.2349078639868706) + | > loss_0: 2.4347281455993652 (2.474243701450409) + | > grad_norm_0: tensor(20.0272, device='cuda:0') (tensor(11.8596, device='cuda:0')) + | > loss_gen: 2.420858383178711 (2.349830468495686) + | > loss_kl: 1.9236094951629639 (1.8790579722041176) + | > loss_feat: 8.329336166381836 (7.072558141890026) + | > loss_mel: 19.08748435974121 (17.71332942114936) + | > loss_duration: 1.5121417045593262 (1.4736808973645408) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.27342987060547 (30.48845690772647) + | > grad_norm_1: tensor(435.5945, device='cuda:0') (tensor(279.7750, device='cuda:0')) + | > current_lr_0: 0.00019442871670171237 + | > current_lr_1: 0.00019442871670171237 + | > step_time: 0.4384 (0.3814647103112842) + | > loader_time: 0.0075 (0.006058236909291102) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4708824157714844 (2.4708824157714844) + | > loss_disc_real_0: 0.10533690452575684 (0.10533690452575684) + | > loss_disc_real_1: 0.2956632375717163 (0.2956632375717163) + | > loss_disc_real_2: 0.26798927783966064 (0.26798927783966064) + | > loss_disc_real_3: 0.24681733548641205 (0.24681733548641205) + | > loss_disc_real_4: 0.25767213106155396 (0.25767213106155396) + | > loss_disc_real_5: 0.2534262239933014 (0.2534262239933014) + | > loss_0: 2.4708824157714844 (2.4708824157714844) + | > loss_gen: 2.549806594848633 (2.549806594848633) + | > loss_kl: 2.1793413162231445 (2.1793413162231445) + | > loss_feat: 7.08980131149292 (7.08980131149292) + | > loss_mel: 18.289968490600586 (18.289968490600586) + | > loss_duration: 1.6865410804748535 (1.6865410804748535) + | > loss_1: 31.79545783996582 (31.79545783996582) + + --> STEP: 1 + | > loss_disc: 2.469994068145752 (2.469994068145752) + | > loss_disc_real_0: 0.11222871392965317 (0.11222871392965317) + | > loss_disc_real_1: 0.2613063454627991 (0.2613063454627991) + | > loss_disc_real_2: 0.24708929657936096 (0.24708929657936096) + | > loss_disc_real_3: 0.2241130918264389 (0.2241130918264389) + | > loss_disc_real_4: 0.21698202192783356 (0.21698202192783356) + | > loss_disc_real_5: 0.2566041052341461 (0.2566041052341461) + | > loss_0: 2.469994068145752 (2.469994068145752) + | > loss_gen: 2.3660802841186523 (2.3660802841186523) + | > loss_kl: 2.627427577972412 (2.627427577972412) + | > loss_feat: 7.459712982177734 (7.459712982177734) + | > loss_mel: 19.84284210205078 (19.84284210205078) + | > loss_duration: 1.7377851009368896 (1.7377851009368896) + | > loss_1: 34.03384780883789 (34.03384780883789) + + --> STEP: 2 + | > loss_disc: 2.4127421379089355 (2.4413681030273438) + | > loss_disc_real_0: 0.0905451774597168 (0.10138694569468498) + | > loss_disc_real_1: 0.27825841307640076 (0.2697823792695999) + | > loss_disc_real_2: 0.23681484162807465 (0.2419520691037178) + | > loss_disc_real_3: 0.20640039443969727 (0.21525674313306808) + | > loss_disc_real_4: 0.20899993181228638 (0.21299097687005997) + | > loss_disc_real_5: 0.237230122089386 (0.24691711366176605) + | > loss_0: 2.4127421379089355 (2.4413681030273438) + | > loss_gen: 2.366067409515381 (2.3660738468170166) + | > loss_kl: 2.3334362506866455 (2.480431914329529) + | > loss_feat: 7.825668811798096 (7.642690896987915) + | > loss_mel: 17.645660400390625 (18.744251251220703) + | > loss_duration: 1.6835508346557617 (1.7106679677963257) + | > loss_1: 31.854385375976562 (32.94411659240723) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003777027130126953 (+0.0002294778823852539) + | > avg_loss_disc: 2.4413681030273438 (-0.19260776042938232) + | > avg_loss_disc_real_0: 0.10138694569468498 (-0.09827148541808128) + | > avg_loss_disc_real_1: 0.2697823792695999 (+0.01254156231880188) + | > avg_loss_disc_real_2: 0.2419520691037178 (-0.0023458153009414673) + | > avg_loss_disc_real_3: 0.21525674313306808 (-0.05195329338312149) + | > avg_loss_disc_real_4: 0.21299097687005997 (-0.04865396022796631) + | > avg_loss_disc_real_5: 0.24691711366176605 (-0.05383901298046112) + | > avg_loss_0: 2.4413681030273438 (-0.19260776042938232) + | > avg_loss_gen: 2.3660738468170166 (-0.16730248928070068) + | > avg_loss_kl: 2.480431914329529 (+0.04648566246032715) + | > avg_loss_feat: 7.642690896987915 (+1.466761589050293) + | > avg_loss_mel: 18.744251251220703 (+1.4953422546386719) + | > avg_loss_duration: 1.7106679677963257 (-0.01039820909500122) + | > avg_loss_1: 32.94411659240723 (+2.8308897018432617) + + + > EPOCH: 227/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:43:38)  + + --> TIME: 2025-04-19 02:43:42 -- STEP: 9/142 -- GLOBAL_STEP: 411525 + | > loss_disc: 2.6639134883880615 (2.4757403002844915) + | > loss_disc_real_0: 0.19039086997509003 (0.12090508722596699) + | > loss_disc_real_1: 0.20173349976539612 (0.20180181993378532) + | > loss_disc_real_2: 0.2311200350522995 (0.23135370512803397) + | > loss_disc_real_3: 0.2277672439813614 (0.23165898025035858) + | > loss_disc_real_4: 0.22465673089027405 (0.22855363289515176) + | > loss_disc_real_5: 0.28209665417671204 (0.26089328196313644) + | > loss_0: 2.6639134883880615 (2.4757403002844915) + | > grad_norm_0: tensor(17.3357, device='cuda:0') (tensor(15.2546, device='cuda:0')) + | > loss_gen: 2.2605695724487305 (2.4038423432244196) + | > loss_kl: 1.7141882181167603 (1.83216392993927) + | > loss_feat: 6.422549247741699 (7.1449752913581) + | > loss_mel: 15.986275672912598 (17.323319011264378) + | > loss_duration: 1.4173831939697266 (1.4283667670355902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.800966262817383 (30.132666693793404) + | > grad_norm_1: tensor(257.3226, device='cuda:0') (tensor(303.5740, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.3232 (0.3238487243652344) + | > loader_time: 0.0044 (0.004566537009345161) + + + --> TIME: 2025-04-19 02:43:51 -- STEP: 34/142 -- GLOBAL_STEP: 411550 + | > loss_disc: 2.4161252975463867 (2.4330953499850105) + | > loss_disc_real_0: 0.11920459568500519 (0.11421067399137161) + | > loss_disc_real_1: 0.2036062628030777 (0.2036120856509489) + | > loss_disc_real_2: 0.2404356151819229 (0.22712958472616532) + | > loss_disc_real_3: 0.22618290781974792 (0.22649343092651927) + | > loss_disc_real_4: 0.23876160383224487 (0.22627223765148835) + | > loss_disc_real_5: 0.22770236432552338 (0.23795109286027796) + | > loss_0: 2.4161252975463867 (2.4330953499850105) + | > grad_norm_0: tensor(4.1768, device='cuda:0') (tensor(13.4444, device='cuda:0')) + | > loss_gen: 2.4226837158203125 (2.3812507180606617) + | > loss_kl: 1.838513970375061 (1.8477751647724825) + | > loss_feat: 6.6654205322265625 (7.112488704569199) + | > loss_mel: 16.957271575927734 (17.427680155810187) + | > loss_duration: 1.4680862426757812 (1.4372278276611776) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.35197639465332 (30.206422413096707) + | > grad_norm_1: tensor(116.7859, device='cuda:0') (tensor(295.4604, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.3585 (0.3497489480411305) + | > loader_time: 0.0054 (0.005266757572398466) + + + --> TIME: 2025-04-19 02:44:00 -- STEP: 59/142 -- GLOBAL_STEP: 411575 + | > loss_disc: 2.4336321353912354 (2.438117956711074) + | > loss_disc_real_0: 0.10351306200027466 (0.11685693428172904) + | > loss_disc_real_1: 0.19253987073898315 (0.20380855844182483) + | > loss_disc_real_2: 0.202534019947052 (0.22678449421615923) + | > loss_disc_real_3: 0.1997363269329071 (0.22664324181564785) + | > loss_disc_real_4: 0.2537190020084381 (0.22902492345389674) + | > loss_disc_real_5: 0.2469707727432251 (0.23303140321020352) + | > loss_0: 2.4336321353912354 (2.438117956711074) + | > grad_norm_0: tensor(6.9625, device='cuda:0') (tensor(11.3656, device='cuda:0')) + | > loss_gen: 2.3705387115478516 (2.3677033084934034) + | > loss_kl: 1.8388497829437256 (1.853951880487345) + | > loss_feat: 6.922677516937256 (7.11228885488995) + | > loss_mel: 16.33238983154297 (17.521001670320157) + | > loss_duration: 1.4714369773864746 (1.448769300670947) + | > amp_scaler: 512.0 (290.7118644067797) + | > loss_1: 28.935894012451172 (30.303714946164924) + | > grad_norm_1: tensor(157.7877, device='cuda:0') (tensor(254.9633, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.3763 (0.35670404515023957) + | > loader_time: 0.006 (0.0055277994123555845) + + + --> TIME: 2025-04-19 02:44:10 -- STEP: 84/142 -- GLOBAL_STEP: 411600 + | > loss_disc: 2.4487154483795166 (2.4546252148492) + | > loss_disc_real_0: 0.13088354468345642 (0.12256621888705663) + | > loss_disc_real_1: 0.18992382287979126 (0.2075097796817621) + | > loss_disc_real_2: 0.20584718883037567 (0.22604089478651682) + | > loss_disc_real_3: 0.22007837891578674 (0.22720700200824512) + | > loss_disc_real_4: 0.23437775671482086 (0.22933419527752058) + | > loss_disc_real_5: 0.22395694255828857 (0.23403940882001603) + | > loss_0: 2.4487154483795166 (2.4546252148492) + | > grad_norm_0: tensor(15.2579, device='cuda:0') (tensor(11.0400, device='cuda:0')) + | > loss_gen: 2.285541296005249 (2.3580028982389543) + | > loss_kl: 2.0432610511779785 (1.8794578384785425) + | > loss_feat: 7.875851154327393 (7.051294071333749) + | > loss_mel: 18.77978515625 (17.503108796619237) + | > loss_duration: 1.5255446434020996 (1.460187704790206) + | > amp_scaler: 512.0 (356.57142857142856) + | > loss_1: 32.50998306274414 (30.25205135345459) + | > grad_norm_1: tensor(341.6193, device='cuda:0') (tensor(239.0866, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.3941 (0.36676853327524095) + | > loader_time: 0.0063 (0.00581488155183338) + + + --> TIME: 2025-04-19 02:44:21 -- STEP: 109/142 -- GLOBAL_STEP: 411625 + | > loss_disc: 2.469367027282715 (2.452162274526896) + | > loss_disc_real_0: 0.1242247223854065 (0.1220016804173452) + | > loss_disc_real_1: 0.2275766283273697 (0.2067496344583844) + | > loss_disc_real_2: 0.2397763878107071 (0.22688005632216776) + | > loss_disc_real_3: 0.24600574374198914 (0.22675008593349283) + | > loss_disc_real_4: 0.24085494875907898 (0.2295022066853462) + | > loss_disc_real_5: 0.2553468346595764 (0.23287210469945854) + | > loss_0: 2.469367027282715 (2.452162274526896) + | > grad_norm_0: tensor(13.3166, device='cuda:0') (tensor(10.7869, device='cuda:0')) + | > loss_gen: 2.443955183029175 (2.353928185384208) + | > loss_kl: 1.9770288467407227 (1.88563701209672) + | > loss_feat: 6.656260013580322 (7.04944908947026) + | > loss_mel: 17.943408966064453 (17.60082136381656) + | > loss_duration: 1.5113916397094727 (1.468379074280415) + | > amp_scaler: 256.0 (342.89908256880733) + | > loss_1: 30.53204345703125 (30.358214737078466) + | > grad_norm_1: tensor(212.7717, device='cuda:0') (tensor(235.3867, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.4127 (0.3760900672422637) + | > loader_time: 0.007 (0.0060616589467459864) + + + --> TIME: 2025-04-19 02:44:32 -- STEP: 134/142 -- GLOBAL_STEP: 411650 + | > loss_disc: 2.602525472640991 (2.456477492602905) + | > loss_disc_real_0: 0.10527248680591583 (0.12479382742251922) + | > loss_disc_real_1: 0.2408260703086853 (0.20739867738378584) + | > loss_disc_real_2: 0.26204025745391846 (0.22626673074355766) + | > loss_disc_real_3: 0.2356545627117157 (0.22727716947669413) + | > loss_disc_real_4: 0.21564701199531555 (0.2290752347725541) + | > loss_disc_real_5: 0.2500593960285187 (0.23265080543151542) + | > loss_0: 2.602525472640991 (2.456477492602905) + | > grad_norm_0: tensor(8.7065, device='cuda:0') (tensor(11.4019, device='cuda:0')) + | > loss_gen: 2.221992254257202 (2.3495694808105925) + | > loss_kl: 1.9988157749176025 (1.8823659428909643) + | > loss_feat: 6.196990013122559 (7.043257958853422) + | > loss_mel: 16.3828125 (17.665686080704866) + | > loss_duration: 1.532972812652588 (1.4749363296067537) + | > amp_scaler: 256.0 (326.6865671641791) + | > loss_1: 28.333581924438477 (30.415815723476125) + | > grad_norm_1: tensor(267.4645, device='cuda:0') (tensor(240.4752, device='cuda:0')) + | > current_lr_0: 0.00019440441311212466 + | > current_lr_1: 0.00019440441311212466 + | > step_time: 0.4418 (0.3875169166878088) + | > loader_time: 0.0077 (0.006314848785969748) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6872079372406006 (2.6872079372406006) + | > loss_disc_real_0: 0.28109750151634216 (0.28109750151634216) + | > loss_disc_real_1: 0.29202333092689514 (0.29202333092689514) + | > loss_disc_real_2: 0.26453420519828796 (0.26453420519828796) + | > loss_disc_real_3: 0.24255353212356567 (0.24255353212356567) + | > loss_disc_real_4: 0.27847906947135925 (0.27847906947135925) + | > loss_disc_real_5: 0.29192599654197693 (0.29192599654197693) + | > loss_0: 2.6872079372406006 (2.6872079372406006) + | > loss_gen: 2.3897671699523926 (2.3897671699523926) + | > loss_kl: 2.190182685852051 (2.190182685852051) + | > loss_feat: 4.226986885070801 (4.226986885070801) + | > loss_mel: 12.272011756896973 (12.272011756896973) + | > loss_duration: 1.6702297925949097 (1.6702297925949097) + | > loss_1: 22.749177932739258 (22.749177932739258) + + --> STEP: 1 + | > loss_disc: 2.5077407360076904 (2.5077407360076904) + | > loss_disc_real_0: 0.144565150141716 (0.144565150141716) + | > loss_disc_real_1: 0.33841967582702637 (0.33841967582702637) + | > loss_disc_real_2: 0.25883370637893677 (0.25883370637893677) + | > loss_disc_real_3: 0.23583170771598816 (0.23583170771598816) + | > loss_disc_real_4: 0.2487742304801941 (0.2487742304801941) + | > loss_disc_real_5: 0.2069241851568222 (0.2069241851568222) + | > loss_0: 2.5077407360076904 (2.5077407360076904) + | > loss_gen: 2.5051281452178955 (2.5051281452178955) + | > loss_kl: 2.6514511108398438 (2.6514511108398438) + | > loss_feat: 6.848313331604004 (6.848313331604004) + | > loss_mel: 18.590044021606445 (18.590044021606445) + | > loss_duration: 1.738457202911377 (1.738457202911377) + | > loss_1: 32.333396911621094 (32.333396911621094) + + --> STEP: 2 + | > loss_disc: 2.4862754344940186 (2.4970080852508545) + | > loss_disc_real_0: 0.2184358686208725 (0.18150050938129425) + | > loss_disc_real_1: 0.3043467402458191 (0.32138320803642273) + | > loss_disc_real_2: 0.23302224278450012 (0.24592797458171844) + | > loss_disc_real_3: 0.20218263566493988 (0.21900717169046402) + | > loss_disc_real_4: 0.267751544713974 (0.25826288759708405) + | > loss_disc_real_5: 0.20920881628990173 (0.20806650072336197) + | > loss_0: 2.4862754344940186 (2.4970080852508545) + | > loss_gen: 2.4647157192230225 (2.484921932220459) + | > loss_kl: 2.482194185256958 (2.566822648048401) + | > loss_feat: 6.732052803039551 (6.790183067321777) + | > loss_mel: 16.90663719177246 (17.748340606689453) + | > loss_duration: 1.6788151264190674 (1.7086361646652222) + | > loss_1: 30.264415740966797 (31.298906326293945) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035924911499023438 (-0.00018453598022460938) + | > avg_loss_disc: 2.4970080852508545 (+0.05563998222351074) + | > avg_loss_disc_real_0: 0.18150050938129425 (+0.08011356368660927) + | > avg_loss_disc_real_1: 0.32138320803642273 (+0.051600828766822815) + | > avg_loss_disc_real_2: 0.24592797458171844 (+0.003975905478000641) + | > avg_loss_disc_real_3: 0.21900717169046402 (+0.003750428557395935) + | > avg_loss_disc_real_4: 0.25826288759708405 (+0.04527191072702408) + | > avg_loss_disc_real_5: 0.20806650072336197 (-0.03885061293840408) + | > avg_loss_0: 2.4970080852508545 (+0.05563998222351074) + | > avg_loss_gen: 2.484921932220459 (+0.11884808540344238) + | > avg_loss_kl: 2.566822648048401 (+0.08639073371887207) + | > avg_loss_feat: 6.790183067321777 (-0.8525078296661377) + | > avg_loss_mel: 17.748340606689453 (-0.99591064453125) + | > avg_loss_duration: 1.7086361646652222 (-0.0020318031311035156) + | > avg_loss_1: 31.298906326293945 (-1.6452102661132812) + + + > EPOCH: 228/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:44:37)  + + --> TIME: 2025-04-19 02:44:43 -- STEP: 17/142 -- GLOBAL_STEP: 411675 + | > loss_disc: 2.380265235900879 (2.402169774560367) + | > loss_disc_real_0: 0.0972951203584671 (0.1103496748734923) + | > loss_disc_real_1: 0.18056228756904602 (0.20416564976467805) + | > loss_disc_real_2: 0.23362717032432556 (0.23002770017175114) + | > loss_disc_real_3: 0.2011871039867401 (0.22766965101746953) + | > loss_disc_real_4: 0.18477711081504822 (0.22664379372316248) + | > loss_disc_real_5: 0.21424996852874756 (0.2340888021623387) + | > loss_0: 2.380265235900879 (2.402169774560367) + | > grad_norm_0: tensor(6.7993, device='cuda:0') (tensor(10.2739, device='cuda:0')) + | > loss_gen: 2.4044456481933594 (2.4325478778165928) + | > loss_kl: 1.6205190420150757 (1.7908450014450972) + | > loss_feat: 7.204809665679932 (7.436288048239315) + | > loss_mel: 17.138893127441406 (17.83823854783002) + | > loss_duration: 1.4266362190246582 (1.422746917780708) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.795303344726562 (30.920666189754712) + | > grad_norm_1: tensor(166.2984, device='cuda:0') (tensor(215.7125, device='cuda:0')) + | > current_lr_0: 0.00019438011256048564 + | > current_lr_1: 0.00019438011256048564 + | > step_time: 0.3376 (0.3284437235663919) + | > loader_time: 0.0051 (0.004877988029928768) + + + --> TIME: 2025-04-19 02:44:52 -- STEP: 42/142 -- GLOBAL_STEP: 411700 + | > loss_disc: 2.391462802886963 (2.4225158975237897) + | > loss_disc_real_0: 0.09043657779693604 (0.1157711712377412) + | > loss_disc_real_1: 0.2240338921546936 (0.20935201254628955) + | > loss_disc_real_2: 0.24403737485408783 (0.22631274305638813) + | > loss_disc_real_3: 0.23233631253242493 (0.22500858846164884) + | > loss_disc_real_4: 0.2108847200870514 (0.22665221634365262) + | > loss_disc_real_5: 0.22006064653396606 (0.22723362985111417) + | > loss_0: 2.391462802886963 (2.4225158975237897) + | > grad_norm_0: tensor(8.8808, device='cuda:0') (tensor(9.8959, device='cuda:0')) + | > loss_gen: 2.5333504676818848 (2.4111166908627464) + | > loss_kl: 1.8724563121795654 (1.8372243813105993) + | > loss_feat: 7.116474628448486 (7.2809232870737715) + | > loss_mel: 18.456356048583984 (17.877885364350817) + | > loss_duration: 1.4792224168777466 (1.4371045913015095) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.45785903930664 (30.844254130408878) + | > grad_norm_1: tensor(179.5375, device='cuda:0') (tensor(228.2165, device='cuda:0')) + | > current_lr_0: 0.00019438011256048564 + | > current_lr_1: 0.00019438011256048564 + | > step_time: 0.3747 (0.3418921629587809) + | > loader_time: 0.0059 (0.005323097819373722) + + + --> TIME: 2025-04-19 02:45:02 -- STEP: 67/142 -- GLOBAL_STEP: 411725 + | > loss_disc: 2.5511507987976074 (2.431150674819947) + | > loss_disc_real_0: 0.06361036002635956 (0.1203549216265109) + | > loss_disc_real_1: 0.20653487741947174 (0.21035663249777325) + | > loss_disc_real_2: 0.20390084385871887 (0.22570895014414147) + | > loss_disc_real_3: 0.23463156819343567 (0.22664798707214753) + | > loss_disc_real_4: 0.21803905069828033 (0.22544573158470552) + | > loss_disc_real_5: 0.2756783664226532 (0.2263151393897498) + | > loss_0: 2.5511507987976074 (2.431150674819947) + | > grad_norm_0: tensor(19.8095, device='cuda:0') (tensor(9.5363, device='cuda:0')) + | > loss_gen: 2.2420907020568848 (2.3918754520700944) + | > loss_kl: 1.8395707607269287 (1.8600262581412472) + | > loss_feat: 7.075896263122559 (7.216463736633756) + | > loss_mel: 17.780122756958008 (17.846256142231958) + | > loss_duration: 1.4220242500305176 (1.4498747355902375) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.359704971313477 (30.764496276627725) + | > grad_norm_1: tensor(337.1095, device='cuda:0') (tensor(186.7253, device='cuda:0')) + | > current_lr_0: 0.00019438011256048564 + | > current_lr_1: 0.00019438011256048564 + | > step_time: 0.3835 (0.35421262570281514) + | > loader_time: 0.0063 (0.00561020267543508) + + + --> TIME: 2025-04-19 02:45:12 -- STEP: 92/142 -- GLOBAL_STEP: 411750 + | > loss_disc: 2.4546258449554443 (2.445567032565241) + | > loss_disc_real_0: 0.17335854470729828 (0.12391076609492302) + | > loss_disc_real_1: 0.23536312580108643 (0.20934312661056934) + | > loss_disc_real_2: 0.25367534160614014 (0.22645602553434993) + | > loss_disc_real_3: 0.21333830058574677 (0.22692645049613455) + | > loss_disc_real_4: 0.21418726444244385 (0.22717407523937846) + | > loss_disc_real_5: 0.23395158350467682 (0.22826029862398686) + | > loss_0: 2.4546258449554443 (2.445567032565241) + | > grad_norm_0: tensor(6.5904, device='cuda:0') (tensor(9.9402, device='cuda:0')) + | > loss_gen: 2.4260315895080566 (2.3847481737966123) + | > loss_kl: 2.0358469486236572 (1.8763780671617258) + | > loss_feat: 7.159862995147705 (7.172048858974291) + | > loss_mel: 18.231271743774414 (17.851046085357677) + | > loss_duration: 1.4486347436904907 (1.462904245957085) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.30164909362793 (30.74712545975395) + | > grad_norm_1: tensor(305.9074, device='cuda:0') (tensor(205.4895, device='cuda:0')) + | > current_lr_0: 0.00019438011256048564 + | > current_lr_1: 0.00019438011256048564 + | > step_time: 0.4092 (0.3655179428017657) + | > loader_time: 0.0068 (0.005833099717679231) + + + --> TIME: 2025-04-19 02:45:23 -- STEP: 117/142 -- GLOBAL_STEP: 411775 + | > loss_disc: 2.543347120285034 (2.448425967469175) + | > loss_disc_real_0: 0.14984352886676788 (0.12441598700407223) + | > loss_disc_real_1: 0.2580464780330658 (0.20886483928586683) + | > loss_disc_real_2: 0.2763971984386444 (0.2281386071545446) + | > loss_disc_real_3: 0.24017393589019775 (0.2271183848890484) + | > loss_disc_real_4: 0.25531506538391113 (0.22798702044364733) + | > loss_disc_real_5: 0.22747905552387238 (0.22653425911552885) + | > loss_0: 2.543347120285034 (2.448425967469175) + | > grad_norm_0: tensor(9.6649, device='cuda:0') (tensor(10.3271, device='cuda:0')) + | > loss_gen: 2.4116530418395996 (2.3738700935983252) + | > loss_kl: 1.8112552165985107 (1.881645501169384) + | > loss_feat: 7.0264787673950195 (7.1226127534850034) + | > loss_mel: 18.026933670043945 (17.86461187020327) + | > loss_duration: 1.4956437349319458 (1.470209554729299) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.77196502685547 (30.71294978541187) + | > grad_norm_1: tensor(291.0316, device='cuda:0') (tensor(224.1054, device='cuda:0')) + | > current_lr_0: 0.00019438011256048564 + | > current_lr_1: 0.00019438011256048564 + | > step_time: 0.4328 (0.37615128867646563) + | > loader_time: 0.0068 (0.006046794418595795) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3614754676818848 (2.3614754676818848) + | > loss_disc_real_0: 0.0938548818230629 (0.0938548818230629) + | > loss_disc_real_1: 0.2503904104232788 (0.2503904104232788) + | > loss_disc_real_2: 0.23985739052295685 (0.23985739052295685) + | > loss_disc_real_3: 0.2249174863100052 (0.2249174863100052) + | > loss_disc_real_4: 0.21314318478107452 (0.21314318478107452) + | > loss_disc_real_5: 0.22051076591014862 (0.22051076591014862) + | > loss_0: 2.3614754676818848 (2.3614754676818848) + | > loss_gen: 2.4210500717163086 (2.4210500717163086) + | > loss_kl: 2.3620755672454834 (2.3620755672454834) + | > loss_feat: 7.366026878356934 (7.366026878356934) + | > loss_mel: 17.47556495666504 (17.47556495666504) + | > loss_duration: 1.7170617580413818 (1.7170617580413818) + | > loss_1: 31.341779708862305 (31.341779708862305) + + --> STEP: 1 + | > loss_disc: 2.395599365234375 (2.395599365234375) + | > loss_disc_real_0: 0.14921002089977264 (0.14921002089977264) + | > loss_disc_real_1: 0.2641926109790802 (0.2641926109790802) + | > loss_disc_real_2: 0.2670472264289856 (0.2670472264289856) + | > loss_disc_real_3: 0.23176315426826477 (0.23176315426826477) + | > loss_disc_real_4: 0.21370579302310944 (0.21370579302310944) + | > loss_disc_real_5: 0.2438105344772339 (0.2438105344772339) + | > loss_0: 2.395599365234375 (2.395599365234375) + | > loss_gen: 2.657411575317383 (2.657411575317383) + | > loss_kl: 2.7320048809051514 (2.7320048809051514) + | > loss_feat: 7.3577141761779785 (7.3577141761779785) + | > loss_mel: 18.919408798217773 (18.919408798217773) + | > loss_duration: 1.7453134059906006 (1.7453134059906006) + | > loss_1: 33.4118537902832 (33.4118537902832) + + --> STEP: 2 + | > loss_disc: 2.484374523162842 (2.4399869441986084) + | > loss_disc_real_0: 0.16485890746116638 (0.1570344641804695) + | > loss_disc_real_1: 0.21368330717086792 (0.23893795907497406) + | > loss_disc_real_2: 0.26110589504241943 (0.2640765607357025) + | > loss_disc_real_3: 0.26753711700439453 (0.24965013563632965) + | > loss_disc_real_4: 0.21518468856811523 (0.21444524079561234) + | > loss_disc_real_5: 0.22188478708267212 (0.232847660779953) + | > loss_0: 2.484374523162842 (2.4399869441986084) + | > loss_gen: 2.348421335220337 (2.50291645526886) + | > loss_kl: 2.1901862621307373 (2.4610955715179443) + | > loss_feat: 7.402911186218262 (7.38031268119812) + | > loss_mel: 17.375761032104492 (18.147584915161133) + | > loss_duration: 1.6757087707519531 (1.7105110883712769) + | > loss_1: 30.99298858642578 (32.20242118835449) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003579854965209961 (-1.2636184692382812e-05) + | > avg_loss_disc: 2.4399869441986084 (-0.057021141052246094) + | > avg_loss_disc_real_0: 0.1570344641804695 (-0.024466045200824738) + | > avg_loss_disc_real_1: 0.23893795907497406 (-0.08244524896144867) + | > avg_loss_disc_real_2: 0.2640765607357025 (+0.01814858615398407) + | > avg_loss_disc_real_3: 0.24965013563632965 (+0.03064296394586563) + | > avg_loss_disc_real_4: 0.21444524079561234 (-0.04381764680147171) + | > avg_loss_disc_real_5: 0.232847660779953 (+0.024781160056591034) + | > avg_loss_0: 2.4399869441986084 (-0.057021141052246094) + | > avg_loss_gen: 2.50291645526886 (+0.01799452304840088) + | > avg_loss_kl: 2.4610955715179443 (-0.10572707653045654) + | > avg_loss_feat: 7.38031268119812 (+0.5901296138763428) + | > avg_loss_mel: 18.147584915161133 (+0.3992443084716797) + | > avg_loss_duration: 1.7105110883712769 (+0.0018749237060546875) + | > avg_loss_1: 32.20242118835449 (+0.9035148620605469) + + + > EPOCH: 229/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:45:36)  + + --> TIME: 2025-04-19 02:45:36 -- STEP: 0/142 -- GLOBAL_STEP: 411800 + | > loss_disc: 2.3585727214813232 (2.3585727214813232) + | > loss_disc_real_0: 0.12007732689380646 (0.12007732689380646) + | > loss_disc_real_1: 0.24833886325359344 (0.24833886325359344) + | > loss_disc_real_2: 0.24123434722423553 (0.24123434722423553) + | > loss_disc_real_3: 0.2364865243434906 (0.2364865243434906) + | > loss_disc_real_4: 0.2136397361755371 (0.2136397361755371) + | > loss_disc_real_5: 0.2382313758134842 (0.2382313758134842) + | > loss_0: 2.3585727214813232 (2.3585727214813232) + | > grad_norm_0: tensor(5.1118, device='cuda:0') (tensor(5.1118, device='cuda:0')) + | > loss_gen: 2.5030784606933594 (2.5030784606933594) + | > loss_kl: 1.6566755771636963 (1.6566755771636963) + | > loss_feat: 7.648078441619873 (7.648078441619873) + | > loss_mel: 18.162586212158203 (18.162586212158203) + | > loss_duration: 1.4534237384796143 (1.4534237384796143) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.423843383789062 (31.423843383789062) + | > grad_norm_1: tensor(223.6902, device='cuda:0') (tensor(223.6902, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.356 (0.3559610843658447) + | > loader_time: 0.314 (0.31400012969970703) + + + --> TIME: 2025-04-19 02:45:45 -- STEP: 25/142 -- GLOBAL_STEP: 411825 + | > loss_disc: 2.219874382019043 (2.401043081283569) + | > loss_disc_real_0: 0.08814837783575058 (0.11236097276210785) + | > loss_disc_real_1: 0.18863454461097717 (0.20070238053798675) + | > loss_disc_real_2: 0.1984667181968689 (0.22715366125106812) + | > loss_disc_real_3: 0.19058720767498016 (0.22285451173782347) + | > loss_disc_real_4: 0.18538926541805267 (0.22641707241535186) + | > loss_disc_real_5: 0.21607060730457306 (0.22825927793979645) + | > loss_0: 2.219874382019043 (2.401043081283569) + | > grad_norm_0: tensor(4.2067, device='cuda:0') (tensor(6.4974, device='cuda:0')) + | > loss_gen: 2.459995746612549 (2.4050719738006596) + | > loss_kl: 1.8245385885238647 (1.8658613109588622) + | > loss_feat: 8.389748573303223 (7.367962265014649) + | > loss_mel: 18.808429718017578 (17.899837417602534) + | > loss_duration: 1.463473916053772 (1.4282335662841796) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.94618606567383 (30.966966552734377) + | > grad_norm_1: tensor(273.1441, device='cuda:0') (tensor(168.9312, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.345 (0.3340765953063965) + | > loader_time: 0.0054 (0.005057239532470703) + + + --> TIME: 2025-04-19 02:45:54 -- STEP: 50/142 -- GLOBAL_STEP: 411850 + | > loss_disc: 2.5001614093780518 (2.4274723434448244) + | > loss_disc_real_0: 0.1055401936173439 (0.11757412239909172) + | > loss_disc_real_1: 0.18829382956027985 (0.20493044078350067) + | > loss_disc_real_2: 0.21958903968334198 (0.22608213186264037) + | > loss_disc_real_3: 0.19642622768878937 (0.22674290359020233) + | > loss_disc_real_4: 0.20808611810207367 (0.2260015431046486) + | > loss_disc_real_5: 0.20056456327438354 (0.22982816010713578) + | > loss_0: 2.5001614093780518 (2.4274723434448244) + | > grad_norm_0: tensor(14.4730, device='cuda:0') (tensor(7.7703, device='cuda:0')) + | > loss_gen: 2.2223637104034424 (2.3925838851928716) + | > loss_kl: 1.9666595458984375 (1.8801429128646852) + | > loss_feat: 7.165289878845215 (7.254443798065186) + | > loss_mel: 17.214780807495117 (17.89578197479248) + | > loss_duration: 1.4837262630462646 (1.4388063406944276) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.052820205688477 (30.861759033203125) + | > grad_norm_1: tensor(279.4473, device='cuda:0') (tensor(205.8607, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.3899 (0.3493911647796631) + | > loader_time: 0.009 (0.005503730773925781) + + + --> TIME: 2025-04-19 02:46:04 -- STEP: 75/142 -- GLOBAL_STEP: 411875 + | > loss_disc: 2.3739521503448486 (2.4303063646952316) + | > loss_disc_real_0: 0.12084479629993439 (0.11941521803538005) + | > loss_disc_real_1: 0.22731950879096985 (0.20443103313446043) + | > loss_disc_real_2: 0.18581528961658478 (0.22510334273179372) + | > loss_disc_real_3: 0.23072540760040283 (0.22670396625995637) + | > loss_disc_real_4: 0.21292302012443542 (0.22615421096483868) + | > loss_disc_real_5: 0.23678578436374664 (0.22860363960266114) + | > loss_0: 2.3739521503448486 (2.4303063646952316) + | > grad_norm_0: tensor(11.3814, device='cuda:0') (tensor(8.6551, device='cuda:0')) + | > loss_gen: 2.468813896179199 (2.39224284807841) + | > loss_kl: 1.8238158226013184 (1.8917334334055582) + | > loss_feat: 7.75600004196167 (7.28759267171224) + | > loss_mel: 18.227455139160156 (17.91426628112793) + | > loss_duration: 1.444730520248413 (1.4527484242121378) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.720815658569336 (30.93858378092448) + | > grad_norm_1: tensor(481.0298, device='cuda:0') (tensor(222.3731, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.3979 (0.3608199055989583) + | > loader_time: 0.0067 (0.005777591069539388) + + + --> TIME: 2025-04-19 02:46:14 -- STEP: 100/142 -- GLOBAL_STEP: 411900 + | > loss_disc: 2.3879692554473877 (2.4449582695961007) + | > loss_disc_real_0: 0.11585447937250137 (0.12252040795981883) + | > loss_disc_real_1: 0.16336165368556976 (0.20619873881340026) + | > loss_disc_real_2: 0.22545379400253296 (0.2259204037487507) + | > loss_disc_real_3: 0.21433480083942413 (0.22786550492048263) + | > loss_disc_real_4: 0.21700023114681244 (0.22784197509288787) + | > loss_disc_real_5: 0.24348723888397217 (0.22963071435689927) + | > loss_0: 2.3879692554473877 (2.4449582695961007) + | > grad_norm_0: tensor(7.4054, device='cuda:0') (tensor(9.1483, device='cuda:0')) + | > loss_gen: 2.4940032958984375 (2.372812304496765) + | > loss_kl: 1.9557560682296753 (1.9043549156188966) + | > loss_feat: 8.322948455810547 (7.163381972312927) + | > loss_mel: 18.749359130859375 (17.795774898529046) + | > loss_duration: 1.5038909912109375 (1.4650728011131284) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.02595901489258 (30.70139711380005) + | > grad_norm_1: tensor(147.4319, device='cuda:0') (tensor(220.6005, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.416 (0.37072049140930174) + | > loader_time: 0.0071 (0.006051261425018311) + + + --> TIME: 2025-04-19 02:46:25 -- STEP: 125/142 -- GLOBAL_STEP: 411925 + | > loss_disc: 2.431915760040283 (2.449450736999512) + | > loss_disc_real_0: 0.1585380882024765 (0.12449786245822907) + | > loss_disc_real_1: 0.19726507365703583 (0.20584293711185456) + | > loss_disc_real_2: 0.24973390996456146 (0.2257141877412796) + | > loss_disc_real_3: 0.2546845078468323 (0.22891637825965883) + | > loss_disc_real_4: 0.22520427405834198 (0.22796431803703307) + | > loss_disc_real_5: 0.23200400173664093 (0.23002025973796844) + | > loss_0: 2.431915760040283 (2.449450736999512) + | > grad_norm_0: tensor(6.7514, device='cuda:0') (tensor(8.7486, device='cuda:0')) + | > loss_gen: 2.4705898761749268 (2.3619986591339113) + | > loss_kl: 1.8220640420913696 (1.9156857576370239) + | > loss_feat: 7.107917308807373 (7.070520244598389) + | > loss_mel: 18.135276794433594 (17.694321495056144) + | > loss_duration: 1.536025047302246 (1.4729377517700193) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.07187271118164 (30.515464111328125) + | > grad_norm_1: tensor(190.1883, device='cuda:0') (tensor(203.9025, device='cuda:0')) + | > current_lr_0: 0.00019435581504641556 + | > current_lr_1: 0.00019435581504641556 + | > step_time: 0.4364 (0.3818652610778809) + | > loader_time: 0.0078 (0.006288009643554688) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5281524658203125 (2.5281524658203125) + | > loss_disc_real_0: 0.17597921192646027 (0.17597921192646027) + | > loss_disc_real_1: 0.231358140707016 (0.231358140707016) + | > loss_disc_real_2: 0.2109007090330124 (0.2109007090330124) + | > loss_disc_real_3: 0.19259047508239746 (0.19259047508239746) + | > loss_disc_real_4: 0.21300117671489716 (0.21300117671489716) + | > loss_disc_real_5: 0.23498579859733582 (0.23498579859733582) + | > loss_0: 2.5281524658203125 (2.5281524658203125) + | > loss_gen: 2.2749125957489014 (2.2749125957489014) + | > loss_kl: 2.3234517574310303 (2.3234517574310303) + | > loss_feat: 7.037133693695068 (7.037133693695068) + | > loss_mel: 19.11511993408203 (19.11511993408203) + | > loss_duration: 1.6487380266189575 (1.6487380266189575) + | > loss_1: 32.39935302734375 (32.39935302734375) + + --> STEP: 1 + | > loss_disc: 2.5552735328674316 (2.5552735328674316) + | > loss_disc_real_0: 0.23497216403484344 (0.23497216403484344) + | > loss_disc_real_1: 0.23361679911613464 (0.23361679911613464) + | > loss_disc_real_2: 0.24494647979736328 (0.24494647979736328) + | > loss_disc_real_3: 0.18368841707706451 (0.18368841707706451) + | > loss_disc_real_4: 0.2642766535282135 (0.2642766535282135) + | > loss_disc_real_5: 0.288897305727005 (0.288897305727005) + | > loss_0: 2.5552735328674316 (2.5552735328674316) + | > loss_gen: 2.386049270629883 (2.386049270629883) + | > loss_kl: 2.6061441898345947 (2.6061441898345947) + | > loss_feat: 6.248952388763428 (6.248952388763428) + | > loss_mel: 17.336347579956055 (17.336347579956055) + | > loss_duration: 1.720139503479004 (1.720139503479004) + | > loss_1: 30.29763412475586 (30.29763412475586) + + --> STEP: 2 + | > loss_disc: 2.464897632598877 (2.5100855827331543) + | > loss_disc_real_0: 0.21386611461639404 (0.22441913932561874) + | > loss_disc_real_1: 0.26003575325012207 (0.24682627618312836) + | > loss_disc_real_2: 0.24446845054626465 (0.24470746517181396) + | > loss_disc_real_3: 0.1685471385717392 (0.17611777782440186) + | > loss_disc_real_4: 0.217606320977211 (0.24094148725271225) + | > loss_disc_real_5: 0.19817420840263367 (0.24353575706481934) + | > loss_0: 2.464897632598877 (2.5100855827331543) + | > loss_gen: 2.347900629043579 (2.366974949836731) + | > loss_kl: 2.0771501064300537 (2.341647148132324) + | > loss_feat: 6.963179588317871 (6.606065988540649) + | > loss_mel: 16.79733657836914 (17.066842079162598) + | > loss_duration: 1.6880388259887695 (1.7040891647338867) + | > loss_1: 29.87360382080078 (30.08561897277832) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038121938705444336 (+0.00023233890533447266) + | > avg_loss_disc: 2.5100855827331543 (+0.0700986385345459) + | > avg_loss_disc_real_0: 0.22441913932561874 (+0.06738467514514923) + | > avg_loss_disc_real_1: 0.24682627618312836 (+0.007888317108154297) + | > avg_loss_disc_real_2: 0.24470746517181396 (-0.01936909556388855) + | > avg_loss_disc_real_3: 0.17611777782440186 (-0.0735323578119278) + | > avg_loss_disc_real_4: 0.24094148725271225 (+0.026496246457099915) + | > avg_loss_disc_real_5: 0.24353575706481934 (+0.010688096284866333) + | > avg_loss_0: 2.5100855827331543 (+0.0700986385345459) + | > avg_loss_gen: 2.366974949836731 (-0.1359415054321289) + | > avg_loss_kl: 2.341647148132324 (-0.11944842338562012) + | > avg_loss_feat: 6.606065988540649 (-0.7742466926574707) + | > avg_loss_mel: 17.066842079162598 (-1.0807428359985352) + | > avg_loss_duration: 1.7040891647338867 (-0.006421923637390137) + | > avg_loss_1: 30.08561897277832 (-2.116802215576172) + + + > EPOCH: 230/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:46:34)  + + --> TIME: 2025-04-19 02:46:38 -- STEP: 8/142 -- GLOBAL_STEP: 411950 + | > loss_disc: 2.3105955123901367 (2.434893161058426) + | > loss_disc_real_0: 0.08499717712402344 (0.11515927128493786) + | > loss_disc_real_1: 0.2104061245918274 (0.21057098172605038) + | > loss_disc_real_2: 0.2225741297006607 (0.22780399210751057) + | > loss_disc_real_3: 0.1926499307155609 (0.22597285732626915) + | > loss_disc_real_4: 0.22221116721630096 (0.21796988509595394) + | > loss_disc_real_5: 0.20497865974903107 (0.22195893712341785) + | > loss_0: 2.3105955123901367 (2.434893161058426) + | > grad_norm_0: tensor(6.2489, device='cuda:0') (tensor(7.3078, device='cuda:0')) + | > loss_gen: 2.574557304382324 (2.3735022246837616) + | > loss_kl: 1.81110417842865 (1.7504405081272125) + | > loss_feat: 7.772827625274658 (7.296448647975922) + | > loss_mel: 18.211233139038086 (17.674594163894653) + | > loss_duration: 1.433500051498413 (1.434559851884842) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.803224563598633 (30.529545545578003) + | > grad_norm_1: tensor(77.1792, device='cuda:0') (tensor(139.4789, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.3266 (0.3247823417186737) + | > loader_time: 0.0053 (0.004560530185699463) + + + --> TIME: 2025-04-19 02:46:46 -- STEP: 33/142 -- GLOBAL_STEP: 411975 + | > loss_disc: 2.3096861839294434 (2.4225910793651235) + | > loss_disc_real_0: 0.12130384892225266 (0.11706910214640877) + | > loss_disc_real_1: 0.20302343368530273 (0.19918520161599823) + | > loss_disc_real_2: 0.23098516464233398 (0.22530077713908572) + | > loss_disc_real_3: 0.21116231381893158 (0.23013797221761761) + | > loss_disc_real_4: 0.22341801226139069 (0.22668991757161688) + | > loss_disc_real_5: 0.2506546378135681 (0.23129069127819754) + | > loss_0: 2.3096861839294434 (2.4225910793651235) + | > grad_norm_0: tensor(10.3826, device='cuda:0') (tensor(7.5816, device='cuda:0')) + | > loss_gen: 2.520688533782959 (2.3949140418659565) + | > loss_kl: 1.8657976388931274 (1.8315438032150269) + | > loss_feat: 7.701928615570068 (7.345163937770959) + | > loss_mel: 18.18238639831543 (17.896685831474535) + | > loss_duration: 1.4332127571105957 (1.4385271830992266) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.70401382446289 (30.90683468905362) + | > grad_norm_1: tensor(407.4263, device='cuda:0') (tensor(155.0322, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.3646 (0.3407736908305775) + | > loader_time: 0.006 (0.005191810203321053) + + + --> TIME: 2025-04-19 02:46:56 -- STEP: 58/142 -- GLOBAL_STEP: 412000 + | > loss_disc: 2.5539252758026123 (2.430510767574968) + | > loss_disc_real_0: 0.1788710653781891 (0.11998080783363047) + | > loss_disc_real_1: 0.24244056642055511 (0.20431188391200428) + | > loss_disc_real_2: 0.23560500144958496 (0.22428228603354816) + | > loss_disc_real_3: 0.24334238469600677 (0.22909573773885594) + | > loss_disc_real_4: 0.23729732632637024 (0.22559852975195852) + | > loss_disc_real_5: 0.21639105677604675 (0.22854490269874705) + | > loss_0: 2.5539252758026123 (2.430510767574968) + | > grad_norm_0: tensor(24.5748, device='cuda:0') (tensor(9.6600, device='cuda:0')) + | > loss_gen: 2.202756404876709 (2.4005384609617058) + | > loss_kl: 1.9573341608047485 (1.861666675271659) + | > loss_feat: 5.544066429138184 (7.231273240056531) + | > loss_mel: 16.051555633544922 (17.79026935840475) + | > loss_duration: 1.4912798404693604 (1.450613728884993) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.246992111206055 (30.73436135259168) + | > grad_norm_1: tensor(229.0967, device='cuda:0') (tensor(224.7980, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.3736 (0.3556435724784588) + | > loader_time: 0.006 (0.005452225948202199) + + + --> TIME: 2025-04-19 02:47:06 -- STEP: 83/142 -- GLOBAL_STEP: 412025 + | > loss_disc: 2.515620470046997 (2.4486518997743905) + | > loss_disc_real_0: 0.16208945214748383 (0.11939667937267258) + | > loss_disc_real_1: 0.16731920838356018 (0.20546676685293036) + | > loss_disc_real_2: 0.2024695724248886 (0.22830551144588423) + | > loss_disc_real_3: 0.19933971762657166 (0.22910284188138433) + | > loss_disc_real_4: 0.2186259776353836 (0.22770903501884046) + | > loss_disc_real_5: 0.2237476259469986 (0.2288077330014792) + | > loss_0: 2.515620470046997 (2.4486518997743905) + | > grad_norm_0: tensor(19.1416, device='cuda:0') (tensor(12.0349, device='cuda:0')) + | > loss_gen: 2.154374361038208 (2.409906091460264) + | > loss_kl: 1.8225408792495728 (1.878566746252129) + | > loss_feat: 5.68311071395874 (7.190327454762286) + | > loss_mel: 15.473312377929688 (17.72775951063777) + | > loss_duration: 1.5488901138305664 (1.4616141477263116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.682228088378906 (30.668173824448182) + | > grad_norm_1: tensor(213.7403, device='cuda:0') (tensor(250.0476, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.3976 (0.36592110955571555) + | > loader_time: 0.0064 (0.00571958128228245) + + + --> TIME: 2025-04-19 02:47:16 -- STEP: 108/142 -- GLOBAL_STEP: 412050 + | > loss_disc: 2.388205051422119 (2.4566950444821964) + | > loss_disc_real_0: 0.14143161475658417 (0.12362788510681302) + | > loss_disc_real_1: 0.2179376780986786 (0.20643135905265808) + | > loss_disc_real_2: 0.2286718636751175 (0.22725827853988717) + | > loss_disc_real_3: 0.23711350560188293 (0.22940754821455037) + | > loss_disc_real_4: 0.20667825639247894 (0.22833304810855123) + | > loss_disc_real_5: 0.2147354632616043 (0.22823175539573035) + | > loss_0: 2.388205051422119 (2.4566950444821964) + | > grad_norm_0: tensor(25.6432, device='cuda:0') (tensor(14.4757, device='cuda:0')) + | > loss_gen: 2.3892500400543213 (2.401652071211075) + | > loss_kl: 1.9069432020187378 (1.8834490069636591) + | > loss_feat: 7.159440040588379 (7.123657844684742) + | > loss_mel: 19.22895622253418 (17.669340928395588) + | > loss_duration: 1.5098049640655518 (1.470021543679414) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.194393157958984 (30.54812122274328) + | > grad_norm_1: tensor(272.8683, device='cuda:0') (tensor(273.0941, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.4214 (0.3754902128820066) + | > loader_time: 0.0066 (0.00596774728209884) + + + --> TIME: 2025-04-19 02:47:28 -- STEP: 133/142 -- GLOBAL_STEP: 412075 + | > loss_disc: 2.5352320671081543 (2.457407144675578) + | > loss_disc_real_0: 0.12252328544855118 (0.12314876939233084) + | > loss_disc_real_1: 0.2473507672548294 (0.20742705912518322) + | > loss_disc_real_2: 0.23375476896762848 (0.2279469298016756) + | > loss_disc_real_3: 0.21900665760040283 (0.2286982930692515) + | > loss_disc_real_4: 0.2337721586227417 (0.2291251745887269) + | > loss_disc_real_5: 0.21495436131954193 (0.22792018994801028) + | > loss_0: 2.5352320671081543 (2.457407144675578) + | > grad_norm_0: tensor(8.5481, device='cuda:0') (tensor(14.2265, device='cuda:0')) + | > loss_gen: 2.263766050338745 (2.3939316918079134) + | > loss_kl: 2.0684967041015625 (1.8943532346782828) + | > loss_feat: 6.657097339630127 (7.10989717254065) + | > loss_mel: 16.822988510131836 (17.6791704364289) + | > loss_duration: 1.5595057010650635 (1.476167811486954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.37185287475586 (30.553520188295753) + | > grad_norm_1: tensor(230.1442, device='cuda:0') (tensor(270.2787, device='cuda:0')) + | > current_lr_0: 0.00019433152056953475 + | > current_lr_1: 0.00019433152056953475 + | > step_time: 0.4362 (0.3866773135680005) + | > loader_time: 0.0074 (0.006197388010813777) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.415390729904175 (2.415390729904175) + | > loss_disc_real_0: 0.1713540405035019 (0.1713540405035019) + | > loss_disc_real_1: 0.17835068702697754 (0.17835068702697754) + | > loss_disc_real_2: 0.225188747048378 (0.225188747048378) + | > loss_disc_real_3: 0.2180010974407196 (0.2180010974407196) + | > loss_disc_real_4: 0.2577279210090637 (0.2577279210090637) + | > loss_disc_real_5: 0.22327706217765808 (0.22327706217765808) + | > loss_0: 2.415390729904175 (2.415390729904175) + | > loss_gen: 2.420103073120117 (2.420103073120117) + | > loss_kl: 2.2693910598754883 (2.2693910598754883) + | > loss_feat: 6.231335163116455 (6.231335163116455) + | > loss_mel: 18.236770629882812 (18.236770629882812) + | > loss_duration: 1.678687334060669 (1.678687334060669) + | > loss_1: 30.836288452148438 (30.836288452148438) + + --> STEP: 1 + | > loss_disc: 2.5750153064727783 (2.5750153064727783) + | > loss_disc_real_0: 0.16973738372325897 (0.16973738372325897) + | > loss_disc_real_1: 0.23330272734165192 (0.23330272734165192) + | > loss_disc_real_2: 0.2531610429286957 (0.2531610429286957) + | > loss_disc_real_3: 0.20231588184833527 (0.20231588184833527) + | > loss_disc_real_4: 0.23090024292469025 (0.23090024292469025) + | > loss_disc_real_5: 0.24878007173538208 (0.24878007173538208) + | > loss_0: 2.5750153064727783 (2.5750153064727783) + | > loss_gen: 2.3396263122558594 (2.3396263122558594) + | > loss_kl: 2.669541358947754 (2.669541358947754) + | > loss_feat: 7.021729946136475 (7.021729946136475) + | > loss_mel: 18.93587303161621 (18.93587303161621) + | > loss_duration: 1.729675054550171 (1.729675054550171) + | > loss_1: 32.69644546508789 (32.69644546508789) + + --> STEP: 2 + | > loss_disc: 2.504979133605957 (2.5399972200393677) + | > loss_disc_real_0: 0.16290274262428284 (0.1663200631737709) + | > loss_disc_real_1: 0.1926039159297943 (0.21295332163572311) + | > loss_disc_real_2: 0.22347185015678406 (0.23831644654273987) + | > loss_disc_real_3: 0.21225965023040771 (0.2072877660393715) + | > loss_disc_real_4: 0.2833099067211151 (0.2571050748229027) + | > loss_disc_real_5: 0.24339035153388977 (0.24608521163463593) + | > loss_0: 2.504979133605957 (2.5399972200393677) + | > loss_gen: 2.3506381511688232 (2.3451322317123413) + | > loss_kl: 2.319688320159912 (2.494614839553833) + | > loss_feat: 6.259632110595703 (6.640681028366089) + | > loss_mel: 17.672163009643555 (18.304018020629883) + | > loss_duration: 1.6629993915557861 (1.6963372230529785) + | > loss_1: 30.265121459960938 (31.480783462524414) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00372922420501709 (-8.296966552734375e-05) + | > avg_loss_disc: 2.5399972200393677 (+0.02991163730621338) + | > avg_loss_disc_real_0: 0.1663200631737709 (-0.05809907615184784) + | > avg_loss_disc_real_1: 0.21295332163572311 (-0.03387295454740524) + | > avg_loss_disc_real_2: 0.23831644654273987 (-0.006391018629074097) + | > avg_loss_disc_real_3: 0.2072877660393715 (+0.031169988214969635) + | > avg_loss_disc_real_4: 0.2571050748229027 (+0.01616358757019043) + | > avg_loss_disc_real_5: 0.24608521163463593 (+0.0025494545698165894) + | > avg_loss_0: 2.5399972200393677 (+0.02991163730621338) + | > avg_loss_gen: 2.3451322317123413 (-0.02184271812438965) + | > avg_loss_kl: 2.494614839553833 (+0.1529676914215088) + | > avg_loss_feat: 6.640681028366089 (+0.03461503982543945) + | > avg_loss_mel: 18.304018020629883 (+1.2371759414672852) + | > avg_loss_duration: 1.6963372230529785 (-0.007751941680908203) + | > avg_loss_1: 31.480783462524414 (+1.3951644897460938) + + + > EPOCH: 231/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:47:33)  + + --> TIME: 2025-04-19 02:47:39 -- STEP: 16/142 -- GLOBAL_STEP: 412100 + | > loss_disc: 2.4282140731811523 (2.418007120490074) + | > loss_disc_real_0: 0.10970864444971085 (0.12003287207335234) + | > loss_disc_real_1: 0.20295153558254242 (0.20809551607817411) + | > loss_disc_real_2: 0.2505651116371155 (0.22236873861402273) + | > loss_disc_real_3: 0.24292488396167755 (0.22612101677805185) + | > loss_disc_real_4: 0.22047990560531616 (0.2287412090227008) + | > loss_disc_real_5: 0.20006097853183746 (0.21684350818395615) + | > loss_0: 2.4282140731811523 (2.418007120490074) + | > grad_norm_0: tensor(7.9141, device='cuda:0') (tensor(9.6364, device='cuda:0')) + | > loss_gen: 2.51033878326416 (2.4141079783439636) + | > loss_kl: 1.8399351835250854 (1.8413098976016045) + | > loss_feat: 7.662294864654541 (7.265801191329956) + | > loss_mel: 18.224821090698242 (18.069206714630127) + | > loss_duration: 1.412076711654663 (1.4176970645785332) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.649465560913086 (31.00812304019928) + | > grad_norm_1: tensor(248.7170, device='cuda:0') (tensor(213.1495, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.3403 (0.3298527002334595) + | > loader_time: 0.005 (0.004556089639663696) + + + --> TIME: 2025-04-19 02:47:48 -- STEP: 41/142 -- GLOBAL_STEP: 412125 + | > loss_disc: 2.4679834842681885 (2.4291374101871397) + | > loss_disc_real_0: 0.12059807777404785 (0.12165533024363401) + | > loss_disc_real_1: 0.2167789787054062 (0.20691372526855004) + | > loss_disc_real_2: 0.21534670889377594 (0.2238470930151823) + | > loss_disc_real_3: 0.2447585016489029 (0.22537255577924775) + | > loss_disc_real_4: 0.24958400428295135 (0.226490294061056) + | > loss_disc_real_5: 0.2188333421945572 (0.22470470863144573) + | > loss_0: 2.4679834842681885 (2.4291374101871397) + | > grad_norm_0: tensor(4.7117, device='cuda:0') (tensor(7.9906, device='cuda:0')) + | > loss_gen: 2.277604579925537 (2.371449453074758) + | > loss_kl: 1.7327075004577637 (1.8543959303600033) + | > loss_feat: 6.728116035461426 (7.049154188574814) + | > loss_mel: 16.620168685913086 (17.617994983021806) + | > loss_duration: 1.4391562938690186 (1.4338509454959776) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.797752380371094 (30.326845448191573) + | > grad_norm_1: tensor(62.3466, device='cuda:0') (tensor(176.3081, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.3612 (0.34199020920730216) + | > loader_time: 0.0052 (0.004984774240633336) + + + --> TIME: 2025-04-19 02:47:58 -- STEP: 66/142 -- GLOBAL_STEP: 412150 + | > loss_disc: 2.389993906021118 (2.4391608888452705) + | > loss_disc_real_0: 0.08827798813581467 (0.12517900200504245) + | > loss_disc_real_1: 0.21549728512763977 (0.20688680311044058) + | > loss_disc_real_2: 0.17759113013744354 (0.2257674237092336) + | > loss_disc_real_3: 0.23609763383865356 (0.22713103371136117) + | > loss_disc_real_4: 0.26637837290763855 (0.22788919463302149) + | > loss_disc_real_5: 0.16730442643165588 (0.2262434819431016) + | > loss_0: 2.389993906021118 (2.4391608888452705) + | > grad_norm_0: tensor(8.1141, device='cuda:0') (tensor(7.4481, device='cuda:0')) + | > loss_gen: 2.340663194656372 (2.3687440698797055) + | > loss_kl: 1.928148627281189 (1.8656572771794868) + | > loss_feat: 8.079948425292969 (7.1064865372397685) + | > loss_mel: 18.63387680053711 (17.688974221547447) + | > loss_duration: 1.4428237676620483 (1.4481409065651172) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.42546081542969 (30.47800280831077) + | > grad_norm_1: tensor(180.6111, device='cuda:0') (tensor(152.2341, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.3896 (0.3541482578624379) + | > loader_time: 0.0056 (0.005283250953211929) + + + --> TIME: 2025-04-19 02:48:08 -- STEP: 91/142 -- GLOBAL_STEP: 412175 + | > loss_disc: 2.4314846992492676 (2.4471881206219024) + | > loss_disc_real_0: 0.14984945952892303 (0.12575681221026638) + | > loss_disc_real_1: 0.20435170829296112 (0.20723894451345717) + | > loss_disc_real_2: 0.25375401973724365 (0.2275335852588926) + | > loss_disc_real_3: 0.20838677883148193 (0.2276097107064593) + | > loss_disc_real_4: 0.2024814635515213 (0.22787305548950865) + | > loss_disc_real_5: 0.2233988493680954 (0.22722676740242884) + | > loss_0: 2.4314846992492676 (2.4471881206219024) + | > grad_norm_0: tensor(6.5836, device='cuda:0') (tensor(7.5182, device='cuda:0')) + | > loss_gen: 2.296050786972046 (2.3640458164634284) + | > loss_kl: 2.052372455596924 (1.887469761974209) + | > loss_feat: 7.009243965148926 (7.076928998087789) + | > loss_mel: 17.267473220825195 (17.724228439750252) + | > loss_duration: 1.4846787452697754 (1.4598614226330768) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.109819412231445 (30.512534361619217) + | > grad_norm_1: tensor(202.7832, device='cuda:0') (tensor(137.7931, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.4072 (0.366585128909939) + | > loader_time: 0.0061 (0.005609575208726823) + + + --> TIME: 2025-04-19 02:48:19 -- STEP: 116/142 -- GLOBAL_STEP: 412200 + | > loss_disc: 2.5019235610961914 (2.452498407199467) + | > loss_disc_real_0: 0.1743212193250656 (0.12759077208566252) + | > loss_disc_real_1: 0.1972874253988266 (0.20736493401486297) + | > loss_disc_real_2: 0.23370566964149475 (0.2268324407010243) + | > loss_disc_real_3: 0.21894216537475586 (0.22747069648627577) + | > loss_disc_real_4: 0.22444990277290344 (0.22831236802298446) + | > loss_disc_real_5: 0.2300858199596405 (0.2274941581076589) + | > loss_0: 2.5019235610961914 (2.452498407199467) + | > grad_norm_0: tensor(6.5746, device='cuda:0') (tensor(7.3992, device='cuda:0')) + | > loss_gen: 2.3355214595794678 (2.351843191631909) + | > loss_kl: 1.944604754447937 (1.8951419756330292) + | > loss_feat: 6.7579169273376465 (7.026427902024368) + | > loss_mel: 16.882658004760742 (17.677744750318865) + | > loss_duration: 1.4645705223083496 (1.4671846813169018) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.385271072387695 (30.418342425905426) + | > grad_norm_1: tensor(133.7625, device='cuda:0') (tensor(130.2045, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.4293 (0.377035716484333) + | > loader_time: 0.0075 (0.005902960382658862) + + + --> TIME: 2025-04-19 02:48:30 -- STEP: 141/142 -- GLOBAL_STEP: 412225 + | > loss_disc: 2.4358534812927246 (2.4513590589482748) + | > loss_disc_real_0: 0.12932080030441284 (0.1274294001941986) + | > loss_disc_real_1: 0.1796196699142456 (0.2066048156496481) + | > loss_disc_real_2: 0.2188839614391327 (0.2274556076484369) + | > loss_disc_real_3: 0.21494266390800476 (0.22724142818586202) + | > loss_disc_real_4: 0.2098899781703949 (0.22858939113769125) + | > loss_disc_real_5: 0.1940692514181137 (0.22700430294300647) + | > loss_0: 2.4358534812927246 (2.4513590589482748) + | > grad_norm_0: tensor(15.8027, device='cuda:0') (tensor(7.2955, device='cuda:0')) + | > loss_gen: 2.6351373195648193 (2.3552237546190304) + | > loss_kl: 1.748765230178833 (1.8994539629483054) + | > loss_feat: 8.23077392578125 (7.016822246795005) + | > loss_mel: 18.723661422729492 (17.6309665923423) + | > loss_duration: 1.326867699623108 (1.4730147512246532) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.66520690917969 (30.375481301165642) + | > grad_norm_1: tensor(521.6926, device='cuda:0') (tensor(135.4069, device='cuda:0')) + | > current_lr_0: 0.00019430722912946354 + | > current_lr_1: 0.00019430722912946354 + | > step_time: 0.2028 (0.3884069192494062) + | > loader_time: 0.0028 (0.006107477431601668) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4437618255615234 (2.4437618255615234) + | > loss_disc_real_0: 0.14035825431346893 (0.14035825431346893) + | > loss_disc_real_1: 0.23982062935829163 (0.23982062935829163) + | > loss_disc_real_2: 0.23689696192741394 (0.23689696192741394) + | > loss_disc_real_3: 0.2236020714044571 (0.2236020714044571) + | > loss_disc_real_4: 0.22661063075065613 (0.22661063075065613) + | > loss_disc_real_5: 0.17552918195724487 (0.17552918195724487) + | > loss_0: 2.4437618255615234 (2.4437618255615234) + | > loss_gen: 2.4147870540618896 (2.4147870540618896) + | > loss_kl: 2.4949686527252197 (2.4949686527252197) + | > loss_feat: 7.344385147094727 (7.344385147094727) + | > loss_mel: 17.620319366455078 (17.620319366455078) + | > loss_duration: 1.6578007936477661 (1.6578007936477661) + | > loss_1: 31.53226089477539 (31.53226089477539) + + --> STEP: 1 + | > loss_disc: 2.460667848587036 (2.460667848587036) + | > loss_disc_real_0: 0.1351814717054367 (0.1351814717054367) + | > loss_disc_real_1: 0.23724108934402466 (0.23724108934402466) + | > loss_disc_real_2: 0.2730270326137543 (0.2730270326137543) + | > loss_disc_real_3: 0.2556396722793579 (0.2556396722793579) + | > loss_disc_real_4: 0.22271333634853363 (0.22271333634853363) + | > loss_disc_real_5: 0.23025134205818176 (0.23025134205818176) + | > loss_0: 2.460667848587036 (2.460667848587036) + | > loss_gen: 2.4504220485687256 (2.4504220485687256) + | > loss_kl: 2.5907700061798096 (2.5907700061798096) + | > loss_feat: 7.471012592315674 (7.471012592315674) + | > loss_mel: 18.68722915649414 (18.68722915649414) + | > loss_duration: 1.7278106212615967 (1.7278106212615967) + | > loss_1: 32.92724609375 (32.92724609375) + + --> STEP: 2 + | > loss_disc: 2.497659206390381 (2.4791635274887085) + | > loss_disc_real_0: 0.1790185272693634 (0.15709999948740005) + | > loss_disc_real_1: 0.25255078077316284 (0.24489593505859375) + | > loss_disc_real_2: 0.2112491875886917 (0.242138110101223) + | > loss_disc_real_3: 0.25239309668540955 (0.25401638448238373) + | > loss_disc_real_4: 0.24795155227184296 (0.2353324443101883) + | > loss_disc_real_5: 0.25954726338386536 (0.24489930272102356) + | > loss_0: 2.497659206390381 (2.4791635274887085) + | > loss_gen: 2.548204183578491 (2.4993131160736084) + | > loss_kl: 2.268195867538452 (2.429482936859131) + | > loss_feat: 7.799316883087158 (7.635164737701416) + | > loss_mel: 18.597442626953125 (18.642335891723633) + | > loss_duration: 1.6632170677185059 (1.6955138444900513) + | > loss_1: 32.87637710571289 (32.901811599731445) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034433603286743164 (-0.00028586387634277344) + | > avg_loss_disc: 2.4791635274887085 (-0.06083369255065918) + | > avg_loss_disc_real_0: 0.15709999948740005 (-0.00922006368637085) + | > avg_loss_disc_real_1: 0.24489593505859375 (+0.031942613422870636) + | > avg_loss_disc_real_2: 0.242138110101223 (+0.0038216635584831238) + | > avg_loss_disc_real_3: 0.25401638448238373 (+0.04672861844301224) + | > avg_loss_disc_real_4: 0.2353324443101883 (-0.021772630512714386) + | > avg_loss_disc_real_5: 0.24489930272102356 (-0.0011859089136123657) + | > avg_loss_0: 2.4791635274887085 (-0.06083369255065918) + | > avg_loss_gen: 2.4993131160736084 (+0.1541808843612671) + | > avg_loss_kl: 2.429482936859131 (-0.06513190269470215) + | > avg_loss_feat: 7.635164737701416 (+0.9944837093353271) + | > avg_loss_mel: 18.642335891723633 (+0.33831787109375) + | > avg_loss_duration: 1.6955138444900513 (-0.0008233785629272461) + | > avg_loss_1: 32.901811599731445 (+1.4210281372070312) + + + > EPOCH: 232/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:48:32)  + + --> TIME: 2025-04-19 02:48:41 -- STEP: 24/142 -- GLOBAL_STEP: 412250 + | > loss_disc: 2.4790170192718506 (2.4689892828464504) + | > loss_disc_real_0: 0.1975657194852829 (0.12300540134310722) + | > loss_disc_real_1: 0.2216343730688095 (0.21150257314244905) + | > loss_disc_real_2: 0.2437514215707779 (0.23160264330605665) + | > loss_disc_real_3: 0.2378447949886322 (0.22958661802113056) + | > loss_disc_real_4: 0.26443177461624146 (0.22797822083036104) + | > loss_disc_real_5: 0.2497197389602661 (0.2304298815627893) + | > loss_0: 2.4790170192718506 (2.4689892828464504) + | > grad_norm_0: tensor(6.4130, device='cuda:0') (tensor(7.9065, device='cuda:0')) + | > loss_gen: 2.523831844329834 (2.370066632827123) + | > loss_kl: 1.8117077350616455 (1.8228546281655629) + | > loss_feat: 7.556882381439209 (7.218443950017293) + | > loss_mel: 19.02191162109375 (17.808316906293232) + | > loss_duration: 1.4713919162750244 (1.427872588237127) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.38572692871094 (30.64755455652873) + | > grad_norm_1: tensor(317.6355, device='cuda:0') (tensor(181.9615, device='cuda:0')) + | > current_lr_0: 0.00019428294072582235 + | > current_lr_1: 0.00019428294072582235 + | > step_time: 0.3419 (0.3326089680194855) + | > loader_time: 0.005 (0.004651705423990885) + + + --> TIME: 2025-04-19 02:48:50 -- STEP: 49/142 -- GLOBAL_STEP: 412275 + | > loss_disc: 2.459940195083618 (2.4547118167487936) + | > loss_disc_real_0: 0.14435918629169464 (0.12521041245484837) + | > loss_disc_real_1: 0.18724341690540314 (0.20763638129039685) + | > loss_disc_real_2: 0.23536355793476105 (0.2276672623595413) + | > loss_disc_real_3: 0.26711902022361755 (0.22914231036390578) + | > loss_disc_real_4: 0.24569173157215118 (0.2262837172162776) + | > loss_disc_real_5: 0.26533281803131104 (0.22963619445051467) + | > loss_0: 2.459940195083618 (2.4547118167487936) + | > grad_norm_0: tensor(4.0822, device='cuda:0') (tensor(9.1673, device='cuda:0')) + | > loss_gen: 2.3844778537750244 (2.362136962462445) + | > loss_kl: 2.0060527324676514 (1.8609754431004426) + | > loss_feat: 6.498673439025879 (7.079999018688591) + | > loss_mel: 17.798500061035156 (17.68525701639604) + | > loss_duration: 1.4551739692687988 (1.4397093519872548) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.14287757873535 (30.428077697753906) + | > grad_norm_1: tensor(148.9473, device='cuda:0') (tensor(201.9792, device='cuda:0')) + | > current_lr_0: 0.00019428294072582235 + | > current_lr_1: 0.00019428294072582235 + | > step_time: 0.3895 (0.3463523679850051) + | > loader_time: 0.0059 (0.005089652781583826) + + + --> TIME: 2025-04-19 02:49:00 -- STEP: 74/142 -- GLOBAL_STEP: 412300 + | > loss_disc: 2.5125904083251953 (2.4496985963872953) + | > loss_disc_real_0: 0.08367862552404404 (0.12508454123461568) + | > loss_disc_real_1: 0.1803978532552719 (0.20601004201012688) + | > loss_disc_real_2: 0.20723755657672882 (0.2258407133253845) + | > loss_disc_real_3: 0.21777775883674622 (0.22883624281432177) + | > loss_disc_real_4: 0.2548849880695343 (0.22697835355191617) + | > loss_disc_real_5: 0.2711661159992218 (0.2294739295501967) + | > loss_0: 2.5125904083251953 (2.4496985963872953) + | > grad_norm_0: tensor(11.0767, device='cuda:0') (tensor(9.5301, device='cuda:0')) + | > loss_gen: 2.2937350273132324 (2.359853567303838) + | > loss_kl: 1.9386106729507446 (1.8798912183658496) + | > loss_feat: 7.017085075378418 (7.082624880043236) + | > loss_mel: 17.177047729492188 (17.694267208511764) + | > loss_duration: 1.4683055877685547 (1.4529207155511186) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.894784927368164 (30.469557530171162) + | > grad_norm_1: tensor(250.1597, device='cuda:0') (tensor(209.9491, device='cuda:0')) + | > current_lr_0: 0.00019428294072582235 + | > current_lr_1: 0.00019428294072582235 + | > step_time: 0.385 (0.3583932148443685) + | > loader_time: 0.0067 (0.0054318292720897795) + + + --> TIME: 2025-04-19 02:49:10 -- STEP: 99/142 -- GLOBAL_STEP: 412325 + | > loss_disc: 2.461181163787842 (2.464068436863445) + | > loss_disc_real_0: 0.2111760675907135 (0.12809547450807363) + | > loss_disc_real_1: 0.21364319324493408 (0.20641667614079484) + | > loss_disc_real_2: 0.24252517521381378 (0.22693308992217284) + | > loss_disc_real_3: 0.21491499245166779 (0.2287670485299043) + | > loss_disc_real_4: 0.22176440060138702 (0.2285089601169933) + | > loss_disc_real_5: 0.21181143820285797 (0.23117462267177274) + | > loss_0: 2.461181163787842 (2.464068436863445) + | > grad_norm_0: tensor(23.0638, device='cuda:0') (tensor(10.1950, device='cuda:0')) + | > loss_gen: 2.3917832374572754 (2.3541540063992903) + | > loss_kl: 1.9973419904708862 (1.8975971238781708) + | > loss_feat: 7.319518566131592 (6.994067509969075) + | > loss_mel: 18.113380432128906 (17.608523879388365) + | > loss_duration: 1.500426173210144 (1.463411722520385) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.32244873046875 (30.31775412896667) + | > grad_norm_1: tensor(398.3379, device='cuda:0') (tensor(217.2170, device='cuda:0')) + | > current_lr_0: 0.00019428294072582235 + | > current_lr_1: 0.00019428294072582235 + | > step_time: 0.411 (0.36908728426153) + | > loader_time: 0.0077 (0.005731654889655837) + + + --> TIME: 2025-04-19 02:49:21 -- STEP: 124/142 -- GLOBAL_STEP: 412350 + | > loss_disc: 2.4576802253723145 (2.468142782488176) + | > loss_disc_real_0: 0.13301494717597961 (0.12846757765979538) + | > loss_disc_real_1: 0.23112976551055908 (0.20750535387665994) + | > loss_disc_real_2: 0.22483959794044495 (0.22741010616863927) + | > loss_disc_real_3: 0.21732878684997559 (0.22842068753896222) + | > loss_disc_real_4: 0.22757300734519958 (0.22785945416938874) + | > loss_disc_real_5: 0.23475758731365204 (0.23094959508988164) + | > loss_0: 2.4576802253723145 (2.468142782488176) + | > grad_norm_0: tensor(11.6498, device='cuda:0') (tensor(10.7771, device='cuda:0')) + | > loss_gen: 2.4515671730041504 (2.3467001568886547) + | > loss_kl: 2.0799880027770996 (1.9033412366144118) + | > loss_feat: 6.944301605224609 (6.964314049290072) + | > loss_mel: 17.769737243652344 (17.581817565425744) + | > loss_duration: 1.523240327835083 (1.470915110841874) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.768835067749023 (30.267088013310588) + | > grad_norm_1: tensor(437.5948, device='cuda:0') (tensor(230.3797, device='cuda:0')) + | > current_lr_0: 0.00019428294072582235 + | > current_lr_1: 0.00019428294072582235 + | > step_time: 0.4388 (0.38067338543553497) + | > loader_time: 0.007 (0.005981610667320991) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4250359535217285 (2.4250359535217285) + | > loss_disc_real_0: 0.20618633925914764 (0.20618633925914764) + | > loss_disc_real_1: 0.23029331862926483 (0.23029331862926483) + | > loss_disc_real_2: 0.2151348739862442 (0.2151348739862442) + | > loss_disc_real_3: 0.24316413700580597 (0.24316413700580597) + | > loss_disc_real_4: 0.24332790076732635 (0.24332790076732635) + | > loss_disc_real_5: 0.2640840411186218 (0.2640840411186218) + | > loss_0: 2.4250359535217285 (2.4250359535217285) + | > loss_gen: 2.5061306953430176 (2.5061306953430176) + | > loss_kl: 2.50032639503479 (2.50032639503479) + | > loss_feat: 6.88871955871582 (6.88871955871582) + | > loss_mel: 17.94268226623535 (17.94268226623535) + | > loss_duration: 1.6932084560394287 (1.6932084560394287) + | > loss_1: 31.53106689453125 (31.53106689453125) + + --> STEP: 1 + | > loss_disc: 2.60994815826416 (2.60994815826416) + | > loss_disc_real_0: 0.223598912358284 (0.223598912358284) + | > loss_disc_real_1: 0.24156999588012695 (0.24156999588012695) + | > loss_disc_real_2: 0.2532862424850464 (0.2532862424850464) + | > loss_disc_real_3: 0.2757861614227295 (0.2757861614227295) + | > loss_disc_real_4: 0.29237931966781616 (0.29237931966781616) + | > loss_disc_real_5: 0.2827204465866089 (0.2827204465866089) + | > loss_0: 2.60994815826416 (2.60994815826416) + | > loss_gen: 2.476449728012085 (2.476449728012085) + | > loss_kl: 2.7001657485961914 (2.7001657485961914) + | > loss_feat: 6.266285419464111 (6.266285419464111) + | > loss_mel: 18.463550567626953 (18.463550567626953) + | > loss_duration: 1.7497456073760986 (1.7497456073760986) + | > loss_1: 31.656198501586914 (31.656198501586914) + + --> STEP: 2 + | > loss_disc: 2.606757164001465 (2.6083526611328125) + | > loss_disc_real_0: 0.2147689163684845 (0.21918391436338425) + | > loss_disc_real_1: 0.2724136710166931 (0.25699183344841003) + | > loss_disc_real_2: 0.22448895871639252 (0.23888760060071945) + | > loss_disc_real_3: 0.24521572887897491 (0.2605009451508522) + | > loss_disc_real_4: 0.23883193731307983 (0.265605628490448) + | > loss_disc_real_5: 0.27630168199539185 (0.27951106429100037) + | > loss_0: 2.606757164001465 (2.6083526611328125) + | > loss_gen: 2.2668354511260986 (2.371642589569092) + | > loss_kl: 2.267334222793579 (2.4837499856948853) + | > loss_feat: 4.847629070281982 (5.556957244873047) + | > loss_mel: 13.159703254699707 (15.81162691116333) + | > loss_duration: 1.6430845260620117 (1.6964150667190552) + | > loss_1: 24.184585571289062 (27.92039203643799) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032939910888671875 (-0.0001493692398071289) + | > avg_loss_disc: 2.6083526611328125 (+0.129189133644104) + | > avg_loss_disc_real_0: 0.21918391436338425 (+0.06208391487598419) + | > avg_loss_disc_real_1: 0.25699183344841003 (+0.012095898389816284) + | > avg_loss_disc_real_2: 0.23888760060071945 (-0.00325050950050354) + | > avg_loss_disc_real_3: 0.2605009451508522 (+0.006484560668468475) + | > avg_loss_disc_real_4: 0.265605628490448 (+0.030273184180259705) + | > avg_loss_disc_real_5: 0.27951106429100037 (+0.03461176156997681) + | > avg_loss_0: 2.6083526611328125 (+0.129189133644104) + | > avg_loss_gen: 2.371642589569092 (-0.1276705265045166) + | > avg_loss_kl: 2.4837499856948853 (+0.054267048835754395) + | > avg_loss_feat: 5.556957244873047 (-2.078207492828369) + | > avg_loss_mel: 15.81162691116333 (-2.8307089805603027) + | > avg_loss_duration: 1.6964150667190552 (+0.0009012222290039062) + | > avg_loss_1: 27.92039203643799 (-4.981419563293457) + + + > EPOCH: 233/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:49:31)  + + --> TIME: 2025-04-19 02:49:34 -- STEP: 7/142 -- GLOBAL_STEP: 412375 + | > loss_disc: 2.462994337081909 (2.430655070713588) + | > loss_disc_real_0: 0.09372185915708542 (0.1179725495832307) + | > loss_disc_real_1: 0.21909768879413605 (0.20275980872767313) + | > loss_disc_real_2: 0.2167825549840927 (0.23157542305333273) + | > loss_disc_real_3: 0.23635268211364746 (0.2231194462094988) + | > loss_disc_real_4: 0.2110145539045334 (0.22563165639128005) + | > loss_disc_real_5: 0.22876624763011932 (0.23065893990652903) + | > loss_0: 2.462994337081909 (2.430655070713588) + | > grad_norm_0: tensor(6.3380, device='cuda:0') (tensor(11.8235, device='cuda:0')) + | > loss_gen: 2.377810001373291 (2.3725172110966275) + | > loss_kl: 1.6853280067443848 (1.7827459233147758) + | > loss_feat: 7.816491603851318 (7.526713507516043) + | > loss_mel: 18.409109115600586 (18.07792064121791) + | > loss_duration: 1.4213603734970093 (1.430750949042184) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.710098266601562 (31.19064739772252) + | > grad_norm_1: tensor(55.6965, device='cuda:0') (tensor(272.8754, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.3255 (0.3265369619641985) + | > loader_time: 0.0039 (0.003961426871163505) + + + --> TIME: 2025-04-19 02:49:42 -- STEP: 32/142 -- GLOBAL_STEP: 412400 + | > loss_disc: 2.429591655731201 (2.4474807977676396) + | > loss_disc_real_0: 0.10755989700555801 (0.11954802880063653) + | > loss_disc_real_1: 0.2380339801311493 (0.20936721237376332) + | > loss_disc_real_2: 0.2268337905406952 (0.22825286677107215) + | > loss_disc_real_3: 0.2602421045303345 (0.2268875758163631) + | > loss_disc_real_4: 0.2841877341270447 (0.22722282493487) + | > loss_disc_real_5: 0.23713819682598114 (0.23043249547481537) + | > loss_0: 2.429591655731201 (2.4474807977676396) + | > grad_norm_0: tensor(11.0812, device='cuda:0') (tensor(10.1676, device='cuda:0')) + | > loss_gen: 2.6520142555236816 (2.3660114109516144) + | > loss_kl: 1.9391210079193115 (1.8268574625253677) + | > loss_feat: 7.22584867477417 (7.237824618816376) + | > loss_mel: 17.978084564208984 (17.942686975002285) + | > loss_duration: 1.4307315349578857 (1.4384890273213387) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.225799560546875 (30.811869144439697) + | > grad_norm_1: tensor(324.2662, device='cuda:0') (tensor(248.1796, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.3545 (0.3380795493721962) + | > loader_time: 0.0057 (0.0048606544733047485) + + + --> TIME: 2025-04-19 02:49:52 -- STEP: 57/142 -- GLOBAL_STEP: 412425 + | > loss_disc: 2.324077606201172 (2.440060866506476) + | > loss_disc_real_0: 0.07450894266366959 (0.12053440981789638) + | > loss_disc_real_1: 0.21010573208332062 (0.20791262466656535) + | > loss_disc_real_2: 0.22994723916053772 (0.2247276476077866) + | > loss_disc_real_3: 0.2058604508638382 (0.22657492992125058) + | > loss_disc_real_4: 0.2458840161561966 (0.22767075242703422) + | > loss_disc_real_5: 0.25639352202415466 (0.23328007496239847) + | > loss_0: 2.324077606201172 (2.440060866506476) + | > grad_norm_0: tensor(17.1216, device='cuda:0') (tensor(10.6149, device='cuda:0')) + | > loss_gen: 2.4968459606170654 (2.370762034466392) + | > loss_kl: 1.8631184101104736 (1.8517145880481654) + | > loss_feat: 7.840212821960449 (7.221686170812239) + | > loss_mel: 18.16118049621582 (17.94402383503161) + | > loss_duration: 1.5034672021865845 (1.4465943365766292) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.864824295043945 (30.83478097748338) + | > grad_norm_1: tensor(460.8112, device='cuda:0') (tensor(256.3678, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.3761 (0.3510304166559588) + | > loader_time: 0.0058 (0.005235412664580765) + + + --> TIME: 2025-04-19 02:50:02 -- STEP: 82/142 -- GLOBAL_STEP: 412450 + | > loss_disc: 2.3644917011260986 (2.444132319310815) + | > loss_disc_real_0: 0.09718974679708481 (0.12174590440785013) + | > loss_disc_real_1: 0.15893951058387756 (0.2063470391965494) + | > loss_disc_real_2: 0.2577577531337738 (0.22499812757823526) + | > loss_disc_real_3: 0.25571388006210327 (0.22635883419978908) + | > loss_disc_real_4: 0.2402251958847046 (0.22767920210594084) + | > loss_disc_real_5: 0.20117808878421783 (0.2324081573907922) + | > loss_0: 2.3644917011260986 (2.444132319310815) + | > grad_norm_0: tensor(4.8610, device='cuda:0') (tensor(9.3531, device='cuda:0')) + | > loss_gen: 2.426332712173462 (2.3602665139407644) + | > loss_kl: 1.9637328386306763 (1.8655746433793046) + | > loss_feat: 7.432745933532715 (7.150592315487746) + | > loss_mel: 19.76128578186035 (17.885561373175644) + | > loss_duration: 1.5250059366226196 (1.457312014044785) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.10910415649414 (30.71930694580078) + | > grad_norm_1: tensor(79.0030, device='cuda:0') (tensor(230.7411, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.3941 (0.36342820888612326) + | > loader_time: 0.0063 (0.005538920076881968) + + + --> TIME: 2025-04-19 02:50:12 -- STEP: 107/142 -- GLOBAL_STEP: 412475 + | > loss_disc: 2.492558717727661 (2.4524226099531234) + | > loss_disc_real_0: 0.13683083653450012 (0.1238578306598084) + | > loss_disc_real_1: 0.20866930484771729 (0.2062328168164904) + | > loss_disc_real_2: 0.20661257207393646 (0.22496743305264233) + | > loss_disc_real_3: 0.21658873558044434 (0.2268435921624442) + | > loss_disc_real_4: 0.1974894404411316 (0.22768406834557792) + | > loss_disc_real_5: 0.20547649264335632 (0.23364627862645088) + | > loss_0: 2.492558717727661 (2.4524226099531234) + | > grad_norm_0: tensor(6.9937, device='cuda:0') (tensor(9.0619, device='cuda:0')) + | > loss_gen: 2.315809488296509 (2.3540360548785917) + | > loss_kl: 1.843539834022522 (1.8725745221164738) + | > loss_feat: 6.945436000823975 (7.121295042127092) + | > loss_mel: 16.977920532226562 (17.86627826512416) + | > loss_duration: 1.4775199890136719 (1.4670411704856658) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.560224533081055 (30.68122507255768) + | > grad_norm_1: tensor(197.0278, device='cuda:0') (tensor(223.7186, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.4212 (0.3752054700227541) + | > loader_time: 0.007 (0.005846081493056823) + + + --> TIME: 2025-04-19 02:50:24 -- STEP: 132/142 -- GLOBAL_STEP: 412500 + | > loss_disc: 2.470050573348999 (2.4560736038468107) + | > loss_disc_real_0: 0.11595499515533447 (0.12590287733032848) + | > loss_disc_real_1: 0.25131288170814514 (0.20688732307065616) + | > loss_disc_real_2: 0.24349501729011536 (0.22549779792175148) + | > loss_disc_real_3: 0.21783259510993958 (0.22746107896620577) + | > loss_disc_real_4: 0.21646054089069366 (0.2278864152277961) + | > loss_disc_real_5: 0.24575769901275635 (0.23482150415127928) + | > loss_0: 2.470050573348999 (2.4560736038468107) + | > grad_norm_0: tensor(11.7366, device='cuda:0') (tensor(8.9452, device='cuda:0')) + | > loss_gen: 2.4035463333129883 (2.353712794455615) + | > loss_kl: 1.9221142530441284 (1.8813494913505786) + | > loss_feat: 6.839183807373047 (7.078451850197532) + | > loss_mel: 17.157787322998047 (17.824195760669124) + | > loss_duration: 1.4801822900772095 (1.4731294747554895) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.802814483642578 (30.61083935246323) + | > grad_norm_1: tensor(345.4128, device='cuda:0') (tensor(215.6198, device='cuda:0')) + | > current_lr_0: 0.0001942586553582316 + | > current_lr_1: 0.0001942586553582316 + | > step_time: 0.4481 (0.38762644746086816) + | > loader_time: 0.0071 (0.006137468598105691) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6420741081237793 (2.6420741081237793) + | > loss_disc_real_0: 0.1780107468366623 (0.1780107468366623) + | > loss_disc_real_1: 0.26379895210266113 (0.26379895210266113) + | > loss_disc_real_2: 0.24514086544513702 (0.24514086544513702) + | > loss_disc_real_3: 0.24254266917705536 (0.24254266917705536) + | > loss_disc_real_4: 0.31220367550849915 (0.31220367550849915) + | > loss_disc_real_5: 0.30045750737190247 (0.30045750737190247) + | > loss_0: 2.6420741081237793 (2.6420741081237793) + | > loss_gen: 2.4290568828582764 (2.4290568828582764) + | > loss_kl: 2.211435556411743 (2.211435556411743) + | > loss_feat: 7.072605133056641 (7.072605133056641) + | > loss_mel: 19.106891632080078 (19.106891632080078) + | > loss_duration: 1.6625301837921143 (1.6625301837921143) + | > loss_1: 32.48251724243164 (32.48251724243164) + + --> STEP: 1 + | > loss_disc: 2.6118011474609375 (2.6118011474609375) + | > loss_disc_real_0: 0.1854274421930313 (0.1854274421930313) + | > loss_disc_real_1: 0.26367583870887756 (0.26367583870887756) + | > loss_disc_real_2: 0.25265151262283325 (0.25265151262283325) + | > loss_disc_real_3: 0.23385578393936157 (0.23385578393936157) + | > loss_disc_real_4: 0.2988241910934448 (0.2988241910934448) + | > loss_disc_real_5: 0.224527969956398 (0.224527969956398) + | > loss_0: 2.6118011474609375 (2.6118011474609375) + | > loss_gen: 2.3036794662475586 (2.3036794662475586) + | > loss_kl: 2.8023452758789062 (2.8023452758789062) + | > loss_feat: 5.850729942321777 (5.850729942321777) + | > loss_mel: 14.956052780151367 (14.956052780151367) + | > loss_duration: 1.7641425132751465 (1.7641425132751465) + | > loss_1: 27.676950454711914 (27.676950454711914) + + --> STEP: 2 + | > loss_disc: 2.6553940773010254 (2.6335976123809814) + | > loss_disc_real_0: 0.15597519278526306 (0.1707013174891472) + | > loss_disc_real_1: 0.3350992500782013 (0.29938754439353943) + | > loss_disc_real_2: 0.2657373547554016 (0.25919443368911743) + | > loss_disc_real_3: 0.2581652104854584 (0.24601049721240997) + | > loss_disc_real_4: 0.29239898920059204 (0.29561159014701843) + | > loss_disc_real_5: 0.2828213572502136 (0.2536746636033058) + | > loss_0: 2.6553940773010254 (2.6335976123809814) + | > loss_gen: 2.4780452251434326 (2.3908623456954956) + | > loss_kl: 2.101665496826172 (2.452005386352539) + | > loss_feat: 7.7884955406188965 (6.819612741470337) + | > loss_mel: 19.155332565307617 (17.055692672729492) + | > loss_duration: 1.6495513916015625 (1.7068469524383545) + | > loss_1: 33.17308807373047 (30.42501926422119) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004213094711303711 (+0.0009191036224365234) + | > avg_loss_disc: 2.6335976123809814 (+0.025244951248168945) + | > avg_loss_disc_real_0: 0.1707013174891472 (-0.04848259687423706) + | > avg_loss_disc_real_1: 0.29938754439353943 (+0.042395710945129395) + | > avg_loss_disc_real_2: 0.25919443368911743 (+0.02030683308839798) + | > avg_loss_disc_real_3: 0.24601049721240997 (-0.01449044793844223) + | > avg_loss_disc_real_4: 0.29561159014701843 (+0.030005961656570435) + | > avg_loss_disc_real_5: 0.2536746636033058 (-0.02583640068769455) + | > avg_loss_0: 2.6335976123809814 (+0.025244951248168945) + | > avg_loss_gen: 2.3908623456954956 (+0.01921975612640381) + | > avg_loss_kl: 2.452005386352539 (-0.03174459934234619) + | > avg_loss_feat: 6.819612741470337 (+1.26265549659729) + | > avg_loss_mel: 17.055692672729492 (+1.244065761566162) + | > avg_loss_duration: 1.7068469524383545 (+0.010431885719299316) + | > avg_loss_1: 30.42501926422119 (+2.504627227783203) + + + > EPOCH: 234/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:50:30)  + + --> TIME: 2025-04-19 02:50:35 -- STEP: 15/142 -- GLOBAL_STEP: 412525 + | > loss_disc: 2.473341464996338 (2.4406938552856445) + | > loss_disc_real_0: 0.1041112169623375 (0.12107504109541575) + | > loss_disc_real_1: 0.20447933673858643 (0.20405468742052715) + | > loss_disc_real_2: 0.21373119950294495 (0.22608419557412465) + | > loss_disc_real_3: 0.24532383680343628 (0.22457600732644398) + | > loss_disc_real_4: 0.22231975197792053 (0.23479326367378234) + | > loss_disc_real_5: 0.25216197967529297 (0.23120745321114858) + | > loss_0: 2.473341464996338 (2.4406938552856445) + | > grad_norm_0: tensor(10.6472, device='cuda:0') (tensor(10.9469, device='cuda:0')) + | > loss_gen: 2.4073829650878906 (2.412121566136678) + | > loss_kl: 1.8173424005508423 (1.8285876592000325) + | > loss_feat: 7.502439498901367 (7.6042202313741045) + | > loss_mel: 18.356489181518555 (18.64760882059733) + | > loss_duration: 1.4309444427490234 (1.42458975315094) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.514598846435547 (31.91712773640951) + | > grad_norm_1: tensor(115.4611, device='cuda:0') (tensor(269.4664, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.3343 (0.3276028633117675) + | > loader_time: 0.0049 (0.0045010884602864586) + + + --> TIME: 2025-04-19 02:50:44 -- STEP: 40/142 -- GLOBAL_STEP: 412550 + | > loss_disc: 2.4627461433410645 (2.4370518267154693) + | > loss_disc_real_0: 0.14080677926540375 (0.11963766105473042) + | > loss_disc_real_1: 0.20281417667865753 (0.20795642845332624) + | > loss_disc_real_2: 0.20739038288593292 (0.22461418695747853) + | > loss_disc_real_3: 0.22588162124156952 (0.2251273959875107) + | > loss_disc_real_4: 0.24046644568443298 (0.2275523442775011) + | > loss_disc_real_5: 0.22318103909492493 (0.22977774105966092) + | > loss_0: 2.4627461433410645 (2.4370518267154693) + | > grad_norm_0: tensor(17.4147, device='cuda:0') (tensor(11.0729, device='cuda:0')) + | > loss_gen: 2.3176896572113037 (2.387498301267624) + | > loss_kl: 1.739503264427185 (1.8144094228744507) + | > loss_feat: 7.042709827423096 (7.351937711238861) + | > loss_mel: 17.62321662902832 (18.058013200759888) + | > loss_duration: 1.432997226715088 (1.4375980198383331) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.156116485595703 (31.04945659637451) + | > grad_norm_1: tensor(375.2786, device='cuda:0') (tensor(253.6120, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.3612 (0.34061095118522644) + | > loader_time: 0.0058 (0.005108273029327391) + + + --> TIME: 2025-04-19 02:50:54 -- STEP: 65/142 -- GLOBAL_STEP: 412575 + | > loss_disc: 2.461505889892578 (2.4498222314394438) + | > loss_disc_real_0: 0.0772508904337883 (0.1193595248919267) + | > loss_disc_real_1: 0.19644570350646973 (0.2071366216127689) + | > loss_disc_real_2: 0.20045939087867737 (0.22581726748209732) + | > loss_disc_real_3: 0.22347576916217804 (0.22620067183788006) + | > loss_disc_real_4: 0.20356149971485138 (0.22983285394998698) + | > loss_disc_real_5: 0.20193956792354584 (0.23192558540747715) + | > loss_0: 2.461505889892578 (2.4498222314394438) + | > grad_norm_0: tensor(8.5113, device='cuda:0') (tensor(11.3296, device='cuda:0')) + | > loss_gen: 2.305540084838867 (2.3829495980189397) + | > loss_kl: 1.961351752281189 (1.8461224959446834) + | > loss_feat: 6.602453708648682 (7.244483918410081) + | > loss_mel: 16.62187385559082 (18.03005369626558) + | > loss_duration: 1.4528545141220093 (1.4489225625991822) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.944074630737305 (30.95253225473257) + | > grad_norm_1: tensor(238.4099, device='cuda:0') (tensor(267.8471, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.3905 (0.3528329665844257) + | > loader_time: 0.0068 (0.005447937892033502) + + + --> TIME: 2025-04-19 02:51:04 -- STEP: 90/142 -- GLOBAL_STEP: 412600 + | > loss_disc: 2.504749059677124 (2.4563632276323104) + | > loss_disc_real_0: 0.11637642234563828 (0.12350490896238221) + | > loss_disc_real_1: 0.1930486410856247 (0.20774504741032915) + | > loss_disc_real_2: 0.22748832404613495 (0.227189658747779) + | > loss_disc_real_3: 0.22508178651332855 (0.22671355472670662) + | > loss_disc_real_4: 0.21401293575763702 (0.22864831537008284) + | > loss_disc_real_5: 0.19586290419101715 (0.2312922986017333) + | > loss_0: 2.504749059677124 (2.4563632276323104) + | > grad_norm_0: tensor(12.1325, device='cuda:0') (tensor(10.8466, device='cuda:0')) + | > loss_gen: 2.157848596572876 (2.371472205056085) + | > loss_kl: 1.8447517156600952 (1.8625940163930257) + | > loss_feat: 6.209549903869629 (7.1405405892266165) + | > loss_mel: 17.11302947998047 (17.92477136188083) + | > loss_duration: 1.4948296546936035 (1.461114362875621) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.820009231567383 (30.760492494371203) + | > grad_norm_1: tensor(206.8027, device='cuda:0') (tensor(248.6706, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.4076 (0.36514411237504746) + | > loader_time: 0.0068 (0.005744732750786673) + + + --> TIME: 2025-04-19 02:51:15 -- STEP: 115/142 -- GLOBAL_STEP: 412625 + | > loss_disc: 2.4355194568634033 (2.462645829242209) + | > loss_disc_real_0: 0.1054200679063797 (0.12372742664554845) + | > loss_disc_real_1: 0.23063471913337708 (0.20704724775708236) + | > loss_disc_real_2: 0.23509225249290466 (0.22667641756327256) + | > loss_disc_real_3: 0.22818002104759216 (0.22771134506101193) + | > loss_disc_real_4: 0.22107668220996857 (0.22942788847114728) + | > loss_disc_real_5: 0.2175794243812561 (0.23183469500230705) + | > loss_0: 2.4355194568634033 (2.462645829242209) + | > grad_norm_0: tensor(16.4139, device='cuda:0') (tensor(10.6623, device='cuda:0')) + | > loss_gen: 2.3803272247314453 (2.3566953534665314) + | > loss_kl: 2.042634963989258 (1.8768966830295064) + | > loss_feat: 7.399481773376465 (7.110314410665761) + | > loss_mel: 18.063962936401367 (17.866980668772815) + | > loss_duration: 1.492487907409668 (1.4682523727416994) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.378894805908203 (30.679139460688052) + | > grad_norm_1: tensor(281.8291, device='cuda:0') (tensor(245.9323, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.4283 (0.3766658658566682) + | > loader_time: 0.0072 (0.006059231965438177) + + + --> TIME: 2025-04-19 02:51:26 -- STEP: 140/142 -- GLOBAL_STEP: 412650 + | > loss_disc: 2.5193846225738525 (2.4625165564673286) + | > loss_disc_real_0: 0.20048461854457855 (0.12506406706358705) + | > loss_disc_real_1: 0.18292969465255737 (0.20661497722779001) + | > loss_disc_real_2: 0.22269630432128906 (0.22641525907175883) + | > loss_disc_real_3: 0.21895834803581238 (0.22719258474452156) + | > loss_disc_real_4: 0.22521786391735077 (0.2292111510676997) + | > loss_disc_real_5: 0.19958677887916565 (0.23265330567955972) + | > loss_0: 2.5193846225738525 (2.4625165564673286) + | > grad_norm_0: tensor(9.3107, device='cuda:0') (tensor(10.4119, device='cuda:0')) + | > loss_gen: 2.301846504211426 (2.356822603089469) + | > loss_kl: 1.9631543159484863 (1.882067837033953) + | > loss_feat: 5.807660102844238 (7.102711084910801) + | > loss_mel: 15.235538482666016 (17.824541466576704) + | > loss_duration: 1.545292854309082 (1.475529910836901) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.853492736816406 (30.641672897338868) + | > grad_norm_1: tensor(273.8748, device='cuda:0') (tensor(239.1382, device='cuda:0')) + | > current_lr_0: 0.00019423437302631182 + | > current_lr_1: 0.00019423437302631182 + | > step_time: 0.4568 (0.38862681559153966) + | > loader_time: 0.0078 (0.00630099262510027) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.472963809967041 (2.472963809967041) + | > loss_disc_real_0: 0.12985558807849884 (0.12985558807849884) + | > loss_disc_real_1: 0.19638937711715698 (0.19638937711715698) + | > loss_disc_real_2: 0.2661149203777313 (0.2661149203777313) + | > loss_disc_real_3: 0.29140347242355347 (0.29140347242355347) + | > loss_disc_real_4: 0.20005106925964355 (0.20005106925964355) + | > loss_disc_real_5: 0.2589820921421051 (0.2589820921421051) + | > loss_0: 2.472963809967041 (2.472963809967041) + | > loss_gen: 2.387299060821533 (2.387299060821533) + | > loss_kl: 2.4609618186950684 (2.4609618186950684) + | > loss_feat: 7.831206798553467 (7.831206798553467) + | > loss_mel: 19.001291275024414 (19.001291275024414) + | > loss_duration: 1.7089686393737793 (1.7089686393737793) + | > loss_1: 33.38972854614258 (33.38972854614258) + + --> STEP: 1 + | > loss_disc: 2.577786684036255 (2.577786684036255) + | > loss_disc_real_0: 0.1727456897497177 (0.1727456897497177) + | > loss_disc_real_1: 0.21380572021007538 (0.21380572021007538) + | > loss_disc_real_2: 0.2835954427719116 (0.2835954427719116) + | > loss_disc_real_3: 0.22255493700504303 (0.22255493700504303) + | > loss_disc_real_4: 0.22947809100151062 (0.22947809100151062) + | > loss_disc_real_5: 0.25518515706062317 (0.25518515706062317) + | > loss_0: 2.577786684036255 (2.577786684036255) + | > loss_gen: 2.351961851119995 (2.351961851119995) + | > loss_kl: 2.6247031688690186 (2.6247031688690186) + | > loss_feat: 6.996429920196533 (6.996429920196533) + | > loss_mel: 17.941171646118164 (17.941171646118164) + | > loss_duration: 1.7321945428848267 (1.7321945428848267) + | > loss_1: 31.646461486816406 (31.646461486816406) + + --> STEP: 2 + | > loss_disc: 2.4453606605529785 (2.5115736722946167) + | > loss_disc_real_0: 0.15644428133964539 (0.16459498554468155) + | > loss_disc_real_1: 0.1951388567686081 (0.20447228848934174) + | > loss_disc_real_2: 0.2943134009838104 (0.288954421877861) + | > loss_disc_real_3: 0.2190934270620346 (0.22082418203353882) + | > loss_disc_real_4: 0.1922239512205124 (0.2108510211110115) + | > loss_disc_real_5: 0.23327673971652985 (0.2442309483885765) + | > loss_0: 2.4453606605529785 (2.5115736722946167) + | > loss_gen: 2.4908437728881836 (2.4214028120040894) + | > loss_kl: 2.2061989307403564 (2.4154510498046875) + | > loss_feat: 7.992885112762451 (7.494657516479492) + | > loss_mel: 19.274860382080078 (18.60801601409912) + | > loss_duration: 1.6650217771530151 (1.698608160018921) + | > loss_1: 33.62981033325195 (32.63813591003418) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038979053497314453 (-0.0003151893615722656) + | > avg_loss_disc: 2.5115736722946167 (-0.12202394008636475) + | > avg_loss_disc_real_0: 0.16459498554468155 (-0.006106331944465637) + | > avg_loss_disc_real_1: 0.20447228848934174 (-0.09491525590419769) + | > avg_loss_disc_real_2: 0.288954421877861 (+0.02975998818874359) + | > avg_loss_disc_real_3: 0.22082418203353882 (-0.025186315178871155) + | > avg_loss_disc_real_4: 0.2108510211110115 (-0.08476056903600693) + | > avg_loss_disc_real_5: 0.2442309483885765 (-0.009443715214729309) + | > avg_loss_0: 2.5115736722946167 (-0.12202394008636475) + | > avg_loss_gen: 2.4214028120040894 (+0.03054046630859375) + | > avg_loss_kl: 2.4154510498046875 (-0.03655433654785156) + | > avg_loss_feat: 7.494657516479492 (+0.6750447750091553) + | > avg_loss_mel: 18.60801601409912 (+1.552323341369629) + | > avg_loss_duration: 1.698608160018921 (-0.008238792419433594) + | > avg_loss_1: 32.63813591003418 (+2.2131166458129883) + + + > EPOCH: 235/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:51:28)  + + --> TIME: 2025-04-19 02:51:37 -- STEP: 23/142 -- GLOBAL_STEP: 412675 + | > loss_disc: 2.4890549182891846 (2.460070143575254) + | > loss_disc_real_0: 0.14326810836791992 (0.1146599912773008) + | > loss_disc_real_1: 0.22529442608356476 (0.2101324297811674) + | > loss_disc_real_2: 0.24620075523853302 (0.23008982124535934) + | > loss_disc_real_3: 0.19386319816112518 (0.227354659334473) + | > loss_disc_real_4: 0.22717183828353882 (0.2321580408707909) + | > loss_disc_real_5: 0.2536158263683319 (0.2370305482460105) + | > loss_0: 2.4890549182891846 (2.460070143575254) + | > grad_norm_0: tensor(4.4528, device='cuda:0') (tensor(8.5155, device='cuda:0')) + | > loss_gen: 2.3724215030670166 (2.41757899781932) + | > loss_kl: 1.9102060794830322 (1.8062936326731807) + | > loss_feat: 7.2228899002075195 (7.260492552881655) + | > loss_mel: 17.31658363342285 (17.709352866463043) + | > loss_duration: 1.4554214477539062 (1.431301008100095) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.277523040771484 (30.625019073486328) + | > grad_norm_1: tensor(125.4136, device='cuda:0') (tensor(222.6546, device='cuda:0')) + | > current_lr_0: 0.00019421009372968352 + | > current_lr_1: 0.00019421009372968352 + | > step_time: 0.3448 (0.33213327241980517) + | > loader_time: 0.0052 (0.004718345144520634) + + + --> TIME: 2025-04-19 02:51:46 -- STEP: 48/142 -- GLOBAL_STEP: 412700 + | > loss_disc: 2.430473566055298 (2.4634597003459917) + | > loss_disc_real_0: 0.12532496452331543 (0.11916657878706853) + | > loss_disc_real_1: 0.2065141499042511 (0.20967076842983565) + | > loss_disc_real_2: 0.21638573706150055 (0.22670766276617846) + | > loss_disc_real_3: 0.21484637260437012 (0.22753852947304645) + | > loss_disc_real_4: 0.23279671370983124 (0.22849885219087204) + | > loss_disc_real_5: 0.22764690220355988 (0.23734648991376162) + | > loss_0: 2.430473566055298 (2.4634597003459917) + | > grad_norm_0: tensor(6.6182, device='cuda:0') (tensor(8.8319, device='cuda:0')) + | > loss_gen: 2.3056650161743164 (2.3700404763221745) + | > loss_kl: 1.8017637729644775 (1.8362204283475876) + | > loss_feat: 7.653138637542725 (7.139879673719406) + | > loss_mel: 17.642702102661133 (17.723506629467014) + | > loss_duration: 1.4471970796585083 (1.4417654772599537) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.850467681884766 (30.51141281922658) + | > grad_norm_1: tensor(246.0377, device='cuda:0') (tensor(228.5266, device='cuda:0')) + | > current_lr_0: 0.00019421009372968352 + | > current_lr_1: 0.00019421009372968352 + | > step_time: 0.3639 (0.34500090777873993) + | > loader_time: 0.0054 (0.005097652475039164) + + + --> TIME: 2025-04-19 02:51:56 -- STEP: 73/142 -- GLOBAL_STEP: 412725 + | > loss_disc: 2.5605432987213135 (2.4620554022593035) + | > loss_disc_real_0: 0.14518503844738007 (0.12189586828016255) + | > loss_disc_real_1: 0.23215609788894653 (0.2083269194380878) + | > loss_disc_real_2: 0.21690256893634796 (0.22614144666554176) + | > loss_disc_real_3: 0.22934430837631226 (0.22824813180590328) + | > loss_disc_real_4: 0.19960449635982513 (0.22829113063746936) + | > loss_disc_real_5: 0.23306801915168762 (0.23577961999259583) + | > loss_0: 2.5605432987213135 (2.4620554022593035) + | > grad_norm_0: tensor(5.9792, device='cuda:0') (tensor(8.5804, device='cuda:0')) + | > loss_gen: 2.2268879413604736 (2.355518448842716) + | > loss_kl: 1.925317406654358 (1.8571712807433245) + | > loss_feat: 5.2822346687316895 (7.120022100945041) + | > loss_mel: 13.231226921081543 (17.711302535174646) + | > loss_duration: 1.5039012432098389 (1.4540671747024745) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 24.169567108154297 (30.498081546940217) + | > grad_norm_1: tensor(91.7790, device='cuda:0') (tensor(210.5232, device='cuda:0')) + | > current_lr_0: 0.00019421009372968352 + | > current_lr_1: 0.00019421009372968352 + | > step_time: 0.3939 (0.357103213871995) + | > loader_time: 0.0065 (0.005409315840838706) + + + --> TIME: 2025-04-19 02:52:06 -- STEP: 98/142 -- GLOBAL_STEP: 412750 + | > loss_disc: 2.55918025970459 (2.4610097408294664) + | > loss_disc_real_0: 0.1970619112253189 (0.1246877952801938) + | > loss_disc_real_1: 0.21409733593463898 (0.20701949937003) + | > loss_disc_real_2: 0.21124495565891266 (0.22510389269006495) + | > loss_disc_real_3: 0.23091213405132294 (0.22804347349672902) + | > loss_disc_real_4: 0.22569097578525543 (0.2286028016586693) + | > loss_disc_real_5: 0.22888854146003723 (0.23359370839839078) + | > loss_0: 2.55918025970459 (2.4610097408294664) + | > grad_norm_0: tensor(7.9707, device='cuda:0') (tensor(8.6660, device='cuda:0')) + | > loss_gen: 2.2371459007263184 (2.346620360199288) + | > loss_kl: 1.9337542057037354 (1.8751777653791466) + | > loss_feat: 6.24686861038208 (7.083008873219392) + | > loss_mel: 15.885772705078125 (17.677896139573086) + | > loss_duration: 1.46886146068573 (1.4645602946378746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.772401809692383 (30.447263464635732) + | > grad_norm_1: tensor(196.6337, device='cuda:0') (tensor(209.7174, device='cuda:0')) + | > current_lr_0: 0.00019421009372968352 + | > current_lr_1: 0.00019421009372968352 + | > step_time: 0.4077 (0.3678591835255526) + | > loader_time: 0.0065 (0.005724816906208895) + + + --> TIME: 2025-04-19 02:52:17 -- STEP: 123/142 -- GLOBAL_STEP: 412775 + | > loss_disc: 2.457885265350342 (2.462100525212481) + | > loss_disc_real_0: 0.11347457021474838 (0.1268922215009608) + | > loss_disc_real_1: 0.254009485244751 (0.20804748266208462) + | > loss_disc_real_2: 0.20647314190864563 (0.22620377610854017) + | > loss_disc_real_3: 0.19328437745571136 (0.22840293381756882) + | > loss_disc_real_4: 0.2084989696741104 (0.22834153873164478) + | > loss_disc_real_5: 0.2522542476654053 (0.23171254992485046) + | > loss_0: 2.457885265350342 (2.462100525212481) + | > grad_norm_0: tensor(16.8556, device='cuda:0') (tensor(9.7937, device='cuda:0')) + | > loss_gen: 2.245198965072632 (2.352961319248852) + | > loss_kl: 1.961351752281189 (1.888582123004324) + | > loss_feat: 7.208974361419678 (7.088587276334685) + | > loss_mel: 17.619556427001953 (17.715029406353718) + | > loss_duration: 1.481197714805603 (1.471685532631913) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.516279220581055 (30.516845703125) + | > grad_norm_1: tensor(185.0302, device='cuda:0') (tensor(221.3933, device='cuda:0')) + | > current_lr_0: 0.00019421009372968352 + | > current_lr_1: 0.00019421009372968352 + | > step_time: 0.4676 (0.3800942180602531) + | > loader_time: 0.0072 (0.005974228789166707) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4120492935180664 (2.4120492935180664) + | > loss_disc_real_0: 0.14017939567565918 (0.14017939567565918) + | > loss_disc_real_1: 0.17751331627368927 (0.17751331627368927) + | > loss_disc_real_2: 0.24695998430252075 (0.24695998430252075) + | > loss_disc_real_3: 0.2923352122306824 (0.2923352122306824) + | > loss_disc_real_4: 0.22598086297512054 (0.22598086297512054) + | > loss_disc_real_5: 0.23512540757656097 (0.23512540757656097) + | > loss_0: 2.4120492935180664 (2.4120492935180664) + | > loss_gen: 2.5525386333465576 (2.5525386333465576) + | > loss_kl: 2.1599841117858887 (2.1599841117858887) + | > loss_feat: 7.265089988708496 (7.265089988708496) + | > loss_mel: 19.12985610961914 (19.12985610961914) + | > loss_duration: 1.6614534854888916 (1.6614534854888916) + | > loss_1: 32.7689208984375 (32.7689208984375) + + --> STEP: 1 + | > loss_disc: 2.5817461013793945 (2.5817461013793945) + | > loss_disc_real_0: 0.17193421721458435 (0.17193421721458435) + | > loss_disc_real_1: 0.2302941381931305 (0.2302941381931305) + | > loss_disc_real_2: 0.24002385139465332 (0.24002385139465332) + | > loss_disc_real_3: 0.27563247084617615 (0.27563247084617615) + | > loss_disc_real_4: 0.22539491951465607 (0.22539491951465607) + | > loss_disc_real_5: 0.2349078357219696 (0.2349078357219696) + | > loss_0: 2.5817461013793945 (2.5817461013793945) + | > loss_gen: 2.236222743988037 (2.236222743988037) + | > loss_kl: 2.7355685234069824 (2.7355685234069824) + | > loss_feat: 6.705212116241455 (6.705212116241455) + | > loss_mel: 17.925294876098633 (17.925294876098633) + | > loss_duration: 1.7676725387573242 (1.7676725387573242) + | > loss_1: 31.369972229003906 (31.369972229003906) + + --> STEP: 2 + | > loss_disc: 2.613997459411621 (2.597871780395508) + | > loss_disc_real_0: 0.18413132429122925 (0.1780327707529068) + | > loss_disc_real_1: 0.21723024547100067 (0.22376219183206558) + | > loss_disc_real_2: 0.20218148827552795 (0.22110266983509064) + | > loss_disc_real_3: 0.26328739523887634 (0.26945993304252625) + | > loss_disc_real_4: 0.18581314384937286 (0.20560403168201447) + | > loss_disc_real_5: 0.25120213627815247 (0.24305498600006104) + | > loss_0: 2.613997459411621 (2.597871780395508) + | > loss_gen: 2.0965890884399414 (2.1664059162139893) + | > loss_kl: 2.312749147415161 (2.5241588354110718) + | > loss_feat: 6.735831260681152 (6.720521688461304) + | > loss_mel: 18.078535079956055 (18.001914978027344) + | > loss_duration: 1.6760108470916748 (1.7218416929244995) + | > loss_1: 30.899715423583984 (31.134843826293945) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003512263298034668 (-0.00038564205169677734) + | > avg_loss_disc: 2.597871780395508 (+0.08629810810089111) + | > avg_loss_disc_real_0: 0.1780327707529068 (+0.01343778520822525) + | > avg_loss_disc_real_1: 0.22376219183206558 (+0.019289903342723846) + | > avg_loss_disc_real_2: 0.22110266983509064 (-0.06785175204277039) + | > avg_loss_disc_real_3: 0.26945993304252625 (+0.04863575100898743) + | > avg_loss_disc_real_4: 0.20560403168201447 (-0.00524698942899704) + | > avg_loss_disc_real_5: 0.24305498600006104 (-0.0011759623885154724) + | > avg_loss_0: 2.597871780395508 (+0.08629810810089111) + | > avg_loss_gen: 2.1664059162139893 (-0.2549968957901001) + | > avg_loss_kl: 2.5241588354110718 (+0.10870778560638428) + | > avg_loss_feat: 6.720521688461304 (-0.7741358280181885) + | > avg_loss_mel: 18.001914978027344 (-0.6061010360717773) + | > avg_loss_duration: 1.7218416929244995 (+0.023233532905578613) + | > avg_loss_1: 31.134843826293945 (-1.5032920837402344) + + + > EPOCH: 236/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:52:27)  + + --> TIME: 2025-04-19 02:52:30 -- STEP: 6/142 -- GLOBAL_STEP: 412800 + | > loss_disc: 2.3610877990722656 (2.426741639773051) + | > loss_disc_real_0: 0.14564472436904907 (0.1280993645389875) + | > loss_disc_real_1: 0.19530712068080902 (0.19534851610660553) + | > loss_disc_real_2: 0.21301710605621338 (0.21827037384112677) + | > loss_disc_real_3: 0.1995301991701126 (0.23251982778310776) + | > loss_disc_real_4: 0.21405525505542755 (0.23852664728959402) + | > loss_disc_real_5: 0.22928014397621155 (0.2539892743031184) + | > loss_0: 2.3610877990722656 (2.426741639773051) + | > grad_norm_0: tensor(19.1424, device='cuda:0') (tensor(17.8206, device='cuda:0')) + | > loss_gen: 2.5507161617279053 (2.4962031841278076) + | > loss_kl: 1.6805819272994995 (1.7884154319763184) + | > loss_feat: 8.072373390197754 (7.665652513504028) + | > loss_mel: 19.125770568847656 (18.610324541727703) + | > loss_duration: 1.4058408737182617 (1.4298005898793538) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.83528518676758 (31.99039618174235) + | > grad_norm_1: tensor(354.4350, device='cuda:0') (tensor(352.9139, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.3298 (0.33471977710723877) + | > loader_time: 0.0039 (0.00443875789642334) + + + --> TIME: 2025-04-19 02:52:39 -- STEP: 31/142 -- GLOBAL_STEP: 412825 + | > loss_disc: 2.4070026874542236 (2.4189754301501867) + | > loss_disc_real_0: 0.10628105700016022 (0.117544584937634) + | > loss_disc_real_1: 0.2127520889043808 (0.2011584913538348) + | > loss_disc_real_2: 0.22414323687553406 (0.2238559530627343) + | > loss_disc_real_3: 0.19744771718978882 (0.22508113903384055) + | > loss_disc_real_4: 0.19604773819446564 (0.2232498598675574) + | > loss_disc_real_5: 0.19526314735412598 (0.23001796776248562) + | > loss_0: 2.4070026874542236 (2.4189754301501867) + | > grad_norm_0: tensor(12.5804, device='cuda:0') (tensor(13.3889, device='cuda:0')) + | > loss_gen: 2.2820775508880615 (2.4048333783303537) + | > loss_kl: 1.7589715719223022 (1.8203325079333397) + | > loss_feat: 7.795299530029297 (7.381087487743747) + | > loss_mel: 18.76299285888672 (18.09129653438445) + | > loss_duration: 1.4327640533447266 (1.4349001876769527) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.0321044921875 (31.132449857650265) + | > grad_norm_1: tensor(397.9827, device='cuda:0') (tensor(323.9500, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.3605 (0.3431594910160188) + | > loader_time: 0.0056 (0.0049847864335583105) + + + --> TIME: 2025-04-19 02:52:48 -- STEP: 56/142 -- GLOBAL_STEP: 412850 + | > loss_disc: 2.514683246612549 (2.4351560771465306) + | > loss_disc_real_0: 0.09654756635427475 (0.12024033788059439) + | > loss_disc_real_1: 0.19686239957809448 (0.20504343642720155) + | > loss_disc_real_2: 0.24977122247219086 (0.22618062208805764) + | > loss_disc_real_3: 0.2642175853252411 (0.22843698598444462) + | > loss_disc_real_4: 0.2252628356218338 (0.22590847366622516) + | > loss_disc_real_5: 0.2420235425233841 (0.2335614107017006) + | > loss_0: 2.514683246612549 (2.4351560771465306) + | > grad_norm_0: tensor(6.9457, device='cuda:0') (tensor(13.2335, device='cuda:0')) + | > loss_gen: 2.408294677734375 (2.40790182352066) + | > loss_kl: 1.740863561630249 (1.8361308595963888) + | > loss_feat: 6.976535797119141 (7.305503146989005) + | > loss_mel: 17.71118927001953 (18.0177310194288) + | > loss_duration: 1.4757654666900635 (1.4437780720846993) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.31264877319336 (31.01104477473668) + | > grad_norm_1: tensor(283.0694, device='cuda:0') (tensor(332.5382, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.3741 (0.3583445506436484) + | > loader_time: 0.0062 (0.005383487258638657) + + + --> TIME: 2025-04-19 02:52:58 -- STEP: 81/142 -- GLOBAL_STEP: 412875 + | > loss_disc: 2.4535508155822754 (2.4469091686201687) + | > loss_disc_real_0: 0.17968064546585083 (0.12397979420644266) + | > loss_disc_real_1: 0.2324436753988266 (0.2058954601302559) + | > loss_disc_real_2: 0.2352352738380432 (0.22649250133537951) + | > loss_disc_real_3: 0.2598859369754791 (0.22815282845202786) + | > loss_disc_real_4: 0.23461507260799408 (0.22684720894436777) + | > loss_disc_real_5: 0.22874975204467773 (0.23265796403090158) + | > loss_0: 2.4535508155822754 (2.4469091686201687) + | > grad_norm_0: tensor(21.3535, device='cuda:0') (tensor(13.4208, device='cuda:0')) + | > loss_gen: 2.481619119644165 (2.383734043733573) + | > loss_kl: 1.7800779342651367 (1.8476331896252103) + | > loss_feat: 6.999264240264893 (7.1592093632545) + | > loss_mel: 18.341175079345703 (17.789574952773105) + | > loss_duration: 1.4601141214370728 (1.4551088574491904) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.0622501373291 (30.63526021698375) + | > grad_norm_1: tensor(48.6912, device='cuda:0') (tensor(318.8883, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.3921 (0.3675818884814227) + | > loader_time: 0.0066 (0.005685968163572711) + + + --> TIME: 2025-04-19 02:53:08 -- STEP: 106/142 -- GLOBAL_STEP: 412900 + | > loss_disc: 2.4515039920806885 (2.4552226111574003) + | > loss_disc_real_0: 0.12601076066493988 (0.12518640783316684) + | > loss_disc_real_1: 0.1761273890733719 (0.20631661223915387) + | > loss_disc_real_2: 0.210905984044075 (0.22570170195035213) + | > loss_disc_real_3: 0.20519137382507324 (0.2290298671654935) + | > loss_disc_real_4: 0.230454683303833 (0.22854232113316375) + | > loss_disc_real_5: 0.234785795211792 (0.2331521121679612) + | > loss_0: 2.4515039920806885 (2.4552226111574003) + | > grad_norm_0: tensor(9.5520, device='cuda:0') (tensor(13.1523, device='cuda:0')) + | > loss_gen: 2.2997636795043945 (2.3682760809952357) + | > loss_kl: 1.8409136533737183 (1.8616258040913995) + | > loss_feat: 7.200881004333496 (7.076714439212151) + | > loss_mel: 17.391983032226562 (17.705064413682464) + | > loss_duration: 1.4943351745605469 (1.4652164252299185) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.227874755859375 (30.476896987771088) + | > grad_norm_1: tensor(294.2813, device='cuda:0') (tensor(312.8664, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.4106 (0.3765811785212103) + | > loader_time: 0.0079 (0.005970496051716355) + + + --> TIME: 2025-04-19 02:53:20 -- STEP: 131/142 -- GLOBAL_STEP: 412925 + | > loss_disc: 2.475303888320923 (2.4580338165050253) + | > loss_disc_real_0: 0.09869209676980972 (0.12552764283791756) + | > loss_disc_real_1: 0.21589967608451843 (0.206913182753643) + | > loss_disc_real_2: 0.2500635087490082 (0.22632652225385186) + | > loss_disc_real_3: 0.2066500186920166 (0.2288306224664659) + | > loss_disc_real_4: 0.21389912068843842 (0.22788661854867717) + | > loss_disc_real_5: 0.2228381633758545 (0.23258305005444826) + | > loss_0: 2.475303888320923 (2.4580338165050253) + | > grad_norm_0: tensor(6.1667, device='cuda:0') (tensor(13.4380, device='cuda:0')) + | > loss_gen: 2.3432021141052246 (2.357131240932086) + | > loss_kl: 1.903007984161377 (1.8692277861005477) + | > loss_feat: 7.1090168952941895 (7.030484345123058) + | > loss_mel: 17.21448516845703 (17.652359168948102) + | > loss_duration: 1.4950886964797974 (1.4710686816513996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.064800262451172 (30.380271096265954) + | > grad_norm_1: tensor(188.6362, device='cuda:0') (tensor(310.9987, device='cuda:0')) + | > current_lr_0: 0.0001941858174679673 + | > current_lr_1: 0.0001941858174679673 + | > step_time: 0.4501 (0.38745871937001936) + | > loader_time: 0.0077 (0.00626271917619778) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6364188194274902 (2.6364188194274902) + | > loss_disc_real_0: 0.15117879211902618 (0.15117879211902618) + | > loss_disc_real_1: 0.28427842259407043 (0.28427842259407043) + | > loss_disc_real_2: 0.2412760853767395 (0.2412760853767395) + | > loss_disc_real_3: 0.25668492913246155 (0.25668492913246155) + | > loss_disc_real_4: 0.2264508157968521 (0.2264508157968521) + | > loss_disc_real_5: 0.26538580656051636 (0.26538580656051636) + | > loss_0: 2.6364188194274902 (2.6364188194274902) + | > loss_gen: 2.2701470851898193 (2.2701470851898193) + | > loss_kl: 2.1577491760253906 (2.1577491760253906) + | > loss_feat: 6.171550273895264 (6.171550273895264) + | > loss_mel: 16.671430587768555 (16.671430587768555) + | > loss_duration: 1.6788349151611328 (1.6788349151611328) + | > loss_1: 28.9497127532959 (28.9497127532959) + + --> STEP: 1 + | > loss_disc: 2.313102960586548 (2.313102960586548) + | > loss_disc_real_0: 0.11731410026550293 (0.11731410026550293) + | > loss_disc_real_1: 0.24969875812530518 (0.24969875812530518) + | > loss_disc_real_2: 0.20775295794010162 (0.20775295794010162) + | > loss_disc_real_3: 0.21592113375663757 (0.21592113375663757) + | > loss_disc_real_4: 0.2236071527004242 (0.2236071527004242) + | > loss_disc_real_5: 0.25307589769363403 (0.25307589769363403) + | > loss_0: 2.313102960586548 (2.313102960586548) + | > loss_gen: 2.4663925170898438 (2.4663925170898438) + | > loss_kl: 2.930605888366699 (2.930605888366699) + | > loss_feat: 7.55465841293335 (7.55465841293335) + | > loss_mel: 19.189193725585938 (19.189193725585938) + | > loss_duration: 1.7262499332427979 (1.7262499332427979) + | > loss_1: 33.86709976196289 (33.86709976196289) + + --> STEP: 2 + | > loss_disc: 2.398088216781616 (2.355595588684082) + | > loss_disc_real_0: 0.13014645874500275 (0.12373027950525284) + | > loss_disc_real_1: 0.2646602392196655 (0.25717949867248535) + | > loss_disc_real_2: 0.20902961492538452 (0.20839128643274307) + | > loss_disc_real_3: 0.2571340501308441 (0.23652759194374084) + | > loss_disc_real_4: 0.18534858524799347 (0.20447786897420883) + | > loss_disc_real_5: 0.20519593358039856 (0.2291359156370163) + | > loss_0: 2.398088216781616 (2.355595588684082) + | > loss_gen: 2.4072046279907227 (2.436798572540283) + | > loss_kl: 2.220055341720581 (2.57533061504364) + | > loss_feat: 8.498039245605469 (8.02634882926941) + | > loss_mel: 18.129236221313477 (18.659214973449707) + | > loss_duration: 1.7045392990112305 (1.7153946161270142) + | > loss_1: 32.959075927734375 (33.41308784484863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035779476165771484 (+6.568431854248047e-05) + | > avg_loss_disc: 2.355595588684082 (-0.24227619171142578) + | > avg_loss_disc_real_0: 0.12373027950525284 (-0.05430249124765396) + | > avg_loss_disc_real_1: 0.25717949867248535 (+0.03341730684041977) + | > avg_loss_disc_real_2: 0.20839128643274307 (-0.012711383402347565) + | > avg_loss_disc_real_3: 0.23652759194374084 (-0.0329323410987854) + | > avg_loss_disc_real_4: 0.20447786897420883 (-0.0011261627078056335) + | > avg_loss_disc_real_5: 0.2291359156370163 (-0.013919070363044739) + | > avg_loss_0: 2.355595588684082 (-0.24227619171142578) + | > avg_loss_gen: 2.436798572540283 (+0.27039265632629395) + | > avg_loss_kl: 2.57533061504364 (+0.05117177963256836) + | > avg_loss_feat: 8.02634882926941 (+1.3058271408081055) + | > avg_loss_mel: 18.659214973449707 (+0.6572999954223633) + | > avg_loss_duration: 1.7153946161270142 (-0.0064470767974853516) + | > avg_loss_1: 33.41308784484863 (+2.2782440185546875) + + + > EPOCH: 237/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:53:26)  + + --> TIME: 2025-04-19 02:53:31 -- STEP: 14/142 -- GLOBAL_STEP: 412950 + | > loss_disc: 2.447720766067505 (2.4362172739846364) + | > loss_disc_real_0: 0.13108587265014648 (0.1233590692281723) + | > loss_disc_real_1: 0.19927561283111572 (0.20316316613129207) + | > loss_disc_real_2: 0.23051510751247406 (0.22411234783274786) + | > loss_disc_real_3: 0.21900203824043274 (0.22672249802521297) + | > loss_disc_real_4: 0.26071062684059143 (0.22869301374469483) + | > loss_disc_real_5: 0.21897579729557037 (0.22836045707975114) + | > loss_0: 2.447720766067505 (2.4362172739846364) + | > grad_norm_0: tensor(11.7470, device='cuda:0') (tensor(10.1301, device='cuda:0')) + | > loss_gen: 2.3656599521636963 (2.3314917598451887) + | > loss_kl: 1.8892748355865479 (1.8320497529847282) + | > loss_feat: 7.8107194900512695 (7.066320725849697) + | > loss_mel: 17.89103889465332 (17.176464693886892) + | > loss_duration: 1.4288848638534546 (1.420727823461805) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.385578155517578 (29.827054568699428) + | > grad_norm_1: tensor(310.5628, device='cuda:0') (tensor(188.1034, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.3336 (0.3277058941977365) + | > loader_time: 0.0049 (0.004601785114833287) + + + --> TIME: 2025-04-19 02:53:40 -- STEP: 39/142 -- GLOBAL_STEP: 412975 + | > loss_disc: 2.405738592147827 (2.4292391630319448) + | > loss_disc_real_0: 0.07732011377811432 (0.12042918648475255) + | > loss_disc_real_1: 0.19391028583049774 (0.20279404062491196) + | > loss_disc_real_2: 0.23261995613574982 (0.22395546848957354) + | > loss_disc_real_3: 0.26340582966804504 (0.2272137586887066) + | > loss_disc_real_4: 0.23251447081565857 (0.22877125480236152) + | > loss_disc_real_5: 0.21766337752342224 (0.22783692524983332) + | > loss_0: 2.405738592147827 (2.4292391630319448) + | > grad_norm_0: tensor(8.2154, device='cuda:0') (tensor(8.3156, device='cuda:0')) + | > loss_gen: 2.3585846424102783 (2.3567063686175223) + | > loss_kl: 1.913761854171753 (1.856356477126097) + | > loss_feat: 6.9029412269592285 (7.194062514182849) + | > loss_mel: 17.589099884033203 (17.6880691479414) + | > loss_duration: 1.4542282819747925 (1.4367415874432294) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.218616485595703 (30.531936009724934) + | > grad_norm_1: tensor(98.8385, device='cuda:0') (tensor(156.9014, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.3597 (0.34232786985544056) + | > loader_time: 0.0053 (0.005167936667417869) + + + --> TIME: 2025-04-19 02:53:50 -- STEP: 64/142 -- GLOBAL_STEP: 413000 + | > loss_disc: 2.4104745388031006 (2.445162169635296) + | > loss_disc_real_0: 0.1511101871728897 (0.12350956193404272) + | > loss_disc_real_1: 0.18708230555057526 (0.2035257986281067) + | > loss_disc_real_2: 0.2393401563167572 (0.22536219912581146) + | > loss_disc_real_3: 0.19108273088932037 (0.22728704055771232) + | > loss_disc_real_4: 0.20087794959545135 (0.23009154642932117) + | > loss_disc_real_5: 0.20448577404022217 (0.22915434394963086) + | > loss_0: 2.4104745388031006 (2.445162169635296) + | > grad_norm_0: tensor(19.0449, device='cuda:0') (tensor(8.9176, device='cuda:0')) + | > loss_gen: 2.4553894996643066 (2.3596088103950024) + | > loss_kl: 1.9305881261825562 (1.876143792644143) + | > loss_feat: 7.162014484405518 (7.140262380242348) + | > loss_mel: 18.40903091430664 (17.77344010770321) + | > loss_duration: 1.5020371675491333 (1.451252844184637) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.45905876159668 (30.600707918405533) + | > grad_norm_1: tensor(210.8260, device='cuda:0') (tensor(183.6652, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.3834 (0.35398080199956894) + | > loader_time: 0.0063 (0.005469631403684616) + + + --> TIME: 2025-04-19 02:54:00 -- STEP: 89/142 -- GLOBAL_STEP: 413025 + | > loss_disc: 2.4634416103363037 (2.4565146130122484) + | > loss_disc_real_0: 0.08128920942544937 (0.12468057665764617) + | > loss_disc_real_1: 0.1977469027042389 (0.2045611831244458) + | > loss_disc_real_2: 0.2165987640619278 (0.22668923019023424) + | > loss_disc_real_3: 0.22764267027378082 (0.22809224852015464) + | > loss_disc_real_4: 0.22890660166740417 (0.22985033473272) + | > loss_disc_real_5: 0.25004127621650696 (0.23145956085639052) + | > loss_0: 2.4634416103363037 (2.4565146130122484) + | > grad_norm_0: tensor(13.4680, device='cuda:0') (tensor(9.8261, device='cuda:0')) + | > loss_gen: 2.295053005218506 (2.3563478073377286) + | > loss_kl: 1.9994893074035645 (1.8914629770128915) + | > loss_feat: 7.473382949829102 (7.090568922878651) + | > loss_mel: 18.06571388244629 (17.815371663382884) + | > loss_duration: 1.473571538925171 (1.4622140587045904) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.30721092224121 (30.615965382436688) + | > grad_norm_1: tensor(293.9790, device='cuda:0') (tensor(207.8696, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.404 (0.36550251821453666) + | > loader_time: 0.0062 (0.0057209636388200055) + + + --> TIME: 2025-04-19 02:54:11 -- STEP: 114/142 -- GLOBAL_STEP: 413050 + | > loss_disc: 2.3798420429229736 (2.46058177947998) + | > loss_disc_real_0: 0.1073792576789856 (0.12588808460063056) + | > loss_disc_real_1: 0.18754252791404724 (0.20431193578661533) + | > loss_disc_real_2: 0.21444593369960785 (0.22595507756137012) + | > loss_disc_real_3: 0.21457929909229279 (0.2273983836435435) + | > loss_disc_real_4: 0.2333696335554123 (0.22971904669937335) + | > loss_disc_real_5: 0.23043924570083618 (0.23165378432001985) + | > loss_0: 2.3798420429229736 (2.46058177947998) + | > grad_norm_0: tensor(8.2420, device='cuda:0') (tensor(11.0697, device='cuda:0')) + | > loss_gen: 2.55509090423584 (2.3480314790156847) + | > loss_kl: 1.9409334659576416 (1.892978492536043) + | > loss_feat: 7.374556064605713 (7.062985236184639) + | > loss_mel: 19.474727630615234 (17.773070670010753) + | > loss_duration: 1.4996280670166016 (1.4695946741522403) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.844940185546875 (30.546660557127836) + | > grad_norm_1: tensor(151.0353, device='cuda:0') (tensor(227.2955, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.4218 (0.37589726322575634) + | > loader_time: 0.008 (0.005950388155485454) + + + --> TIME: 2025-04-19 02:54:22 -- STEP: 139/142 -- GLOBAL_STEP: 413075 + | > loss_disc: 2.456233024597168 (2.4665752160463397) + | > loss_disc_real_0: 0.16686061024665833 (0.12703106694405888) + | > loss_disc_real_1: 0.21062500774860382 (0.20554924933172816) + | > loss_disc_real_2: 0.22186270356178284 (0.22591950964155816) + | > loss_disc_real_3: 0.21182438731193542 (0.22789111851359442) + | > loss_disc_real_4: 0.2283080518245697 (0.23092146142781209) + | > loss_disc_real_5: 0.2402968555688858 (0.23295048211547112) + | > loss_0: 2.456233024597168 (2.4665752160463397) + | > grad_norm_0: tensor(6.3698, device='cuda:0') (tensor(10.9103, device='cuda:0')) + | > loss_gen: 2.4082553386688232 (2.343020847375444) + | > loss_kl: 1.90153169631958 (1.9025432583239439) + | > loss_feat: 7.3530449867248535 (7.020950677583543) + | > loss_mel: 18.666759490966797 (17.714940036801124) + | > loss_duration: 1.5054652690887451 (1.4758843346465407) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.83505630493164 (30.45733917016777) + | > grad_norm_1: tensor(215.6587, device='cuda:0') (tensor(232.3573, device='cuda:0')) + | > current_lr_0: 0.0001941615442407838 + | > current_lr_1: 0.0001941615442407838 + | > step_time: 0.4658 (0.3898078091710593) + | > loader_time: 0.0092 (0.006205275761995384) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4415407180786133 (2.4415407180786133) + | > loss_disc_real_0: 0.1661018431186676 (0.1661018431186676) + | > loss_disc_real_1: 0.21353605389595032 (0.21353605389595032) + | > loss_disc_real_2: 0.24036842584609985 (0.24036842584609985) + | > loss_disc_real_3: 0.2303076833486557 (0.2303076833486557) + | > loss_disc_real_4: 0.1976831555366516 (0.1976831555366516) + | > loss_disc_real_5: 0.20301997661590576 (0.20301997661590576) + | > loss_0: 2.4415407180786133 (2.4415407180786133) + | > loss_gen: 2.3504137992858887 (2.3504137992858887) + | > loss_kl: 2.3156354427337646 (2.3156354427337646) + | > loss_feat: 7.726467132568359 (7.726467132568359) + | > loss_mel: 18.75774383544922 (18.75774383544922) + | > loss_duration: 1.6760410070419312 (1.6760410070419312) + | > loss_1: 32.82630157470703 (32.82630157470703) + + --> STEP: 1 + | > loss_disc: 2.5216469764709473 (2.5216469764709473) + | > loss_disc_real_0: 0.13451579213142395 (0.13451579213142395) + | > loss_disc_real_1: 0.26039016246795654 (0.26039016246795654) + | > loss_disc_real_2: 0.24887961149215698 (0.24887961149215698) + | > loss_disc_real_3: 0.24562464654445648 (0.24562464654445648) + | > loss_disc_real_4: 0.26869019865989685 (0.26869019865989685) + | > loss_disc_real_5: 0.22241663932800293 (0.22241663932800293) + | > loss_0: 2.5216469764709473 (2.5216469764709473) + | > loss_gen: 2.3198654651641846 (2.3198654651641846) + | > loss_kl: 2.652418851852417 (2.652418851852417) + | > loss_feat: 7.190390110015869 (7.190390110015869) + | > loss_mel: 19.16904067993164 (19.16904067993164) + | > loss_duration: 1.7288535833358765 (1.7288535833358765) + | > loss_1: 33.060569763183594 (33.060569763183594) + + --> STEP: 2 + | > loss_disc: 2.402513027191162 (2.4620800018310547) + | > loss_disc_real_0: 0.14178399741649628 (0.1381498947739601) + | > loss_disc_real_1: 0.23970389366149902 (0.2500470280647278) + | > loss_disc_real_2: 0.25192755460739136 (0.25040358304977417) + | > loss_disc_real_3: 0.16909687221050262 (0.20736075937747955) + | > loss_disc_real_4: 0.22793328762054443 (0.24831174314022064) + | > loss_disc_real_5: 0.21049824357032776 (0.21645744144916534) + | > loss_0: 2.402513027191162 (2.4620800018310547) + | > loss_gen: 2.2157373428344727 (2.2678014039993286) + | > loss_kl: 2.1929638385772705 (2.4226913452148438) + | > loss_feat: 6.954885959625244 (7.072638034820557) + | > loss_mel: 18.559410095214844 (18.864225387573242) + | > loss_duration: 1.6637141704559326 (1.6962838768959045) + | > loss_1: 31.58670997619629 (32.32363986968994) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031795501708984375 (-0.00039839744567871094) + | > avg_loss_disc: 2.4620800018310547 (+0.10648441314697266) + | > avg_loss_disc_real_0: 0.1381498947739601 (+0.014419615268707275) + | > avg_loss_disc_real_1: 0.2500470280647278 (-0.007132470607757568) + | > avg_loss_disc_real_2: 0.25040358304977417 (+0.0420122966170311) + | > avg_loss_disc_real_3: 0.20736075937747955 (-0.02916683256626129) + | > avg_loss_disc_real_4: 0.24831174314022064 (+0.04383387416601181) + | > avg_loss_disc_real_5: 0.21645744144916534 (-0.012678474187850952) + | > avg_loss_0: 2.4620800018310547 (+0.10648441314697266) + | > avg_loss_gen: 2.2678014039993286 (-0.1689971685409546) + | > avg_loss_kl: 2.4226913452148438 (-0.1526392698287964) + | > avg_loss_feat: 7.072638034820557 (-0.9537107944488525) + | > avg_loss_mel: 18.864225387573242 (+0.20501041412353516) + | > avg_loss_duration: 1.6962838768959045 (-0.01911073923110962) + | > avg_loss_1: 32.32363986968994 (-1.0894479751586914) + + + > EPOCH: 238/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:54:25)  + + --> TIME: 2025-04-19 02:54:33 -- STEP: 22/142 -- GLOBAL_STEP: 413100 + | > loss_disc: 2.509014844894409 (2.4365826736796987) + | > loss_disc_real_0: 0.14653730392456055 (0.12355701252818108) + | > loss_disc_real_1: 0.19978396594524384 (0.20633820579810577) + | > loss_disc_real_2: 0.25816309452056885 (0.22759138318625363) + | > loss_disc_real_3: 0.26578187942504883 (0.22959452935240485) + | > loss_disc_real_4: 0.2545046806335449 (0.22938234088095752) + | > loss_disc_real_5: 0.24301224946975708 (0.22561803527853705) + | > loss_0: 2.509014844894409 (2.4365826736796987) + | > grad_norm_0: tensor(10.0067, device='cuda:0') (tensor(8.5329, device='cuda:0')) + | > loss_gen: 2.2854268550872803 (2.369664788246155) + | > loss_kl: 1.864030122756958 (1.8296921795064753) + | > loss_feat: 6.125905513763428 (7.213122476230968) + | > loss_mel: 15.340828895568848 (17.909323085438118) + | > loss_duration: 1.4117943048477173 (1.4288802851330151) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.027986526489258 (30.75068291750821) + | > grad_norm_1: tensor(84.6207, device='cuda:0') (tensor(158.6241, device='cuda:0')) + | > current_lr_0: 0.0001941372740477537 + | > current_lr_1: 0.0001941372740477537 + | > step_time: 0.3451 (0.3356083198027177) + | > loader_time: 0.0055 (0.004925619472156872) + + + --> TIME: 2025-04-19 02:54:42 -- STEP: 47/142 -- GLOBAL_STEP: 413125 + | > loss_disc: 2.4878339767456055 (2.439175118791296) + | > loss_disc_real_0: 0.11918630450963974 (0.1233865371409883) + | > loss_disc_real_1: 0.22046369314193726 (0.20505075600553066) + | > loss_disc_real_2: 0.22081880271434784 (0.22538438693005988) + | > loss_disc_real_3: 0.2062375396490097 (0.22668119599210454) + | > loss_disc_real_4: 0.21993611752986908 (0.22685651386037786) + | > loss_disc_real_5: 0.2208901196718216 (0.2300758311089049) + | > loss_0: 2.4878339767456055 (2.439175118791296) + | > grad_norm_0: tensor(5.2063, device='cuda:0') (tensor(7.2774, device='cuda:0')) + | > loss_gen: 2.354624032974243 (2.3747405194221662) + | > loss_kl: 1.9731240272521973 (1.8666049521020118) + | > loss_feat: 7.409427642822266 (7.221937991203146) + | > loss_mel: 17.675127029418945 (17.944113629929564) + | > loss_duration: 1.4054888486862183 (1.4414911422323673) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.817790985107422 (30.84888827547114) + | > grad_norm_1: tensor(60.7305, device='cuda:0') (tensor(129.3961, device='cuda:0')) + | > current_lr_0: 0.0001941372740477537 + | > current_lr_1: 0.0001941372740477537 + | > step_time: 0.3767 (0.3470314563588893) + | > loader_time: 0.0059 (0.005275898791374043) + + + --> TIME: 2025-04-19 02:54:52 -- STEP: 72/142 -- GLOBAL_STEP: 413150 + | > loss_disc: 2.5117251873016357 (2.4566515154308743) + | > loss_disc_real_0: 0.13929994404315948 (0.12971175720708242) + | > loss_disc_real_1: 0.2210671603679657 (0.20606503263115883) + | > loss_disc_real_2: 0.2091267704963684 (0.22500158970554673) + | > loss_disc_real_3: 0.2642311155796051 (0.2278841082006693) + | > loss_disc_real_4: 0.22349481284618378 (0.2296498879376385) + | > loss_disc_real_5: 0.27990418672561646 (0.2319527351193958) + | > loss_0: 2.5117251873016357 (2.4566515154308743) + | > grad_norm_0: tensor(10.4995, device='cuda:0') (tensor(7.5728, device='cuda:0')) + | > loss_gen: 2.2489452362060547 (2.363817264636357) + | > loss_kl: 1.7693841457366943 (1.8743848933113947) + | > loss_feat: 7.012925624847412 (7.162366410096486) + | > loss_mel: 18.56438446044922 (18.01424005296495) + | > loss_duration: 1.4523237943649292 (1.4534265382422342) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.047964096069336 (30.86823519070943) + | > grad_norm_1: tensor(321.4968, device='cuda:0') (tensor(131.6089, device='cuda:0')) + | > current_lr_0: 0.0001941372740477537 + | > current_lr_1: 0.0001941372740477537 + | > step_time: 0.3964 (0.35833150148391724) + | > loader_time: 0.0064 (0.005582150485780504) + + + --> TIME: 2025-04-19 02:55:02 -- STEP: 97/142 -- GLOBAL_STEP: 413175 + | > loss_disc: 2.6758270263671875 (2.469129382949515) + | > loss_disc_real_0: 0.10237004607915878 (0.1323590966844067) + | > loss_disc_real_1: 0.2165031135082245 (0.20806865209771186) + | > loss_disc_real_2: 0.2400818020105362 (0.22501559079307876) + | > loss_disc_real_3: 0.2163562923669815 (0.22640906965609678) + | > loss_disc_real_4: 0.2895403206348419 (0.2298059841406714) + | > loss_disc_real_5: 0.2554129958152771 (0.23066868364196463) + | > loss_0: 2.6758270263671875 (2.469129382949515) + | > grad_norm_0: tensor(7.4790, device='cuda:0') (tensor(7.5227, device='cuda:0')) + | > loss_gen: 2.270878791809082 (2.3521278666466765) + | > loss_kl: 1.741385579109192 (1.8728451224946485) + | > loss_feat: 7.069899559020996 (7.078263007488447) + | > loss_mel: 17.478090286254883 (17.97306255458557) + | > loss_duration: 1.4948961734771729 (1.4646367154170556) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05514907836914 (30.740935197810536) + | > grad_norm_1: tensor(71.8481, device='cuda:0') (tensor(137.0652, device='cuda:0')) + | > current_lr_0: 0.0001941372740477537 + | > current_lr_1: 0.0001941372740477537 + | > step_time: 0.4075 (0.3689229537531271) + | > loader_time: 0.0069 (0.005872849336604482) + + + --> TIME: 2025-04-19 02:55:13 -- STEP: 122/142 -- GLOBAL_STEP: 413200 + | > loss_disc: 2.520477294921875 (2.47373990152703) + | > loss_disc_real_0: 0.14044013619422913 (0.13281714152850085) + | > loss_disc_real_1: 0.2221578061580658 (0.20824019427670806) + | > loss_disc_real_2: 0.20634222030639648 (0.22550227883897847) + | > loss_disc_real_3: 0.1990663707256317 (0.22745352788049666) + | > loss_disc_real_4: 0.20138715207576752 (0.22906996102118102) + | > loss_disc_real_5: 0.21988283097743988 (0.2311287253606515) + | > loss_0: 2.520477294921875 (2.47373990152703) + | > grad_norm_0: tensor(10.1469, device='cuda:0') (tensor(7.7404, device='cuda:0')) + | > loss_gen: 2.2030222415924072 (2.3445718640186737) + | > loss_kl: 1.7555803060531616 (1.8838311980982296) + | > loss_feat: 6.422892093658447 (7.047887477718416) + | > loss_mel: 15.761613845825195 (17.927929041815595) + | > loss_duration: 1.4892587661743164 (1.4718620132227413) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.632366180419922 (30.676081516703622) + | > grad_norm_1: tensor(301.9820, device='cuda:0') (tensor(145.7213, device='cuda:0')) + | > current_lr_0: 0.0001941372740477537 + | > current_lr_1: 0.0001941372740477537 + | > step_time: 0.4517 (0.3798968381569033) + | > loader_time: 0.0073 (0.006130470604193015) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5494370460510254 (2.5494370460510254) + | > loss_disc_real_0: 0.2077987641096115 (0.2077987641096115) + | > loss_disc_real_1: 0.22653262317180634 (0.22653262317180634) + | > loss_disc_real_2: 0.2561332583427429 (0.2561332583427429) + | > loss_disc_real_3: 0.26520097255706787 (0.26520097255706787) + | > loss_disc_real_4: 0.24470752477645874 (0.24470752477645874) + | > loss_disc_real_5: 0.3204204738140106 (0.3204204738140106) + | > loss_0: 2.5494370460510254 (2.5494370460510254) + | > loss_gen: 2.650068998336792 (2.650068998336792) + | > loss_kl: 2.2152082920074463 (2.2152082920074463) + | > loss_feat: 6.7324137687683105 (6.7324137687683105) + | > loss_mel: 19.466108322143555 (19.466108322143555) + | > loss_duration: 1.686058521270752 (1.686058521270752) + | > loss_1: 32.74985885620117 (32.74985885620117) + + --> STEP: 1 + | > loss_disc: 2.5594825744628906 (2.5594825744628906) + | > loss_disc_real_0: 0.25647446513175964 (0.25647446513175964) + | > loss_disc_real_1: 0.24522936344146729 (0.24522936344146729) + | > loss_disc_real_2: 0.23189795017242432 (0.23189795017242432) + | > loss_disc_real_3: 0.2279430478811264 (0.2279430478811264) + | > loss_disc_real_4: 0.23820747435092926 (0.23820747435092926) + | > loss_disc_real_5: 0.23165400326251984 (0.23165400326251984) + | > loss_0: 2.5594825744628906 (2.5594825744628906) + | > loss_gen: 2.43232798576355 (2.43232798576355) + | > loss_kl: 2.5992586612701416 (2.5992586612701416) + | > loss_feat: 6.721980571746826 (6.721980571746826) + | > loss_mel: 17.705854415893555 (17.705854415893555) + | > loss_duration: 1.7245111465454102 (1.7245111465454102) + | > loss_1: 31.18393325805664 (31.18393325805664) + + --> STEP: 2 + | > loss_disc: 2.746640205383301 (2.6530613899230957) + | > loss_disc_real_0: 0.30378079414367676 (0.2801276296377182) + | > loss_disc_real_1: 0.26298046112060547 (0.2541049122810364) + | > loss_disc_real_2: 0.2704739272594452 (0.25118593871593475) + | > loss_disc_real_3: 0.2739614248275757 (0.25095223635435104) + | > loss_disc_real_4: 0.23846469819545746 (0.23833608627319336) + | > loss_disc_real_5: 0.22755542397499084 (0.22960471361875534) + | > loss_0: 2.746640205383301 (2.6530613899230957) + | > loss_gen: 2.4433505535125732 (2.4378392696380615) + | > loss_kl: 2.3552377223968506 (2.477248191833496) + | > loss_feat: 6.817935466766357 (6.769958019256592) + | > loss_mel: 17.58245277404785 (17.644153594970703) + | > loss_duration: 1.6421005725860596 (1.6833058595657349) + | > loss_1: 30.84107780456543 (31.012505531311035) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037015676498413086 (+0.0005220174789428711) + | > avg_loss_disc: 2.6530613899230957 (+0.19098138809204102) + | > avg_loss_disc_real_0: 0.2801276296377182 (+0.1419777348637581) + | > avg_loss_disc_real_1: 0.2541049122810364 (+0.004057884216308594) + | > avg_loss_disc_real_2: 0.25118593871593475 (+0.0007823556661605835) + | > avg_loss_disc_real_3: 0.25095223635435104 (+0.04359147697687149) + | > avg_loss_disc_real_4: 0.23833608627319336 (-0.009975656867027283) + | > avg_loss_disc_real_5: 0.22960471361875534 (+0.013147272169589996) + | > avg_loss_0: 2.6530613899230957 (+0.19098138809204102) + | > avg_loss_gen: 2.4378392696380615 (+0.1700378656387329) + | > avg_loss_kl: 2.477248191833496 (+0.054556846618652344) + | > avg_loss_feat: 6.769958019256592 (-0.30268001556396484) + | > avg_loss_mel: 17.644153594970703 (-1.220071792602539) + | > avg_loss_duration: 1.6833058595657349 (-0.012978017330169678) + | > avg_loss_1: 31.012505531311035 (-1.3111343383789062) + + + > EPOCH: 239/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:55:24)  + + --> TIME: 2025-04-19 02:55:26 -- STEP: 5/142 -- GLOBAL_STEP: 413225 + | > loss_disc: 2.359034776687622 (2.406018018722534) + | > loss_disc_real_0: 0.17824040353298187 (0.11744005531072617) + | > loss_disc_real_1: 0.1646084189414978 (0.19491395056247712) + | > loss_disc_real_2: 0.18526144325733185 (0.22122092843055724) + | > loss_disc_real_3: 0.210395947098732 (0.2284278988838196) + | > loss_disc_real_4: 0.2239745557308197 (0.24368185997009278) + | > loss_disc_real_5: 0.22606714069843292 (0.23998510837554932) + | > loss_0: 2.359034776687622 (2.406018018722534) + | > grad_norm_0: tensor(12.5163, device='cuda:0') (tensor(12.9082, device='cuda:0')) + | > loss_gen: 2.4818150997161865 (2.4422035694122313) + | > loss_kl: 1.5247420072555542 (1.7109347581863403) + | > loss_feat: 7.8265485763549805 (7.536667537689209) + | > loss_mel: 17.719776153564453 (17.977353286743163) + | > loss_duration: 1.4350100755691528 (1.4348783493041992) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.98789405822754 (31.102037811279295) + | > grad_norm_1: tensor(187.7377, device='cuda:0') (tensor(283.8778, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.3253 (0.32496938705444334) + | > loader_time: 0.004 (0.004176282882690429) + + + --> TIME: 2025-04-19 02:55:35 -- STEP: 30/142 -- GLOBAL_STEP: 413250 + | > loss_disc: 2.331073760986328 (2.483069411913553) + | > loss_disc_real_0: 0.09181379526853561 (0.12042079617579778) + | > loss_disc_real_1: 0.2197589874267578 (0.21222069611152014) + | > loss_disc_real_2: 0.23530885577201843 (0.22783749004205067) + | > loss_disc_real_3: 0.22808954119682312 (0.23243213891983033) + | > loss_disc_real_4: 0.22694054245948792 (0.23096862186988196) + | > loss_disc_real_5: 0.24017687141895294 (0.24134633541107178) + | > loss_0: 2.331073760986328 (2.483069411913553) + | > grad_norm_0: tensor(17.5531, device='cuda:0') (tensor(13.3020, device='cuda:0')) + | > loss_gen: 2.5176455974578857 (2.381770157814026) + | > loss_kl: 1.9832866191864014 (1.8106964786847433) + | > loss_feat: 7.735066890716553 (7.222612396876017) + | > loss_mel: 18.102508544921875 (17.643363380432127) + | > loss_duration: 1.460920810699463 (1.4361793597539265) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.799427032470703 (30.494621404012044) + | > grad_norm_1: tensor(347.2167, device='cuda:0') (tensor(306.5613, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.351 (0.337294602394104) + | > loader_time: 0.0051 (0.004809697469075522) + + + --> TIME: 2025-04-19 02:55:44 -- STEP: 55/142 -- GLOBAL_STEP: 413275 + | > loss_disc: 2.5698041915893555 (2.462082173607566) + | > loss_disc_real_0: 0.10264088958501816 (0.12202512134205211) + | > loss_disc_real_1: 0.24432365596294403 (0.21012582453814418) + | > loss_disc_real_2: 0.2507582902908325 (0.22694311954758384) + | > loss_disc_real_3: 0.26775163412094116 (0.2287356113845652) + | > loss_disc_real_4: 0.27002277970314026 (0.22793160378932953) + | > loss_disc_real_5: 0.2602599561214447 (0.2338048135692423) + | > loss_0: 2.5698041915893555 (2.462082173607566) + | > grad_norm_0: tensor(7.8037, device='cuda:0') (tensor(11.7328, device='cuda:0')) + | > loss_gen: 2.4171955585479736 (2.3670659802176734) + | > loss_kl: 1.893423080444336 (1.8343590823086826) + | > loss_feat: 7.073835849761963 (7.17053110816262) + | > loss_mel: 16.86690902709961 (17.64204538518732) + | > loss_duration: 1.4990105628967285 (1.444416169686751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.75037384033203 (30.45841747630726) + | > grad_norm_1: tensor(226.0493, device='cuda:0') (tensor(261.3289, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.3715 (0.34871445135636764) + | > loader_time: 0.0059 (0.00514893965287642) + + + --> TIME: 2025-04-19 02:55:54 -- STEP: 80/142 -- GLOBAL_STEP: 413300 + | > loss_disc: 2.3670055866241455 (2.479225081205368) + | > loss_disc_real_0: 0.09347521513700485 (0.12415555464103818) + | > loss_disc_real_1: 0.15234005451202393 (0.21008903589099645) + | > loss_disc_real_2: 0.23095160722732544 (0.2280245766043663) + | > loss_disc_real_3: 0.21930097043514252 (0.23112777527421713) + | > loss_disc_real_4: 0.2169092446565628 (0.22863622661679983) + | > loss_disc_real_5: 0.2357194721698761 (0.2358342783525586) + | > loss_0: 2.3670055866241455 (2.479225081205368) + | > grad_norm_0: tensor(4.9333, device='cuda:0') (tensor(11.5119, device='cuda:0')) + | > loss_gen: 2.514709949493408 (2.349879264831543) + | > loss_kl: 1.9126685857772827 (1.8533470064401627) + | > loss_feat: 7.518300533294678 (7.082466769218445) + | > loss_mel: 18.402873992919922 (17.69820103645325) + | > loss_duration: 1.4760288000106812 (1.4558602556586269) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.824581146240234 (30.439754152297972) + | > grad_norm_1: tensor(152.5878, device='cuda:0') (tensor(250.5464, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.4084 (0.3613856434822083) + | > loader_time: 0.0063 (0.005456563830375672) + + + --> TIME: 2025-04-19 02:56:04 -- STEP: 105/142 -- GLOBAL_STEP: 413325 + | > loss_disc: 2.4136462211608887 (2.473707666851225) + | > loss_disc_real_0: 0.09882143139839172 (0.12644278996047514) + | > loss_disc_real_1: 0.2111833542585373 (0.20925361372175671) + | > loss_disc_real_2: 0.23567752540111542 (0.22701008717219034) + | > loss_disc_real_3: 0.22648915648460388 (0.23047661724544707) + | > loss_disc_real_4: 0.1952659636735916 (0.22790674794287907) + | > loss_disc_real_5: 0.2283572405576706 (0.23383958155200596) + | > loss_0: 2.4136462211608887 (2.473707666851225) + | > grad_norm_0: tensor(9.8707, device='cuda:0') (tensor(12.4475, device='cuda:0')) + | > loss_gen: 2.501814842224121 (2.3563030015854602) + | > loss_kl: 2.0699009895324707 (1.8758653402328491) + | > loss_feat: 7.363569259643555 (7.054268287477039) + | > loss_mel: 18.541501998901367 (17.679501342773438) + | > loss_duration: 1.505169153213501 (1.4658516849790304) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.981956481933594 (30.431789416358583) + | > grad_norm_1: tensor(151.2849, device='cuda:0') (tensor(258.2845, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.4226 (0.3726291565668015) + | > loader_time: 0.0068 (0.005729032698131745) + + + --> TIME: 2025-04-19 02:56:16 -- STEP: 130/142 -- GLOBAL_STEP: 413350 + | > loss_disc: 2.4081549644470215 (2.4793183601819546) + | > loss_disc_real_0: 0.1439211666584015 (0.12714879128795395) + | > loss_disc_real_1: 0.20212464034557343 (0.20935176278536136) + | > loss_disc_real_2: 0.21227507293224335 (0.22875138979691725) + | > loss_disc_real_3: 0.23323652148246765 (0.23147452427790716) + | > loss_disc_real_4: 0.20725415647029877 (0.22897867892797177) + | > loss_disc_real_5: 0.21829979121685028 (0.23458105360086148) + | > loss_0: 2.4081549644470215 (2.4793183601819546) + | > grad_norm_0: tensor(5.4538, device='cuda:0') (tensor(11.8747, device='cuda:0')) + | > loss_gen: 2.410128593444824 (2.3500205076657807) + | > loss_kl: 1.8073735237121582 (1.8760371354910044) + | > loss_feat: 7.264434337615967 (7.0071383329538195) + | > loss_mel: 17.768280029296875 (17.65917195540208) + | > loss_duration: 1.4675073623657227 (1.472761654853821) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.717723846435547 (30.36512926541842) + | > grad_norm_1: tensor(314.4297, device='cuda:0') (tensor(255.8780, device='cuda:0')) + | > current_lr_0: 0.0001941130068884977 + | > current_lr_1: 0.0001941130068884977 + | > step_time: 0.4694 (0.38508161764878496) + | > loader_time: 0.0078 (0.006007858423086315) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3240177631378174 (2.3240177631378174) + | > loss_disc_real_0: 0.10363402217626572 (0.10363402217626572) + | > loss_disc_real_1: 0.2263851761817932 (0.2263851761817932) + | > loss_disc_real_2: 0.28314849734306335 (0.28314849734306335) + | > loss_disc_real_3: 0.2411404550075531 (0.2411404550075531) + | > loss_disc_real_4: 0.2096368968486786 (0.2096368968486786) + | > loss_disc_real_5: 0.2643145024776459 (0.2643145024776459) + | > loss_0: 2.3240177631378174 (2.3240177631378174) + | > loss_gen: 2.578248977661133 (2.578248977661133) + | > loss_kl: 2.4691002368927 (2.4691002368927) + | > loss_feat: 7.081856727600098 (7.081856727600098) + | > loss_mel: 18.883968353271484 (18.883968353271484) + | > loss_duration: 1.7094610929489136 (1.7094610929489136) + | > loss_1: 32.722633361816406 (32.722633361816406) + + --> STEP: 1 + | > loss_disc: 2.5866539478302 (2.5866539478302) + | > loss_disc_real_0: 0.1148684173822403 (0.1148684173822403) + | > loss_disc_real_1: 0.26806581020355225 (0.26806581020355225) + | > loss_disc_real_2: 0.29162994027137756 (0.29162994027137756) + | > loss_disc_real_3: 0.25374630093574524 (0.25374630093574524) + | > loss_disc_real_4: 0.258883535861969 (0.258883535861969) + | > loss_disc_real_5: 0.2830631136894226 (0.2830631136894226) + | > loss_0: 2.5866539478302 (2.5866539478302) + | > loss_gen: 2.3162715435028076 (2.3162715435028076) + | > loss_kl: 2.7259128093719482 (2.7259128093719482) + | > loss_feat: 6.236515045166016 (6.236515045166016) + | > loss_mel: 15.647435188293457 (15.647435188293457) + | > loss_duration: 1.725957989692688 (1.725957989692688) + | > loss_1: 28.65209197998047 (28.65209197998047) + + --> STEP: 2 + | > loss_disc: 2.2872252464294434 (2.4369395971298218) + | > loss_disc_real_0: 0.10912731289863586 (0.11199786514043808) + | > loss_disc_real_1: 0.21864914894104004 (0.24335747957229614) + | > loss_disc_real_2: 0.2723610997200012 (0.2819955199956894) + | > loss_disc_real_3: 0.21143920719623566 (0.23259275406599045) + | > loss_disc_real_4: 0.2410878986120224 (0.2499857172369957) + | > loss_disc_real_5: 0.2200934737920761 (0.25157829374074936) + | > loss_0: 2.2872252464294434 (2.4369395971298218) + | > loss_gen: 2.524747848510742 (2.420509696006775) + | > loss_kl: 2.1386077404022217 (2.432260274887085) + | > loss_feat: 8.689556121826172 (7.463035583496094) + | > loss_mel: 18.81557846069336 (17.231506824493408) + | > loss_duration: 1.6673295497894287 (1.6966437697410583) + | > loss_1: 33.8358154296875 (31.243953704833984) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036367177963256836 (-6.4849853515625e-05) + | > avg_loss_disc: 2.4369395971298218 (-0.21612179279327393) + | > avg_loss_disc_real_0: 0.11199786514043808 (-0.16812976449728012) + | > avg_loss_disc_real_1: 0.24335747957229614 (-0.010747432708740234) + | > avg_loss_disc_real_2: 0.2819955199956894 (+0.03080958127975464) + | > avg_loss_disc_real_3: 0.23259275406599045 (-0.018359482288360596) + | > avg_loss_disc_real_4: 0.2499857172369957 (+0.011649630963802338) + | > avg_loss_disc_real_5: 0.25157829374074936 (+0.02197358012199402) + | > avg_loss_0: 2.4369395971298218 (-0.21612179279327393) + | > avg_loss_gen: 2.420509696006775 (-0.01732957363128662) + | > avg_loss_kl: 2.432260274887085 (-0.04498791694641113) + | > avg_loss_feat: 7.463035583496094 (+0.693077564239502) + | > avg_loss_mel: 17.231506824493408 (-0.4126467704772949) + | > avg_loss_duration: 1.6966437697410583 (+0.013337910175323486) + | > avg_loss_1: 31.243953704833984 (+0.23144817352294922) + + + > EPOCH: 240/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:56:22)  + + --> TIME: 2025-04-19 02:56:28 -- STEP: 13/142 -- GLOBAL_STEP: 413375 + | > loss_disc: 2.4725959300994873 (2.414677986731896) + | > loss_disc_real_0: 0.09950798004865646 (0.11010380261219464) + | > loss_disc_real_1: 0.21503902971744537 (0.20325557772929853) + | > loss_disc_real_2: 0.23021076619625092 (0.22555030538485601) + | > loss_disc_real_3: 0.23966529965400696 (0.2244674964593007) + | > loss_disc_real_4: 0.22013641893863678 (0.22920607259640327) + | > loss_disc_real_5: 0.2355911284685135 (0.22822248477202195) + | > loss_0: 2.4725959300994873 (2.414677986731896) + | > grad_norm_0: tensor(10.5574, device='cuda:0') (tensor(9.5721, device='cuda:0')) + | > loss_gen: 2.379162311553955 (2.396498074898353) + | > loss_kl: 1.8428727388381958 (1.7805879757954524) + | > loss_feat: 6.864802360534668 (7.345583328833947) + | > loss_mel: 16.501644134521484 (17.76011555011456) + | > loss_duration: 1.4462409019470215 (1.4231408742757945) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.03472328186035 (30.705926161545975) + | > grad_norm_1: tensor(37.7470, device='cuda:0') (tensor(257.2700, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.332 (0.33687276106614333) + | > loader_time: 0.0048 (0.004570759259737455) + + + --> TIME: 2025-04-19 02:56:37 -- STEP: 38/142 -- GLOBAL_STEP: 413400 + | > loss_disc: 2.5042338371276855 (2.433231792951885) + | > loss_disc_real_0: 0.13056576251983643 (0.11652344680930439) + | > loss_disc_real_1: 0.1909380704164505 (0.2083176699908156) + | > loss_disc_real_2: 0.2012939304113388 (0.22541126334353498) + | > loss_disc_real_3: 0.22397924959659576 (0.22540900895470067) + | > loss_disc_real_4: 0.21694359183311462 (0.22651531547307968) + | > loss_disc_real_5: 0.23266379535198212 (0.23341914972192362) + | > loss_0: 2.5042338371276855 (2.433231792951885) + | > grad_norm_0: tensor(15.9008, device='cuda:0') (tensor(10.6648, device='cuda:0')) + | > loss_gen: 2.255728006362915 (2.391308113148338) + | > loss_kl: 1.79172945022583 (1.8097839041760093) + | > loss_feat: 7.20206356048584 (7.238743455786454) + | > loss_mel: 16.47637176513672 (17.84560610118665) + | > loss_duration: 1.4595179557800293 (1.4379244321270994) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.18541145324707 (30.7233661350451) + | > grad_norm_1: tensor(314.1009, device='cuda:0') (tensor(269.6732, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.3632 (0.34569300475873443) + | > loader_time: 0.0059 (0.004962444305419923) + + + --> TIME: 2025-04-19 02:56:46 -- STEP: 63/142 -- GLOBAL_STEP: 413425 + | > loss_disc: 2.5351858139038086 (2.444718213308424) + | > loss_disc_real_0: 0.1582392454147339 (0.11900036900289475) + | > loss_disc_real_1: 0.21208485960960388 (0.21057113342814976) + | > loss_disc_real_2: 0.25325533747673035 (0.22514664819316257) + | > loss_disc_real_3: 0.23108091950416565 (0.22676432653078957) + | > loss_disc_real_4: 0.23289300501346588 (0.22623609928857713) + | > loss_disc_real_5: 0.266400545835495 (0.23332346171613724) + | > loss_0: 2.5351858139038086 (2.444718213308424) + | > grad_norm_0: tensor(8.4231, device='cuda:0') (tensor(10.3870, device='cuda:0')) + | > loss_gen: 2.266777515411377 (2.376159013263763) + | > loss_kl: 1.8680176734924316 (1.817141436395191) + | > loss_feat: 6.167044162750244 (7.151824466765873) + | > loss_mel: 16.992446899414062 (17.73050444466727) + | > loss_duration: 1.5122733116149902 (1.4500080763347565) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.80655860900879 (30.525637550959512) + | > grad_norm_1: tensor(261.6485, device='cuda:0') (tensor(255.9370, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.3913 (0.3558783039214119) + | > loader_time: 0.0061 (0.005386674214923192) + + + --> TIME: 2025-04-19 02:56:56 -- STEP: 88/142 -- GLOBAL_STEP: 413450 + | > loss_disc: 2.3770697116851807 (2.4603055146607473) + | > loss_disc_real_0: 0.13075484335422516 (0.12248415457592769) + | > loss_disc_real_1: 0.18752525746822357 (0.20918321067636664) + | > loss_disc_real_2: 0.21658623218536377 (0.2265597852793607) + | > loss_disc_real_3: 0.21772132813930511 (0.22836722924628042) + | > loss_disc_real_4: 0.23905904591083527 (0.2287332813509486) + | > loss_disc_real_5: 0.21441100537776947 (0.23385656489567322) + | > loss_0: 2.3770697116851807 (2.4603055146607473) + | > grad_norm_0: tensor(14.3928, device='cuda:0') (tensor(11.0626, device='cuda:0')) + | > loss_gen: 2.365126848220825 (2.362585842609406) + | > loss_kl: 1.9496755599975586 (1.8345178934660824) + | > loss_feat: 6.717456340789795 (7.1236573891206225) + | > loss_mel: 18.127113342285156 (17.789173462174155) + | > loss_duration: 1.549952745437622 (1.4621705968271606) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.709325790405273 (30.572105234319512) + | > grad_norm_1: tensor(296.4677, device='cuda:0') (tensor(254.3062, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.4023 (0.36769233508543553) + | > loader_time: 0.0066 (0.005701785737817938) + + + --> TIME: 2025-04-19 02:57:07 -- STEP: 113/142 -- GLOBAL_STEP: 413475 + | > loss_disc: 2.490771532058716 (2.460624471174931) + | > loss_disc_real_0: 0.09234770387411118 (0.12455498090887492) + | > loss_disc_real_1: 0.20552700757980347 (0.20876297737117364) + | > loss_disc_real_2: 0.22511275112628937 (0.22583076687513198) + | > loss_disc_real_3: 0.2002333253622055 (0.22789625520199802) + | > loss_disc_real_4: 0.25323250889778137 (0.22883721777295643) + | > loss_disc_real_5: 0.2120671272277832 (0.23319752382493653) + | > loss_0: 2.490771532058716 (2.460624471174931) + | > grad_norm_0: tensor(10.0558, device='cuda:0') (tensor(12.1928, device='cuda:0')) + | > loss_gen: 2.328514575958252 (2.3636629286065576) + | > loss_kl: 1.9847654104232788 (1.8448401858321333) + | > loss_feat: 7.362174987792969 (7.114780219255295) + | > loss_mel: 16.743852615356445 (17.734163208345393) + | > loss_duration: 1.5060780048370361 (1.4697822511723613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.925386428833008 (30.527228870223055) + | > grad_norm_1: tensor(457.9828, device='cuda:0') (tensor(270.8861, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.4258 (0.37953981678042803) + | > loader_time: 0.0068 (0.005947562445581487) + + + --> TIME: 2025-04-19 02:57:18 -- STEP: 138/142 -- GLOBAL_STEP: 413500 + | > loss_disc: 2.4827136993408203 (2.467158661372419) + | > loss_disc_real_0: 0.17821422219276428 (0.1273509366572767) + | > loss_disc_real_1: 0.218987837433815 (0.2100065904467002) + | > loss_disc_real_2: 0.23945192992687225 (0.22554976374342822) + | > loss_disc_real_3: 0.2498631328344345 (0.22875864674215732) + | > loss_disc_real_4: 0.22974686324596405 (0.2280140034314515) + | > loss_disc_real_5: 0.24072127044200897 (0.2333076059602309) + | > loss_0: 2.4827136993408203 (2.467158661372419) + | > grad_norm_0: tensor(14.0905, device='cuda:0') (tensor(13.4312, device='cuda:0')) + | > loss_gen: 2.4442646503448486 (2.365303506021915) + | > loss_kl: 1.8876525163650513 (1.854970582153486) + | > loss_feat: 6.5586981773376465 (7.084357942360034) + | > loss_mel: 17.984025955200195 (17.693053839863197) + | > loss_duration: 1.5456554889678955 (1.4759351844372959) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.420299530029297 (30.473621188730434) + | > grad_norm_1: tensor(250.4669, device='cuda:0') (tensor(279.7697, device='cuda:0')) + | > current_lr_0: 0.00019408874276263664 + | > current_lr_1: 0.00019408874276263664 + | > step_time: 0.453 (0.3906461743340977) + | > loader_time: 0.0073 (0.00617964544158051) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.567756414413452 (2.567756414413452) + | > loss_disc_real_0: 0.18674030900001526 (0.18674030900001526) + | > loss_disc_real_1: 0.2139599770307541 (0.2139599770307541) + | > loss_disc_real_2: 0.24824364483356476 (0.24824364483356476) + | > loss_disc_real_3: 0.28061041235923767 (0.28061041235923767) + | > loss_disc_real_4: 0.2171342372894287 (0.2171342372894287) + | > loss_disc_real_5: 0.2258642613887787 (0.2258642613887787) + | > loss_0: 2.567756414413452 (2.567756414413452) + | > loss_gen: 2.3859002590179443 (2.3859002590179443) + | > loss_kl: 2.2798941135406494 (2.2798941135406494) + | > loss_feat: 7.083303451538086 (7.083303451538086) + | > loss_mel: 19.265689849853516 (19.265689849853516) + | > loss_duration: 1.6974358558654785 (1.6974358558654785) + | > loss_1: 32.712223052978516 (32.712223052978516) + + --> STEP: 1 + | > loss_disc: 2.5501346588134766 (2.5501346588134766) + | > loss_disc_real_0: 0.19883516430854797 (0.19883516430854797) + | > loss_disc_real_1: 0.194073885679245 (0.194073885679245) + | > loss_disc_real_2: 0.264540433883667 (0.264540433883667) + | > loss_disc_real_3: 0.25657910108566284 (0.25657910108566284) + | > loss_disc_real_4: 0.2260400503873825 (0.2260400503873825) + | > loss_disc_real_5: 0.2354619801044464 (0.2354619801044464) + | > loss_0: 2.5501346588134766 (2.5501346588134766) + | > loss_gen: 2.2508482933044434 (2.2508482933044434) + | > loss_kl: 2.5834033489227295 (2.5834033489227295) + | > loss_feat: 5.52562952041626 (5.52562952041626) + | > loss_mel: 15.49000072479248 (15.49000072479248) + | > loss_duration: 1.708800196647644 (1.708800196647644) + | > loss_1: 27.55868148803711 (27.55868148803711) + + --> STEP: 2 + | > loss_disc: 2.5101888179779053 (2.530161738395691) + | > loss_disc_real_0: 0.21080723404884338 (0.20482119917869568) + | > loss_disc_real_1: 0.19959233701229095 (0.19683311134576797) + | > loss_disc_real_2: 0.24835801124572754 (0.25644922256469727) + | > loss_disc_real_3: 0.2974674701690674 (0.2770232856273651) + | > loss_disc_real_4: 0.2400175780057907 (0.2330288141965866) + | > loss_disc_real_5: 0.22417674958705902 (0.22981936484575272) + | > loss_0: 2.5101888179779053 (2.530161738395691) + | > loss_gen: 2.407566547393799 (2.329207420349121) + | > loss_kl: 2.451535940170288 (2.517469644546509) + | > loss_feat: 6.761418342590332 (6.143523931503296) + | > loss_mel: 18.102109909057617 (16.79605531692505) + | > loss_duration: 1.661086082458496 (1.68494313955307) + | > loss_1: 31.383716583251953 (29.47119903564453) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0039539337158203125 (+0.0003172159194946289) + | > avg_loss_disc: 2.530161738395691 (+0.09322214126586914) + | > avg_loss_disc_real_0: 0.20482119917869568 (+0.0928233340382576) + | > avg_loss_disc_real_1: 0.19683311134576797 (-0.04652436822652817) + | > avg_loss_disc_real_2: 0.25644922256469727 (-0.025546297430992126) + | > avg_loss_disc_real_3: 0.2770232856273651 (+0.044430531561374664) + | > avg_loss_disc_real_4: 0.2330288141965866 (-0.016956903040409088) + | > avg_loss_disc_real_5: 0.22981936484575272 (-0.021758928894996643) + | > avg_loss_0: 2.530161738395691 (+0.09322214126586914) + | > avg_loss_gen: 2.329207420349121 (-0.09130227565765381) + | > avg_loss_kl: 2.517469644546509 (+0.08520936965942383) + | > avg_loss_feat: 6.143523931503296 (-1.3195116519927979) + | > avg_loss_mel: 16.79605531692505 (-0.4354515075683594) + | > avg_loss_duration: 1.68494313955307 (-0.011700630187988281) + | > avg_loss_1: 29.47119903564453 (-1.7727546691894531) + + + > EPOCH: 241/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:57:22)  + + --> TIME: 2025-04-19 02:57:29 -- STEP: 21/142 -- GLOBAL_STEP: 413525 + | > loss_disc: 2.6164445877075195 (2.4380784261794313) + | > loss_disc_real_0: 0.18071357905864716 (0.12524126186257317) + | > loss_disc_real_1: 0.23552241921424866 (0.2073236448424203) + | > loss_disc_real_2: 0.2321263551712036 (0.22306309782323383) + | > loss_disc_real_3: 0.22072304785251617 (0.22004798622358412) + | > loss_disc_real_4: 0.23342636227607727 (0.22676774149849302) + | > loss_disc_real_5: 0.2535449266433716 (0.2285494939202354) + | > loss_0: 2.6164445877075195 (2.4380784261794313) + | > grad_norm_0: tensor(10.2346, device='cuda:0') (tensor(7.2993, device='cuda:0')) + | > loss_gen: 2.2109436988830566 (2.368277674629575) + | > loss_kl: 1.7911689281463623 (1.8234228917530604) + | > loss_feat: 5.93353271484375 (7.311847777593703) + | > loss_mel: 15.74179744720459 (17.74290316445487) + | > loss_duration: 1.451462745666504 (1.4290314458665394) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.12890625 (30.675482886178152) + | > grad_norm_1: tensor(198.7056, device='cuda:0') (tensor(152.5052, device='cuda:0')) + | > current_lr_0: 0.0001940644816697913 + | > current_lr_1: 0.0001940644816697913 + | > step_time: 0.3411 (0.33249114808582125) + | > loader_time: 0.0054 (0.004840828123546782) + + + --> TIME: 2025-04-19 02:57:38 -- STEP: 46/142 -- GLOBAL_STEP: 413550 + | > loss_disc: 2.3884353637695312 (2.4316840534624844) + | > loss_disc_real_0: 0.0895785316824913 (0.11907947921882504) + | > loss_disc_real_1: 0.23732103407382965 (0.2076014053562413) + | > loss_disc_real_2: 0.22881311178207397 (0.22374854722748633) + | > loss_disc_real_3: 0.21762780845165253 (0.22347380480040674) + | > loss_disc_real_4: 0.19778284430503845 (0.2279326102655867) + | > loss_disc_real_5: 0.26223739981651306 (0.2297715975538544) + | > loss_0: 2.3884353637695312 (2.4316840534624844) + | > grad_norm_0: tensor(6.1551, device='cuda:0') (tensor(7.2329, device='cuda:0')) + | > loss_gen: 2.581782817840576 (2.3775086973024453) + | > loss_kl: 1.7077490091323853 (1.8461867363556572) + | > loss_feat: 7.469367027282715 (7.277438360711803) + | > loss_mel: 18.058332443237305 (17.797110205111295) + | > loss_duration: 1.4296963214874268 (1.440327323001364) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.246929168701172 (30.738571415776793) + | > grad_norm_1: tensor(137.6640, device='cuda:0') (tensor(148.4328, device='cuda:0')) + | > current_lr_0: 0.0001940644816697913 + | > current_lr_1: 0.0001940644816697913 + | > step_time: 0.3651 (0.34726093644681183) + | > loader_time: 0.0058 (0.005162379016046941) + + + --> TIME: 2025-04-19 02:57:48 -- STEP: 71/142 -- GLOBAL_STEP: 413575 + | > loss_disc: 2.434030532836914 (2.4441654379938687) + | > loss_disc_real_0: 0.21837544441223145 (0.1244614839973584) + | > loss_disc_real_1: 0.1921101063489914 (0.20636332391853063) + | > loss_disc_real_2: 0.24588604271411896 (0.22429712221656048) + | > loss_disc_real_3: 0.21679387986660004 (0.2254277914762497) + | > loss_disc_real_4: 0.2045639455318451 (0.22849293577838953) + | > loss_disc_real_5: 0.23315265774726868 (0.23071478181321856) + | > loss_0: 2.434030532836914 (2.4441654379938687) + | > grad_norm_0: tensor(8.3404, device='cuda:0') (tensor(7.4029, device='cuda:0')) + | > loss_gen: 2.2703182697296143 (2.366664943560748) + | > loss_kl: 1.9607341289520264 (1.8700047865719862) + | > loss_feat: 6.330572128295898 (7.176181900669151) + | > loss_mel: 16.715097427368164 (17.710828713967768) + | > loss_duration: 1.4946906566619873 (1.4513880450960617) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.771413803100586 (30.575068514112015) + | > grad_norm_1: tensor(110.0547, device='cuda:0') (tensor(142.8763, device='cuda:0')) + | > current_lr_0: 0.0001940644816697913 + | > current_lr_1: 0.0001940644816697913 + | > step_time: 0.3856 (0.3579380915198528) + | > loader_time: 0.006 (0.005473022729578154) + + + --> TIME: 2025-04-19 02:57:58 -- STEP: 96/142 -- GLOBAL_STEP: 413600 + | > loss_disc: 2.470512866973877 (2.4621294488509498) + | > loss_disc_real_0: 0.11306023597717285 (0.12596881304246682) + | > loss_disc_real_1: 0.18733985722064972 (0.2066628062166274) + | > loss_disc_real_2: 0.2535671591758728 (0.22582365479320288) + | > loss_disc_real_3: 0.23826733231544495 (0.22651830175891519) + | > loss_disc_real_4: 0.25337162613868713 (0.23038667161017656) + | > loss_disc_real_5: 0.2683742940425873 (0.23257752088829875) + | > loss_0: 2.470512866973877 (2.4621294488509498) + | > grad_norm_0: tensor(5.3679, device='cuda:0') (tensor(7.6015, device='cuda:0')) + | > loss_gen: 2.3990581035614014 (2.3460504797597728) + | > loss_kl: 1.8727329969406128 (1.8803094637890656) + | > loss_feat: 7.081639289855957 (7.106418405969937) + | > loss_mel: 18.488466262817383 (17.734753876924504) + | > loss_duration: 1.4897677898406982 (1.4622720306118329) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.331663131713867 (30.529804329077404) + | > grad_norm_1: tensor(185.3867, device='cuda:0') (tensor(147.1719, device='cuda:0')) + | > current_lr_0: 0.0001940644816697913 + | > current_lr_1: 0.0001940644816697913 + | > step_time: 0.4044 (0.36823956171671557) + | > loader_time: 0.007 (0.005730010569095612) + + + --> TIME: 2025-04-19 02:58:09 -- STEP: 121/142 -- GLOBAL_STEP: 413625 + | > loss_disc: 2.5473175048828125 (2.4626134919726157) + | > loss_disc_real_0: 0.15916138887405396 (0.12621440473666865) + | > loss_disc_real_1: 0.22884207963943481 (0.20700883027935818) + | > loss_disc_real_2: 0.20340675115585327 (0.22530690241943707) + | > loss_disc_real_3: 0.2530653178691864 (0.22728016344476337) + | > loss_disc_real_4: 0.24480968713760376 (0.22982050983373783) + | > loss_disc_real_5: 0.2778201699256897 (0.23224310503025686) + | > loss_0: 2.5473175048828125 (2.4626134919726157) + | > grad_norm_0: tensor(7.0460, device='cuda:0') (tensor(7.8124, device='cuda:0')) + | > loss_gen: 2.4199609756469727 (2.346982820960116) + | > loss_kl: 1.9278316497802734 (1.8856017904833329) + | > loss_feat: 6.7647929191589355 (7.07439145963054) + | > loss_mel: 17.925678253173828 (17.707774288398173) + | > loss_duration: 1.5286335945129395 (1.4700178449804133) + | > amp_scaler: 512.0 (300.4297520661157) + | > loss_1: 30.566898345947266 (30.484768244845807) + | > grad_norm_1: tensor(192.3820, device='cuda:0') (tensor(153.5722, device='cuda:0')) + | > current_lr_0: 0.0001940644816697913 + | > current_lr_1: 0.0001940644816697913 + | > step_time: 0.4437 (0.37915349400733134) + | > loader_time: 0.0072 (0.005975906513939219) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3361358642578125 (2.3361358642578125) + | > loss_disc_real_0: 0.15899111330509186 (0.15899111330509186) + | > loss_disc_real_1: 0.2093319594860077 (0.2093319594860077) + | > loss_disc_real_2: 0.2500852048397064 (0.2500852048397064) + | > loss_disc_real_3: 0.2344932109117508 (0.2344932109117508) + | > loss_disc_real_4: 0.22226035594940186 (0.22226035594940186) + | > loss_disc_real_5: 0.2511812448501587 (0.2511812448501587) + | > loss_0: 2.3361358642578125 (2.3361358642578125) + | > loss_gen: 2.6999261379241943 (2.6999261379241943) + | > loss_kl: 2.347886085510254 (2.347886085510254) + | > loss_feat: 6.3792901039123535 (6.3792901039123535) + | > loss_mel: 18.673137664794922 (18.673137664794922) + | > loss_duration: 1.6848909854888916 (1.6848909854888916) + | > loss_1: 31.785131454467773 (31.785131454467773) + + --> STEP: 1 + | > loss_disc: 2.4008162021636963 (2.4008162021636963) + | > loss_disc_real_0: 0.1352580040693283 (0.1352580040693283) + | > loss_disc_real_1: 0.22110120952129364 (0.22110120952129364) + | > loss_disc_real_2: 0.2711697220802307 (0.2711697220802307) + | > loss_disc_real_3: 0.23439320921897888 (0.23439320921897888) + | > loss_disc_real_4: 0.28030818700790405 (0.28030818700790405) + | > loss_disc_real_5: 0.21146541833877563 (0.21146541833877563) + | > loss_0: 2.4008162021636963 (2.4008162021636963) + | > loss_gen: 2.5645949840545654 (2.5645949840545654) + | > loss_kl: 2.690312147140503 (2.690312147140503) + | > loss_feat: 6.643019199371338 (6.643019199371338) + | > loss_mel: 19.232440948486328 (19.232440948486328) + | > loss_duration: 1.722553014755249 (1.722553014755249) + | > loss_1: 32.85292053222656 (32.85292053222656) + + --> STEP: 2 + | > loss_disc: 2.5100433826446533 (2.455429792404175) + | > loss_disc_real_0: 0.20049071311950684 (0.16787435859441757) + | > loss_disc_real_1: 0.19875290989875793 (0.2099270597100258) + | > loss_disc_real_2: 0.23488843441009521 (0.25302907824516296) + | > loss_disc_real_3: 0.23594728112220764 (0.23517024517059326) + | > loss_disc_real_4: 0.2209581732749939 (0.250633180141449) + | > loss_disc_real_5: 0.2594626843929291 (0.23546405136585236) + | > loss_0: 2.5100433826446533 (2.455429792404175) + | > loss_gen: 2.431072235107422 (2.4978336095809937) + | > loss_kl: 2.099208354949951 (2.394760251045227) + | > loss_feat: 7.667058944702148 (7.155039072036743) + | > loss_mel: 20.043020248413086 (19.637730598449707) + | > loss_duration: 1.6617752313613892 (1.692164123058319) + | > loss_1: 33.90213394165039 (33.37752723693848) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003222942352294922 (-0.0007309913635253906) + | > avg_loss_disc: 2.455429792404175 (-0.07473194599151611) + | > avg_loss_disc_real_0: 0.16787435859441757 (-0.03694684058427811) + | > avg_loss_disc_real_1: 0.2099270597100258 (+0.013093948364257812) + | > avg_loss_disc_real_2: 0.25302907824516296 (-0.0034201443195343018) + | > avg_loss_disc_real_3: 0.23517024517059326 (-0.04185304045677185) + | > avg_loss_disc_real_4: 0.250633180141449 (+0.017604365944862366) + | > avg_loss_disc_real_5: 0.23546405136585236 (+0.00564468652009964) + | > avg_loss_0: 2.455429792404175 (-0.07473194599151611) + | > avg_loss_gen: 2.4978336095809937 (+0.16862618923187256) + | > avg_loss_kl: 2.394760251045227 (-0.12270939350128174) + | > avg_loss_feat: 7.155039072036743 (+1.0115151405334473) + | > avg_loss_mel: 19.637730598449707 (+2.841675281524658) + | > avg_loss_duration: 1.692164123058319 (+0.0072209835052490234) + | > avg_loss_1: 33.37752723693848 (+3.9063282012939453) + + + > EPOCH: 242/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:58:20)  + + --> TIME: 2025-04-19 02:58:22 -- STEP: 4/142 -- GLOBAL_STEP: 413650 + | > loss_disc: 2.314105987548828 (2.3544341325759888) + | > loss_disc_real_0: 0.08272527903318405 (0.10474047251045704) + | > loss_disc_real_1: 0.19887837767601013 (0.20869144424796104) + | > loss_disc_real_2: 0.18393546342849731 (0.2263261340558529) + | > loss_disc_real_3: 0.21008643507957458 (0.232666976749897) + | > loss_disc_real_4: 0.22103403508663177 (0.23487534001469612) + | > loss_disc_real_5: 0.22603806853294373 (0.24043038487434387) + | > loss_0: 2.314105987548828 (2.3544341325759888) + | > grad_norm_0: tensor(12.9643, device='cuda:0') (tensor(10.6674, device='cuda:0')) + | > loss_gen: 2.5101683139801025 (2.44680655002594) + | > loss_kl: 1.9015529155731201 (1.8257595300674438) + | > loss_feat: 7.858023643493652 (7.3917635679244995) + | > loss_mel: 19.637475967407227 (18.501676082611084) + | > loss_duration: 1.4198713302612305 (1.4213621616363525) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.327091217041016 (31.58736801147461) + | > grad_norm_1: tensor(364.4630, device='cuda:0') (tensor(207.7480, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.3202 (0.3236255645751953) + | > loader_time: 0.0055 (0.0042760372161865234) + + + --> TIME: 2025-04-19 02:58:31 -- STEP: 29/142 -- GLOBAL_STEP: 413675 + | > loss_disc: 2.3989405632019043 (2.409768762259647) + | > loss_disc_real_0: 0.07536628097295761 (0.1100198510905792) + | > loss_disc_real_1: 0.18782398104667664 (0.20407544127826033) + | > loss_disc_real_2: 0.2018110454082489 (0.22567989908415695) + | > loss_disc_real_3: 0.23918020725250244 (0.22694626656071892) + | > loss_disc_real_4: 0.22468359768390656 (0.22781410145348516) + | > loss_disc_real_5: 0.22422489523887634 (0.2265654920504011) + | > loss_0: 2.3989405632019043 (2.409768762259647) + | > grad_norm_0: tensor(11.0755, device='cuda:0') (tensor(14.1636, device='cuda:0')) + | > loss_gen: 2.360804319381714 (2.4112942958700248) + | > loss_kl: 1.8981510400772095 (1.8435150919289425) + | > loss_feat: 7.156654357910156 (7.239022781108988) + | > loss_mel: 17.08106803894043 (17.96429469667632) + | > loss_duration: 1.4355214834213257 (1.4347169399261475) + | > amp_scaler: 256.0 (353.1034482758621) + | > loss_1: 29.932199478149414 (30.892843772625103) + | > grad_norm_1: tensor(339.9834, device='cuda:0') (tensor(304.5556, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.3731 (0.3469985271322316) + | > loader_time: 0.0052 (0.004917013234105603) + + + --> TIME: 2025-04-19 02:58:40 -- STEP: 54/142 -- GLOBAL_STEP: 413700 + | > loss_disc: 2.472486734390259 (2.43550431728363) + | > loss_disc_real_0: 0.17152944207191467 (0.11528971170385678) + | > loss_disc_real_1: 0.21592910587787628 (0.2048077235619227) + | > loss_disc_real_2: 0.2264719158411026 (0.2266121374236213) + | > loss_disc_real_3: 0.19651570916175842 (0.2257229812719204) + | > loss_disc_real_4: 0.2354513704776764 (0.22837434350340455) + | > loss_disc_real_5: 0.20939074456691742 (0.23020547021318366) + | > loss_0: 2.472486734390259 (2.43550431728363) + | > grad_norm_0: tensor(21.7418, device='cuda:0') (tensor(13.3583, device='cuda:0')) + | > loss_gen: 2.585296154022217 (2.399085521697998) + | > loss_kl: 1.8560267686843872 (1.8608213199509516) + | > loss_feat: 7.75333309173584 (7.21811490588718) + | > loss_mel: 18.37836265563965 (17.956663891121195) + | > loss_duration: 1.4653046131134033 (1.4420490419423138) + | > amp_scaler: 256.0 (308.14814814814827) + | > loss_1: 32.03832244873047 (30.876734556975187) + | > grad_norm_1: tensor(550.4679, device='cuda:0') (tensor(316.4547, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.3711 (0.35404472439377394) + | > loader_time: 0.006 (0.00524937665020978) + + + --> TIME: 2025-04-19 02:58:50 -- STEP: 79/142 -- GLOBAL_STEP: 413725 + | > loss_disc: 2.4198341369628906 (2.450773728044727) + | > loss_disc_real_0: 0.17649655044078827 (0.1198269526792478) + | > loss_disc_real_1: 0.19580602645874023 (0.20534591471092611) + | > loss_disc_real_2: 0.26191720366477966 (0.22729029572462733) + | > loss_disc_real_3: 0.24910007417201996 (0.22870528754554212) + | > loss_disc_real_4: 0.21940840780735016 (0.22859051634993735) + | > loss_disc_real_5: 0.23672275245189667 (0.23234121444859082) + | > loss_0: 2.4198341369628906 (2.450773728044727) + | > grad_norm_0: tensor(5.3095, device='cuda:0') (tensor(11.9255, device='cuda:0')) + | > loss_gen: 2.366368532180786 (2.3744049555138704) + | > loss_kl: 1.8683637380599976 (1.8705660723432709) + | > loss_feat: 6.944889068603516 (7.114033457599109) + | > loss_mel: 18.554750442504883 (17.866884195351904) + | > loss_duration: 1.5116796493530273 (1.4541423230231558) + | > amp_scaler: 256.0 (291.6455696202532) + | > loss_1: 31.246051788330078 (30.680030895184867) + | > grad_norm_1: tensor(53.8247, device='cuda:0') (tensor(287.5828, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.3855 (0.3649694406533544) + | > loader_time: 0.0068 (0.005611618862876411) + + + --> TIME: 2025-04-19 02:59:01 -- STEP: 104/142 -- GLOBAL_STEP: 413750 + | > loss_disc: 2.4708240032196045 (2.46438004878851) + | > loss_disc_real_0: 0.13253460824489594 (0.123417679578639) + | > loss_disc_real_1: 0.21880434453487396 (0.20639280716960245) + | > loss_disc_real_2: 0.2140503227710724 (0.22830763058020517) + | > loss_disc_real_3: 0.23050270974636078 (0.22919188482830155) + | > loss_disc_real_4: 0.22200153768062592 (0.22965197007243449) + | > loss_disc_real_5: 0.22366824746131897 (0.23219547248803651) + | > loss_0: 2.4708240032196045 (2.46438004878851) + | > grad_norm_0: tensor(10.3450, device='cuda:0') (tensor(12.3262, device='cuda:0')) + | > loss_gen: 2.3199779987335205 (2.3603875384880952) + | > loss_kl: 1.9454896450042725 (1.883014208995379) + | > loss_feat: 6.208048343658447 (7.065519415415251) + | > loss_mel: 16.530059814453125 (17.84402465820313) + | > loss_duration: 1.5220247507095337 (1.464699371502949) + | > amp_scaler: 256.0 (283.0769230769231) + | > loss_1: 28.52560043334961 (30.61764511695275) + | > grad_norm_1: tensor(44.3951, device='cuda:0') (tensor(285.5772, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.4115 (0.37476134070983314) + | > loader_time: 0.0068 (0.00588174737416781) + + + --> TIME: 2025-04-19 02:59:12 -- STEP: 129/142 -- GLOBAL_STEP: 413775 + | > loss_disc: 2.380570411682129 (2.46444019605947) + | > loss_disc_real_0: 0.13594704866409302 (0.12495722829602486) + | > loss_disc_real_1: 0.19437286257743835 (0.2064731323210768) + | > loss_disc_real_2: 0.19216836988925934 (0.2278863079557123) + | > loss_disc_real_3: 0.214964359998703 (0.228848177102185) + | > loss_disc_real_4: 0.23875290155410767 (0.2293185693349025) + | > loss_disc_real_5: 0.2286035567522049 (0.231967545756998) + | > loss_0: 2.380570411682129 (2.46444019605947) + | > grad_norm_0: tensor(7.5524, device='cuda:0') (tensor(12.0548, device='cuda:0')) + | > loss_gen: 2.412343978881836 (2.3560551635978757) + | > loss_kl: 2.019105911254883 (1.8901700927305591) + | > loss_feat: 7.003458023071289 (7.026995488839556) + | > loss_mel: 18.347484588623047 (17.75988806495371) + | > loss_duration: 1.503063678741455 (1.471860479938891) + | > amp_scaler: 256.0 (277.82945736434107) + | > loss_1: 31.28545570373535 (30.504969308542652) + | > grad_norm_1: tensor(154.4044, device='cuda:0') (tensor(272.7436, device='cuda:0')) + | > current_lr_0: 0.00019404022360958257 + | > current_lr_1: 0.00019404022360958257 + | > step_time: 0.443 (0.3857229994248975) + | > loader_time: 0.0075 (0.0061733778132948775) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.468156099319458 (2.468156099319458) + | > loss_disc_real_0: 0.16509422659873962 (0.16509422659873962) + | > loss_disc_real_1: 0.19891487061977386 (0.19891487061977386) + | > loss_disc_real_2: 0.26047348976135254 (0.26047348976135254) + | > loss_disc_real_3: 0.2473050057888031 (0.2473050057888031) + | > loss_disc_real_4: 0.23817817866802216 (0.23817817866802216) + | > loss_disc_real_5: 0.22679071128368378 (0.22679071128368378) + | > loss_0: 2.468156099319458 (2.468156099319458) + | > loss_gen: 2.280273914337158 (2.280273914337158) + | > loss_kl: 2.427356719970703 (2.427356719970703) + | > loss_feat: 5.929080009460449 (5.929080009460449) + | > loss_mel: 16.590713500976562 (16.590713500976562) + | > loss_duration: 1.701709270477295 (1.701709270477295) + | > loss_1: 28.929134368896484 (28.929134368896484) + + --> STEP: 1 + | > loss_disc: 2.476956367492676 (2.476956367492676) + | > loss_disc_real_0: 0.12496916204690933 (0.12496916204690933) + | > loss_disc_real_1: 0.2175533026456833 (0.2175533026456833) + | > loss_disc_real_2: 0.21257425844669342 (0.21257425844669342) + | > loss_disc_real_3: 0.22019080817699432 (0.22019080817699432) + | > loss_disc_real_4: 0.2298402637243271 (0.2298402637243271) + | > loss_disc_real_5: 0.24742740392684937 (0.24742740392684937) + | > loss_0: 2.476956367492676 (2.476956367492676) + | > loss_gen: 2.2391836643218994 (2.2391836643218994) + | > loss_kl: 2.581498384475708 (2.581498384475708) + | > loss_feat: 7.904027938842773 (7.904027938842773) + | > loss_mel: 19.682958602905273 (19.682958602905273) + | > loss_duration: 1.7265655994415283 (1.7265655994415283) + | > loss_1: 34.13423538208008 (34.13423538208008) + + --> STEP: 2 + | > loss_disc: 2.259808301925659 (2.3683823347091675) + | > loss_disc_real_0: 0.12244945764541626 (0.1237093098461628) + | > loss_disc_real_1: 0.15694299340248108 (0.18724814802408218) + | > loss_disc_real_2: 0.1847914457321167 (0.19868285208940506) + | > loss_disc_real_3: 0.22263725101947784 (0.22141402959823608) + | > loss_disc_real_4: 0.1999291032552719 (0.2148846834897995) + | > loss_disc_real_5: 0.19430875778198242 (0.2208680808544159) + | > loss_0: 2.259808301925659 (2.3683823347091675) + | > loss_gen: 2.242929220199585 (2.241056442260742) + | > loss_kl: 2.179997205734253 (2.3807477951049805) + | > loss_feat: 7.180374622344971 (7.542201280593872) + | > loss_mel: 17.62248420715332 (18.652721405029297) + | > loss_duration: 1.6746761798858643 (1.7006208896636963) + | > loss_1: 30.900463104248047 (32.51734924316406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033365488052368164 (+0.00011360645294189453) + | > avg_loss_disc: 2.3683823347091675 (-0.08704745769500732) + | > avg_loss_disc_real_0: 0.1237093098461628 (-0.044165048748254776) + | > avg_loss_disc_real_1: 0.18724814802408218 (-0.022678911685943604) + | > avg_loss_disc_real_2: 0.19868285208940506 (-0.054346226155757904) + | > avg_loss_disc_real_3: 0.22141402959823608 (-0.013756215572357178) + | > avg_loss_disc_real_4: 0.2148846834897995 (-0.035748496651649475) + | > avg_loss_disc_real_5: 0.2208680808544159 (-0.014595970511436462) + | > avg_loss_0: 2.3683823347091675 (-0.08704745769500732) + | > avg_loss_gen: 2.241056442260742 (-0.25677716732025146) + | > avg_loss_kl: 2.3807477951049805 (-0.014012455940246582) + | > avg_loss_feat: 7.542201280593872 (+0.3871622085571289) + | > avg_loss_mel: 18.652721405029297 (-0.9850091934204102) + | > avg_loss_duration: 1.7006208896636963 (+0.008456766605377197) + | > avg_loss_1: 32.51734924316406 (-0.8601779937744141) + + + > EPOCH: 243/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 02:59:19)  + + --> TIME: 2025-04-19 02:59:24 -- STEP: 12/142 -- GLOBAL_STEP: 413800 + | > loss_disc: 2.4524879455566406 (2.4866684079170227) + | > loss_disc_real_0: 0.11724650859832764 (0.1293232161551714) + | > loss_disc_real_1: 0.2079058438539505 (0.21504369378089905) + | > loss_disc_real_2: 0.2479015290737152 (0.23029987762371698) + | > loss_disc_real_3: 0.23584434390068054 (0.22672932222485542) + | > loss_disc_real_4: 0.21819986402988434 (0.23066678022344908) + | > loss_disc_real_5: 0.22737202048301697 (0.2337617501616478) + | > loss_0: 2.4524879455566406 (2.4866684079170227) + | > grad_norm_0: tensor(15.6008, device='cuda:0') (tensor(14.0885, device='cuda:0')) + | > loss_gen: 2.239149808883667 (2.3521060148874917) + | > loss_kl: 1.9176294803619385 (1.8237624963124592) + | > loss_feat: 6.512486457824707 (7.148961901664734) + | > loss_mel: 17.479909896850586 (17.910594304402668) + | > loss_duration: 1.4530692100524902 (1.4260904391606648) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.602245330810547 (30.66151507695516) + | > grad_norm_1: tensor(213.4316, device='cuda:0') (tensor(309.3228, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.3382 (0.3340640068054199) + | > loader_time: 0.0047 (0.004638532797495524) + + + --> TIME: 2025-04-19 02:59:33 -- STEP: 37/142 -- GLOBAL_STEP: 413825 + | > loss_disc: 2.422147750854492 (2.452973372227437) + | > loss_disc_real_0: 0.09712766855955124 (0.11999808010217305) + | > loss_disc_real_1: 0.17503854632377625 (0.20734160816347277) + | > loss_disc_real_2: 0.24817384779453278 (0.22892271465546377) + | > loss_disc_real_3: 0.2442273497581482 (0.22672019616977587) + | > loss_disc_real_4: 0.20505492389202118 (0.22604823313854835) + | > loss_disc_real_5: 0.22293080389499664 (0.232477022183908) + | > loss_0: 2.422147750854492 (2.452973372227437) + | > grad_norm_0: tensor(8.2455, device='cuda:0') (tensor(13.6564, device='cuda:0')) + | > loss_gen: 2.4104061126708984 (2.3844193059044914) + | > loss_kl: 1.9637237787246704 (1.849349273217691) + | > loss_feat: 6.740151405334473 (7.119720458984375) + | > loss_mel: 16.459129333496094 (17.615197748751253) + | > loss_duration: 1.4668713808059692 (1.436454863161654) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.040283203125 (30.405141675794447) + | > grad_norm_1: tensor(190.9474, device='cuda:0') (tensor(334.6033, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.3544 (0.34257491214855296) + | > loader_time: 0.006 (0.005084050668252481) + + + --> TIME: 2025-04-19 02:59:42 -- STEP: 62/142 -- GLOBAL_STEP: 413850 + | > loss_disc: 2.5280613899230957 (2.4533122162665086) + | > loss_disc_real_0: 0.10000264644622803 (0.12384578657727088) + | > loss_disc_real_1: 0.20953501760959625 (0.20897689605912856) + | > loss_disc_real_2: 0.21013878285884857 (0.22499477983482422) + | > loss_disc_real_3: 0.20731285214424133 (0.2282056515255282) + | > loss_disc_real_4: 0.22096969187259674 (0.22775279898797313) + | > loss_disc_real_5: 0.23275823891162872 (0.23305692043035262) + | > loss_0: 2.5280613899230957 (2.4533122162665086) + | > grad_norm_0: tensor(5.9528, device='cuda:0') (tensor(12.0474, device='cuda:0')) + | > loss_gen: 2.264836549758911 (2.3806581189555507) + | > loss_kl: 1.946798324584961 (1.8664041084627951) + | > loss_feat: 7.617870330810547 (7.146726685185587) + | > loss_mel: 17.79034423828125 (17.743089752812548) + | > loss_duration: 1.49570894241333 (1.4487391575690238) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.115558624267578 (30.585617772994503) + | > grad_norm_1: tensor(64.9722, device='cuda:0') (tensor(303.2993, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.3813 (0.3528857769504671) + | > loader_time: 0.0059 (0.005404087804978895) + + + --> TIME: 2025-04-19 02:59:52 -- STEP: 87/142 -- GLOBAL_STEP: 413875 + | > loss_disc: 2.4507884979248047 (2.4630457998692297) + | > loss_disc_real_0: 0.1326165646314621 (0.12467598298500324) + | > loss_disc_real_1: 0.1771618276834488 (0.20783372788593688) + | > loss_disc_real_2: 0.21250197291374207 (0.2265391423441898) + | > loss_disc_real_3: 0.2398310750722885 (0.22926961365102352) + | > loss_disc_real_4: 0.23116493225097656 (0.22891086459844961) + | > loss_disc_real_5: 0.2811542749404907 (0.2344434910464561) + | > loss_0: 2.4507884979248047 (2.4630457998692297) + | > grad_norm_0: tensor(14.5723, device='cuda:0') (tensor(11.9141, device='cuda:0')) + | > loss_gen: 2.2293665409088135 (2.3580160250608957) + | > loss_kl: 1.9211055040359497 (1.879829256013892) + | > loss_feat: 6.848196506500244 (7.010813844615016) + | > loss_mel: 17.750885009765625 (17.621732569288934) + | > loss_duration: 1.5078160762786865 (1.4600768733298637) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.257369995117188 (30.330468539533943) + | > grad_norm_1: tensor(397.7046, device='cuda:0') (tensor(294.5911, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.3991 (0.3642597061464156) + | > loader_time: 0.0068 (0.005681144780126111) + + + --> TIME: 2025-04-19 03:00:03 -- STEP: 112/142 -- GLOBAL_STEP: 413900 + | > loss_disc: 2.5553171634674072 (2.4694286244256154) + | > loss_disc_real_0: 0.12646713852882385 (0.12531823551814472) + | > loss_disc_real_1: 0.23593348264694214 (0.2075469439317073) + | > loss_disc_real_2: 0.22989381849765778 (0.22675518704844372) + | > loss_disc_real_3: 0.23533569276332855 (0.2294706299102732) + | > loss_disc_real_4: 0.23884791135787964 (0.23018208066267626) + | > loss_disc_real_5: 0.24255435168743134 (0.23440961952188186) + | > loss_0: 2.5553171634674072 (2.4694286244256154) + | > grad_norm_0: tensor(4.8477, device='cuda:0') (tensor(11.3528, device='cuda:0')) + | > loss_gen: 2.1974611282348633 (2.3466761090925763) + | > loss_kl: 1.9017614126205444 (1.8928294543709074) + | > loss_feat: 6.209373474121094 (6.989903522389276) + | > loss_mel: 16.532032012939453 (17.68730639559882) + | > loss_duration: 1.4704340696334839 (1.4678940421768591) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.311059951782227 (30.384609494890487) + | > grad_norm_1: tensor(74.4590, device='cuda:0') (tensor(280.8117, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.4238 (0.37472962055887493) + | > loader_time: 0.0068 (0.005955023424965995) + + + --> TIME: 2025-04-19 03:00:14 -- STEP: 137/142 -- GLOBAL_STEP: 413925 + | > loss_disc: 2.5685651302337646 (2.467215485816454) + | > loss_disc_real_0: 0.18665309250354767 (0.12653936364137344) + | > loss_disc_real_1: 0.20236587524414062 (0.20641165993509503) + | > loss_disc_real_2: 0.2233932912349701 (0.22635988205888846) + | > loss_disc_real_3: 0.21212968230247498 (0.22970625464498562) + | > loss_disc_real_4: 0.22701525688171387 (0.22947706651948666) + | > loss_disc_real_5: 0.2378440946340561 (0.23404210533973943) + | > loss_0: 2.5685651302337646 (2.467215485816454) + | > grad_norm_0: tensor(5.0591, device='cuda:0') (tensor(11.0083, device='cuda:0')) + | > loss_gen: 2.1105525493621826 (2.340078200737055) + | > loss_kl: 1.850745677947998 (1.896815230376529) + | > loss_feat: 6.421097755432129 (6.971870937486635) + | > loss_mel: 17.724475860595703 (17.65711448140388) + | > loss_duration: 1.4934513568878174 (1.473997658186585) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.600322723388672 (30.339876467294065) + | > grad_norm_1: tensor(131.8819, device='cuda:0') (tensor(269.1388, device='cuda:0')) + | > current_lr_0: 0.00019401596858163137 + | > current_lr_1: 0.00019401596858163137 + | > step_time: 0.4561 (0.3864869705951997) + | > loader_time: 0.0074 (0.006212787906618884) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.347027540206909 (2.347027540206909) + | > loss_disc_real_0: 0.14630703628063202 (0.14630703628063202) + | > loss_disc_real_1: 0.2136513590812683 (0.2136513590812683) + | > loss_disc_real_2: 0.2506292760372162 (0.2506292760372162) + | > loss_disc_real_3: 0.223282590508461 (0.223282590508461) + | > loss_disc_real_4: 0.2565382122993469 (0.2565382122993469) + | > loss_disc_real_5: 0.24381467700004578 (0.24381467700004578) + | > loss_0: 2.347027540206909 (2.347027540206909) + | > loss_gen: 2.5519163608551025 (2.5519163608551025) + | > loss_kl: 2.8445980548858643 (2.8445980548858643) + | > loss_feat: 7.191152572631836 (7.191152572631836) + | > loss_mel: 19.43000030517578 (19.43000030517578) + | > loss_duration: 1.7022669315338135 (1.7022669315338135) + | > loss_1: 33.71993637084961 (33.71993637084961) + + --> STEP: 1 + | > loss_disc: 2.465179920196533 (2.465179920196533) + | > loss_disc_real_0: 0.15374264121055603 (0.15374264121055603) + | > loss_disc_real_1: 0.25168365240097046 (0.25168365240097046) + | > loss_disc_real_2: 0.26089245080947876 (0.26089245080947876) + | > loss_disc_real_3: 0.259290486574173 (0.259290486574173) + | > loss_disc_real_4: 0.23469166457653046 (0.23469166457653046) + | > loss_disc_real_5: 0.26616889238357544 (0.26616889238357544) + | > loss_0: 2.465179920196533 (2.465179920196533) + | > loss_gen: 2.5184521675109863 (2.5184521675109863) + | > loss_kl: 2.5337274074554443 (2.5337274074554443) + | > loss_feat: 7.026495933532715 (7.026495933532715) + | > loss_mel: 17.594146728515625 (17.594146728515625) + | > loss_duration: 1.732211947441101 (1.732211947441101) + | > loss_1: 31.405033111572266 (31.405033111572266) + + --> STEP: 2 + | > loss_disc: 2.598422050476074 (2.5318009853363037) + | > loss_disc_real_0: 0.18474888801574707 (0.16924576461315155) + | > loss_disc_real_1: 0.26537343859672546 (0.25852854549884796) + | > loss_disc_real_2: 0.2681964039802551 (0.26454442739486694) + | > loss_disc_real_3: 0.26064953207969666 (0.2599700093269348) + | > loss_disc_real_4: 0.27842554450035095 (0.2565586045384407) + | > loss_disc_real_5: 0.3029226064682007 (0.28454574942588806) + | > loss_0: 2.598422050476074 (2.5318009853363037) + | > loss_gen: 2.6197264194488525 (2.5690892934799194) + | > loss_kl: 2.210860013961792 (2.372293710708618) + | > loss_feat: 6.184008598327637 (6.605252265930176) + | > loss_mel: 16.3184814453125 (16.956314086914062) + | > loss_duration: 1.687427282333374 (1.7098196148872375) + | > loss_1: 29.020503997802734 (30.2127685546875) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034371614456176758 (+0.00010061264038085938) + | > avg_loss_disc: 2.5318009853363037 (+0.16341865062713623) + | > avg_loss_disc_real_0: 0.16924576461315155 (+0.045536454766988754) + | > avg_loss_disc_real_1: 0.25852854549884796 (+0.07128039747476578) + | > avg_loss_disc_real_2: 0.26454442739486694 (+0.06586157530546188) + | > avg_loss_disc_real_3: 0.2599700093269348 (+0.03855597972869873) + | > avg_loss_disc_real_4: 0.2565586045384407 (+0.041673921048641205) + | > avg_loss_disc_real_5: 0.28454574942588806 (+0.06367766857147217) + | > avg_loss_0: 2.5318009853363037 (+0.16341865062713623) + | > avg_loss_gen: 2.5690892934799194 (+0.32803285121917725) + | > avg_loss_kl: 2.372293710708618 (-0.008454084396362305) + | > avg_loss_feat: 6.605252265930176 (-0.9369490146636963) + | > avg_loss_mel: 16.956314086914062 (-1.6964073181152344) + | > avg_loss_duration: 1.7098196148872375 (+0.00919872522354126) + | > avg_loss_1: 30.2127685546875 (-2.3045806884765625) + + + > EPOCH: 244/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:00:18)  + + --> TIME: 2025-04-19 03:00:25 -- STEP: 20/142 -- GLOBAL_STEP: 413950 + | > loss_disc: 2.2771658897399902 (2.403072714805603) + | > loss_disc_real_0: 0.08071031421422958 (0.11037792079150677) + | > loss_disc_real_1: 0.18713796138763428 (0.20288578048348427) + | > loss_disc_real_2: 0.21181870996952057 (0.22610546723008157) + | > loss_disc_real_3: 0.2100408971309662 (0.22973477393388747) + | > loss_disc_real_4: 0.21463125944137573 (0.2236717648804188) + | > loss_disc_real_5: 0.2129756361246109 (0.23253533542156218) + | > loss_0: 2.2771658897399902 (2.403072714805603) + | > grad_norm_0: tensor(5.4403, device='cuda:0') (tensor(7.6256, device='cuda:0')) + | > loss_gen: 2.526512384414673 (2.4124597668647767) + | > loss_kl: 1.8592746257781982 (1.8205933213233947) + | > loss_feat: 8.468385696411133 (7.566631102561951) + | > loss_mel: 18.73042869567871 (18.17824430465698) + | > loss_duration: 1.478649616241455 (1.428959995508194) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.06325149536133 (31.406888580322267) + | > grad_norm_1: tensor(158.2157, device='cuda:0') (tensor(209.9281, device='cuda:0')) + | > current_lr_0: 0.00019399171658555865 + | > current_lr_1: 0.00019399171658555865 + | > step_time: 0.3407 (0.33019189834594725) + | > loader_time: 0.0055 (0.004704868793487549) + + + --> TIME: 2025-04-19 03:00:34 -- STEP: 45/142 -- GLOBAL_STEP: 413975 + | > loss_disc: 2.48681378364563 (2.416137684716119) + | > loss_disc_real_0: 0.1275145709514618 (0.11880781236622069) + | > loss_disc_real_1: 0.2759818732738495 (0.2061498012807634) + | > loss_disc_real_2: 0.2543807923793793 (0.22539611160755157) + | > loss_disc_real_3: 0.2022799402475357 (0.22699425253603195) + | > loss_disc_real_4: 0.24025443196296692 (0.22375880711608462) + | > loss_disc_real_5: 0.19169996678829193 (0.22938274873627557) + | > loss_0: 2.48681378364563 (2.416137684716119) + | > grad_norm_0: tensor(10.7299, device='cuda:0') (tensor(9.0314, device='cuda:0')) + | > loss_gen: 2.195774555206299 (2.3924993250105118) + | > loss_kl: 1.8475557565689087 (1.832887151506212) + | > loss_feat: 5.9385666847229 (7.214988761478001) + | > loss_mel: 17.925207138061523 (17.79982638888889) + | > loss_duration: 1.4575053453445435 (1.4393503030141195) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.36461067199707 (30.67955199347602) + | > grad_norm_1: tensor(146.0341, device='cuda:0') (tensor(227.4418, device='cuda:0')) + | > current_lr_0: 0.00019399171658555865 + | > current_lr_1: 0.00019399171658555865 + | > step_time: 0.3712 (0.3432849195268419) + | > loader_time: 0.006 (0.005142243703206381) + + + --> TIME: 2025-04-19 03:00:44 -- STEP: 70/142 -- GLOBAL_STEP: 414000 + | > loss_disc: 2.4974124431610107 (2.441975058828082) + | > loss_disc_real_0: 0.10621457546949387 (0.12099099638206619) + | > loss_disc_real_1: 0.22109010815620422 (0.20718214980193547) + | > loss_disc_real_2: 0.2736816704273224 (0.2255483471921512) + | > loss_disc_real_3: 0.2075059562921524 (0.22918023467063903) + | > loss_disc_real_4: 0.23516930639743805 (0.22814135019268308) + | > loss_disc_real_5: 0.2492249608039856 (0.23112362708364215) + | > loss_0: 2.4974124431610107 (2.441975058828082) + | > grad_norm_0: tensor(9.3706, device='cuda:0') (tensor(10.5174, device='cuda:0')) + | > loss_gen: 2.320923328399658 (2.388018410546439) + | > loss_kl: 1.8522995710372925 (1.8586225066866193) + | > loss_feat: 7.0824737548828125 (7.22605310848781) + | > loss_mel: 17.492067337036133 (17.888411699022555) + | > loss_duration: 1.4871749877929688 (1.4518327713012695) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.234939575195312 (30.812938472202845) + | > grad_norm_1: tensor(290.0464, device='cuda:0') (tensor(242.7428, device='cuda:0')) + | > current_lr_0: 0.00019399171658555865 + | > current_lr_1: 0.00019399171658555865 + | > step_time: 0.3896 (0.35585740293775286) + | > loader_time: 0.0062 (0.005478286743164062) + + + --> TIME: 2025-04-19 03:00:54 -- STEP: 95/142 -- GLOBAL_STEP: 414025 + | > loss_disc: 2.5457537174224854 (2.4517057745080257) + | > loss_disc_real_0: 0.17382271587848663 (0.12143890920438265) + | > loss_disc_real_1: 0.21456892788410187 (0.20700644411538774) + | > loss_disc_real_2: 0.22253574430942535 (0.22699639091366217) + | > loss_disc_real_3: 0.23718400299549103 (0.22931819887537705) + | > loss_disc_real_4: 0.2349923700094223 (0.22976585454062412) + | > loss_disc_real_5: 0.2307148277759552 (0.23187785964263113) + | > loss_0: 2.5457537174224854 (2.4517057745080257) + | > grad_norm_0: tensor(16.8448, device='cuda:0') (tensor(12.2450, device='cuda:0')) + | > loss_gen: 2.079376697540283 (2.3802211284637447) + | > loss_kl: 2.011173725128174 (1.8745341363706087) + | > loss_feat: 5.700496673583984 (7.108863168013723) + | > loss_mel: 16.88490104675293 (17.78946630578291) + | > loss_duration: 1.5001022815704346 (1.4621504156213059) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.176050186157227 (30.615234997397973) + | > grad_norm_1: tensor(296.2944, device='cuda:0') (tensor(272.0701, device='cuda:0')) + | > current_lr_0: 0.00019399171658555865 + | > current_lr_1: 0.00019399171658555865 + | > step_time: 0.4136 (0.3684438705444336) + | > loader_time: 0.0069 (0.005782707113968699) + + + --> TIME: 2025-04-19 03:01:05 -- STEP: 120/142 -- GLOBAL_STEP: 414050 + | > loss_disc: 2.4715733528137207 (2.4625690400600444) + | > loss_disc_real_0: 0.17207996547222137 (0.12429706752300262) + | > loss_disc_real_1: 0.21243368089199066 (0.2080473261574904) + | > loss_disc_real_2: 0.21298454701900482 (0.2267240283389886) + | > loss_disc_real_3: 0.2647329568862915 (0.23011351687212786) + | > loss_disc_real_4: 0.250130832195282 (0.23036382744709652) + | > loss_disc_real_5: 0.2887696325778961 (0.23368775496880215) + | > loss_0: 2.4715733528137207 (2.4625690400600444) + | > grad_norm_0: tensor(11.6892, device='cuda:0') (tensor(13.6273, device='cuda:0')) + | > loss_gen: 2.521104335784912 (2.3723829607168825) + | > loss_kl: 1.8289564847946167 (1.8871170749266943) + | > loss_feat: 7.266539573669434 (7.056507464249929) + | > loss_mel: 18.908506393432617 (17.75745871067047) + | > loss_duration: 1.5200620889663696 (1.4694725304841996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.045169830322266 (30.542938661575317) + | > grad_norm_1: tensor(483.6272, device='cuda:0') (tensor(289.4944, device='cuda:0')) + | > current_lr_0: 0.00019399171658555865 + | > current_lr_1: 0.00019399171658555865 + | > step_time: 0.4339 (0.3791111330191294) + | > loader_time: 0.0073 (0.006079280376434326) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3652570247650146 (2.3652570247650146) + | > loss_disc_real_0: 0.12240221351385117 (0.12240221351385117) + | > loss_disc_real_1: 0.19490106403827667 (0.19490106403827667) + | > loss_disc_real_2: 0.2625259757041931 (0.2625259757041931) + | > loss_disc_real_3: 0.22206659615039825 (0.22206659615039825) + | > loss_disc_real_4: 0.19294974207878113 (0.19294974207878113) + | > loss_disc_real_5: 0.23229412734508514 (0.23229412734508514) + | > loss_0: 2.3652570247650146 (2.3652570247650146) + | > loss_gen: 2.4066407680511475 (2.4066407680511475) + | > loss_kl: 2.6215333938598633 (2.6215333938598633) + | > loss_feat: 7.97831392288208 (7.97831392288208) + | > loss_mel: 19.048994064331055 (19.048994064331055) + | > loss_duration: 1.70048189163208 (1.70048189163208) + | > loss_1: 33.75596237182617 (33.75596237182617) + + --> STEP: 1 + | > loss_disc: 2.4802188873291016 (2.4802188873291016) + | > loss_disc_real_0: 0.19894947111606598 (0.19894947111606598) + | > loss_disc_real_1: 0.1759447157382965 (0.1759447157382965) + | > loss_disc_real_2: 0.27130940556526184 (0.27130940556526184) + | > loss_disc_real_3: 0.24553737044334412 (0.24553737044334412) + | > loss_disc_real_4: 0.23424585163593292 (0.23424585163593292) + | > loss_disc_real_5: 0.22975915670394897 (0.22975915670394897) + | > loss_0: 2.4802188873291016 (2.4802188873291016) + | > loss_gen: 2.516078233718872 (2.516078233718872) + | > loss_kl: 2.7110841274261475 (2.7110841274261475) + | > loss_feat: 7.543941020965576 (7.543941020965576) + | > loss_mel: 19.43726921081543 (19.43726921081543) + | > loss_duration: 1.746138095855713 (1.746138095855713) + | > loss_1: 33.95451354980469 (33.95451354980469) + + --> STEP: 2 + | > loss_disc: 2.3551738262176514 (2.4176963567733765) + | > loss_disc_real_0: 0.17736344039440155 (0.18815645575523376) + | > loss_disc_real_1: 0.21889424324035645 (0.19741947948932648) + | > loss_disc_real_2: 0.22880035638809204 (0.25005488097667694) + | > loss_disc_real_3: 0.18137581646442413 (0.21345659345388412) + | > loss_disc_real_4: 0.22343024611473083 (0.22883804887533188) + | > loss_disc_real_5: 0.2065351903438568 (0.2181471735239029) + | > loss_0: 2.3551738262176514 (2.4176963567733765) + | > loss_gen: 2.4883317947387695 (2.502205014228821) + | > loss_kl: 2.2706809043884277 (2.4908825159072876) + | > loss_feat: 7.963917255401611 (7.753929138183594) + | > loss_mel: 20.36972999572754 (19.903499603271484) + | > loss_duration: 1.6866885423660278 (1.7164133191108704) + | > loss_1: 34.77934646606445 (34.36693000793457) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003885507583618164 (+0.0004483461380004883) + | > avg_loss_disc: 2.4176963567733765 (-0.11410462856292725) + | > avg_loss_disc_real_0: 0.18815645575523376 (+0.018910691142082214) + | > avg_loss_disc_real_1: 0.19741947948932648 (-0.061109066009521484) + | > avg_loss_disc_real_2: 0.25005488097667694 (-0.014489546418190002) + | > avg_loss_disc_real_3: 0.21345659345388412 (-0.04651341587305069) + | > avg_loss_disc_real_4: 0.22883804887533188 (-0.027720555663108826) + | > avg_loss_disc_real_5: 0.2181471735239029 (-0.06639857590198517) + | > avg_loss_0: 2.4176963567733765 (-0.11410462856292725) + | > avg_loss_gen: 2.502205014228821 (-0.06688427925109863) + | > avg_loss_kl: 2.4908825159072876 (+0.11858880519866943) + | > avg_loss_feat: 7.753929138183594 (+1.148676872253418) + | > avg_loss_mel: 19.903499603271484 (+2.947185516357422) + | > avg_loss_duration: 1.7164133191108704 (+0.0065937042236328125) + | > avg_loss_1: 34.36693000793457 (+4.15416145324707) + + + > EPOCH: 245/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:01:16)  + + --> TIME: 2025-04-19 03:01:18 -- STEP: 3/142 -- GLOBAL_STEP: 414075 + | > loss_disc: 2.440464973449707 (2.4344950517018638) + | > loss_disc_real_0: 0.08983826637268066 (0.10071004678805669) + | > loss_disc_real_1: 0.19094817340373993 (0.1969636082649231) + | > loss_disc_real_2: 0.22550570964813232 (0.24839582045873007) + | > loss_disc_real_3: 0.2377757877111435 (0.24286986390749613) + | > loss_disc_real_4: 0.23301534354686737 (0.24027594923973083) + | > loss_disc_real_5: 0.23081468045711517 (0.21718230843544006) + | > loss_0: 2.440464973449707 (2.4344950517018638) + | > grad_norm_0: tensor(14.4971, device='cuda:0') (tensor(12.0679, device='cuda:0')) + | > loss_gen: 2.4026613235473633 (2.3576626777648926) + | > loss_kl: 1.9740898609161377 (1.8816368977228801) + | > loss_feat: 6.736226558685303 (7.065457661946614) + | > loss_mel: 18.509641647338867 (17.70503298441569) + | > loss_duration: 1.4348554611206055 (1.4270492792129517) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.057476043701172 (30.436840057373047) + | > grad_norm_1: tensor(264.0874, device='cuda:0') (tensor(229.5857, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.3232 (0.3264319101969401) + | > loader_time: 0.0038 (0.004111448923746745) + + + --> TIME: 2025-04-19 03:01:27 -- STEP: 28/142 -- GLOBAL_STEP: 414100 + | > loss_disc: 2.396393299102783 (2.428438561303275) + | > loss_disc_real_0: 0.12837152183055878 (0.11676297150552273) + | > loss_disc_real_1: 0.22418971359729767 (0.20387521173272813) + | > loss_disc_real_2: 0.2106885462999344 (0.22635844190205848) + | > loss_disc_real_3: 0.22671908140182495 (0.22525167731302126) + | > loss_disc_real_4: 0.19830790162086487 (0.22316001089555876) + | > loss_disc_real_5: 0.24828208982944489 (0.22855068636792047) + | > loss_0: 2.396393299102783 (2.428438561303275) + | > grad_norm_0: tensor(7.4285, device='cuda:0') (tensor(9.8366, device='cuda:0')) + | > loss_gen: 2.4722049236297607 (2.3731221641813005) + | > loss_kl: 1.8800684213638306 (1.7912756843226296) + | > loss_feat: 7.020720958709717 (7.18268895149231) + | > loss_mel: 18.685623168945312 (17.891094275883265) + | > loss_duration: 1.462519884109497 (1.4331312009266444) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.521137237548828 (30.671312740870885) + | > grad_norm_1: tensor(168.7272, device='cuda:0') (tensor(235.0438, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.3516 (0.3371086461203439) + | > loader_time: 0.0054 (0.0047622663634164) + + + --> TIME: 2025-04-19 03:01:36 -- STEP: 53/142 -- GLOBAL_STEP: 414125 + | > loss_disc: 2.4258599281311035 (2.443801349064089) + | > loss_disc_real_0: 0.11984965950250626 (0.12042087204051467) + | > loss_disc_real_1: 0.18784277141094208 (0.20355306144030588) + | > loss_disc_real_2: 0.2252989411354065 (0.22688273523213728) + | > loss_disc_real_3: 0.23584724962711334 (0.22931455272548604) + | > loss_disc_real_4: 0.1963651329278946 (0.2286931079513622) + | > loss_disc_real_5: 0.21326427161693573 (0.23087848160626753) + | > loss_0: 2.4258599281311035 (2.443801349064089) + | > grad_norm_0: tensor(15.7919, device='cuda:0') (tensor(10.7734, device='cuda:0')) + | > loss_gen: 2.3885726928710938 (2.3653769043256645) + | > loss_kl: 1.922350287437439 (1.824346753786195) + | > loss_feat: 8.290440559387207 (7.0886235686967956) + | > loss_mel: 18.840776443481445 (17.708817805884014) + | > loss_duration: 1.4486823081970215 (1.4417883270191696) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.89081954956055 (30.428953566641177) + | > grad_norm_1: tensor(322.8945, device='cuda:0') (tensor(251.1125, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.3774 (0.3519601641960864) + | > loader_time: 0.0057 (0.005200754921391325) + + + --> TIME: 2025-04-19 03:01:46 -- STEP: 78/142 -- GLOBAL_STEP: 414150 + | > loss_disc: 2.5751636028289795 (2.457253012901697) + | > loss_disc_real_0: 0.12460185587406158 (0.12426058928935955) + | > loss_disc_real_1: 0.2239425778388977 (0.20505951669735786) + | > loss_disc_real_2: 0.23952855169773102 (0.2282591876693261) + | > loss_disc_real_3: 0.26188182830810547 (0.22917983260674354) + | > loss_disc_real_4: 0.24626141786575317 (0.22879169995968157) + | > loss_disc_real_5: 0.22815781831741333 (0.2314795121932641) + | > loss_0: 2.5751636028289795 (2.457253012901697) + | > grad_norm_0: tensor(14.2563, device='cuda:0') (tensor(10.3396, device='cuda:0')) + | > loss_gen: 2.211012601852417 (2.3515427173712307) + | > loss_kl: 1.8760231733322144 (1.8476850726665595) + | > loss_feat: 7.096932888031006 (7.067464565619444) + | > loss_mel: 18.158367156982422 (17.651890351222107) + | > loss_duration: 1.512974739074707 (1.4541141329667506) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.855308532714844 (30.372696925432255) + | > grad_norm_1: tensor(276.7185, device='cuda:0') (tensor(244.2358, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.4056 (0.36607551880371875) + | > loader_time: 0.0069 (0.005553352527129345) + + + --> TIME: 2025-04-19 03:01:57 -- STEP: 103/142 -- GLOBAL_STEP: 414175 + | > loss_disc: 2.389763355255127 (2.466003732773865) + | > loss_disc_real_0: 0.10081513226032257 (0.12551947639694494) + | > loss_disc_real_1: 0.16030655801296234 (0.20639153953316142) + | > loss_disc_real_2: 0.19050434231758118 (0.2272855060771831) + | > loss_disc_real_3: 0.2093166708946228 (0.22910493901632364) + | > loss_disc_real_4: 0.19596123695373535 (0.22868474276320447) + | > loss_disc_real_5: 0.23125188052654266 (0.23228376004302387) + | > loss_0: 2.389763355255127 (2.466003732773865) + | > grad_norm_0: tensor(6.2426, device='cuda:0') (tensor(10.7548, device='cuda:0')) + | > loss_gen: 2.3873846530914307 (2.336474164018352) + | > loss_kl: 1.9026696681976318 (1.859385533240235) + | > loss_feat: 7.078981876373291 (7.0011413768657205) + | > loss_mel: 18.52684211730957 (17.67310613798864) + | > loss_duration: 1.5204885005950928 (1.4644832391183353) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.416366577148438 (30.334590560024225) + | > grad_norm_1: tensor(198.6462, device='cuda:0') (tensor(249.2058, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.4243 (0.3782938776664363) + | > loader_time: 0.0081 (0.005883948316851866) + + + --> TIME: 2025-04-19 03:02:08 -- STEP: 128/142 -- GLOBAL_STEP: 414200 + | > loss_disc: 2.4971706867218018 (2.4644664414227018) + | > loss_disc_real_0: 0.12496509402990341 (0.12553799507441007) + | > loss_disc_real_1: 0.244898721575737 (0.20609367988072336) + | > loss_disc_real_2: 0.21415095031261444 (0.22690543055068702) + | > loss_disc_real_3: 0.23730535805225372 (0.22905496903695166) + | > loss_disc_real_4: 0.21629400551319122 (0.229458088404499) + | > loss_disc_real_5: 0.2606014013290405 (0.23230653523933142) + | > loss_0: 2.4971706867218018 (2.4644664414227018) + | > grad_norm_0: tensor(5.6478, device='cuda:0') (tensor(10.5661, device='cuda:0')) + | > loss_gen: 2.266512393951416 (2.338937144726514) + | > loss_kl: 1.810992956161499 (1.8685669489204884) + | > loss_feat: 6.603237628936768 (6.983110658824444) + | > loss_mel: 17.12547492980957 (17.62227195501328) + | > loss_duration: 1.519897699356079 (1.4713226687163115) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.326114654541016 (30.28420938551426) + | > grad_norm_1: tensor(117.4267, device='cuda:0') (tensor(251.7312, device='cuda:0')) + | > current_lr_0: 0.00019396746762098544 + | > current_lr_1: 0.00019396746762098544 + | > step_time: 0.4353 (0.3903929498046636) + | > loader_time: 0.0069 (0.006111735478043556) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5170392990112305 (2.5170392990112305) + | > loss_disc_real_0: 0.16126707196235657 (0.16126707196235657) + | > loss_disc_real_1: 0.23692311346530914 (0.23692311346530914) + | > loss_disc_real_2: 0.27074718475341797 (0.27074718475341797) + | > loss_disc_real_3: 0.2945553660392761 (0.2945553660392761) + | > loss_disc_real_4: 0.2416892647743225 (0.2416892647743225) + | > loss_disc_real_5: 0.2504447102546692 (0.2504447102546692) + | > loss_0: 2.5170392990112305 (2.5170392990112305) + | > loss_gen: 2.4968531131744385 (2.4968531131744385) + | > loss_kl: 2.311251640319824 (2.311251640319824) + | > loss_feat: 7.4764404296875 (7.4764404296875) + | > loss_mel: 19.451156616210938 (19.451156616210938) + | > loss_duration: 1.695578932762146 (1.695578932762146) + | > loss_1: 33.43128204345703 (33.43128204345703) + + --> STEP: 1 + | > loss_disc: 2.544931650161743 (2.544931650161743) + | > loss_disc_real_0: 0.18971697986125946 (0.18971697986125946) + | > loss_disc_real_1: 0.25174400210380554 (0.25174400210380554) + | > loss_disc_real_2: 0.22912298142910004 (0.22912298142910004) + | > loss_disc_real_3: 0.25325730443000793 (0.25325730443000793) + | > loss_disc_real_4: 0.2545805275440216 (0.2545805275440216) + | > loss_disc_real_5: 0.2684450149536133 (0.2684450149536133) + | > loss_0: 2.544931650161743 (2.544931650161743) + | > loss_gen: 2.4222733974456787 (2.4222733974456787) + | > loss_kl: 2.5191991329193115 (2.5191991329193115) + | > loss_feat: 6.239912509918213 (6.239912509918213) + | > loss_mel: 17.801027297973633 (17.801027297973633) + | > loss_duration: 1.7377941608428955 (1.7377941608428955) + | > loss_1: 30.72020721435547 (30.72020721435547) + + --> STEP: 2 + | > loss_disc: 2.590998649597168 (2.5679651498794556) + | > loss_disc_real_0: 0.1877640187740326 (0.18874049931764603) + | > loss_disc_real_1: 0.22675107419490814 (0.23924753814935684) + | > loss_disc_real_2: 0.24571415781974792 (0.23741856962442398) + | > loss_disc_real_3: 0.2747271656990051 (0.26399223506450653) + | > loss_disc_real_4: 0.2617121934890747 (0.25814636051654816) + | > loss_disc_real_5: 0.27979177236557007 (0.2741183936595917) + | > loss_0: 2.590998649597168 (2.5679651498794556) + | > loss_gen: 2.42922306060791 (2.4257482290267944) + | > loss_kl: 2.2996275424957275 (2.4094133377075195) + | > loss_feat: 7.563518047332764 (6.901715278625488) + | > loss_mel: 18.971050262451172 (18.386038780212402) + | > loss_duration: 1.6822397708892822 (1.7100169658660889) + | > loss_1: 32.94565963745117 (31.83293342590332) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033850669860839844 (-0.0005004405975341797) + | > avg_loss_disc: 2.5679651498794556 (+0.1502687931060791) + | > avg_loss_disc_real_0: 0.18874049931764603 (+0.000584043562412262) + | > avg_loss_disc_real_1: 0.23924753814935684 (+0.041828058660030365) + | > avg_loss_disc_real_2: 0.23741856962442398 (-0.01263631135225296) + | > avg_loss_disc_real_3: 0.26399223506450653 (+0.050535641610622406) + | > avg_loss_disc_real_4: 0.25814636051654816 (+0.029308311641216278) + | > avg_loss_disc_real_5: 0.2741183936595917 (+0.05597122013568878) + | > avg_loss_0: 2.5679651498794556 (+0.1502687931060791) + | > avg_loss_gen: 2.4257482290267944 (-0.07645678520202637) + | > avg_loss_kl: 2.4094133377075195 (-0.08146917819976807) + | > avg_loss_feat: 6.901715278625488 (-0.8522138595581055) + | > avg_loss_mel: 18.386038780212402 (-1.517460823059082) + | > avg_loss_duration: 1.7100169658660889 (-0.006396353244781494) + | > avg_loss_1: 31.83293342590332 (-2.53399658203125) + + + > EPOCH: 246/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:02:16)  + + --> TIME: 2025-04-19 03:02:20 -- STEP: 11/142 -- GLOBAL_STEP: 414225 + | > loss_disc: 2.4227797985076904 (2.387273680080067) + | > loss_disc_real_0: 0.11563964188098907 (0.11872618916359814) + | > loss_disc_real_1: 0.186440572142601 (0.20504085313190112) + | > loss_disc_real_2: 0.22116482257843018 (0.22365284643389963) + | > loss_disc_real_3: 0.20166771113872528 (0.22402074391191656) + | > loss_disc_real_4: 0.23875530064105988 (0.2238527996973558) + | > loss_disc_real_5: 0.23088745772838593 (0.2280539247122678) + | > loss_0: 2.4227797985076904 (2.387273680080067) + | > grad_norm_0: tensor(4.9475, device='cuda:0') (tensor(5.2647, device='cuda:0')) + | > loss_gen: 2.301220417022705 (2.4066273732618852) + | > loss_kl: 1.6771273612976074 (1.7755370140075684) + | > loss_feat: 6.920193195343018 (7.217099016362971) + | > loss_mel: 17.90167808532715 (17.676041516390715) + | > loss_duration: 1.422758936882019 (1.4235569672151045) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.222976684570312 (30.498861833052203) + | > grad_norm_1: tensor(136.0374, device='cuda:0') (tensor(100.3354, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.3308 (0.3303442434831099) + | > loader_time: 0.0044 (0.004389632831920277) + + + --> TIME: 2025-04-19 03:02:29 -- STEP: 36/142 -- GLOBAL_STEP: 414250 + | > loss_disc: 2.4758620262145996 (2.431722276740604) + | > loss_disc_real_0: 0.056971680372953415 (0.1173364111325807) + | > loss_disc_real_1: 0.21332117915153503 (0.20563024365239674) + | > loss_disc_real_2: 0.2369690090417862 (0.22624184315403303) + | > loss_disc_real_3: 0.2011902779340744 (0.22502548744281134) + | > loss_disc_real_4: 0.24822799861431122 (0.22462976723909378) + | > loss_disc_real_5: 0.2086697220802307 (0.2300856862631109) + | > loss_0: 2.4758620262145996 (2.431722276740604) + | > grad_norm_0: tensor(14.7748, device='cuda:0') (tensor(10.6175, device='cuda:0')) + | > loss_gen: 2.470888376235962 (2.400204042593638) + | > loss_kl: 1.9191399812698364 (1.828207446469201) + | > loss_feat: 7.248188495635986 (7.25175044271681) + | > loss_mel: 17.427427291870117 (17.846314191818237) + | > loss_duration: 1.4335949420928955 (1.4362023042307959) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.49924087524414 (30.76267851723565) + | > grad_norm_1: tensor(389.4791, device='cuda:0') (tensor(278.3012, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.3546 (0.34034857485029435) + | > loader_time: 0.0057 (0.004949013392130534) + + + --> TIME: 2025-04-19 03:02:38 -- STEP: 61/142 -- GLOBAL_STEP: 414275 + | > loss_disc: 2.538618803024292 (2.4385042894081996) + | > loss_disc_real_0: 0.1349288523197174 (0.12085927662546517) + | > loss_disc_real_1: 0.21313071250915527 (0.2055151811877235) + | > loss_disc_real_2: 0.2234918177127838 (0.22574366971117552) + | > loss_disc_real_3: 0.23249003291130066 (0.22596386539154364) + | > loss_disc_real_4: 0.24583396315574646 (0.22728085151461305) + | > loss_disc_real_5: 0.2535255551338196 (0.23202834192846641) + | > loss_0: 2.538618803024292 (2.4385042894081996) + | > grad_norm_0: tensor(11.6838, device='cuda:0') (tensor(11.6339, device='cuda:0')) + | > loss_gen: 2.2196555137634277 (2.4012154672966624) + | > loss_kl: 1.9133349657058716 (1.8484484367683285) + | > loss_feat: 6.411775588989258 (7.205445031650731) + | > loss_mel: 16.825233459472656 (17.71508765611492) + | > loss_duration: 1.4646787643432617 (1.44809257007036) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.834678649902344 (30.618289197077516) + | > grad_norm_1: tensor(252.9077, device='cuda:0') (tensor(282.0002, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.3819 (0.3522801203805892) + | > loader_time: 0.0058 (0.0052509307861328125) + + + --> TIME: 2025-04-19 03:02:48 -- STEP: 86/142 -- GLOBAL_STEP: 414300 + | > loss_disc: 2.3732247352600098 (2.45104072814764) + | > loss_disc_real_0: 0.09564293920993805 (0.1222882136096095) + | > loss_disc_real_1: 0.19876234233379364 (0.20733892813671467) + | > loss_disc_real_2: 0.20801052451133728 (0.22611239483190138) + | > loss_disc_real_3: 0.23943211138248444 (0.22709623100452644) + | > loss_disc_real_4: 0.23298144340515137 (0.2282271107961965) + | > loss_disc_real_5: 0.2164120227098465 (0.2324737708928973) + | > loss_0: 2.3732247352600098 (2.45104072814764) + | > grad_norm_0: tensor(12.6077, device='cuda:0') (tensor(13.2552, device='cuda:0')) + | > loss_gen: 2.3986523151397705 (2.3884972971539162) + | > loss_kl: 1.8974761962890625 (1.8606160879135132) + | > loss_feat: 7.849760055541992 (7.148085516552593) + | > loss_mel: 19.241531372070312 (17.742578384488127) + | > loss_duration: 1.5177816152572632 (1.460171102091323) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.90520095825195 (30.59994841730872) + | > grad_norm_1: tensor(447.0635, device='cuda:0') (tensor(300.4020, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.3998 (0.36379319013551237) + | > loader_time: 0.0062 (0.005525300669115643) + + + --> TIME: 2025-04-19 03:02:59 -- STEP: 111/142 -- GLOBAL_STEP: 414325 + | > loss_disc: 2.497112512588501 (2.4591209072250497) + | > loss_disc_real_0: 0.13534004986286163 (0.12297198743567811) + | > loss_disc_real_1: 0.2192040979862213 (0.2069977854823207) + | > loss_disc_real_2: 0.25198137760162354 (0.2262561901195629) + | > loss_disc_real_3: 0.2177981734275818 (0.2282223782023868) + | > loss_disc_real_4: 0.243446484208107 (0.2292819679588885) + | > loss_disc_real_5: 0.2024899274110794 (0.23376518004649394) + | > loss_0: 2.497112512588501 (2.4591209072250497) + | > grad_norm_0: tensor(5.8834, device='cuda:0') (tensor(13.0392, device='cuda:0')) + | > loss_gen: 2.294796943664551 (2.3765780861313286) + | > loss_kl: 1.9300732612609863 (1.8788525746749327) + | > loss_feat: 6.4647674560546875 (7.116812882122693) + | > loss_mel: 17.50271224975586 (17.713073318069036) + | > loss_duration: 1.5174436569213867 (1.467827656247594) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.709793090820312 (30.55314454087266) + | > grad_norm_1: tensor(281.2900, device='cuda:0') (tensor(301.4444, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.4265 (0.3751533439567499) + | > loader_time: 0.007 (0.005800940968968845) + + + --> TIME: 2025-04-19 03:03:10 -- STEP: 136/142 -- GLOBAL_STEP: 414350 + | > loss_disc: 2.3799710273742676 (2.4637978041873256) + | > loss_disc_real_0: 0.09015745669603348 (0.12549978113897584) + | > loss_disc_real_1: 0.20456315577030182 (0.20724189741646545) + | > loss_disc_real_2: 0.2286616712808609 (0.22633441274656968) + | > loss_disc_real_3: 0.227599635720253 (0.22781604462686708) + | > loss_disc_real_4: 0.22940874099731445 (0.22893375673276536) + | > loss_disc_real_5: 0.23761677742004395 (0.23337736276581006) + | > loss_0: 2.3799710273742676 (2.4637978041873256) + | > grad_norm_0: tensor(11.7976, device='cuda:0') (tensor(13.5832, device='cuda:0')) + | > loss_gen: 2.349130868911743 (2.368354017243666) + | > loss_kl: 1.95392644405365 (1.8858269032310038) + | > loss_feat: 7.312756538391113 (7.05229862647898) + | > loss_mel: 17.214828491210938 (17.634821492082924) + | > loss_duration: 1.5162478685379028 (1.4741842343526725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.34688949584961 (30.4154852698831) + | > grad_norm_1: tensor(343.2682, device='cuda:0') (tensor(301.6029, device='cuda:0')) + | > current_lr_0: 0.0001939432216875328 + | > current_lr_1: 0.0001939432216875328 + | > step_time: 0.4503 (0.38679908829576837) + | > loader_time: 0.0071 (0.006108664414461919) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.464144706726074 (2.464144706726074) + | > loss_disc_real_0: 0.1795196533203125 (0.1795196533203125) + | > loss_disc_real_1: 0.27825531363487244 (0.27825531363487244) + | > loss_disc_real_2: 0.25626876950263977 (0.25626876950263977) + | > loss_disc_real_3: 0.2616194784641266 (0.2616194784641266) + | > loss_disc_real_4: 0.26693883538246155 (0.26693883538246155) + | > loss_disc_real_5: 0.19541005790233612 (0.19541005790233612) + | > loss_0: 2.464144706726074 (2.464144706726074) + | > loss_gen: 2.524610996246338 (2.524610996246338) + | > loss_kl: 2.3106257915496826 (2.3106257915496826) + | > loss_feat: 6.8404364585876465 (6.8404364585876465) + | > loss_mel: 16.809635162353516 (16.809635162353516) + | > loss_duration: 1.6912975311279297 (1.6912975311279297) + | > loss_1: 30.176605224609375 (30.176605224609375) + + --> STEP: 1 + | > loss_disc: 2.5680530071258545 (2.5680530071258545) + | > loss_disc_real_0: 0.2022748738527298 (0.2022748738527298) + | > loss_disc_real_1: 0.26107993721961975 (0.26107993721961975) + | > loss_disc_real_2: 0.24981310963630676 (0.24981310963630676) + | > loss_disc_real_3: 0.2486129254102707 (0.2486129254102707) + | > loss_disc_real_4: 0.23580925166606903 (0.23580925166606903) + | > loss_disc_real_5: 0.1822599619626999 (0.1822599619626999) + | > loss_0: 2.5680530071258545 (2.5680530071258545) + | > loss_gen: 2.3081228733062744 (2.3081228733062744) + | > loss_kl: 2.5593385696411133 (2.5593385696411133) + | > loss_feat: 6.855290412902832 (6.855290412902832) + | > loss_mel: 18.813587188720703 (18.813587188720703) + | > loss_duration: 1.7128558158874512 (1.7128558158874512) + | > loss_1: 32.24919509887695 (32.24919509887695) + + --> STEP: 2 + | > loss_disc: 2.6057722568511963 (2.5869126319885254) + | > loss_disc_real_0: 0.23072583973407745 (0.21650035679340363) + | > loss_disc_real_1: 0.23915718495845795 (0.25011856108903885) + | > loss_disc_real_2: 0.24495573341846466 (0.2473844215273857) + | > loss_disc_real_3: 0.2418566644191742 (0.24523479491472244) + | > loss_disc_real_4: 0.2690216302871704 (0.2524154409766197) + | > loss_disc_real_5: 0.22802379727363586 (0.20514187961816788) + | > loss_0: 2.6057722568511963 (2.5869126319885254) + | > loss_gen: 2.3408493995666504 (2.3244861364364624) + | > loss_kl: 2.3416154384613037 (2.4504770040512085) + | > loss_feat: 6.771160125732422 (6.813225269317627) + | > loss_mel: 18.000446319580078 (18.40701675415039) + | > loss_duration: 1.6900426149368286 (1.70144921541214) + | > loss_1: 31.144113540649414 (31.696654319763184) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003521561622619629 (+0.00013649463653564453) + | > avg_loss_disc: 2.5869126319885254 (+0.018947482109069824) + | > avg_loss_disc_real_0: 0.21650035679340363 (+0.0277598574757576) + | > avg_loss_disc_real_1: 0.25011856108903885 (+0.010871022939682007) + | > avg_loss_disc_real_2: 0.2473844215273857 (+0.009965851902961731) + | > avg_loss_disc_real_3: 0.24523479491472244 (-0.018757440149784088) + | > avg_loss_disc_real_4: 0.2524154409766197 (-0.005730919539928436) + | > avg_loss_disc_real_5: 0.20514187961816788 (-0.0689765140414238) + | > avg_loss_0: 2.5869126319885254 (+0.018947482109069824) + | > avg_loss_gen: 2.3244861364364624 (-0.10126209259033203) + | > avg_loss_kl: 2.4504770040512085 (+0.041063666343688965) + | > avg_loss_feat: 6.813225269317627 (-0.08849000930786133) + | > avg_loss_mel: 18.40701675415039 (+0.02097797393798828) + | > avg_loss_duration: 1.70144921541214 (-0.008567750453948975) + | > avg_loss_1: 31.696654319763184 (-0.13627910614013672) + + + > EPOCH: 247/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:03:15)  + + --> TIME: 2025-04-19 03:03:22 -- STEP: 19/142 -- GLOBAL_STEP: 414375 + | > loss_disc: 2.617007255554199 (2.476830745998182) + | > loss_disc_real_0: 0.08061438798904419 (0.12379174012886851) + | > loss_disc_real_1: 0.21717515587806702 (0.20944783248399435) + | > loss_disc_real_2: 0.26030492782592773 (0.23173688352108002) + | > loss_disc_real_3: 0.30748558044433594 (0.23408242354267522) + | > loss_disc_real_4: 0.285597562789917 (0.2371199789800142) + | > loss_disc_real_5: 0.2514967620372772 (0.23042608326987216) + | > loss_0: 2.617007255554199 (2.476830745998182) + | > grad_norm_0: tensor(15.2461, device='cuda:0') (tensor(13.5969, device='cuda:0')) + | > loss_gen: 2.188856601715088 (2.4059995224601343) + | > loss_kl: 1.6301559209823608 (1.8212702838998092) + | > loss_feat: 6.811844825744629 (7.255226988541453) + | > loss_mel: 18.041156768798828 (17.748014851620322) + | > loss_duration: 1.4480435848236084 (1.4200540530054193) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.120058059692383 (30.650565498753597) + | > grad_norm_1: tensor(196.3274, device='cuda:0') (tensor(294.7447, device='cuda:0')) + | > current_lr_0: 0.00019391897878482186 + | > current_lr_1: 0.00019391897878482186 + | > step_time: 0.3399 (0.3304536217137387) + | > loader_time: 0.0052 (0.004780430542795281) + + + --> TIME: 2025-04-19 03:03:31 -- STEP: 44/142 -- GLOBAL_STEP: 414400 + | > loss_disc: 2.5249359607696533 (2.4718032912774524) + | > loss_disc_real_0: 0.1292438954114914 (0.12388604523783381) + | > loss_disc_real_1: 0.1941203773021698 (0.210021165961569) + | > loss_disc_real_2: 0.24101151525974274 (0.23005800186233086) + | > loss_disc_real_3: 0.2271650731563568 (0.22860911251469093) + | > loss_disc_real_4: 0.2368163764476776 (0.23149662295525725) + | > loss_disc_real_5: 0.232038676738739 (0.2291746363043785) + | > loss_0: 2.5249359607696533 (2.4718032912774524) + | > grad_norm_0: tensor(10.6131, device='cuda:0') (tensor(10.5506, device='cuda:0')) + | > loss_gen: 2.324645757675171 (2.355518834157424) + | > loss_kl: 1.8589818477630615 (1.841940316286954) + | > loss_feat: 7.1237664222717285 (7.112290469082919) + | > loss_mel: 17.707679748535156 (17.785417556762695) + | > loss_duration: 1.4659539461135864 (1.4387291426008397) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.481027603149414 (30.533896402879193) + | > grad_norm_1: tensor(155.3895, device='cuda:0') (tensor(232.9454, device='cuda:0')) + | > current_lr_0: 0.00019391897878482186 + | > current_lr_1: 0.00019391897878482186 + | > step_time: 0.3638 (0.3433227701620623) + | > loader_time: 0.0055 (0.005242358554493297) + + + --> TIME: 2025-04-19 03:03:40 -- STEP: 69/142 -- GLOBAL_STEP: 414425 + | > loss_disc: 2.432671308517456 (2.466110143108645) + | > loss_disc_real_0: 0.16214703023433685 (0.12466573445261389) + | > loss_disc_real_1: 0.23393334448337555 (0.2105229915916056) + | > loss_disc_real_2: 0.21452927589416504 (0.22828417562920114) + | > loss_disc_real_3: 0.2460237592458725 (0.23006145353766455) + | > loss_disc_real_4: 0.21977472305297852 (0.23001940138098123) + | > loss_disc_real_5: 0.24733074009418488 (0.22967062415420145) + | > loss_0: 2.432671308517456 (2.466110143108645) + | > grad_norm_0: tensor(9.8514, device='cuda:0') (tensor(9.5695, device='cuda:0')) + | > loss_gen: 2.4145705699920654 (2.356427807738817) + | > loss_kl: 1.900370478630066 (1.8581611004428586) + | > loss_feat: 7.708897590637207 (7.128298441569011) + | > loss_mel: 18.705873489379883 (17.812093831490778) + | > loss_duration: 1.5308432579040527 (1.449940750564354) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.260555267333984 (30.604921921439793) + | > grad_norm_1: tensor(241.1945, device='cuda:0') (tensor(203.2719, device='cuda:0')) + | > current_lr_0: 0.00019391897878482186 + | > current_lr_1: 0.00019391897878482186 + | > step_time: 0.4059 (0.3557068714197132) + | > loader_time: 0.0063 (0.005590542503025221) + + + --> TIME: 2025-04-19 03:03:51 -- STEP: 94/142 -- GLOBAL_STEP: 414450 + | > loss_disc: 2.4992730617523193 (2.4668515235819717) + | > loss_disc_real_0: 0.13513338565826416 (0.12609188393392462) + | > loss_disc_real_1: 0.1929371953010559 (0.20981702906020144) + | > loss_disc_real_2: 0.23258407413959503 (0.2274019331374067) + | > loss_disc_real_3: 0.24414315819740295 (0.2285416194416107) + | > loss_disc_real_4: 0.2345396727323532 (0.2291322371110003) + | > loss_disc_real_5: 0.24672357738018036 (0.22921806351935609) + | > loss_0: 2.4992730617523193 (2.4668515235819717) + | > grad_norm_0: tensor(13.8509, device='cuda:0') (tensor(9.1708, device='cuda:0')) + | > loss_gen: 2.51116681098938 (2.351143365210677) + | > loss_kl: 1.8091084957122803 (1.8806747134695663) + | > loss_feat: 6.671265602111816 (7.087233862978347) + | > loss_mel: 18.08108139038086 (17.83486648316079) + | > loss_duration: 1.505547046661377 (1.4608355532301234) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.578168869018555 (30.614753966635845) + | > grad_norm_1: tensor(238.1471, device='cuda:0') (tensor(180.7303, device='cuda:0')) + | > current_lr_0: 0.00019391897878482186 + | > current_lr_1: 0.00019391897878482186 + | > step_time: 0.4039 (0.36756959113668886) + | > loader_time: 0.0067 (0.005851347395714293) + + + --> TIME: 2025-04-19 03:04:01 -- STEP: 119/142 -- GLOBAL_STEP: 414475 + | > loss_disc: 2.442288398742676 (2.466092634601753) + | > loss_disc_real_0: 0.08984438329935074 (0.12708397435040034) + | > loss_disc_real_1: 0.24168604612350464 (0.2093675935468754) + | > loss_disc_real_2: 0.23326429724693298 (0.22817161842053676) + | > loss_disc_real_3: 0.22661417722702026 (0.2283426352909633) + | > loss_disc_real_4: 0.23679840564727783 (0.22938969656198965) + | > loss_disc_real_5: 0.2726067006587982 (0.23133993136281727) + | > loss_0: 2.442288398742676 (2.466092634601753) + | > grad_norm_0: tensor(7.2351, device='cuda:0') (tensor(9.6527, device='cuda:0')) + | > loss_gen: 2.2507314682006836 (2.3541513871745905) + | > loss_kl: 2.1443891525268555 (1.8937947689986028) + | > loss_feat: 7.124314785003662 (7.092965390501904) + | > loss_mel: 18.291906356811523 (17.884254728044787) + | > loss_duration: 1.5536487102508545 (1.4688033366403659) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.364988327026367 (30.69396963039366) + | > grad_norm_1: tensor(101.4950, device='cuda:0') (tensor(196.5472, device='cuda:0')) + | > current_lr_0: 0.00019391897878482186 + | > current_lr_1: 0.00019391897878482186 + | > step_time: 0.4305 (0.37926583530522195) + | > loader_time: 0.0069 (0.006075604623105346) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.465479850769043 (2.465479850769043) + | > loss_disc_real_0: 0.1286270171403885 (0.1286270171403885) + | > loss_disc_real_1: 0.23687611520290375 (0.23687611520290375) + | > loss_disc_real_2: 0.2352529764175415 (0.2352529764175415) + | > loss_disc_real_3: 0.20914173126220703 (0.20914173126220703) + | > loss_disc_real_4: 0.18262970447540283 (0.18262970447540283) + | > loss_disc_real_5: 0.23023653030395508 (0.23023653030395508) + | > loss_0: 2.465479850769043 (2.465479850769043) + | > loss_gen: 2.1648404598236084 (2.1648404598236084) + | > loss_kl: 2.3757708072662354 (2.3757708072662354) + | > loss_feat: 7.234337329864502 (7.234337329864502) + | > loss_mel: 18.051189422607422 (18.051189422607422) + | > loss_duration: 1.6789746284484863 (1.6789746284484863) + | > loss_1: 31.505111694335938 (31.505111694335938) + + --> STEP: 1 + | > loss_disc: 2.5542948246002197 (2.5542948246002197) + | > loss_disc_real_0: 0.1540125459432602 (0.1540125459432602) + | > loss_disc_real_1: 0.22204414010047913 (0.22204414010047913) + | > loss_disc_real_2: 0.27360665798187256 (0.27360665798187256) + | > loss_disc_real_3: 0.21021291613578796 (0.21021291613578796) + | > loss_disc_real_4: 0.24354146420955658 (0.24354146420955658) + | > loss_disc_real_5: 0.2155028134584427 (0.2155028134584427) + | > loss_0: 2.5542948246002197 (2.5542948246002197) + | > loss_gen: 2.121626377105713 (2.121626377105713) + | > loss_kl: 2.6698102951049805 (2.6698102951049805) + | > loss_feat: 6.280104637145996 (6.280104637145996) + | > loss_mel: 15.460948944091797 (15.460948944091797) + | > loss_duration: 1.7249956130981445 (1.7249956130981445) + | > loss_1: 28.257484436035156 (28.257484436035156) + + --> STEP: 2 + | > loss_disc: 2.432231903076172 (2.493263363838196) + | > loss_disc_real_0: 0.13946205377578735 (0.14673729985952377) + | > loss_disc_real_1: 0.22413545846939087 (0.223089799284935) + | > loss_disc_real_2: 0.238026425242424 (0.2558165416121483) + | > loss_disc_real_3: 0.216921865940094 (0.21356739103794098) + | > loss_disc_real_4: 0.23250725865364075 (0.23802436143159866) + | > loss_disc_real_5: 0.22439074516296387 (0.21994677931070328) + | > loss_0: 2.432231903076172 (2.493263363838196) + | > loss_gen: 2.2949516773223877 (2.2082890272140503) + | > loss_kl: 2.3432114124298096 (2.506510853767395) + | > loss_feat: 6.407660007476807 (6.343882322311401) + | > loss_mel: 16.429079055786133 (15.945013999938965) + | > loss_duration: 1.6701676845550537 (1.6975816488265991) + | > loss_1: 29.145071029663086 (28.70127773284912) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031911134719848633 (-0.0003304481506347656) + | > avg_loss_disc: 2.493263363838196 (-0.09364926815032959) + | > avg_loss_disc_real_0: 0.14673729985952377 (-0.06976305693387985) + | > avg_loss_disc_real_1: 0.223089799284935 (-0.02702876180410385) + | > avg_loss_disc_real_2: 0.2558165416121483 (+0.008432120084762573) + | > avg_loss_disc_real_3: 0.21356739103794098 (-0.031667403876781464) + | > avg_loss_disc_real_4: 0.23802436143159866 (-0.014391079545021057) + | > avg_loss_disc_real_5: 0.21994677931070328 (+0.0148048996925354) + | > avg_loss_0: 2.493263363838196 (-0.09364926815032959) + | > avg_loss_gen: 2.2082890272140503 (-0.11619710922241211) + | > avg_loss_kl: 2.506510853767395 (+0.05603384971618652) + | > avg_loss_feat: 6.343882322311401 (-0.4693429470062256) + | > avg_loss_mel: 15.945013999938965 (-2.462002754211426) + | > avg_loss_duration: 1.6975816488265991 (-0.0038675665855407715) + | > avg_loss_1: 28.70127773284912 (-2.9953765869140625) + + + > EPOCH: 248/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:04:13)  + + --> TIME: 2025-04-19 03:04:15 -- STEP: 2/142 -- GLOBAL_STEP: 414500 + | > loss_disc: 2.5664114952087402 (2.514841079711914) + | > loss_disc_real_0: 0.22517076134681702 (0.20262406766414642) + | > loss_disc_real_1: 0.23680834472179413 (0.256850890815258) + | > loss_disc_real_2: 0.26343294978141785 (0.26178380846977234) + | > loss_disc_real_3: 0.22482755780220032 (0.2208881676197052) + | > loss_disc_real_4: 0.24040740728378296 (0.22813348472118378) + | > loss_disc_real_5: 0.24056240916252136 (0.21584360301494598) + | > loss_0: 2.5664114952087402 (2.514841079711914) + | > grad_norm_0: tensor(11.5058, device='cuda:0') (tensor(11.5574, device='cuda:0')) + | > loss_gen: 2.2176907062530518 (2.359618306159973) + | > loss_kl: 1.8228946924209595 (1.7327573895454407) + | > loss_feat: 6.774160861968994 (6.992783069610596) + | > loss_mel: 16.85407829284668 (16.99406147003174) + | > loss_duration: 1.4372566938400269 (1.4177170991897583) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.106081008911133 (29.49693775177002) + | > grad_norm_1: tensor(197.2381, device='cuda:0') (tensor(251.1811, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.3214 (0.33260786533355713) + | > loader_time: 0.004 (0.004366397857666016) + + + --> TIME: 2025-04-19 03:04:23 -- STEP: 27/142 -- GLOBAL_STEP: 414525 + | > loss_disc: 2.3249154090881348 (2.4514148323624223) + | > loss_disc_real_0: 0.12160153687000275 (0.11934457057052189) + | > loss_disc_real_1: 0.2302839159965515 (0.208415146779131) + | > loss_disc_real_2: 0.20414137840270996 (0.22703757219844395) + | > loss_disc_real_3: 0.19657519459724426 (0.22866294670988013) + | > loss_disc_real_4: 0.1749797761440277 (0.2286918560663859) + | > loss_disc_real_5: 0.2184317260980606 (0.23031723885624497) + | > loss_0: 2.3249154090881348 (2.4514148323624223) + | > grad_norm_0: tensor(8.8445, device='cuda:0') (tensor(12.4601, device='cuda:0')) + | > loss_gen: 2.438015937805176 (2.3772716875429514) + | > loss_kl: 1.8606157302856445 (1.8355618759437844) + | > loss_feat: 7.106364727020264 (7.266018743868227) + | > loss_mel: 17.005605697631836 (17.90514380843551) + | > loss_duration: 1.4630980491638184 (1.4344500921390675) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.873701095581055 (30.818445982756437) + | > grad_norm_1: tensor(103.4027, device='cuda:0') (tensor(315.5626, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.349 (0.3347762160831027) + | > loader_time: 0.0053 (0.004913224114312066) + + + --> TIME: 2025-04-19 03:04:32 -- STEP: 52/142 -- GLOBAL_STEP: 414550 + | > loss_disc: 2.500788450241089 (2.4452507174932046) + | > loss_disc_real_0: 0.13243083655834198 (0.11984055131100692) + | > loss_disc_real_1: 0.18791918456554413 (0.2093547135591507) + | > loss_disc_real_2: 0.24876758456230164 (0.22702506413826576) + | > loss_disc_real_3: 0.22980205714702606 (0.22631751029537275) + | > loss_disc_real_4: 0.25834721326828003 (0.228608410518903) + | > loss_disc_real_5: 0.24506644904613495 (0.23007860923042664) + | > loss_0: 2.500788450241089 (2.4452507174932046) + | > grad_norm_0: tensor(13.5759, device='cuda:0') (tensor(12.5186, device='cuda:0')) + | > loss_gen: 2.339886426925659 (2.3902477759581346) + | > loss_kl: 1.9368555545806885 (1.851355472436318) + | > loss_feat: 6.517289638519287 (7.2566611766815186) + | > loss_mel: 17.272174835205078 (17.91842519319974) + | > loss_duration: 1.5080757141113281 (1.442496214921658) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.574281692504883 (30.85918554892907) + | > grad_norm_1: tensor(339.2299, device='cuda:0') (tensor(320.6136, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.3629 (0.346257915863624) + | > loader_time: 0.0059 (0.005363368071042574) + + + --> TIME: 2025-04-19 03:04:42 -- STEP: 77/142 -- GLOBAL_STEP: 414575 + | > loss_disc: 2.5104870796203613 (2.4527005220388443) + | > loss_disc_real_0: 0.08278277516365051 (0.1213070885701613) + | > loss_disc_real_1: 0.18025806546211243 (0.2092981967446092) + | > loss_disc_real_2: 0.20635712146759033 (0.22599036662609545) + | > loss_disc_real_3: 0.23456212878227234 (0.22719662123686307) + | > loss_disc_real_4: 0.2686091959476471 (0.23055901136491205) + | > loss_disc_real_5: 0.2810218334197998 (0.23074628703005903) + | > loss_0: 2.5104870796203613 (2.4527005220388443) + | > grad_norm_0: tensor(18.5094, device='cuda:0') (tensor(14.8540, device='cuda:0')) + | > loss_gen: 2.254120349884033 (2.3835166119909905) + | > loss_kl: 2.052438974380493 (1.8654344995300491) + | > loss_feat: 7.188082695007324 (7.2223841307999255) + | > loss_mel: 17.82132339477539 (17.87370826671649) + | > loss_duration: 1.4822986125946045 (1.453818590610059) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.798261642456055 (30.798861887547876) + | > grad_norm_1: tensor(328.6118, device='cuda:0') (tensor(332.3488, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.3953 (0.3588220330027791) + | > loader_time: 0.0065 (0.005652310012222885) + + + --> TIME: 2025-04-19 03:04:53 -- STEP: 102/142 -- GLOBAL_STEP: 414600 + | > loss_disc: 2.452518939971924 (2.460216798034368) + | > loss_disc_real_0: 0.1381085216999054 (0.12398748552682352) + | > loss_disc_real_1: 0.24876970052719116 (0.20984099527784422) + | > loss_disc_real_2: 0.2135155200958252 (0.2260442903521014) + | > loss_disc_real_3: 0.23932549357414246 (0.22744803349761403) + | > loss_disc_real_4: 0.25608956813812256 (0.22871252938228495) + | > loss_disc_real_5: 0.26828688383102417 (0.23184243428940868) + | > loss_0: 2.452518939971924 (2.460216798034368) + | > grad_norm_0: tensor(9.8914, device='cuda:0') (tensor(13.5631, device='cuda:0')) + | > loss_gen: 2.3458991050720215 (2.368398825327555) + | > loss_kl: 2.0468554496765137 (1.881258627947639) + | > loss_feat: 6.689112186431885 (7.149915952308505) + | > loss_mel: 17.048219680786133 (17.878985059027563) + | > loss_duration: 1.4745100736618042 (1.4641242436334203) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.604597091674805 (30.742682569167194) + | > grad_norm_1: tensor(66.2001, device='cuda:0') (tensor(287.2719, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.4239 (0.37030935287475586) + | > loader_time: 0.0091 (0.005917862349865483) + + + --> TIME: 2025-04-19 03:05:04 -- STEP: 127/142 -- GLOBAL_STEP: 414625 + | > loss_disc: 2.4832143783569336 (2.4625396202868357) + | > loss_disc_real_0: 0.14939026534557343 (0.12442031807786837) + | > loss_disc_real_1: 0.22695425152778625 (0.20890993665991806) + | > loss_disc_real_2: 0.23434117436408997 (0.22605003275739866) + | > loss_disc_real_3: 0.1968381255865097 (0.22623843071967598) + | > loss_disc_real_4: 0.2250186800956726 (0.2299512033387432) + | > loss_disc_real_5: 0.22774860262870789 (0.232127616959294) + | > loss_0: 2.4832143783569336 (2.4625396202868357) + | > grad_norm_0: tensor(4.9235, device='cuda:0') (tensor(12.3648, device='cuda:0')) + | > loss_gen: 2.3516998291015625 (2.359318950983483) + | > loss_kl: 2.0455856323242188 (1.8877659660624706) + | > loss_feat: 6.347468852996826 (7.083923050737757) + | > loss_mel: 15.960586547851562 (17.78519496767539) + | > loss_duration: 1.501556396484375 (1.4716967973183464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.206897735595703 (30.587899696169877) + | > grad_norm_1: tensor(81.1940, device='cuda:0') (tensor(259.2941, device='cuda:0')) + | > current_lr_0: 0.00019389473891247375 + | > current_lr_1: 0.00019389473891247375 + | > step_time: 0.4465 (0.38265461433590864) + | > loader_time: 0.0089 (0.006183346425454448) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5294950008392334 (2.5294950008392334) + | > loss_disc_real_0: 0.19383881986141205 (0.19383881986141205) + | > loss_disc_real_1: 0.25592300295829773 (0.25592300295829773) + | > loss_disc_real_2: 0.20778138935565948 (0.20778138935565948) + | > loss_disc_real_3: 0.26158812642097473 (0.26158812642097473) + | > loss_disc_real_4: 0.1885794848203659 (0.1885794848203659) + | > loss_disc_real_5: 0.20816969871520996 (0.20816969871520996) + | > loss_0: 2.5294950008392334 (2.5294950008392334) + | > loss_gen: 2.2849411964416504 (2.2849411964416504) + | > loss_kl: 2.467329740524292 (2.467329740524292) + | > loss_feat: 6.427647590637207 (6.427647590637207) + | > loss_mel: 17.53858757019043 (17.53858757019043) + | > loss_duration: 1.7052948474884033 (1.7052948474884033) + | > loss_1: 30.42380142211914 (30.42380142211914) + + --> STEP: 1 + | > loss_disc: 2.6540937423706055 (2.6540937423706055) + | > loss_disc_real_0: 0.18414145708084106 (0.18414145708084106) + | > loss_disc_real_1: 0.27010852098464966 (0.27010852098464966) + | > loss_disc_real_2: 0.21100126206874847 (0.21100126206874847) + | > loss_disc_real_3: 0.2830042541027069 (0.2830042541027069) + | > loss_disc_real_4: 0.21154239773750305 (0.21154239773750305) + | > loss_disc_real_5: 0.2673684358596802 (0.2673684358596802) + | > loss_0: 2.6540937423706055 (2.6540937423706055) + | > loss_gen: 2.232175350189209 (2.232175350189209) + | > loss_kl: 2.7028281688690186 (2.7028281688690186) + | > loss_feat: 5.9182610511779785 (5.9182610511779785) + | > loss_mel: 17.54952621459961 (17.54952621459961) + | > loss_duration: 1.738710641860962 (1.738710641860962) + | > loss_1: 30.141502380371094 (30.141502380371094) + + --> STEP: 2 + | > loss_disc: 2.4778153896331787 (2.565954566001892) + | > loss_disc_real_0: 0.14781174063682556 (0.1659765988588333) + | > loss_disc_real_1: 0.20136594772338867 (0.23573723435401917) + | > loss_disc_real_2: 0.24090859293937683 (0.22595492750406265) + | > loss_disc_real_3: 0.2755701541900635 (0.2792872041463852) + | > loss_disc_real_4: 0.18752777576446533 (0.1995350867509842) + | > loss_disc_real_5: 0.22946253418922424 (0.2484154850244522) + | > loss_0: 2.4778153896331787 (2.565954566001892) + | > loss_gen: 2.4801909923553467 (2.356183171272278) + | > loss_kl: 2.3519790172576904 (2.5274035930633545) + | > loss_feat: 8.263325691223145 (7.0907933712005615) + | > loss_mel: 18.734352111816406 (18.141939163208008) + | > loss_duration: 1.6752161979675293 (1.7069634199142456) + | > loss_1: 33.505062103271484 (31.82328224182129) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003834962844848633 (+0.0006438493728637695) + | > avg_loss_disc: 2.565954566001892 (+0.07269120216369629) + | > avg_loss_disc_real_0: 0.1659765988588333 (+0.01923929899930954) + | > avg_loss_disc_real_1: 0.23573723435401917 (+0.012647435069084167) + | > avg_loss_disc_real_2: 0.22595492750406265 (-0.029861614108085632) + | > avg_loss_disc_real_3: 0.2792872041463852 (+0.06571981310844421) + | > avg_loss_disc_real_4: 0.1995350867509842 (-0.03848927468061447) + | > avg_loss_disc_real_5: 0.2484154850244522 (+0.028468705713748932) + | > avg_loss_0: 2.565954566001892 (+0.07269120216369629) + | > avg_loss_gen: 2.356183171272278 (+0.14789414405822754) + | > avg_loss_kl: 2.5274035930633545 (+0.020892739295959473) + | > avg_loss_feat: 7.0907933712005615 (+0.7469110488891602) + | > avg_loss_mel: 18.141939163208008 (+2.196925163269043) + | > avg_loss_duration: 1.7069634199142456 (+0.009381771087646484) + | > avg_loss_1: 31.82328224182129 (+3.122004508972168) + + + > EPOCH: 249/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:05:12)  + + --> TIME: 2025-04-19 03:05:16 -- STEP: 10/142 -- GLOBAL_STEP: 414650 + | > loss_disc: 2.4593918323516846 (2.456887459754944) + | > loss_disc_real_0: 0.1497693806886673 (0.1264880709350109) + | > loss_disc_real_1: 0.18247826397418976 (0.20336345732212066) + | > loss_disc_real_2: 0.2423105090856552 (0.23020604401826858) + | > loss_disc_real_3: 0.24542655050754547 (0.23014471530914307) + | > loss_disc_real_4: 0.23730771243572235 (0.22772178500890733) + | > loss_disc_real_5: 0.2498367577791214 (0.2330335333943367) + | > loss_0: 2.4593918323516846 (2.456887459754944) + | > grad_norm_0: tensor(13.8184, device='cuda:0') (tensor(8.4234, device='cuda:0')) + | > loss_gen: 2.4200327396392822 (2.37490828037262) + | > loss_kl: 1.903264045715332 (1.81486519575119) + | > loss_feat: 7.039127349853516 (7.265075778961181) + | > loss_mel: 18.350645065307617 (17.96571750640869) + | > loss_duration: 1.3913531303405762 (1.4200346350669861) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.10442352294922 (30.84060115814209) + | > grad_norm_1: tensor(153.3525, device='cuda:0') (tensor(120.5870, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.3347 (0.3264645576477051) + | > loader_time: 0.0045 (0.004465961456298828) + + + --> TIME: 2025-04-19 03:05:25 -- STEP: 35/142 -- GLOBAL_STEP: 414675 + | > loss_disc: 2.397343158721924 (2.4545630114419112) + | > loss_disc_real_0: 0.12388420104980469 (0.12335493671042581) + | > loss_disc_real_1: 0.22154153883457184 (0.2079843785081591) + | > loss_disc_real_2: 0.22693216800689697 (0.22574793057782308) + | > loss_disc_real_3: 0.21544824540615082 (0.22495184625898088) + | > loss_disc_real_4: 0.23320545256137848 (0.22753460662705557) + | > loss_disc_real_5: 0.2326795607805252 (0.23161204286984036) + | > loss_0: 2.397343158721924 (2.4545630114419112) + | > grad_norm_0: tensor(4.0020, device='cuda:0') (tensor(8.9585, device='cuda:0')) + | > loss_gen: 2.535543441772461 (2.3638209342956547) + | > loss_kl: 1.7813137769699097 (1.8522450583321708) + | > loss_feat: 6.886497497558594 (7.188444750649588) + | > loss_mel: 17.011985778808594 (17.77514182499477) + | > loss_duration: 1.4424428939819336 (1.4327822242464339) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65778350830078 (30.612434550694058) + | > grad_norm_1: tensor(141.2069, device='cuda:0') (tensor(152.2158, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.3575 (0.33871478353227885) + | > loader_time: 0.0052 (0.004846995217459542) + + + --> TIME: 2025-04-19 03:05:34 -- STEP: 60/142 -- GLOBAL_STEP: 414700 + | > loss_disc: 2.613581418991089 (2.452966952323913) + | > loss_disc_real_0: 0.1692134290933609 (0.1255580101162195) + | > loss_disc_real_1: 0.1972062885761261 (0.20810571735103925) + | > loss_disc_real_2: 0.2017098069190979 (0.2241763601700465) + | > loss_disc_real_3: 0.21294230222702026 (0.22663902764519056) + | > loss_disc_real_4: 0.22943256795406342 (0.22871432726581892) + | > loss_disc_real_5: 0.2337053120136261 (0.23012024785081545) + | > loss_0: 2.613581418991089 (2.452966952323913) + | > grad_norm_0: tensor(11.2398, device='cuda:0') (tensor(9.6022, device='cuda:0')) + | > loss_gen: 2.100922107696533 (2.3640809257825217) + | > loss_kl: 2.0139312744140625 (1.8665163119633992) + | > loss_feat: 6.353217124938965 (7.193653217951456) + | > loss_mel: 15.961666107177734 (17.75721364021302) + | > loss_duration: 1.4774103164672852 (1.4436621606349944) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.907146453857422 (30.62512626647949) + | > grad_norm_1: tensor(291.9674, device='cuda:0') (tensor(212.8543, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.3751 (0.35005567868550624) + | > loader_time: 0.006 (0.005135250091552735) + + + --> TIME: 2025-04-19 03:05:44 -- STEP: 85/142 -- GLOBAL_STEP: 414725 + | > loss_disc: 2.4549508094787598 (2.461994535782758) + | > loss_disc_real_0: 0.07847603410482407 (0.12566276464392157) + | > loss_disc_real_1: 0.1949501931667328 (0.20790655595414778) + | > loss_disc_real_2: 0.23441803455352783 (0.2256939989679) + | > loss_disc_real_3: 0.24242819845676422 (0.22779948676333708) + | > loss_disc_real_4: 0.23585569858551025 (0.22938189331222983) + | > loss_disc_real_5: 0.24407853186130524 (0.23244439591379726) + | > loss_0: 2.4549508094787598 (2.461994535782758) + | > grad_norm_0: tensor(5.7875, device='cuda:0') (tensor(9.6681, device='cuda:0')) + | > loss_gen: 2.4194459915161133 (2.3558552601758165) + | > loss_kl: 1.8355985879898071 (1.870153509869295) + | > loss_feat: 8.021622657775879 (7.122898343030144) + | > loss_mel: 18.99993896484375 (17.75789870093851) + | > loss_duration: 1.5087312459945679 (1.456333966816173) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.78533935546875 (30.56313976961024) + | > grad_norm_1: tensor(264.7746, device='cuda:0') (tensor(211.6084, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.3962 (0.3619493260103114) + | > loader_time: 0.0065 (0.005429610084084904) + + + --> TIME: 2025-04-19 03:05:55 -- STEP: 110/142 -- GLOBAL_STEP: 414750 + | > loss_disc: 2.3990182876586914 (2.4625572204589834) + | > loss_disc_real_0: 0.1801428496837616 (0.12576922388239345) + | > loss_disc_real_1: 0.19800738990306854 (0.20668197992173107) + | > loss_disc_real_2: 0.21505282819271088 (0.2257000664418394) + | > loss_disc_real_3: 0.21802128851413727 (0.2273338564417579) + | > loss_disc_real_4: 0.22838254272937775 (0.22879825288599187) + | > loss_disc_real_5: 0.1985776126384735 (0.2314720472151583) + | > loss_0: 2.3990182876586914 (2.4625572204589834) + | > grad_norm_0: tensor(30.2071, device='cuda:0') (tensor(10.4636, device='cuda:0')) + | > loss_gen: 2.4817605018615723 (2.354033786600285) + | > loss_kl: 2.034668445587158 (1.8851458657871594) + | > loss_feat: 7.4693169593811035 (7.101151925867254) + | > loss_mel: 18.95124053955078 (17.78142477382313) + | > loss_duration: 1.4849966764450073 (1.46494014154781) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.42198181152344 (30.58669652071866) + | > grad_norm_1: tensor(231.1317, device='cuda:0') (tensor(228.9312, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.4312 (0.3725283536044034) + | > loader_time: 0.0073 (0.005757481401616877) + + + --> TIME: 2025-04-19 03:06:06 -- STEP: 135/142 -- GLOBAL_STEP: 414775 + | > loss_disc: 2.491014242172241 (2.470387105588559) + | > loss_disc_real_0: 0.09581347554922104 (0.12727084441317463) + | > loss_disc_real_1: 0.20563103258609772 (0.20773345276161476) + | > loss_disc_real_2: 0.2155708223581314 (0.22693270985726957) + | > loss_disc_real_3: 0.24404717981815338 (0.2281137298654627) + | > loss_disc_real_4: 0.24475115537643433 (0.2297470360994339) + | > loss_disc_real_5: 0.2124144434928894 (0.23252028480724052) + | > loss_0: 2.491014242172241 (2.470387105588559) + | > grad_norm_0: tensor(9.3390, device='cuda:0') (tensor(10.9522, device='cuda:0')) + | > loss_gen: 2.2753560543060303 (2.3499378716504125) + | > loss_kl: 1.8036912679672241 (1.8871348407533433) + | > loss_feat: 6.301498889923096 (7.075867645828812) + | > loss_mel: 17.044361114501953 (17.741302624455205) + | > loss_duration: 1.4721732139587402 (1.4715901524932296) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.89708137512207 (30.52583322878237) + | > grad_norm_1: tensor(289.9558, device='cuda:0') (tensor(238.9225, device='cuda:0')) + | > current_lr_0: 0.00019387050207010967 + | > current_lr_1: 0.00019387050207010967 + | > step_time: 0.4521 (0.38449164673134134) + | > loader_time: 0.0073 (0.006039112585562246) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.578702926635742 (2.578702926635742) + | > loss_disc_real_0: 0.25844675302505493 (0.25844675302505493) + | > loss_disc_real_1: 0.26000043749809265 (0.26000043749809265) + | > loss_disc_real_2: 0.2460639476776123 (0.2460639476776123) + | > loss_disc_real_3: 0.2364427149295807 (0.2364427149295807) + | > loss_disc_real_4: 0.21360915899276733 (0.21360915899276733) + | > loss_disc_real_5: 0.2443220615386963 (0.2443220615386963) + | > loss_0: 2.578702926635742 (2.578702926635742) + | > loss_gen: 2.7289156913757324 (2.7289156913757324) + | > loss_kl: 2.3956410884857178 (2.3956410884857178) + | > loss_feat: 6.41386079788208 (6.41386079788208) + | > loss_mel: 16.23402976989746 (16.23402976989746) + | > loss_duration: 1.6923012733459473 (1.6923012733459473) + | > loss_1: 29.46474838256836 (29.46474838256836) + + --> STEP: 1 + | > loss_disc: 2.7578563690185547 (2.7578563690185547) + | > loss_disc_real_0: 0.3723093569278717 (0.3723093569278717) + | > loss_disc_real_1: 0.2722340524196625 (0.2722340524196625) + | > loss_disc_real_2: 0.2392519861459732 (0.2392519861459732) + | > loss_disc_real_3: 0.24743472039699554 (0.24743472039699554) + | > loss_disc_real_4: 0.20752522349357605 (0.20752522349357605) + | > loss_disc_real_5: 0.26279664039611816 (0.26279664039611816) + | > loss_0: 2.7578563690185547 (2.7578563690185547) + | > loss_gen: 2.6865713596343994 (2.6865713596343994) + | > loss_kl: 2.5194244384765625 (2.5194244384765625) + | > loss_feat: 6.56574010848999 (6.56574010848999) + | > loss_mel: 17.865724563598633 (17.865724563598633) + | > loss_duration: 1.7379252910614014 (1.7379252910614014) + | > loss_1: 31.375385284423828 (31.375385284423828) + + --> STEP: 2 + | > loss_disc: 2.566086769104004 (2.6619715690612793) + | > loss_disc_real_0: 0.340371698141098 (0.35634052753448486) + | > loss_disc_real_1: 0.20989905297756195 (0.2410665526986122) + | > loss_disc_real_2: 0.23411747813224792 (0.23668473213911057) + | > loss_disc_real_3: 0.2441445291042328 (0.24578962475061417) + | > loss_disc_real_4: 0.2341141551733017 (0.22081968933343887) + | > loss_disc_real_5: 0.31284457445144653 (0.28782060742378235) + | > loss_0: 2.566086769104004 (2.6619715690612793) + | > loss_gen: 2.904141426086426 (2.7953563928604126) + | > loss_kl: 2.167832612991333 (2.3436285257339478) + | > loss_feat: 6.531467914581299 (6.5486040115356445) + | > loss_mel: 17.525253295898438 (17.695488929748535) + | > loss_duration: 1.675070881843567 (1.7064980864524841) + | > loss_1: 30.803768157958984 (31.089576721191406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033032894134521484 (-0.0005316734313964844) + | > avg_loss_disc: 2.6619715690612793 (+0.09601700305938721) + | > avg_loss_disc_real_0: 0.35634052753448486 (+0.19036392867565155) + | > avg_loss_disc_real_1: 0.2410665526986122 (+0.005329318344593048) + | > avg_loss_disc_real_2: 0.23668473213911057 (+0.010729804635047913) + | > avg_loss_disc_real_3: 0.24578962475061417 (-0.03349757939577103) + | > avg_loss_disc_real_4: 0.22081968933343887 (+0.02128460258245468) + | > avg_loss_disc_real_5: 0.28782060742378235 (+0.03940512239933014) + | > avg_loss_0: 2.6619715690612793 (+0.09601700305938721) + | > avg_loss_gen: 2.7953563928604126 (+0.43917322158813477) + | > avg_loss_kl: 2.3436285257339478 (-0.18377506732940674) + | > avg_loss_feat: 6.5486040115356445 (-0.542189359664917) + | > avg_loss_mel: 17.695488929748535 (-0.44645023345947266) + | > avg_loss_duration: 1.7064980864524841 (-0.0004653334617614746) + | > avg_loss_1: 31.089576721191406 (-0.7337055206298828) + + + > EPOCH: 250/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:06:11)  + + --> TIME: 2025-04-19 03:06:17 -- STEP: 18/142 -- GLOBAL_STEP: 414800 + | > loss_disc: 2.4800546169281006 (2.4202899005677967) + | > loss_disc_real_0: 0.13361990451812744 (0.11650443573792775) + | > loss_disc_real_1: 0.21513329446315765 (0.19940239273839527) + | > loss_disc_real_2: 0.21065270900726318 (0.21880282958348593) + | > loss_disc_real_3: 0.1917419135570526 (0.22084762818283504) + | > loss_disc_real_4: 0.21607208251953125 (0.22365098777744505) + | > loss_disc_real_5: 0.2292679399251938 (0.23041397829850516) + | > loss_0: 2.4800546169281006 (2.4202899005677967) + | > grad_norm_0: tensor(20.0041, device='cuda:0') (tensor(14.7869, device='cuda:0')) + | > loss_gen: 2.2650740146636963 (2.351218952072991) + | > loss_kl: 1.732053279876709 (1.8319730228847928) + | > loss_feat: 6.612054347991943 (7.119605196846856) + | > loss_mel: 16.62547492980957 (17.42225996653239) + | > loss_duration: 1.4344990253448486 (1.4234177072842915) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.66915512084961 (30.148474587334526) + | > grad_norm_1: tensor(380.6743, device='cuda:0') (tensor(330.3898, device='cuda:0')) + | > current_lr_0: 0.0001938462682573509 + | > current_lr_1: 0.0001938462682573509 + | > step_time: 0.3367 (0.32992639806535506) + | > loader_time: 0.0046 (0.004585093922085232) + + + --> TIME: 2025-04-19 03:06:26 -- STEP: 43/142 -- GLOBAL_STEP: 414825 + | > loss_disc: 2.3404548168182373 (2.4215683548949487) + | > loss_disc_real_0: 0.11191333830356598 (0.11762009231850158) + | > loss_disc_real_1: 0.23229308426380157 (0.2031569886346196) + | > loss_disc_real_2: 0.2575368881225586 (0.22189280286777852) + | > loss_disc_real_3: 0.22793546319007874 (0.22488562554814095) + | > loss_disc_real_4: 0.22886739671230316 (0.2259474091751631) + | > loss_disc_real_5: 0.21900911629199982 (0.22888794506705085) + | > loss_0: 2.3404548168182373 (2.4215683548949487) + | > grad_norm_0: tensor(4.3482, device='cuda:0') (tensor(11.9354, device='cuda:0')) + | > loss_gen: 2.4987106323242188 (2.3650453977806625) + | > loss_kl: 1.7564127445220947 (1.8527546206185983) + | > loss_feat: 7.501031875610352 (7.139881710673487) + | > loss_mel: 18.873443603515625 (17.627131883488143) + | > loss_duration: 1.4655839204788208 (1.43850240873736) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.095184326171875 (30.423316024070562) + | > grad_norm_1: tensor(294.2983, device='cuda:0') (tensor(283.6167, device='cuda:0')) + | > current_lr_0: 0.0001938462682573509 + | > current_lr_1: 0.0001938462682573509 + | > step_time: 0.3627 (0.34377023231151493) + | > loader_time: 0.0051 (0.004965310872987259) + + + --> TIME: 2025-04-19 03:06:36 -- STEP: 68/142 -- GLOBAL_STEP: 414850 + | > loss_disc: 2.4841060638427734 (2.432780584868263) + | > loss_disc_real_0: 0.12970750033855438 (0.11911407167858937) + | > loss_disc_real_1: 0.19587789475917816 (0.20347674563527107) + | > loss_disc_real_2: 0.21976599097251892 (0.22191524439874818) + | > loss_disc_real_3: 0.22358465194702148 (0.22553508860223434) + | > loss_disc_real_4: 0.21128690242767334 (0.2258053627960822) + | > loss_disc_real_5: 0.2198854237794876 (0.23106323839987025) + | > loss_0: 2.4841060638427734 (2.432780584868263) + | > grad_norm_0: tensor(16.2877, device='cuda:0') (tensor(11.2389, device='cuda:0')) + | > loss_gen: 2.5011353492736816 (2.366650756667642) + | > loss_kl: 2.071312665939331 (1.8682734896154964) + | > loss_feat: 7.548255920410156 (7.16912061326644) + | > loss_mel: 17.03495979309082 (17.685707779491644) + | > loss_duration: 1.4746897220611572 (1.450362873428008) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.630353927612305 (30.540115440593045) + | > grad_norm_1: tensor(377.3374, device='cuda:0') (tensor(262.8520, device='cuda:0')) + | > current_lr_0: 0.0001938462682573509 + | > current_lr_1: 0.0001938462682573509 + | > step_time: 0.3888 (0.3562429897925433) + | > loader_time: 0.006 (0.005265852984260111) + + + --> TIME: 2025-04-19 03:06:46 -- STEP: 93/142 -- GLOBAL_STEP: 414875 + | > loss_disc: 2.3845701217651367 (2.44710362085732) + | > loss_disc_real_0: 0.12442215532064438 (0.1202310491152989) + | > loss_disc_real_1: 0.2196001410484314 (0.20566529868751443) + | > loss_disc_real_2: 0.21206514537334442 (0.2231191752418395) + | > loss_disc_real_3: 0.20851962268352509 (0.22730619756765263) + | > loss_disc_real_4: 0.21732644736766815 (0.22770952906018943) + | > loss_disc_real_5: 0.2363688051700592 (0.23272309876898284) + | > loss_0: 2.3845701217651367 (2.44710362085732) + | > grad_norm_0: tensor(4.8826, device='cuda:0') (tensor(10.4010, device='cuda:0')) + | > loss_gen: 2.4837257862091064 (2.3616457985293473) + | > loss_kl: 1.8546241521835327 (1.877463775296365) + | > loss_feat: 7.418236255645752 (7.146859384352161) + | > loss_mel: 18.809965133666992 (17.757697054134898) + | > loss_duration: 1.4808082580566406 (1.4622891320977154) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.047359466552734 (30.60595506237399) + | > grad_norm_1: tensor(237.5635, device='cuda:0') (tensor(252.6292, device='cuda:0')) + | > current_lr_0: 0.0001938462682573509 + | > current_lr_1: 0.0001938462682573509 + | > step_time: 0.4052 (0.3681335526127969) + | > loader_time: 0.0063 (0.005548369499944869) + + + --> TIME: 2025-04-19 03:06:57 -- STEP: 118/142 -- GLOBAL_STEP: 414900 + | > loss_disc: 2.5091981887817383 (2.452878335774954) + | > loss_disc_real_0: 0.11345946788787842 (0.12246663704261941) + | > loss_disc_real_1: 0.23276399075984955 (0.2061716593423132) + | > loss_disc_real_2: 0.2780453860759735 (0.2238541239146459) + | > loss_disc_real_3: 0.27371200919151306 (0.228153463768757) + | > loss_disc_real_4: 0.2533401846885681 (0.22905431902509624) + | > loss_disc_real_5: 0.2576795220375061 (0.2327928844902475) + | > loss_0: 2.5091981887817383 (2.452878335774954) + | > grad_norm_0: tensor(6.9782, device='cuda:0') (tensor(10.3433, device='cuda:0')) + | > loss_gen: 2.29921293258667 (2.3557029194751027) + | > loss_kl: 1.9608391523361206 (1.8892468123112696) + | > loss_feat: 6.2846598625183105 (7.099051633123624) + | > loss_mel: 17.314836502075195 (17.662592079679833) + | > loss_duration: 1.476731777191162 (1.4680971846742141) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.336280822753906 (30.47469059895661) + | > grad_norm_1: tensor(328.2326, device='cuda:0') (tensor(248.1307, device='cuda:0')) + | > current_lr_0: 0.0001938462682573509 + | > current_lr_1: 0.0001938462682573509 + | > step_time: 0.4444 (0.37956895060458423) + | > loader_time: 0.0068 (0.005821803868827171) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5395710468292236 (2.5395710468292236) + | > loss_disc_real_0: 0.2244974672794342 (0.2244974672794342) + | > loss_disc_real_1: 0.1965007483959198 (0.1965007483959198) + | > loss_disc_real_2: 0.21287944912910461 (0.21287944912910461) + | > loss_disc_real_3: 0.21307572722434998 (0.21307572722434998) + | > loss_disc_real_4: 0.20788922905921936 (0.20788922905921936) + | > loss_disc_real_5: 0.23039385676383972 (0.23039385676383972) + | > loss_0: 2.5395710468292236 (2.5395710468292236) + | > loss_gen: 2.212440252304077 (2.212440252304077) + | > loss_kl: 2.4157583713531494 (2.4157583713531494) + | > loss_feat: 6.598236560821533 (6.598236560821533) + | > loss_mel: 16.839025497436523 (16.839025497436523) + | > loss_duration: 1.6723475456237793 (1.6723475456237793) + | > loss_1: 29.737808227539062 (29.737808227539062) + + --> STEP: 1 + | > loss_disc: 2.60063099861145 (2.60063099861145) + | > loss_disc_real_0: 0.21101294457912445 (0.21101294457912445) + | > loss_disc_real_1: 0.2285291850566864 (0.2285291850566864) + | > loss_disc_real_2: 0.21042072772979736 (0.21042072772979736) + | > loss_disc_real_3: 0.21558085083961487 (0.21558085083961487) + | > loss_disc_real_4: 0.2223193198442459 (0.2223193198442459) + | > loss_disc_real_5: 0.2470872700214386 (0.2470872700214386) + | > loss_0: 2.60063099861145 (2.60063099861145) + | > loss_gen: 2.1420788764953613 (2.1420788764953613) + | > loss_kl: 2.6801676750183105 (2.6801676750183105) + | > loss_feat: 5.547360897064209 (5.547360897064209) + | > loss_mel: 16.92119789123535 (16.92119789123535) + | > loss_duration: 1.7367167472839355 (1.7367167472839355) + | > loss_1: 29.027523040771484 (29.027523040771484) + + --> STEP: 2 + | > loss_disc: 2.4503402709960938 (2.525485634803772) + | > loss_disc_real_0: 0.17578372359275818 (0.19339833408594131) + | > loss_disc_real_1: 0.1764553189277649 (0.20249225199222565) + | > loss_disc_real_2: 0.19313088059425354 (0.20177580416202545) + | > loss_disc_real_3: 0.1936456710100174 (0.20461326092481613) + | > loss_disc_real_4: 0.2311258316040039 (0.2267225757241249) + | > loss_disc_real_5: 0.24073655903339386 (0.24391191452741623) + | > loss_0: 2.4503402709960938 (2.525485634803772) + | > loss_gen: 2.1708102226257324 (2.156444549560547) + | > loss_kl: 2.2204525470733643 (2.4503101110458374) + | > loss_feat: 6.395636558532715 (5.971498727798462) + | > loss_mel: 16.44454002380371 (16.68286895751953) + | > loss_duration: 1.6723389625549316 (1.7045278549194336) + | > loss_1: 28.903778076171875 (28.96565055847168) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.005048990249633789 (+0.0017457008361816406) + | > avg_loss_disc: 2.525485634803772 (-0.13648593425750732) + | > avg_loss_disc_real_0: 0.19339833408594131 (-0.16294219344854355) + | > avg_loss_disc_real_1: 0.20249225199222565 (-0.038574300706386566) + | > avg_loss_disc_real_2: 0.20177580416202545 (-0.034908927977085114) + | > avg_loss_disc_real_3: 0.20461326092481613 (-0.041176363825798035) + | > avg_loss_disc_real_4: 0.2267225757241249 (+0.005902886390686035) + | > avg_loss_disc_real_5: 0.24391191452741623 (-0.04390869289636612) + | > avg_loss_0: 2.525485634803772 (-0.13648593425750732) + | > avg_loss_gen: 2.156444549560547 (-0.6389118432998657) + | > avg_loss_kl: 2.4503101110458374 (+0.10668158531188965) + | > avg_loss_feat: 5.971498727798462 (-0.5771052837371826) + | > avg_loss_mel: 16.68286895751953 (-1.012619972229004) + | > avg_loss_duration: 1.7045278549194336 (-0.001970231533050537) + | > avg_loss_1: 28.96565055847168 (-2.1239261627197266) + + + > EPOCH: 251/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:07:10)  + + --> TIME: 2025-04-19 03:07:11 -- STEP: 1/142 -- GLOBAL_STEP: 414925 + | > loss_disc: 2.3213930130004883 (2.3213930130004883) + | > loss_disc_real_0: 0.08197598904371262 (0.08197598904371262) + | > loss_disc_real_1: 0.17447687685489655 (0.17447687685489655) + | > loss_disc_real_2: 0.19031229615211487 (0.19031229615211487) + | > loss_disc_real_3: 0.19613492488861084 (0.19613492488861084) + | > loss_disc_real_4: 0.2045394778251648 (0.2045394778251648) + | > loss_disc_real_5: 0.21081455051898956 (0.21081455051898956) + | > loss_0: 2.3213930130004883 (2.3213930130004883) + | > grad_norm_0: tensor(11.4065, device='cuda:0') (tensor(11.4065, device='cuda:0')) + | > loss_gen: 2.3312947750091553 (2.3312947750091553) + | > loss_kl: 1.7501956224441528 (1.7501956224441528) + | > loss_feat: 7.462714195251465 (7.462714195251465) + | > loss_mel: 18.331247329711914 (18.331247329711914) + | > loss_duration: 1.4069340229034424 (1.4069340229034424) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.282386779785156 (31.282386779785156) + | > grad_norm_1: tensor(279.6954, device='cuda:0') (tensor(279.6954, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.3381 (0.3380849361419678) + | > loader_time: 0.0039 (0.003930330276489258) + + + --> TIME: 2025-04-19 03:07:19 -- STEP: 26/142 -- GLOBAL_STEP: 414950 + | > loss_disc: 2.4574921131134033 (2.4556881189346313) + | > loss_disc_real_0: 0.15897174179553986 (0.12098589453559655) + | > loss_disc_real_1: 0.19620594382286072 (0.20260470704390451) + | > loss_disc_real_2: 0.22655583918094635 (0.22810737215555632) + | > loss_disc_real_3: 0.2212550938129425 (0.2287835255265236) + | > loss_disc_real_4: 0.23594897985458374 (0.2282189795604119) + | > loss_disc_real_5: 0.24875876307487488 (0.23439371012724364) + | > loss_0: 2.4574921131134033 (2.4556881189346313) + | > grad_norm_0: tensor(8.0986, device='cuda:0') (tensor(13.0956, device='cuda:0')) + | > loss_gen: 2.385436534881592 (2.373724057124211) + | > loss_kl: 1.8584712743759155 (1.8562078017454882) + | > loss_feat: 6.872502326965332 (7.136716017356286) + | > loss_mel: 18.030078887939453 (17.619843116173374) + | > loss_duration: 1.4514153003692627 (1.4331150605128362) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.597904205322266 (30.41960620880127) + | > grad_norm_1: tensor(451.1322, device='cuda:0') (tensor(279.2463, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.3594 (0.3363863321451041) + | > loader_time: 0.0049 (0.00475145303286039) + + + --> TIME: 2025-04-19 03:07:28 -- STEP: 51/142 -- GLOBAL_STEP: 414975 + | > loss_disc: 2.593074083328247 (2.4542929798949005) + | > loss_disc_real_0: 0.12277807295322418 (0.12177774704554502) + | > loss_disc_real_1: 0.23620882630348206 (0.2061198222286561) + | > loss_disc_real_2: 0.2393755167722702 (0.22742423853453467) + | > loss_disc_real_3: 0.2361755669116974 (0.22818512367267235) + | > loss_disc_real_4: 0.23956798017024994 (0.2270917419125052) + | > loss_disc_real_5: 0.22209207713603973 (0.2299260903222888) + | > loss_0: 2.593074083328247 (2.4542929798949005) + | > grad_norm_0: tensor(17.6010, device='cuda:0') (tensor(12.3401, device='cuda:0')) + | > loss_gen: 2.3363044261932373 (2.3773527098636995) + | > loss_kl: 1.9633880853652954 (1.8842639852972591) + | > loss_feat: 7.318697452545166 (7.133213800542495) + | > loss_mel: 17.421947479248047 (17.59391167584587) + | > loss_duration: 1.439460039138794 (1.4410648369321637) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.47979736328125 (30.429807139377967) + | > grad_norm_1: tensor(408.6483, device='cuda:0') (tensor(273.8643, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.3679 (0.34791992224899004) + | > loader_time: 0.0059 (0.005166764352835861) + + + --> TIME: 2025-04-19 03:07:38 -- STEP: 76/142 -- GLOBAL_STEP: 415000 + | > loss_disc: 2.536402702331543 (2.4597817345669397) + | > loss_disc_real_0: 0.07985766977071762 (0.12387966452852676) + | > loss_disc_real_1: 0.2336723357439041 (0.20636100165153803) + | > loss_disc_real_2: 0.23931796848773956 (0.22713398815769897) + | > loss_disc_real_3: 0.2049015313386917 (0.22750269503969894) + | > loss_disc_real_4: 0.2618815004825592 (0.22750354126880043) + | > loss_disc_real_5: 0.24220526218414307 (0.23291496854079397) + | > loss_0: 2.536402702331543 (2.4597817345669397) + | > grad_norm_0: tensor(22.6449, device='cuda:0') (tensor(13.2800, device='cuda:0')) + | > loss_gen: 2.1637930870056152 (2.3733181231900264) + | > loss_kl: 2.035224676132202 (1.8950659867964292) + | > loss_feat: 6.814018249511719 (7.12125761885392) + | > loss_mel: 16.90860366821289 (17.65250370376988) + | > loss_duration: 1.4982061386108398 (1.4529821119810407) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.419845581054688 (30.495127452047246) + | > grad_norm_1: tensor(281.3568, device='cuda:0') (tensor(294.3420, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.398 (0.35997108095570624) + | > loader_time: 0.0066 (0.005527267330571224) + + + --> TIME: 2025-04-19 03:07:49 -- STEP: 101/142 -- GLOBAL_STEP: 415025 + | > loss_disc: 2.586460590362549 (2.4618526213239917) + | > loss_disc_real_0: 0.15266747772693634 (0.12437798156596647) + | > loss_disc_real_1: 0.23780687153339386 (0.2067096081405583) + | > loss_disc_real_2: 0.2693358361721039 (0.22748230869817263) + | > loss_disc_real_3: 0.2541503310203552 (0.22832361055482733) + | > loss_disc_real_4: 0.2577214241027832 (0.2278943561976499) + | > loss_disc_real_5: 0.21934053301811218 (0.23095174561632742) + | > loss_0: 2.586460590362549 (2.4618526213239917) + | > grad_norm_0: tensor(11.8569, device='cuda:0') (tensor(12.6231, device='cuda:0')) + | > loss_gen: 2.3344926834106445 (2.364019797580078) + | > loss_kl: 1.9762423038482666 (1.9072048687698817) + | > loss_feat: 6.813835144042969 (7.061276426409731) + | > loss_mel: 17.408973693847656 (17.626760747173037) + | > loss_duration: 1.5060135126113892 (1.463471050309663) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.0395565032959 (30.422732796999487) + | > grad_norm_1: tensor(216.7041, device='cuda:0') (tensor(277.1080, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.4177 (0.37118876098406195) + | > loader_time: 0.007 (0.005854363488678885) + + + --> TIME: 2025-04-19 03:08:00 -- STEP: 126/142 -- GLOBAL_STEP: 415050 + | > loss_disc: 2.4333505630493164 (2.4639305311536037) + | > loss_disc_real_0: 0.15078948438167572 (0.12596678509125633) + | > loss_disc_real_1: 0.19269999861717224 (0.20636968775874093) + | > loss_disc_real_2: 0.2453346848487854 (0.2272184102071656) + | > loss_disc_real_3: 0.22916601598262787 (0.2284061187316501) + | > loss_disc_real_4: 0.23547500371932983 (0.22758291377907708) + | > loss_disc_real_5: 0.241658553481102 (0.23207946665703305) + | > loss_0: 2.4333505630493164 (2.4639305311536037) + | > grad_norm_0: tensor(11.5070, device='cuda:0') (tensor(12.2221, device='cuda:0')) + | > loss_gen: 2.4672722816467285 (2.3565633315888665) + | > loss_kl: 1.9805452823638916 (1.9126234366780235) + | > loss_feat: 8.240559577941895 (7.003262962613787) + | > loss_mel: 18.69377326965332 (17.599033658466638) + | > loss_duration: 1.5072333812713623 (1.470226257566422) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.88938522338867 (30.34170953054277) + | > grad_norm_1: tensor(393.1407, device='cuda:0') (tensor(261.8826, device='cuda:0')) + | > current_lr_0: 0.00019382203747381871 + | > current_lr_1: 0.00019382203747381871 + | > step_time: 0.4419 (0.3843067241093469) + | > loader_time: 0.0073 (0.006144075166611444) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5542125701904297 (2.5542125701904297) + | > loss_disc_real_0: 0.19539062678813934 (0.19539062678813934) + | > loss_disc_real_1: 0.2256201058626175 (0.2256201058626175) + | > loss_disc_real_2: 0.24643152952194214 (0.24643152952194214) + | > loss_disc_real_3: 0.2654690444469452 (0.2654690444469452) + | > loss_disc_real_4: 0.23515281081199646 (0.23515281081199646) + | > loss_disc_real_5: 0.2576500475406647 (0.2576500475406647) + | > loss_0: 2.5542125701904297 (2.5542125701904297) + | > loss_gen: 2.4125475883483887 (2.4125475883483887) + | > loss_kl: 2.428286075592041 (2.428286075592041) + | > loss_feat: 6.097465515136719 (6.097465515136719) + | > loss_mel: 18.170658111572266 (18.170658111572266) + | > loss_duration: 1.69444739818573 (1.69444739818573) + | > loss_1: 30.803403854370117 (30.803403854370117) + + --> STEP: 1 + | > loss_disc: 2.531766891479492 (2.531766891479492) + | > loss_disc_real_0: 0.20563684403896332 (0.20563684403896332) + | > loss_disc_real_1: 0.25086140632629395 (0.25086140632629395) + | > loss_disc_real_2: 0.2582501173019409 (0.2582501173019409) + | > loss_disc_real_3: 0.25403016805648804 (0.25403016805648804) + | > loss_disc_real_4: 0.2662709653377533 (0.2662709653377533) + | > loss_disc_real_5: 0.25251051783561707 (0.25251051783561707) + | > loss_0: 2.531766891479492 (2.531766891479492) + | > loss_gen: 2.509498119354248 (2.509498119354248) + | > loss_kl: 2.659519910812378 (2.659519910812378) + | > loss_feat: 7.470187664031982 (7.470187664031982) + | > loss_mel: 19.297971725463867 (19.297971725463867) + | > loss_duration: 1.7111932039260864 (1.7111932039260864) + | > loss_1: 33.648372650146484 (33.648372650146484) + + --> STEP: 2 + | > loss_disc: 2.3714513778686523 (2.4516091346740723) + | > loss_disc_real_0: 0.163314551115036 (0.18447569757699966) + | > loss_disc_real_1: 0.25176289677619934 (0.25131215155124664) + | > loss_disc_real_2: 0.23533369600772858 (0.24679190665483475) + | > loss_disc_real_3: 0.2514236271381378 (0.2527268975973129) + | > loss_disc_real_4: 0.24612900614738464 (0.25619998574256897) + | > loss_disc_real_5: 0.22119629383087158 (0.23685340583324432) + | > loss_0: 2.3714513778686523 (2.4516091346740723) + | > loss_gen: 2.669950485229492 (2.58972430229187) + | > loss_kl: 2.317981004714966 (2.488750457763672) + | > loss_feat: 7.774776458740234 (7.622482061386108) + | > loss_mel: 19.38372039794922 (19.340846061706543) + | > loss_duration: 1.6886770725250244 (1.6999351382255554) + | > loss_1: 33.83510971069336 (33.74174118041992) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036432743072509766 (-0.0014057159423828125) + | > avg_loss_disc: 2.4516091346740723 (-0.07387650012969971) + | > avg_loss_disc_real_0: 0.18447569757699966 (-0.00892263650894165) + | > avg_loss_disc_real_1: 0.25131215155124664 (+0.048819899559020996) + | > avg_loss_disc_real_2: 0.24679190665483475 (+0.045016102492809296) + | > avg_loss_disc_real_3: 0.2527268975973129 (+0.048113636672496796) + | > avg_loss_disc_real_4: 0.25619998574256897 (+0.02947741001844406) + | > avg_loss_disc_real_5: 0.23685340583324432 (-0.0070585086941719055) + | > avg_loss_0: 2.4516091346740723 (-0.07387650012969971) + | > avg_loss_gen: 2.58972430229187 (+0.43327975273132324) + | > avg_loss_kl: 2.488750457763672 (+0.03844034671783447) + | > avg_loss_feat: 7.622482061386108 (+1.6509833335876465) + | > avg_loss_mel: 19.340846061706543 (+2.6579771041870117) + | > avg_loss_duration: 1.6999351382255554 (-0.004592716693878174) + | > avg_loss_1: 33.74174118041992 (+4.776090621948242) + + + > EPOCH: 252/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:08:09)  + + --> TIME: 2025-04-19 03:08:12 -- STEP: 9/142 -- GLOBAL_STEP: 415075 + | > loss_disc: 2.422710657119751 (2.4507755438486734) + | > loss_disc_real_0: 0.13162462413311005 (0.12223160515228908) + | > loss_disc_real_1: 0.2521938681602478 (0.21552218662367928) + | > loss_disc_real_2: 0.2252872735261917 (0.2288416309489144) + | > loss_disc_real_3: 0.2508828043937683 (0.2319779379500283) + | > loss_disc_real_4: 0.23131929337978363 (0.21000387767950693) + | > loss_disc_real_5: 0.2322399914264679 (0.24033291637897491) + | > loss_0: 2.422710657119751 (2.4507755438486734) + | > grad_norm_0: tensor(10.4294, device='cuda:0') (tensor(10.0043, device='cuda:0')) + | > loss_gen: 2.428925037384033 (2.3735735416412354) + | > loss_kl: 1.8672500848770142 (1.8584715260399713) + | > loss_feat: 8.024083137512207 (7.236236148410374) + | > loss_mel: 18.137651443481445 (17.857670466105144) + | > loss_duration: 1.4271488189697266 (1.4274375041325886) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.88505744934082 (30.75338914659288) + | > grad_norm_1: tensor(201.1606, device='cuda:0') (tensor(141.1891, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.3256 (0.3267659876081679) + | > loader_time: 0.0049 (0.004390345679389106) + + + --> TIME: 2025-04-19 03:08:21 -- STEP: 34/142 -- GLOBAL_STEP: 415100 + | > loss_disc: 2.490154266357422 (2.456163153928869) + | > loss_disc_real_0: 0.17430876195430756 (0.11893191254314255) + | > loss_disc_real_1: 0.22436551749706268 (0.2044315780786907) + | > loss_disc_real_2: 0.20682524144649506 (0.22418665447655847) + | > loss_disc_real_3: 0.19636982679367065 (0.23030655524309943) + | > loss_disc_real_4: 0.21482810378074646 (0.22637686527827205) + | > loss_disc_real_5: 0.2147018313407898 (0.23874191501561334) + | > loss_0: 2.490154266357422 (2.456163153928869) + | > grad_norm_0: tensor(26.6099, device='cuda:0') (tensor(10.6614, device='cuda:0')) + | > loss_gen: 2.437819480895996 (2.3872723018421844) + | > loss_kl: 1.907671332359314 (1.9005409584325903) + | > loss_feat: 8.086756706237793 (7.306848147336175) + | > loss_mel: 18.377487182617188 (17.902805216172165) + | > loss_duration: 1.4434378147125244 (1.4333513133666094) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.253173828125 (30.93081788455739) + | > grad_norm_1: tensor(438.6013, device='cuda:0') (tensor(252.7709, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.3556 (0.3401545426424812) + | > loader_time: 0.0057 (0.00497287862441119) + + + --> TIME: 2025-04-19 03:08:30 -- STEP: 59/142 -- GLOBAL_STEP: 415125 + | > loss_disc: 2.510531187057495 (2.455913422471386) + | > loss_disc_real_0: 0.13668639957904816 (0.12012684307361053) + | > loss_disc_real_1: 0.21016563475131989 (0.2052826053005154) + | > loss_disc_real_2: 0.24576294422149658 (0.22411070890345816) + | > loss_disc_real_3: 0.24747657775878906 (0.23090821125749814) + | > loss_disc_real_4: 0.246555894613266 (0.22883469245191349) + | > loss_disc_real_5: 0.2346750795841217 (0.2362369581804437) + | > loss_0: 2.510531187057495 (2.455913422471386) + | > grad_norm_0: tensor(9.0551, device='cuda:0') (tensor(10.6244, device='cuda:0')) + | > loss_gen: 2.3615562915802 (2.3801415168632896) + | > loss_kl: 1.9658323526382446 (1.9086899009801575) + | > loss_feat: 6.953456878662109 (7.213676800162105) + | > loss_mel: 17.197288513183594 (17.738191750090007) + | > loss_duration: 1.4652330875396729 (1.4453637014001102) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.94336700439453 (30.686063669495663) + | > grad_norm_1: tensor(223.3373, device='cuda:0') (tensor(268.9359, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.3746 (0.3508021104133735) + | > loader_time: 0.0057 (0.005398237099081782) + + + --> TIME: 2025-04-19 03:08:40 -- STEP: 84/142 -- GLOBAL_STEP: 415150 + | > loss_disc: 2.4642646312713623 (2.4624273720241727) + | > loss_disc_real_0: 0.0995970070362091 (0.12279287131414526) + | > loss_disc_real_1: 0.19816890358924866 (0.2065916247665883) + | > loss_disc_real_2: 0.20759733021259308 (0.22504562387863794) + | > loss_disc_real_3: 0.207508847117424 (0.22920363751195727) + | > loss_disc_real_4: 0.21558687090873718 (0.22829649703843252) + | > loss_disc_real_5: 0.223703533411026 (0.23631564971236957) + | > loss_0: 2.4642646312713623 (2.4624273720241727) + | > grad_norm_0: tensor(14.4115, device='cuda:0') (tensor(10.6125, device='cuda:0')) + | > loss_gen: 2.266291618347168 (2.366353083224523) + | > loss_kl: 1.992698311805725 (1.913560607603618) + | > loss_feat: 7.273889541625977 (7.139995188940139) + | > loss_mel: 18.733386993408203 (17.722581954229444) + | > loss_duration: 1.513741135597229 (1.4583393221809753) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.780006408691406 (30.60083023707072) + | > grad_norm_1: tensor(322.0996, device='cuda:0') (tensor(241.0515, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.3927 (0.3625267176401048) + | > loader_time: 0.0062 (0.005649436087835403) + + + --> TIME: 2025-04-19 03:08:51 -- STEP: 109/142 -- GLOBAL_STEP: 415175 + | > loss_disc: 2.530118942260742 (2.4686938863281815) + | > loss_disc_real_0: 0.08668267726898193 (0.12441511561564349) + | > loss_disc_real_1: 0.20323631167411804 (0.20572343290945813) + | > loss_disc_real_2: 0.25086450576782227 (0.22640596248141123) + | > loss_disc_real_3: 0.2749197781085968 (0.23031102394292113) + | > loss_disc_real_4: 0.25843045115470886 (0.22903034077325) + | > loss_disc_real_5: 0.2646370232105255 (0.23565651284992148) + | > loss_0: 2.530118942260742 (2.4686938863281815) + | > grad_norm_0: tensor(17.1782, device='cuda:0') (tensor(12.1962, device='cuda:0')) + | > loss_gen: 2.4120893478393555 (2.358236509725588) + | > loss_kl: 1.9414323568344116 (1.9173559869101289) + | > loss_feat: 7.400784015655518 (7.07650527166664) + | > loss_mel: 18.638856887817383 (17.700799180827012) + | > loss_duration: 1.5085935592651367 (1.4669421342534756) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.901756286621094 (30.51983906807156) + | > grad_norm_1: tensor(474.2018, device='cuda:0') (tensor(266.0701, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.4114 (0.37271782892559646) + | > loader_time: 0.0066 (0.00590567851285322) + + + --> TIME: 2025-04-19 03:09:02 -- STEP: 134/142 -- GLOBAL_STEP: 415200 + | > loss_disc: 2.5103538036346436 (2.4670577138217533) + | > loss_disc_real_0: 0.16474518179893494 (0.12373499566716935) + | > loss_disc_real_1: 0.22736594080924988 (0.20597647853306875) + | > loss_disc_real_2: 0.24421857297420502 (0.22715459508237554) + | > loss_disc_real_3: 0.24391970038414001 (0.2297773670349548) + | > loss_disc_real_4: 0.21880856156349182 (0.22819434306514796) + | > loss_disc_real_5: 0.24284479022026062 (0.23519986745581697) + | > loss_0: 2.5103538036346436 (2.4670577138217533) + | > grad_norm_0: tensor(20.0057, device='cuda:0') (tensor(12.9617, device='cuda:0')) + | > loss_gen: 2.407776355743408 (2.358481384035367) + | > loss_kl: 1.9968868494033813 (1.9196686771378588) + | > loss_feat: 6.586932182312012 (7.027830949470179) + | > loss_mel: 18.38414192199707 (17.665211314585672) + | > loss_duration: 1.5337166786193848 (1.4730188659767607) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.909452438354492 (30.44421120543978) + | > grad_norm_1: tensor(205.8863, device='cuda:0') (tensor(273.3557, device='cuda:0')) + | > current_lr_0: 0.0001937978097191345 + | > current_lr_1: 0.0001937978097191345 + | > step_time: 0.453 (0.3849016392408912) + | > loader_time: 0.0068 (0.006187668487207213) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.380026340484619 (2.380026340484619) + | > loss_disc_real_0: 0.16293644905090332 (0.16293644905090332) + | > loss_disc_real_1: 0.17737147212028503 (0.17737147212028503) + | > loss_disc_real_2: 0.22947442531585693 (0.22947442531585693) + | > loss_disc_real_3: 0.23330168426036835 (0.23330168426036835) + | > loss_disc_real_4: 0.21382631361484528 (0.21382631361484528) + | > loss_disc_real_5: 0.2174832969903946 (0.2174832969903946) + | > loss_0: 2.380026340484619 (2.380026340484619) + | > loss_gen: 2.289605140686035 (2.289605140686035) + | > loss_kl: 2.3119029998779297 (2.3119029998779297) + | > loss_feat: 7.075397968292236 (7.075397968292236) + | > loss_mel: 17.85246467590332 (17.85246467590332) + | > loss_duration: 1.6978821754455566 (1.6978821754455566) + | > loss_1: 31.227251052856445 (31.227251052856445) + + --> STEP: 1 + | > loss_disc: 2.4271082878112793 (2.4271082878112793) + | > loss_disc_real_0: 0.12306385487318039 (0.12306385487318039) + | > loss_disc_real_1: 0.21654757857322693 (0.21654757857322693) + | > loss_disc_real_2: 0.23905113339424133 (0.23905113339424133) + | > loss_disc_real_3: 0.21468469500541687 (0.21468469500541687) + | > loss_disc_real_4: 0.20977939665317535 (0.20977939665317535) + | > loss_disc_real_5: 0.2408643662929535 (0.2408643662929535) + | > loss_0: 2.4271082878112793 (2.4271082878112793) + | > loss_gen: 2.2216029167175293 (2.2216029167175293) + | > loss_kl: 2.7478787899017334 (2.7478787899017334) + | > loss_feat: 7.069599151611328 (7.069599151611328) + | > loss_mel: 18.86484718322754 (18.86484718322754) + | > loss_duration: 1.7431316375732422 (1.7431316375732422) + | > loss_1: 32.64706039428711 (32.64706039428711) + + --> STEP: 2 + | > loss_disc: 2.468231439590454 (2.4476698637008667) + | > loss_disc_real_0: 0.12535281479358673 (0.12420833483338356) + | > loss_disc_real_1: 0.2144503891468048 (0.21549898386001587) + | > loss_disc_real_2: 0.263904333114624 (0.2514777332544327) + | > loss_disc_real_3: 0.23707665503025055 (0.2258806750178337) + | > loss_disc_real_4: 0.23192740976810455 (0.22085340321063995) + | > loss_disc_real_5: 0.2948135733604431 (0.2678389698266983) + | > loss_0: 2.468231439590454 (2.4476698637008667) + | > loss_gen: 2.3917276859283447 (2.306665301322937) + | > loss_kl: 2.2362658977508545 (2.492072343826294) + | > loss_feat: 6.672512054443359 (6.871055603027344) + | > loss_mel: 17.56464195251465 (18.214744567871094) + | > loss_duration: 1.6786197423934937 (1.710875689983368) + | > loss_1: 30.543766021728516 (31.595413208007812) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035506486892700195 (-9.262561798095703e-05) + | > avg_loss_disc: 2.4476698637008667 (-0.003939270973205566) + | > avg_loss_disc_real_0: 0.12420833483338356 (-0.060267362743616104) + | > avg_loss_disc_real_1: 0.21549898386001587 (-0.035813167691230774) + | > avg_loss_disc_real_2: 0.2514777332544327 (+0.004685826599597931) + | > avg_loss_disc_real_3: 0.2258806750178337 (-0.026846222579479218) + | > avg_loss_disc_real_4: 0.22085340321063995 (-0.035346582531929016) + | > avg_loss_disc_real_5: 0.2678389698266983 (+0.03098556399345398) + | > avg_loss_0: 2.4476698637008667 (-0.003939270973205566) + | > avg_loss_gen: 2.306665301322937 (-0.2830590009689331) + | > avg_loss_kl: 2.492072343826294 (+0.0033218860626220703) + | > avg_loss_feat: 6.871055603027344 (-0.7514264583587646) + | > avg_loss_mel: 18.214744567871094 (-1.1261014938354492) + | > avg_loss_duration: 1.710875689983368 (+0.0109405517578125) + | > avg_loss_1: 31.595413208007812 (-2.1463279724121094) + + + > EPOCH: 253/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:09:07)  + + --> TIME: 2025-04-19 03:09:13 -- STEP: 17/142 -- GLOBAL_STEP: 415225 + | > loss_disc: 2.3916985988616943 (2.419401435291066) + | > loss_disc_real_0: 0.120371975004673 (0.12024735067697133) + | > loss_disc_real_1: 0.17644453048706055 (0.20106860381715438) + | > loss_disc_real_2: 0.23446083068847656 (0.22387719767935135) + | > loss_disc_real_3: 0.2315865308046341 (0.22525163871400497) + | > loss_disc_real_4: 0.20220701396465302 (0.21713414963553934) + | > loss_disc_real_5: 0.1871214658021927 (0.22515901134294622) + | > loss_0: 2.3916985988616943 (2.419401435291066) + | > grad_norm_0: tensor(9.3242, device='cuda:0') (tensor(12.2322, device='cuda:0')) + | > loss_gen: 2.3659284114837646 (2.401835525737089) + | > loss_kl: 1.6030900478363037 (1.8305007079068352) + | > loss_feat: 7.409573554992676 (7.233089643366196) + | > loss_mel: 18.362998962402344 (17.41878812453326) + | > loss_duration: 1.4200520515441895 (1.419166179264293) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.161643981933594 (30.303380293004654) + | > grad_norm_1: tensor(233.7462, device='cuda:0') (tensor(247.3668, device='cuda:0')) + | > current_lr_0: 0.0001937735849929196 + | > current_lr_1: 0.0001937735849929196 + | > step_time: 0.3578 (0.33196715747608857) + | > loader_time: 0.0051 (0.0048986042247099034) + + + --> TIME: 2025-04-19 03:09:23 -- STEP: 42/142 -- GLOBAL_STEP: 415250 + | > loss_disc: 2.434727907180786 (2.4417436349959596) + | > loss_disc_real_0: 0.10330789536237717 (0.11588137517017978) + | > loss_disc_real_1: 0.25105148553848267 (0.2032755290468534) + | > loss_disc_real_2: 0.21672677993774414 (0.22495647448868977) + | > loss_disc_real_3: 0.2209198772907257 (0.22950018303734915) + | > loss_disc_real_4: 0.23847569525241852 (0.2250080963685399) + | > loss_disc_real_5: 0.25504809617996216 (0.2321012697759129) + | > loss_0: 2.434727907180786 (2.4417436349959596) + | > grad_norm_0: tensor(9.9999, device='cuda:0') (tensor(11.9180, device='cuda:0')) + | > loss_gen: 2.4178409576416016 (2.3833672602971387) + | > loss_kl: 1.7428399324417114 (1.8329700231552124) + | > loss_feat: 6.942050457000732 (7.204869508743286) + | > loss_mel: 17.508319854736328 (17.616347880590535) + | > loss_duration: 1.4767446517944336 (1.4346615530195688) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.08779525756836 (30.472216379074823) + | > grad_norm_1: tensor(218.6038, device='cuda:0') (tensor(279.4763, device='cuda:0')) + | > current_lr_0: 0.0001937735849929196 + | > current_lr_1: 0.0001937735849929196 + | > step_time: 0.558 (0.35467491263435) + | > loader_time: 0.006 (0.005329177493140811) + + + --> TIME: 2025-04-19 03:09:32 -- STEP: 67/142 -- GLOBAL_STEP: 415275 + | > loss_disc: 2.4740121364593506 (2.4361555967757944) + | > loss_disc_real_0: 0.13303707540035248 (0.11460796359982063) + | > loss_disc_real_1: 0.19306911528110504 (0.2032952215244521) + | > loss_disc_real_2: 0.20380456745624542 (0.2245592597260404) + | > loss_disc_real_3: 0.24969342350959778 (0.22862860137846933) + | > loss_disc_real_4: 0.21770507097244263 (0.22844121215948418) + | > loss_disc_real_5: 0.269765168428421 (0.2311739926017932) + | > loss_0: 2.4740121364593506 (2.4361555967757944) + | > grad_norm_0: tensor(9.5586, device='cuda:0') (tensor(10.9581, device='cuda:0')) + | > loss_gen: 2.4120535850524902 (2.3769613593371948) + | > loss_kl: 1.9380866289138794 (1.85850811182563) + | > loss_feat: 8.441106796264648 (7.203654552573588) + | > loss_mel: 19.02858543395996 (17.742378533776133) + | > loss_duration: 1.425239086151123 (1.4473094708883938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.24507141113281 (30.62881213515552) + | > grad_norm_1: tensor(59.3663, device='cuda:0') (tensor(262.5915, device='cuda:0')) + | > current_lr_0: 0.0001937735849929196 + | > current_lr_1: 0.0001937735849929196 + | > step_time: 0.3852 (0.3622800698920862) + | > loader_time: 0.0061 (0.005588542169599391) + + + --> TIME: 2025-04-19 03:09:43 -- STEP: 92/142 -- GLOBAL_STEP: 415300 + | > loss_disc: 2.419419288635254 (2.448814498341601) + | > loss_disc_real_0: 0.12424059212207794 (0.11930223655603502) + | > loss_disc_real_1: 0.19037629663944244 (0.20522548031547796) + | > loss_disc_real_2: 0.20848357677459717 (0.2250951598195926) + | > loss_disc_real_3: 0.2129489779472351 (0.22747882552768872) + | > loss_disc_real_4: 0.24412913620471954 (0.22937751156480415) + | > loss_disc_real_5: 0.2248249650001526 (0.23267773393055666) + | > loss_0: 2.419419288635254 (2.448814498341601) + | > grad_norm_0: tensor(5.7479, device='cuda:0') (tensor(11.0501, device='cuda:0')) + | > loss_gen: 2.454871654510498 (2.3609993328218866) + | > loss_kl: 2.0820107460021973 (1.880727751099545) + | > loss_feat: 7.653045177459717 (7.11990809958914) + | > loss_mel: 19.45853042602539 (17.740072395490568) + | > loss_duration: 1.4492833614349365 (1.4593586623668668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.097740173339844 (30.561066233593486) + | > grad_norm_1: tensor(324.1510, device='cuda:0') (tensor(268.0273, device='cuda:0')) + | > current_lr_0: 0.0001937735849929196 + | > current_lr_1: 0.0001937735849929196 + | > step_time: 0.4132 (0.37169962603112927) + | > loader_time: 0.0064 (0.005819178145864736) + + + --> TIME: 2025-04-19 03:09:53 -- STEP: 117/142 -- GLOBAL_STEP: 415325 + | > loss_disc: 2.4040632247924805 (2.4518246895227676) + | > loss_disc_real_0: 0.15509919822216034 (0.12348056606884696) + | > loss_disc_real_1: 0.23542608320713043 (0.2054975926876068) + | > loss_disc_real_2: 0.2572021782398224 (0.22469411866787153) + | > loss_disc_real_3: 0.2513076066970825 (0.22808157722664696) + | > loss_disc_real_4: 0.2209610939025879 (0.2281866999518158) + | > loss_disc_real_5: 0.20214727520942688 (0.23217387166288164) + | > loss_0: 2.4040632247924805 (2.4518246895227676) + | > grad_norm_0: tensor(33.0795, device='cuda:0') (tensor(12.8017, device='cuda:0')) + | > loss_gen: 2.3263132572174072 (2.369745111873006) + | > loss_kl: 1.9879075288772583 (1.8951795294753506) + | > loss_feat: 7.797475814819336 (7.115877302283915) + | > loss_mel: 18.150920867919922 (17.717050267080992) + | > loss_duration: 1.4852356910705566 (1.466711667867807) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.747852325439453 (30.56456399575258) + | > grad_norm_1: tensor(382.4984, device='cuda:0') (tensor(287.2273, device='cuda:0')) + | > current_lr_0: 0.0001937735849929196 + | > current_lr_1: 0.0001937735849929196 + | > step_time: 0.4358 (0.3813065056108002) + | > loader_time: 0.007 (0.0060613155364990234) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.49839448928833 (2.49839448928833) + | > loss_disc_real_0: 0.12890639901161194 (0.12890639901161194) + | > loss_disc_real_1: 0.29660066962242126 (0.29660066962242126) + | > loss_disc_real_2: 0.23392616212368011 (0.23392616212368011) + | > loss_disc_real_3: 0.24778001010417938 (0.24778001010417938) + | > loss_disc_real_4: 0.2331521213054657 (0.2331521213054657) + | > loss_disc_real_5: 0.23764288425445557 (0.23764288425445557) + | > loss_0: 2.49839448928833 (2.49839448928833) + | > loss_gen: 2.4030401706695557 (2.4030401706695557) + | > loss_kl: 2.5233664512634277 (2.5233664512634277) + | > loss_feat: 7.5112786293029785 (7.5112786293029785) + | > loss_mel: 18.716106414794922 (18.716106414794922) + | > loss_duration: 1.7025635242462158 (1.7025635242462158) + | > loss_1: 32.856353759765625 (32.856353759765625) + + --> STEP: 1 + | > loss_disc: 2.6578164100646973 (2.6578164100646973) + | > loss_disc_real_0: 0.23648352921009064 (0.23648352921009064) + | > loss_disc_real_1: 0.28326746821403503 (0.28326746821403503) + | > loss_disc_real_2: 0.2384597808122635 (0.2384597808122635) + | > loss_disc_real_3: 0.24519938230514526 (0.24519938230514526) + | > loss_disc_real_4: 0.25919872522354126 (0.25919872522354126) + | > loss_disc_real_5: 0.2543319761753082 (0.2543319761753082) + | > loss_0: 2.6578164100646973 (2.6578164100646973) + | > loss_gen: 2.2852444648742676 (2.2852444648742676) + | > loss_kl: 2.7402002811431885 (2.7402002811431885) + | > loss_feat: 5.083444118499756 (5.083444118499756) + | > loss_mel: 16.803197860717773 (16.803197860717773) + | > loss_duration: 1.7444170713424683 (1.7444170713424683) + | > loss_1: 28.656503677368164 (28.656503677368164) + + --> STEP: 2 + | > loss_disc: 2.487791061401367 (2.5728037357330322) + | > loss_disc_real_0: 0.1730787754058838 (0.2047811523079872) + | > loss_disc_real_1: 0.25635144114494324 (0.26980945467948914) + | > loss_disc_real_2: 0.2582443058490753 (0.2483520433306694) + | > loss_disc_real_3: 0.2367241382598877 (0.24096176028251648) + | > loss_disc_real_4: 0.18980556726455688 (0.22450214624404907) + | > loss_disc_real_5: 0.25255799293518066 (0.25344498455524445) + | > loss_0: 2.487791061401367 (2.5728037357330322) + | > loss_gen: 2.3388724327087402 (2.312058448791504) + | > loss_kl: 2.179452896118164 (2.4598265886306763) + | > loss_feat: 6.766259670257568 (5.924851894378662) + | > loss_mel: 17.116613388061523 (16.95990562438965) + | > loss_duration: 1.6755602359771729 (1.7099886536598206) + | > loss_1: 30.076759338378906 (29.366631507873535) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038373470306396484 (+0.0002866983413696289) + | > avg_loss_disc: 2.5728037357330322 (+0.12513387203216553) + | > avg_loss_disc_real_0: 0.2047811523079872 (+0.08057281747460365) + | > avg_loss_disc_real_1: 0.26980945467948914 (+0.05431047081947327) + | > avg_loss_disc_real_2: 0.2483520433306694 (-0.003125689923763275) + | > avg_loss_disc_real_3: 0.24096176028251648 (+0.01508108526468277) + | > avg_loss_disc_real_4: 0.22450214624404907 (+0.0036487430334091187) + | > avg_loss_disc_real_5: 0.25344498455524445 (-0.014393985271453857) + | > avg_loss_0: 2.5728037357330322 (+0.12513387203216553) + | > avg_loss_gen: 2.312058448791504 (+0.0053931474685668945) + | > avg_loss_kl: 2.4598265886306763 (-0.032245755195617676) + | > avg_loss_feat: 5.924851894378662 (-0.9462037086486816) + | > avg_loss_mel: 16.95990562438965 (-1.2548389434814453) + | > avg_loss_duration: 1.7099886536598206 (-0.0008870363235473633) + | > avg_loss_1: 29.366631507873535 (-2.2287817001342773) + + + > EPOCH: 254/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:10:06)  + + --> TIME: 2025-04-19 03:10:07 -- STEP: 0/142 -- GLOBAL_STEP: 415350 + | > loss_disc: 2.339139461517334 (2.339139461517334) + | > loss_disc_real_0: 0.10085838288068771 (0.10085838288068771) + | > loss_disc_real_1: 0.2162276804447174 (0.2162276804447174) + | > loss_disc_real_2: 0.21089774370193481 (0.21089774370193481) + | > loss_disc_real_3: 0.2485295683145523 (0.2485295683145523) + | > loss_disc_real_4: 0.2386552095413208 (0.2386552095413208) + | > loss_disc_real_5: 0.26763322949409485 (0.26763322949409485) + | > loss_0: 2.339139461517334 (2.339139461517334) + | > grad_norm_0: tensor(4.0732, device='cuda:0') (tensor(4.0732, device='cuda:0')) + | > loss_gen: 2.624192953109741 (2.624192953109741) + | > loss_kl: 1.729415774345398 (1.729415774345398) + | > loss_feat: 7.664781093597412 (7.664781093597412) + | > loss_mel: 19.27307891845703 (19.27307891845703) + | > loss_duration: 1.4498074054718018 (1.4498074054718018) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.741275787353516 (32.741275787353516) + | > grad_norm_1: tensor(156.5230, device='cuda:0') (tensor(156.5230, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.3563 (0.3562610149383545) + | > loader_time: 0.3275 (0.32746076583862305) + + + --> TIME: 2025-04-19 03:10:15 -- STEP: 25/142 -- GLOBAL_STEP: 415375 + | > loss_disc: 2.368839740753174 (2.411600456237793) + | > loss_disc_real_0: 0.11081460863351822 (0.11548828676342965) + | > loss_disc_real_1: 0.20033761858940125 (0.1980150955915451) + | > loss_disc_real_2: 0.19350317120552063 (0.22503345668315888) + | > loss_disc_real_3: 0.19827473163604736 (0.22272023618221282) + | > loss_disc_real_4: 0.18066954612731934 (0.22675549268722534) + | > loss_disc_real_5: 0.24205471575260162 (0.23336419701576233) + | > loss_0: 2.368839740753174 (2.411600456237793) + | > grad_norm_0: tensor(7.0033, device='cuda:0') (tensor(9.5414, device='cuda:0')) + | > loss_gen: 2.3597097396850586 (2.3776115798950195) + | > loss_kl: 1.9906078577041626 (1.8749179553985595) + | > loss_feat: 7.872394561767578 (7.330927543640136) + | > loss_mel: 18.32172393798828 (17.91705955505371) + | > loss_duration: 1.4571725130081177 (1.4299824714660645) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.001609802246094 (30.930499114990234) + | > grad_norm_1: tensor(246.3454, device='cuda:0') (tensor(176.0158, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.3456 (0.3375462818145752) + | > loader_time: 0.0054 (0.004965143203735352) + + + --> TIME: 2025-04-19 03:10:24 -- STEP: 50/142 -- GLOBAL_STEP: 415400 + | > loss_disc: 2.44480562210083 (2.4353804874420164) + | > loss_disc_real_0: 0.09736287593841553 (0.11827766947448254) + | > loss_disc_real_1: 0.18865767121315002 (0.2053972923755646) + | > loss_disc_real_2: 0.2505418658256531 (0.22639296889305116) + | > loss_disc_real_3: 0.2408713847398758 (0.22623444259166717) + | > loss_disc_real_4: 0.23394113779067993 (0.2269662719964981) + | > loss_disc_real_5: 0.21085956692695618 (0.23315341383218766) + | > loss_0: 2.44480562210083 (2.4353804874420164) + | > grad_norm_0: tensor(6.2349, device='cuda:0') (tensor(8.4519, device='cuda:0')) + | > loss_gen: 2.264568567276001 (2.375963258743286) + | > loss_kl: 1.9112337827682495 (1.8850525403022766) + | > loss_feat: 7.307533264160156 (7.326052560806274) + | > loss_mel: 18.167240142822266 (17.953303222656253) + | > loss_duration: 1.4784290790557861 (1.4384797596931458) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.129005432128906 (30.978851509094238) + | > grad_norm_1: tensor(321.5857, device='cuda:0') (tensor(160.7711, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.3693 (0.34758000373840336) + | > loader_time: 0.0054 (0.005310049057006836) + + + --> TIME: 2025-04-19 03:10:34 -- STEP: 75/142 -- GLOBAL_STEP: 415425 + | > loss_disc: 2.4640908241271973 (2.458301067352295) + | > loss_disc_real_0: 0.11531525105237961 (0.12257224703828494) + | > loss_disc_real_1: 0.1847161501646042 (0.20646347840627036) + | > loss_disc_real_2: 0.20379875600337982 (0.2267321103811264) + | > loss_disc_real_3: 0.2299068719148636 (0.22616160074869793) + | > loss_disc_real_4: 0.2274305522441864 (0.22911978721618653) + | > loss_disc_real_5: 0.18936362862586975 (0.2345543642838796) + | > loss_0: 2.4640908241271973 (2.458301067352295) + | > grad_norm_0: tensor(7.5151, device='cuda:0') (tensor(8.6522, device='cuda:0')) + | > loss_gen: 2.242652416229248 (2.351874901453653) + | > loss_kl: 1.930240273475647 (1.8880542421340942) + | > loss_feat: 6.802910327911377 (7.172448654174804) + | > loss_mel: 16.65245819091797 (17.813950360616047) + | > loss_duration: 1.4427303075790405 (1.451011438369751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.07099151611328 (30.677339731852214) + | > grad_norm_1: tensor(80.0715, device='cuda:0') (tensor(174.5025, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.3951 (0.35883711814880376) + | > loader_time: 0.0063 (0.0055951722462971995) + + + --> TIME: 2025-04-19 03:10:44 -- STEP: 100/142 -- GLOBAL_STEP: 415450 + | > loss_disc: 2.371711492538452 (2.4642228341102586) + | > loss_disc_real_0: 0.15450508892536163 (0.1263601055368781) + | > loss_disc_real_1: 0.19729696214199066 (0.2065980140864849) + | > loss_disc_real_2: 0.2113412618637085 (0.2269453425705433) + | > loss_disc_real_3: 0.19583232700824738 (0.2270851054787636) + | > loss_disc_real_4: 0.20391836762428284 (0.22795362263917923) + | > loss_disc_real_5: 0.19048787653446198 (0.2346523742377758) + | > loss_0: 2.371711492538452 (2.4642228341102586) + | > grad_norm_0: tensor(6.8423, device='cuda:0') (tensor(8.5200, device='cuda:0')) + | > loss_gen: 2.3390135765075684 (2.3408506369590754) + | > loss_kl: 1.9343115091323853 (1.8967979514598847) + | > loss_feat: 7.1128082275390625 (7.104752550125122) + | > loss_mel: 17.145349502563477 (17.782523441314698) + | > loss_duration: 1.5001814365386963 (1.4615151202678682) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.031665802001953 (30.586439819335936) + | > grad_norm_1: tensor(77.8212, device='cuda:0') (tensor(166.8604, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.4125 (0.36907575607299803) + | > loader_time: 0.0071 (0.005850830078124997) + + + --> TIME: 2025-04-19 03:10:55 -- STEP: 125/142 -- GLOBAL_STEP: 415475 + | > loss_disc: 2.5256528854370117 (2.4684046096801744) + | > loss_disc_real_0: 0.12769198417663574 (0.1279081657826901) + | > loss_disc_real_1: 0.16601379215717316 (0.2071627025604248) + | > loss_disc_real_2: 0.23654301464557648 (0.2263402279615402) + | > loss_disc_real_3: 0.21853233873844147 (0.22702662003040314) + | > loss_disc_real_4: 0.2343965768814087 (0.22805778884887695) + | > loss_disc_real_5: 0.19483688473701477 (0.23472137367725374) + | > loss_0: 2.5256528854370117 (2.4684046096801744) + | > grad_norm_0: tensor(10.0485, device='cuda:0') (tensor(8.2855, device='cuda:0')) + | > loss_gen: 2.2119898796081543 (2.3350906543731687) + | > loss_kl: 1.9133585691452026 (1.9041417016983033) + | > loss_feat: 6.316144943237305 (7.059447811126709) + | > loss_mel: 16.320344924926758 (17.749026039123535) + | > loss_duration: 1.5272014141082764 (1.4690075197219852) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.289039611816406 (30.516713897705078) + | > grad_norm_1: tensor(265.2280, device='cuda:0') (tensor(163.5304, device='cuda:0')) + | > current_lr_0: 0.00019374936329479546 + | > current_lr_1: 0.00019374936329479546 + | > step_time: 0.4392 (0.38054038429260256) + | > loader_time: 0.0086 (0.006128730773925779) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.517908811569214 (2.517908811569214) + | > loss_disc_real_0: 0.12024765461683273 (0.12024765461683273) + | > loss_disc_real_1: 0.21484091877937317 (0.21484091877937317) + | > loss_disc_real_2: 0.25195959210395813 (0.25195959210395813) + | > loss_disc_real_3: 0.2712606191635132 (0.2712606191635132) + | > loss_disc_real_4: 0.25889864563941956 (0.25889864563941956) + | > loss_disc_real_5: 0.24392567574977875 (0.24392567574977875) + | > loss_0: 2.517908811569214 (2.517908811569214) + | > loss_gen: 2.4570837020874023 (2.4570837020874023) + | > loss_kl: 2.4131548404693604 (2.4131548404693604) + | > loss_feat: 6.500644207000732 (6.500644207000732) + | > loss_mel: 18.031007766723633 (18.031007766723633) + | > loss_duration: 1.7046163082122803 (1.7046163082122803) + | > loss_1: 31.10650634765625 (31.10650634765625) + + --> STEP: 1 + | > loss_disc: 2.587956428527832 (2.587956428527832) + | > loss_disc_real_0: 0.15806443989276886 (0.15806443989276886) + | > loss_disc_real_1: 0.20919813215732574 (0.20919813215732574) + | > loss_disc_real_2: 0.283811092376709 (0.283811092376709) + | > loss_disc_real_3: 0.2890993654727936 (0.2890993654727936) + | > loss_disc_real_4: 0.27369022369384766 (0.27369022369384766) + | > loss_disc_real_5: 0.24945515394210815 (0.24945515394210815) + | > loss_0: 2.587956428527832 (2.587956428527832) + | > loss_gen: 2.4577674865722656 (2.4577674865722656) + | > loss_kl: 2.5499846935272217 (2.5499846935272217) + | > loss_feat: 6.666204452514648 (6.666204452514648) + | > loss_mel: 18.743478775024414 (18.743478775024414) + | > loss_duration: 1.749545931816101 (1.749545931816101) + | > loss_1: 32.1669807434082 (32.1669807434082) + + --> STEP: 2 + | > loss_disc: 2.5259361267089844 (2.556946277618408) + | > loss_disc_real_0: 0.15729497373104095 (0.1576797068119049) + | > loss_disc_real_1: 0.22602157294750214 (0.21760985255241394) + | > loss_disc_real_2: 0.249611958861351 (0.26671152561903) + | > loss_disc_real_3: 0.22910888493061066 (0.2591041252017021) + | > loss_disc_real_4: 0.265048623085022 (0.2693694233894348) + | > loss_disc_real_5: 0.1496177613735199 (0.19953645765781403) + | > loss_0: 2.5259361267089844 (2.556946277618408) + | > loss_gen: 2.374980926513672 (2.4163742065429688) + | > loss_kl: 2.2287724018096924 (2.389378547668457) + | > loss_feat: 8.676539421081543 (7.671371936798096) + | > loss_mel: 19.368736267089844 (19.05610752105713) + | > loss_duration: 1.7143328189849854 (1.7319393754005432) + | > loss_1: 34.36336135864258 (33.26517105102539) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033513307571411133 (-0.00048601627349853516) + | > avg_loss_disc: 2.556946277618408 (-0.015857458114624023) + | > avg_loss_disc_real_0: 0.1576797068119049 (-0.047101445496082306) + | > avg_loss_disc_real_1: 0.21760985255241394 (-0.052199602127075195) + | > avg_loss_disc_real_2: 0.26671152561903 (+0.018359482288360596) + | > avg_loss_disc_real_3: 0.2591041252017021 (+0.01814236491918564) + | > avg_loss_disc_real_4: 0.2693694233894348 (+0.04486727714538574) + | > avg_loss_disc_real_5: 0.19953645765781403 (-0.05390852689743042) + | > avg_loss_0: 2.556946277618408 (-0.015857458114624023) + | > avg_loss_gen: 2.4163742065429688 (+0.10431575775146484) + | > avg_loss_kl: 2.389378547668457 (-0.07044804096221924) + | > avg_loss_feat: 7.671371936798096 (+1.7465200424194336) + | > avg_loss_mel: 19.05610752105713 (+2.0962018966674805) + | > avg_loss_duration: 1.7319393754005432 (+0.021950721740722656) + | > avg_loss_1: 33.26517105102539 (+3.8985395431518555) + + + > EPOCH: 255/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:11:05)  + + --> TIME: 2025-04-19 03:11:08 -- STEP: 8/142 -- GLOBAL_STEP: 415500 + | > loss_disc: 2.4168331623077393 (2.4517135620117188) + | > loss_disc_real_0: 0.08951839059591293 (0.11526400409638882) + | > loss_disc_real_1: 0.17773029208183289 (0.20121835730969906) + | > loss_disc_real_2: 0.25408339500427246 (0.23749672807753086) + | > loss_disc_real_3: 0.20713144540786743 (0.23341894149780273) + | > loss_disc_real_4: 0.1855500191450119 (0.23835965432226658) + | > loss_disc_real_5: 0.2262154072523117 (0.2592868246138096) + | > loss_0: 2.4168331623077393 (2.4517135620117188) + | > grad_norm_0: tensor(11.2975, device='cuda:0') (tensor(13.1106, device='cuda:0')) + | > loss_gen: 2.47422194480896 (2.4510125517845154) + | > loss_kl: 1.8995676040649414 (1.8421399146318436) + | > loss_feat: 7.953439712524414 (7.620300233364105) + | > loss_mel: 18.759763717651367 (18.206854104995728) + | > loss_duration: 1.416886806488037 (1.4222651422023773) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.50387954711914 (31.54257106781006) + | > grad_norm_1: tensor(398.9904, device='cuda:0') (tensor(364.2062, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.3234 (0.3248821198940277) + | > loader_time: 0.0044 (0.004528462886810303) + + + --> TIME: 2025-04-19 03:11:17 -- STEP: 33/142 -- GLOBAL_STEP: 415525 + | > loss_disc: 2.5052080154418945 (2.4413702704689726) + | > loss_disc_real_0: 0.19796164333820343 (0.12130460775259769) + | > loss_disc_real_1: 0.22313927114009857 (0.20290473871158832) + | > loss_disc_real_2: 0.2177114188671112 (0.22583407073309927) + | > loss_disc_real_3: 0.21874737739562988 (0.22707396700526727) + | > loss_disc_real_4: 0.21626044809818268 (0.22718041457913138) + | > loss_disc_real_5: 0.2627808749675751 (0.23788246241482822) + | > loss_0: 2.5052080154418945 (2.4413702704689726) + | > grad_norm_0: tensor(9.8096, device='cuda:0') (tensor(12.6379, device='cuda:0')) + | > loss_gen: 2.3049943447113037 (2.3966536305167456) + | > loss_kl: 1.7252525091171265 (1.8484267285375884) + | > loss_feat: 6.458226203918457 (7.298404693603516) + | > loss_mel: 17.128490447998047 (17.950168667417586) + | > loss_duration: 1.4370334148406982 (1.432558738824093) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.053997039794922 (30.926212599783234) + | > grad_norm_1: tensor(296.8884, device='cuda:0') (tensor(316.8363, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.3584 (0.34507586739280005) + | > loader_time: 0.0052 (0.004994146751635002) + + + --> TIME: 2025-04-19 03:11:26 -- STEP: 58/142 -- GLOBAL_STEP: 415550 + | > loss_disc: 2.459044933319092 (2.448537275708956) + | > loss_disc_real_0: 0.15515044331550598 (0.12220978338656754) + | > loss_disc_real_1: 0.22901707887649536 (0.20665842346076307) + | > loss_disc_real_2: 0.23128558695316315 (0.22504463997380486) + | > loss_disc_real_3: 0.23378196358680725 (0.22732799680068574) + | > loss_disc_real_4: 0.24461041390895844 (0.22783892653111754) + | > loss_disc_real_5: 0.22435042262077332 (0.23411837143116984) + | > loss_0: 2.459044933319092 (2.448537275708956) + | > grad_norm_0: tensor(9.9361, device='cuda:0') (tensor(11.9984, device='cuda:0')) + | > loss_gen: 2.3930845260620117 (2.3761487212674366) + | > loss_kl: 1.930598497390747 (1.8552451832541104) + | > loss_feat: 6.701972961425781 (7.210418610737242) + | > loss_mel: 17.73157501220703 (17.948821692631164) + | > loss_duration: 1.4739617109298706 (1.4446749933834733) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.23119354248047 (30.835309225937415) + | > grad_norm_1: tensor(197.0034, device='cuda:0') (tensor(295.6303, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.373 (0.35566716769645956) + | > loader_time: 0.0061 (0.005379763142815951) + + + --> TIME: 2025-04-19 03:11:36 -- STEP: 83/142 -- GLOBAL_STEP: 415575 + | > loss_disc: 2.3419272899627686 (2.4603574994098723) + | > loss_disc_real_0: 0.12847790122032166 (0.12547045910214805) + | > loss_disc_real_1: 0.22335371375083923 (0.20550961092293982) + | > loss_disc_real_2: 0.20638257265090942 (0.22651719436588058) + | > loss_disc_real_3: 0.23919521272182465 (0.22850374069558568) + | > loss_disc_real_4: 0.19505789875984192 (0.2278410926999816) + | > loss_disc_real_5: 0.20567244291305542 (0.2347736292215715) + | > loss_0: 2.3419272899627686 (2.4603574994098723) + | > grad_norm_0: tensor(15.8396, device='cuda:0') (tensor(12.3064, device='cuda:0')) + | > loss_gen: 2.3357415199279785 (2.3628032667091095) + | > loss_kl: 1.8889566659927368 (1.8741763973810586) + | > loss_feat: 7.454048156738281 (7.15166838197823) + | > loss_mel: 18.584632873535156 (17.83592532054487) + | > loss_duration: 1.5285000801086426 (1.4563705547746404) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.791879653930664 (30.680944052087256) + | > grad_norm_1: tensor(270.9030, device='cuda:0') (tensor(280.4333, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.3995 (0.366159286843725) + | > loader_time: 0.0066 (0.005681339516697158) + + + --> TIME: 2025-04-19 03:11:47 -- STEP: 108/142 -- GLOBAL_STEP: 415600 + | > loss_disc: 2.4951014518737793 (2.4770199568183346) + | > loss_disc_real_0: 0.08452609926462173 (0.12736487630064847) + | > loss_disc_real_1: 0.27185335755348206 (0.20737215241900198) + | > loss_disc_real_2: 0.2426951825618744 (0.22703570817355756) + | > loss_disc_real_3: 0.23802368342876434 (0.22903021342224544) + | > loss_disc_real_4: 0.2499382644891739 (0.22970535109440485) + | > loss_disc_real_5: 0.24672293663024902 (0.2354421869472221) + | > loss_0: 2.4951014518737793 (2.4770199568183346) + | > grad_norm_0: tensor(10.1072, device='cuda:0') (tensor(12.1435, device='cuda:0')) + | > loss_gen: 2.4109246730804443 (2.3429832900011984) + | > loss_kl: 2.013176202774048 (1.8819264305962458) + | > loss_feat: 6.957405090332031 (7.021201071915804) + | > loss_mel: 18.08168601989746 (17.676913093637534) + | > loss_duration: 1.4927798509597778 (1.4652878432362169) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.95597267150879 (30.388311898266828) + | > grad_norm_1: tensor(323.0870, device='cuda:0') (tensor(277.1649, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.42 (0.37556987117837975) + | > loader_time: 0.0074 (0.005971517827775741) + + + --> TIME: 2025-04-19 03:11:58 -- STEP: 133/142 -- GLOBAL_STEP: 415625 + | > loss_disc: 2.5620269775390625 (2.4832249584054598) + | > loss_disc_real_0: 0.11933992058038712 (0.12848762092285587) + | > loss_disc_real_1: 0.19697077572345734 (0.20819260796209924) + | > loss_disc_real_2: 0.21199148893356323 (0.22686448021042616) + | > loss_disc_real_3: 0.2179330438375473 (0.22956300577274838) + | > loss_disc_real_4: 0.21236155927181244 (0.22986469871567605) + | > loss_disc_real_5: 0.22058795392513275 (0.23621886105913864) + | > loss_0: 2.5620269775390625 (2.4832249584054598) + | > grad_norm_0: tensor(12.1536, device='cuda:0') (tensor(12.7567, device='cuda:0')) + | > loss_gen: 2.299112319946289 (2.3405027568788457) + | > loss_kl: 1.9518251419067383 (1.886207689916281) + | > loss_feat: 7.490954875946045 (6.990457620835842) + | > loss_mel: 18.091781616210938 (17.652819870109838) + | > loss_duration: 1.5454967021942139 (1.4719876328805335) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.379169464111328 (30.341975620814733) + | > grad_norm_1: tensor(364.6947, device='cuda:0') (tensor(279.5876, device='cuda:0')) + | > current_lr_0: 0.0001937251446243836 + | > current_lr_1: 0.0001937251446243836 + | > step_time: 0.4377 (0.38689112663269043) + | > loader_time: 0.0079 (0.006268058504377091) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.502941131591797 (2.502941131591797) + | > loss_disc_real_0: 0.19148004055023193 (0.19148004055023193) + | > loss_disc_real_1: 0.19711919128894806 (0.19711919128894806) + | > loss_disc_real_2: 0.25827667117118835 (0.25827667117118835) + | > loss_disc_real_3: 0.24654699862003326 (0.24654699862003326) + | > loss_disc_real_4: 0.2339237779378891 (0.2339237779378891) + | > loss_disc_real_5: 0.2128511369228363 (0.2128511369228363) + | > loss_0: 2.502941131591797 (2.502941131591797) + | > loss_gen: 2.3818719387054443 (2.3818719387054443) + | > loss_kl: 2.401756525039673 (2.401756525039673) + | > loss_feat: 6.400366306304932 (6.400366306304932) + | > loss_mel: 17.227048873901367 (17.227048873901367) + | > loss_duration: 1.715502381324768 (1.715502381324768) + | > loss_1: 30.126544952392578 (30.126544952392578) + + --> STEP: 1 + | > loss_disc: 2.438251256942749 (2.438251256942749) + | > loss_disc_real_0: 0.17148451507091522 (0.17148451507091522) + | > loss_disc_real_1: 0.2137083262205124 (0.2137083262205124) + | > loss_disc_real_2: 0.2675591707229614 (0.2675591707229614) + | > loss_disc_real_3: 0.2434520572423935 (0.2434520572423935) + | > loss_disc_real_4: 0.23885275423526764 (0.23885275423526764) + | > loss_disc_real_5: 0.2146705985069275 (0.2146705985069275) + | > loss_0: 2.438251256942749 (2.438251256942749) + | > loss_gen: 2.414769172668457 (2.414769172668457) + | > loss_kl: 2.7268776893615723 (2.7268776893615723) + | > loss_feat: 7.011427402496338 (7.011427402496338) + | > loss_mel: 19.46819496154785 (19.46819496154785) + | > loss_duration: 1.7675364017486572 (1.7675364017486572) + | > loss_1: 33.3888053894043 (33.3888053894043) + + --> STEP: 2 + | > loss_disc: 2.5542409420013428 (2.496246099472046) + | > loss_disc_real_0: 0.24430525302886963 (0.20789488404989243) + | > loss_disc_real_1: 0.2491704225540161 (0.23143937438726425) + | > loss_disc_real_2: 0.25186893343925476 (0.2597140520811081) + | > loss_disc_real_3: 0.21538548171520233 (0.2294187694787979) + | > loss_disc_real_4: 0.21454916894435883 (0.22670096158981323) + | > loss_disc_real_5: 0.21198824048042297 (0.21332941949367523) + | > loss_0: 2.5542409420013428 (2.496246099472046) + | > loss_gen: 2.3720004558563232 (2.39338481426239) + | > loss_kl: 2.258880853652954 (2.492879271507263) + | > loss_feat: 3.9903342723846436 (5.500880837440491) + | > loss_mel: 13.392017364501953 (16.430106163024902) + | > loss_duration: 1.680093765258789 (1.7238150835037231) + | > loss_1: 23.69332504272461 (28.541065216064453) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036622285842895508 (+0.0003108978271484375) + | > avg_loss_disc: 2.496246099472046 (-0.060700178146362305) + | > avg_loss_disc_real_0: 0.20789488404989243 (+0.05021517723798752) + | > avg_loss_disc_real_1: 0.23143937438726425 (+0.013829521834850311) + | > avg_loss_disc_real_2: 0.2597140520811081 (-0.0069974735379219055) + | > avg_loss_disc_real_3: 0.2294187694787979 (-0.029685355722904205) + | > avg_loss_disc_real_4: 0.22670096158981323 (-0.04266846179962158) + | > avg_loss_disc_real_5: 0.21332941949367523 (+0.013792961835861206) + | > avg_loss_0: 2.496246099472046 (-0.060700178146362305) + | > avg_loss_gen: 2.39338481426239 (-0.022989392280578613) + | > avg_loss_kl: 2.492879271507263 (+0.10350072383880615) + | > avg_loss_feat: 5.500880837440491 (-2.170491099357605) + | > avg_loss_mel: 16.430106163024902 (-2.6260013580322266) + | > avg_loss_duration: 1.7238150835037231 (-0.008124291896820068) + | > avg_loss_1: 28.541065216064453 (-4.7241058349609375) + + + > EPOCH: 256/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:12:03)  + + --> TIME: 2025-04-19 03:12:09 -- STEP: 16/142 -- GLOBAL_STEP: 415650 + | > loss_disc: 2.448427438735962 (2.440586283802986) + | > loss_disc_real_0: 0.11040208488702774 (0.11504398426041007) + | > loss_disc_real_1: 0.2640073597431183 (0.21201138012111187) + | > loss_disc_real_2: 0.236943781375885 (0.22726420313119888) + | > loss_disc_real_3: 0.23374630510807037 (0.22875971719622612) + | > loss_disc_real_4: 0.23269766569137573 (0.22954534832388163) + | > loss_disc_real_5: 0.2460990846157074 (0.22526971902698278) + | > loss_0: 2.448427438735962 (2.440586283802986) + | > grad_norm_0: tensor(11.4673, device='cuda:0') (tensor(10.8886, device='cuda:0')) + | > loss_gen: 2.339768409729004 (2.3597348481416702) + | > loss_kl: 1.8067128658294678 (1.8436491265892982) + | > loss_feat: 7.311912536621094 (7.261764198541641) + | > loss_mel: 18.250782012939453 (18.008519411087036) + | > loss_duration: 1.4166206121444702 (1.4234315976500511) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.125795364379883 (30.897098898887634) + | > grad_norm_1: tensor(323.2067, device='cuda:0') (tensor(275.7052, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.339 (0.32855258882045746) + | > loader_time: 0.005 (0.0047211796045303345) + + + --> TIME: 2025-04-19 03:12:18 -- STEP: 41/142 -- GLOBAL_STEP: 415675 + | > loss_disc: 2.240025758743286 (2.429119354341088) + | > loss_disc_real_0: 0.08497052639722824 (0.11375883348831316) + | > loss_disc_real_1: 0.1887810379266739 (0.20413059795775065) + | > loss_disc_real_2: 0.19797983765602112 (0.22654332111521466) + | > loss_disc_real_3: 0.2246553599834442 (0.2254621470119895) + | > loss_disc_real_4: 0.22941003739833832 (0.2302711249851599) + | > loss_disc_real_5: 0.22452810406684875 (0.2271533695662894) + | > loss_0: 2.240025758743286 (2.429119354341088) + | > grad_norm_0: tensor(7.0245, device='cuda:0') (tensor(10.2136, device='cuda:0')) + | > loss_gen: 2.689081907272339 (2.3690573180594097) + | > loss_kl: 1.7931385040283203 (1.8469736692382068) + | > loss_feat: 8.08936595916748 (7.239955250809833) + | > loss_mel: 18.314180374145508 (17.835685474116627) + | > loss_duration: 1.4286597967147827 (1.4353810403405167) + | > amp_scaler: 256.0 (318.4390243902439) + | > loss_1: 32.31442642211914 (30.727052642077936) + | > grad_norm_1: tensor(206.6094, device='cuda:0') (tensor(257.0518, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.3634 (0.34158485110213116) + | > loader_time: 0.0056 (0.005239480879248642) + + + --> TIME: 2025-04-19 03:12:28 -- STEP: 66/142 -- GLOBAL_STEP: 415700 + | > loss_disc: 2.6173770427703857 (2.441648450764742) + | > loss_disc_real_0: 0.15821947157382965 (0.11911676609606454) + | > loss_disc_real_1: 0.23066873848438263 (0.20531515461025815) + | > loss_disc_real_2: 0.24033908545970917 (0.2254196344451471) + | > loss_disc_real_3: 0.20474354922771454 (0.22691615919272104) + | > loss_disc_real_4: 0.20865081250667572 (0.23021358567656894) + | > loss_disc_real_5: 0.24271084368228912 (0.23067581156889597) + | > loss_0: 2.6173770427703857 (2.441648450764742) + | > grad_norm_0: tensor(11.3609, device='cuda:0') (tensor(10.0229, device='cuda:0')) + | > loss_gen: 2.0964810848236084 (2.3693769881219575) + | > loss_kl: 1.9304208755493164 (1.8624743006446145) + | > loss_feat: 6.568541526794434 (7.212393702882709) + | > loss_mel: 17.466142654418945 (17.839201421448674) + | > loss_duration: 1.4489359855651855 (1.4483996232350667) + | > amp_scaler: 256.0 (294.7878787878788) + | > loss_1: 29.510522842407227 (30.731845971309777) + | > grad_norm_1: tensor(273.0913, device='cuda:0') (tensor(242.1652, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.3836 (0.3533178965250651) + | > loader_time: 0.0061 (0.00548878221800833) + + + --> TIME: 2025-04-19 03:12:38 -- STEP: 91/142 -- GLOBAL_STEP: 415725 + | > loss_disc: 2.3963301181793213 (2.4510811187408788) + | > loss_disc_real_0: 0.1689925491809845 (0.12347457844477433) + | > loss_disc_real_1: 0.18641291558742523 (0.2061557473389657) + | > loss_disc_real_2: 0.24121829867362976 (0.2241161572081702) + | > loss_disc_real_3: 0.22577020525932312 (0.2287098187339175) + | > loss_disc_real_4: 0.22259250283241272 (0.22856248866070758) + | > loss_disc_real_5: 0.21097637712955475 (0.23165883811620566) + | > loss_0: 2.3963301181793213 (2.4510811187408788) + | > grad_norm_0: tensor(17.3305, device='cuda:0') (tensor(9.8188, device='cuda:0')) + | > loss_gen: 2.3396220207214355 (2.353915869534671) + | > loss_kl: 2.0210442543029785 (1.884484968342624) + | > loss_feat: 8.017117500305176 (7.15051776760227) + | > loss_mel: 19.038423538208008 (17.811089327047153) + | > loss_duration: 1.5028959512710571 (1.4609721623934238) + | > amp_scaler: 256.0 (284.1318681318681) + | > loss_1: 32.919105529785156 (30.66098007789025) + | > grad_norm_1: tensor(266.1802, device='cuda:0') (tensor(223.1186, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.4039 (0.3646484809917408) + | > loader_time: 0.0066 (0.0057336367093599756) + + + --> TIME: 2025-04-19 03:12:49 -- STEP: 116/142 -- GLOBAL_STEP: 415750 + | > loss_disc: 2.568154811859131 (2.4655986736560696) + | > loss_disc_real_0: 0.2153984159231186 (0.12589164681989573) + | > loss_disc_real_1: 0.23754291236400604 (0.20737506076693535) + | > loss_disc_real_2: 0.26224201917648315 (0.22538126208658876) + | > loss_disc_real_3: 0.2587611675262451 (0.22908726694255038) + | > loss_disc_real_4: 0.23831865191459656 (0.2295453192601944) + | > loss_disc_real_5: 0.2381439208984375 (0.23188250987180348) + | > loss_0: 2.568154811859131 (2.4655986736560696) + | > grad_norm_0: tensor(31.3719, device='cuda:0') (tensor(10.1434, device='cuda:0')) + | > loss_gen: 2.4655182361602783 (2.3429146162394825) + | > loss_kl: 1.6425875425338745 (1.8872789339772587) + | > loss_feat: 6.693745136260986 (7.074373635752448) + | > loss_mel: 16.31663703918457 (17.775670824379755) + | > loss_duration: 1.46474027633667 (1.467423614756815) + | > amp_scaler: 256.0 (278.0689655172414) + | > loss_1: 28.583229064941406 (30.54766166621241) + | > grad_norm_1: tensor(566.3420, device='cuda:0') (tensor(225.0978, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.4268 (0.3755840803014821) + | > loader_time: 0.0074 (0.005981506972477354) + + + --> TIME: 2025-04-19 03:13:00 -- STEP: 141/142 -- GLOBAL_STEP: 415775 + | > loss_disc: 2.188546895980835 (2.4669478565243117) + | > loss_disc_real_0: 0.05387578532099724 (0.12587719355174837) + | > loss_disc_real_1: 0.1567608267068863 (0.20754726252234573) + | > loss_disc_real_2: 0.18740391731262207 (0.22551019415787774) + | > loss_disc_real_3: 0.14410346746444702 (0.22856580460747927) + | > loss_disc_real_4: 0.12866848707199097 (0.22955082811362354) + | > loss_disc_real_5: 0.22470928728580475 (0.23276611993498836) + | > loss_0: 2.188546895980835 (2.4669478565243117) + | > grad_norm_0: tensor(16.5964, device='cuda:0') (tensor(11.3357, device='cuda:0')) + | > loss_gen: 2.774374485015869 (2.346136363685554) + | > loss_kl: 1.5652103424072266 (1.8872893921872402) + | > loss_feat: 10.621646881103516 (7.056750872456435) + | > loss_mel: 17.87613868713379 (17.750395754550365) + | > loss_duration: 1.3228440284729004 (1.4728146233457204) + | > amp_scaler: 256.0 (274.15602836879424) + | > loss_1: 34.16021728515625 (30.513387044270832) + | > grad_norm_1: tensor(698.9326, device='cuda:0') (tensor(242.6639, device='cuda:0')) + | > current_lr_0: 0.00019370092898130556 + | > current_lr_1: 0.00019370092898130556 + | > step_time: 0.2041 (0.3859162804082776) + | > loader_time: 0.0028 (0.006188316548124273) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3988842964172363 (2.3988842964172363) + | > loss_disc_real_0: 0.1417725682258606 (0.1417725682258606) + | > loss_disc_real_1: 0.24460797011852264 (0.24460797011852264) + | > loss_disc_real_2: 0.22226464748382568 (0.22226464748382568) + | > loss_disc_real_3: 0.22894251346588135 (0.22894251346588135) + | > loss_disc_real_4: 0.2147722691297531 (0.2147722691297531) + | > loss_disc_real_5: 0.20828521251678467 (0.20828521251678467) + | > loss_0: 2.3988842964172363 (2.3988842964172363) + | > loss_gen: 2.331127643585205 (2.331127643585205) + | > loss_kl: 2.251081943511963 (2.251081943511963) + | > loss_feat: 6.35565710067749 (6.35565710067749) + | > loss_mel: 16.665685653686523 (16.665685653686523) + | > loss_duration: 1.6797029972076416 (1.6797029972076416) + | > loss_1: 29.283254623413086 (29.283254623413086) + + --> STEP: 1 + | > loss_disc: 2.2823715209960938 (2.2823715209960938) + | > loss_disc_real_0: 0.08842016756534576 (0.08842016756534576) + | > loss_disc_real_1: 0.19609342515468597 (0.19609342515468597) + | > loss_disc_real_2: 0.2049771100282669 (0.2049771100282669) + | > loss_disc_real_3: 0.1824556291103363 (0.1824556291103363) + | > loss_disc_real_4: 0.1794721633195877 (0.1794721633195877) + | > loss_disc_real_5: 0.21651947498321533 (0.21651947498321533) + | > loss_0: 2.2823715209960938 (2.2823715209960938) + | > loss_gen: 2.313199043273926 (2.313199043273926) + | > loss_kl: 2.5749895572662354 (2.5749895572662354) + | > loss_feat: 8.043680191040039 (8.043680191040039) + | > loss_mel: 19.616077423095703 (19.616077423095703) + | > loss_duration: 1.7693549394607544 (1.7693549394607544) + | > loss_1: 34.31730270385742 (34.31730270385742) + + --> STEP: 2 + | > loss_disc: 2.426682949066162 (2.354527235031128) + | > loss_disc_real_0: 0.10825429111719131 (0.09833722934126854) + | > loss_disc_real_1: 0.2302384227514267 (0.21316592395305634) + | > loss_disc_real_2: 0.23450203239917755 (0.21973957121372223) + | > loss_disc_real_3: 0.21788600087165833 (0.20017081499099731) + | > loss_disc_real_4: 0.2350264936685562 (0.20724932849407196) + | > loss_disc_real_5: 0.24007761478424072 (0.22829854488372803) + | > loss_0: 2.426682949066162 (2.354527235031128) + | > loss_gen: 2.272156238555908 (2.292677640914917) + | > loss_kl: 2.3136541843414307 (2.444321870803833) + | > loss_feat: 6.099239826202393 (7.071460008621216) + | > loss_mel: 17.585927963256836 (18.60100269317627) + | > loss_duration: 1.7108873128890991 (1.7401211261749268) + | > loss_1: 29.98186492919922 (32.14958381652832) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035161972045898438 (-0.00014603137969970703) + | > avg_loss_disc: 2.354527235031128 (-0.14171886444091797) + | > avg_loss_disc_real_0: 0.09833722934126854 (-0.10955765470862389) + | > avg_loss_disc_real_1: 0.21316592395305634 (-0.018273450434207916) + | > avg_loss_disc_real_2: 0.21973957121372223 (-0.039974480867385864) + | > avg_loss_disc_real_3: 0.20017081499099731 (-0.029247954487800598) + | > avg_loss_disc_real_4: 0.20724932849407196 (-0.019451633095741272) + | > avg_loss_disc_real_5: 0.22829854488372803 (+0.014969125390052795) + | > avg_loss_0: 2.354527235031128 (-0.14171886444091797) + | > avg_loss_gen: 2.292677640914917 (-0.10070717334747314) + | > avg_loss_kl: 2.444321870803833 (-0.048557400703430176) + | > avg_loss_feat: 7.071460008621216 (+1.570579171180725) + | > avg_loss_mel: 18.60100269317627 (+2.170896530151367) + | > avg_loss_duration: 1.7401211261749268 (+0.016306042671203613) + | > avg_loss_1: 32.14958381652832 (+3.608518600463867) + + + > EPOCH: 257/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:13:02)  + + --> TIME: 2025-04-19 03:13:11 -- STEP: 24/142 -- GLOBAL_STEP: 415800 + | > loss_disc: 2.580021858215332 (2.4299607674280796) + | > loss_disc_real_0: 0.11112470924854279 (0.11588532415529092) + | > loss_disc_real_1: 0.23802843689918518 (0.2060180101543665) + | > loss_disc_real_2: 0.2681537866592407 (0.2190996346374353) + | > loss_disc_real_3: 0.22636325657367706 (0.2259362880140543) + | > loss_disc_real_4: 0.23679311573505402 (0.23166021207968393) + | > loss_disc_real_5: 0.21512861549854279 (0.22933953938384852) + | > loss_0: 2.580021858215332 (2.4299607674280796) + | > grad_norm_0: tensor(12.6419, device='cuda:0') (tensor(8.3084, device='cuda:0')) + | > loss_gen: 2.223651647567749 (2.402256111303966) + | > loss_kl: 1.9671486616134644 (1.830301766594251) + | > loss_feat: 6.671922206878662 (7.382811387379964) + | > loss_mel: 17.540925979614258 (17.947746833165485) + | > loss_duration: 1.4640023708343506 (1.4268000721931455) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.867650985717773 (30.989916165669758) + | > grad_norm_1: tensor(28.3611, device='cuda:0') (tensor(204.2092, device='cuda:0')) + | > current_lr_0: 0.00019367671636518288 + | > current_lr_1: 0.00019367671636518288 + | > step_time: 0.3428 (0.33266321818033856) + | > loader_time: 0.0055 (0.005075275897979736) + + + --> TIME: 2025-04-19 03:13:20 -- STEP: 49/142 -- GLOBAL_STEP: 415825 + | > loss_disc: 2.4483137130737305 (2.431126784305184) + | > loss_disc_real_0: 0.1499217301607132 (0.12151819756444619) + | > loss_disc_real_1: 0.20967157185077667 (0.2049462390797479) + | > loss_disc_real_2: 0.21030865609645844 (0.22176319695248894) + | > loss_disc_real_3: 0.20801515877246857 (0.2274236536147643) + | > loss_disc_real_4: 0.23788423836231232 (0.22614268988979105) + | > loss_disc_real_5: 0.25736361742019653 (0.22872936847258588) + | > loss_0: 2.4483137130737305 (2.431126784305184) + | > grad_norm_0: tensor(5.4031, device='cuda:0') (tensor(8.8287, device='cuda:0')) + | > loss_gen: 2.23470401763916 (2.390011729026328) + | > loss_kl: 2.0357816219329834 (1.8545082345300792) + | > loss_feat: 6.6429643630981445 (7.269881199817268) + | > loss_mel: 17.267528533935547 (17.845149760343592) + | > loss_duration: 1.4680684804916382 (1.4370950022522284) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.649045944213867 (30.796645845685685) + | > grad_norm_1: tensor(159.6832, device='cuda:0') (tensor(190.1890, device='cuda:0')) + | > current_lr_0: 0.00019367671636518288 + | > current_lr_1: 0.00019367671636518288 + | > step_time: 0.3699 (0.34555561201913015) + | > loader_time: 0.0057 (0.00541875800307916) + + + --> TIME: 2025-04-19 03:13:30 -- STEP: 74/142 -- GLOBAL_STEP: 415850 + | > loss_disc: 2.5355050563812256 (2.45669551475628) + | > loss_disc_real_0: 0.14466945827007294 (0.12495386147418537) + | > loss_disc_real_1: 0.20977431535720825 (0.20551891544380707) + | > loss_disc_real_2: 0.21447651088237762 (0.22317913617636706) + | > loss_disc_real_3: 0.24249906837940216 (0.22907960273929545) + | > loss_disc_real_4: 0.24425853788852692 (0.2289300621361346) + | > loss_disc_real_5: 0.2505190074443817 (0.23140330371019002) + | > loss_0: 2.5355050563812256 (2.45669551475628) + | > grad_norm_0: tensor(6.8123, device='cuda:0') (tensor(7.9333, device='cuda:0')) + | > loss_gen: 2.428367853164673 (2.3711167283960304) + | > loss_kl: 1.8544631004333496 (1.866737565478763) + | > loss_feat: 6.977029800415039 (7.196886359034358) + | > loss_mel: 17.09017562866211 (17.81210187963537) + | > loss_duration: 1.476322054862976 (1.4510318121394596) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.826358795166016 (30.6978743011887) + | > grad_norm_1: tensor(218.3745, device='cuda:0') (tensor(163.9414, device='cuda:0')) + | > current_lr_0: 0.00019367671636518288 + | > current_lr_1: 0.00019367671636518288 + | > step_time: 0.3877 (0.35921243396965236) + | > loader_time: 0.0063 (0.005740085163631954) + + + --> TIME: 2025-04-19 03:13:40 -- STEP: 99/142 -- GLOBAL_STEP: 415875 + | > loss_disc: 2.5246403217315674 (2.4677189480174673) + | > loss_disc_real_0: 0.11157933622598648 (0.12748928322936556) + | > loss_disc_real_1: 0.20851357281208038 (0.20572680882131214) + | > loss_disc_real_2: 0.23086035251617432 (0.2242667277654012) + | > loss_disc_real_3: 0.21865135431289673 (0.2298861469584282) + | > loss_disc_real_4: 0.24221110343933105 (0.22963879114449626) + | > loss_disc_real_5: 0.24448999762535095 (0.23257859638243011) + | > loss_0: 2.5246403217315674 (2.4677189480174673) + | > grad_norm_0: tensor(4.9888, device='cuda:0') (tensor(7.6617, device='cuda:0')) + | > loss_gen: 2.293421506881714 (2.350031652835885) + | > loss_kl: 1.915242075920105 (1.8773439099090268) + | > loss_feat: 6.960356712341309 (7.104328993595008) + | > loss_mel: 18.004602432250977 (17.76423100750856) + | > loss_duration: 1.5009034872055054 (1.462139112780792) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.67452621459961 (30.55807464291351) + | > grad_norm_1: tensor(93.8419, device='cuda:0') (tensor(150.6394, device='cuda:0')) + | > current_lr_0: 0.00019367671636518288 + | > current_lr_1: 0.00019367671636518288 + | > step_time: 0.4038 (0.3691644644496417) + | > loader_time: 0.0067 (0.005983446583603368) + + + --> TIME: 2025-04-19 03:13:51 -- STEP: 124/142 -- GLOBAL_STEP: 415900 + | > loss_disc: 2.4511077404022217 (2.4758979005198323) + | > loss_disc_real_0: 0.12377376854419708 (0.12896308915749677) + | > loss_disc_real_1: 0.2085777074098587 (0.20654227212071422) + | > loss_disc_real_2: 0.25598689913749695 (0.22478986375274196) + | > loss_disc_real_3: 0.23607371747493744 (0.2297852911776112) + | > loss_disc_real_4: 0.24242651462554932 (0.22998498272030585) + | > loss_disc_real_5: 0.25701960921287537 (0.2330999375591355) + | > loss_0: 2.4511077404022217 (2.4758979005198323) + | > grad_norm_0: tensor(6.0735, device='cuda:0') (tensor(7.5705, device='cuda:0')) + | > loss_gen: 2.497941732406616 (2.3397194589338) + | > loss_kl: 1.9515010118484497 (1.8851201793839854) + | > loss_feat: 7.070523262023926 (7.042091185046781) + | > loss_mel: 18.04798698425293 (17.724499333289362) + | > loss_duration: 1.5286622047424316 (1.46921508735226) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.096614837646484 (30.460645275731242) + | > grad_norm_1: tensor(333.0081, device='cuda:0') (tensor(153.2987, device='cuda:0')) + | > current_lr_0: 0.00019367671636518288 + | > current_lr_1: 0.00019367671636518288 + | > step_time: 0.4392 (0.38040750641976634) + | > loader_time: 0.0077 (0.006248356834534676) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.496089458465576 (2.496089458465576) + | > loss_disc_real_0: 0.23212824761867523 (0.23212824761867523) + | > loss_disc_real_1: 0.27871859073638916 (0.27871859073638916) + | > loss_disc_real_2: 0.22846846282482147 (0.22846846282482147) + | > loss_disc_real_3: 0.2038327306509018 (0.2038327306509018) + | > loss_disc_real_4: 0.22887176275253296 (0.22887176275253296) + | > loss_disc_real_5: 0.2183736115694046 (0.2183736115694046) + | > loss_0: 2.496089458465576 (2.496089458465576) + | > loss_gen: 2.5080928802490234 (2.5080928802490234) + | > loss_kl: 2.105565071105957 (2.105565071105957) + | > loss_feat: 6.617442607879639 (6.617442607879639) + | > loss_mel: 16.700035095214844 (16.700035095214844) + | > loss_duration: 1.7004987001419067 (1.7004987001419067) + | > loss_1: 29.631633758544922 (29.631633758544922) + + --> STEP: 1 + | > loss_disc: 2.4804799556732178 (2.4804799556732178) + | > loss_disc_real_0: 0.1483861654996872 (0.1483861654996872) + | > loss_disc_real_1: 0.28973647952079773 (0.28973647952079773) + | > loss_disc_real_2: 0.27572867274284363 (0.27572867274284363) + | > loss_disc_real_3: 0.23632054030895233 (0.23632054030895233) + | > loss_disc_real_4: 0.31580910086631775 (0.31580910086631775) + | > loss_disc_real_5: 0.27951109409332275 (0.27951109409332275) + | > loss_0: 2.4804799556732178 (2.4804799556732178) + | > loss_gen: 2.820136070251465 (2.820136070251465) + | > loss_kl: 2.543703079223633 (2.543703079223633) + | > loss_feat: 7.1939568519592285 (7.1939568519592285) + | > loss_mel: 19.142711639404297 (19.142711639404297) + | > loss_duration: 1.755627155303955 (1.755627155303955) + | > loss_1: 33.45613479614258 (33.45613479614258) + + --> STEP: 2 + | > loss_disc: 2.497478485107422 (2.48897922039032) + | > loss_disc_real_0: 0.19461415708065033 (0.17150016129016876) + | > loss_disc_real_1: 0.21376274526119232 (0.251749612390995) + | > loss_disc_real_2: 0.26953017711639404 (0.27262942492961884) + | > loss_disc_real_3: 0.23177771270275116 (0.23404912650585175) + | > loss_disc_real_4: 0.22805540263652802 (0.2719322517514229) + | > loss_disc_real_5: 0.22650165855884552 (0.25300637632608414) + | > loss_0: 2.497478485107422 (2.48897922039032) + | > loss_gen: 2.6107406616210938 (2.7154383659362793) + | > loss_kl: 2.1918909549713135 (2.367797017097473) + | > loss_feat: 7.491207599639893 (7.3425822257995605) + | > loss_mel: 18.105289459228516 (18.624000549316406) + | > loss_duration: 1.7203006744384766 (1.7379639148712158) + | > loss_1: 32.11943054199219 (32.78778266906738) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003743886947631836 (+0.0002276897430419922) + | > avg_loss_disc: 2.48897922039032 (+0.1344519853591919) + | > avg_loss_disc_real_0: 0.17150016129016876 (+0.07316293194890022) + | > avg_loss_disc_real_1: 0.251749612390995 (+0.03858368843793869) + | > avg_loss_disc_real_2: 0.27262942492961884 (+0.052889853715896606) + | > avg_loss_disc_real_3: 0.23404912650585175 (+0.03387831151485443) + | > avg_loss_disc_real_4: 0.2719322517514229 (+0.06468292325735092) + | > avg_loss_disc_real_5: 0.25300637632608414 (+0.02470783144235611) + | > avg_loss_0: 2.48897922039032 (+0.1344519853591919) + | > avg_loss_gen: 2.7154383659362793 (+0.4227607250213623) + | > avg_loss_kl: 2.367797017097473 (-0.07652485370635986) + | > avg_loss_feat: 7.3425822257995605 (+0.2711222171783447) + | > avg_loss_mel: 18.624000549316406 (+0.02299785614013672) + | > avg_loss_duration: 1.7379639148712158 (-0.0021572113037109375) + | > avg_loss_1: 32.78778266906738 (+0.6381988525390625) + + + > EPOCH: 258/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:14:00)  + + --> TIME: 2025-04-19 03:14:03 -- STEP: 7/142 -- GLOBAL_STEP: 415925 + | > loss_disc: 2.385166645050049 (2.435566016605922) + | > loss_disc_real_0: 0.08420688658952713 (0.12639100956065313) + | > loss_disc_real_1: 0.19026556611061096 (0.2024807674544198) + | > loss_disc_real_2: 0.2119075506925583 (0.2261848705155509) + | > loss_disc_real_3: 0.21977414190769196 (0.2373463362455368) + | > loss_disc_real_4: 0.20144563913345337 (0.23703033583504812) + | > loss_disc_real_5: 0.21988287568092346 (0.24598089073385512) + | > loss_0: 2.385166645050049 (2.435566016605922) + | > grad_norm_0: tensor(9.7844, device='cuda:0') (tensor(12.4812, device='cuda:0')) + | > loss_gen: 2.292093515396118 (2.405761344092233) + | > loss_kl: 1.8876675367355347 (1.8488127163478307) + | > loss_feat: 7.829831123352051 (7.2314622061593195) + | > loss_mel: 17.678306579589844 (17.5016484941755) + | > loss_duration: 1.4274578094482422 (1.4295938185283117) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.1153564453125 (30.417278289794922) + | > grad_norm_1: tensor(315.1675, device='cuda:0') (tensor(273.1469, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.3245 (0.3248686449868338) + | > loader_time: 0.0042 (0.00452113151550293) + + + --> TIME: 2025-04-19 03:14:12 -- STEP: 32/142 -- GLOBAL_STEP: 415950 + | > loss_disc: 2.491448402404785 (2.44669695943594) + | > loss_disc_real_0: 0.0732678771018982 (0.11986981041263789) + | > loss_disc_real_1: 0.1960534304380417 (0.20481051225215197) + | > loss_disc_real_2: 0.22897687554359436 (0.22806485556066036) + | > loss_disc_real_3: 0.24328191578388214 (0.23006291687488556) + | > loss_disc_real_4: 0.20456922054290771 (0.23018776439130306) + | > loss_disc_real_5: 0.2423880249261856 (0.22976205311715603) + | > loss_0: 2.491448402404785 (2.44669695943594) + | > grad_norm_0: tensor(19.2882, device='cuda:0') (tensor(13.4494, device='cuda:0')) + | > loss_gen: 2.4060516357421875 (2.3746943324804306) + | > loss_kl: 1.9439241886138916 (1.8605114631354809) + | > loss_feat: 7.2790045738220215 (7.181143373250961) + | > loss_mel: 17.755931854248047 (17.65495684742927) + | > loss_duration: 1.4209377765655518 (1.429418869316578) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.805850982666016 (30.500725090503693) + | > grad_norm_1: tensor(242.3321, device='cuda:0') (tensor(293.9321, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.3633 (0.34012454748153687) + | > loader_time: 0.0059 (0.005171045660972595) + + + --> TIME: 2025-04-19 03:14:22 -- STEP: 57/142 -- GLOBAL_STEP: 415975 + | > loss_disc: 2.5219361782073975 (2.4514495531717935) + | > loss_disc_real_0: 0.18586352467536926 (0.12215232522341243) + | > loss_disc_real_1: 0.18906928598880768 (0.20653708828122994) + | > loss_disc_real_2: 0.23849987983703613 (0.22732130032882356) + | > loss_disc_real_3: 0.21919888257980347 (0.2306823704326362) + | > loss_disc_real_4: 0.22294744849205017 (0.22905163231648898) + | > loss_disc_real_5: 0.2294943481683731 (0.23113282338569038) + | > loss_0: 2.5219361782073975 (2.4514495531717935) + | > grad_norm_0: tensor(27.0672, device='cuda:0') (tensor(14.5674, device='cuda:0')) + | > loss_gen: 2.2843856811523438 (2.3939268170741577) + | > loss_kl: 1.9394567012786865 (1.8636377589744435) + | > loss_feat: 7.311033248901367 (7.23123606464319) + | > loss_mel: 18.239858627319336 (17.77672993509393) + | > loss_duration: 1.5009554624557495 (1.4407763041948016) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.27569007873535 (30.706306925991125) + | > grad_norm_1: tensor(172.1242, device='cuda:0') (tensor(286.2217, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.3765 (0.35699391783329476) + | > loader_time: 0.0056 (0.005508958247669956) + + + --> TIME: 2025-04-19 03:14:32 -- STEP: 82/142 -- GLOBAL_STEP: 416000 + | > loss_disc: 2.5294740200042725 (2.465467883319389) + | > loss_disc_real_0: 0.1276531219482422 (0.12268340669390632) + | > loss_disc_real_1: 0.18834947049617767 (0.20772861562124112) + | > loss_disc_real_2: 0.22824475169181824 (0.22780785691447375) + | > loss_disc_real_3: 0.2545047104358673 (0.2311256418867809) + | > loss_disc_real_4: 0.2485722452402115 (0.23086473254895792) + | > loss_disc_real_5: 0.24581480026245117 (0.23358742165856244) + | > loss_0: 2.5294740200042725 (2.465467883319389) + | > grad_norm_0: tensor(5.2924, device='cuda:0') (tensor(14.3737, device='cuda:0')) + | > loss_gen: 2.1777901649475098 (2.375182675152292) + | > loss_kl: 2.0213093757629395 (1.8760614540518783) + | > loss_feat: 6.177295684814453 (7.0963480239961205) + | > loss_mel: 17.429670333862305 (17.664176603642904) + | > loss_duration: 1.529264211654663 (1.452876452992602) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.335329055786133 (30.464645176399046) + | > grad_norm_1: tensor(127.8266, device='cuda:0') (tensor(280.2579, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.3949 (0.36706691834984745) + | > loader_time: 0.0065 (0.005732111814545423) + + + --> TIME: 2025-04-19 03:14:42 -- STEP: 107/142 -- GLOBAL_STEP: 416025 + | > loss_disc: 2.3424742221832275 (2.4687089140170086) + | > loss_disc_real_0: 0.1608182191848755 (0.1227118425837187) + | > loss_disc_real_1: 0.21619559824466705 (0.20760052672056395) + | > loss_disc_real_2: 0.21857742965221405 (0.2278623360896779) + | > loss_disc_real_3: 0.22417008876800537 (0.23000883861122845) + | > loss_disc_real_4: 0.2062833160161972 (0.23085396855233986) + | > loss_disc_real_5: 0.24748726189136505 (0.23335737157090802) + | > loss_0: 2.3424742221832275 (2.4687089140170086) + | > grad_norm_0: tensor(5.5868, device='cuda:0') (tensor(12.7803, device='cuda:0')) + | > loss_gen: 2.5454468727111816 (2.3551560038718122) + | > loss_kl: 1.8855507373809814 (1.8884471142403434) + | > loss_feat: 7.398341655731201 (7.024091818622339) + | > loss_mel: 18.393878936767578 (17.61775177215861) + | > loss_duration: 1.4859706163406372 (1.4635532158557496) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.709190368652344 (30.34899996819897) + | > grad_norm_1: tensor(81.2405, device='cuda:0') (tensor(251.5716, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.4159 (0.3763691509995504) + | > loader_time: 0.0071 (0.0059866103056435286) + + + --> TIME: 2025-04-19 03:14:53 -- STEP: 132/142 -- GLOBAL_STEP: 416050 + | > loss_disc: 2.519117832183838 (2.4717107812563577) + | > loss_disc_real_0: 0.12748919427394867 (0.12379856401997985) + | > loss_disc_real_1: 0.2309267371892929 (0.2081529791608001) + | > loss_disc_real_2: 0.22889330983161926 (0.2280303764749657) + | > loss_disc_real_3: 0.22925658524036407 (0.2306867236215057) + | > loss_disc_real_4: 0.21428513526916504 (0.23118476005214633) + | > loss_disc_real_5: 0.2131822109222412 (0.23395309495654973) + | > loss_0: 2.519117832183838 (2.4717107812563577) + | > grad_norm_0: tensor(13.5902, device='cuda:0') (tensor(11.8146, device='cuda:0')) + | > loss_gen: 2.2320408821105957 (2.3413128861875254) + | > loss_kl: 2.0564708709716797 (1.8933307537526796) + | > loss_feat: 7.402581691741943 (6.975326093760404) + | > loss_mel: 18.58516502380371 (17.571943694894966) + | > loss_duration: 1.4780352115631104 (1.469812702952009) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.75429344177246 (30.251726150512695) + | > grad_norm_1: tensor(400.8248, device='cuda:0') (tensor(236.7592, device='cuda:0')) + | > current_lr_0: 0.0001936525067756372 + | > current_lr_1: 0.0001936525067756372 + | > step_time: 0.448 (0.38755208976341005) + | > loader_time: 0.0072 (0.00626656503388376) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.35876202583313 (2.35876202583313) + | > loss_disc_real_0: 0.13965348899364471 (0.13965348899364471) + | > loss_disc_real_1: 0.19325770437717438 (0.19325770437717438) + | > loss_disc_real_2: 0.21698977053165436 (0.21698977053165436) + | > loss_disc_real_3: 0.2911454737186432 (0.2911454737186432) + | > loss_disc_real_4: 0.18868330121040344 (0.18868330121040344) + | > loss_disc_real_5: 0.2748665511608124 (0.2748665511608124) + | > loss_0: 2.35876202583313 (2.35876202583313) + | > loss_gen: 2.5412638187408447 (2.5412638187408447) + | > loss_kl: 2.5225892066955566 (2.5225892066955566) + | > loss_feat: 7.138503074645996 (7.138503074645996) + | > loss_mel: 18.788114547729492 (18.788114547729492) + | > loss_duration: 1.6670138835906982 (1.6670138835906982) + | > loss_1: 32.65748596191406 (32.65748596191406) + + --> STEP: 1 + | > loss_disc: 2.5672121047973633 (2.5672121047973633) + | > loss_disc_real_0: 0.17934776842594147 (0.17934776842594147) + | > loss_disc_real_1: 0.22767436504364014 (0.22767436504364014) + | > loss_disc_real_2: 0.23036934435367584 (0.23036934435367584) + | > loss_disc_real_3: 0.2830106019973755 (0.2830106019973755) + | > loss_disc_real_4: 0.24350620806217194 (0.24350620806217194) + | > loss_disc_real_5: 0.23867356777191162 (0.23867356777191162) + | > loss_0: 2.5672121047973633 (2.5672121047973633) + | > loss_gen: 2.3387246131896973 (2.3387246131896973) + | > loss_kl: 2.725053548812866 (2.725053548812866) + | > loss_feat: 5.730782985687256 (5.730782985687256) + | > loss_mel: 15.851709365844727 (15.851709365844727) + | > loss_duration: 1.758313536643982 (1.758313536643982) + | > loss_1: 28.404582977294922 (28.404582977294922) + + --> STEP: 2 + | > loss_disc: 2.5438008308410645 (2.555506467819214) + | > loss_disc_real_0: 0.14558671414852142 (0.16246724128723145) + | > loss_disc_real_1: 0.22953563928604126 (0.2286050021648407) + | > loss_disc_real_2: 0.23273965716362 (0.23155450075864792) + | > loss_disc_real_3: 0.2407853752374649 (0.2618979886174202) + | > loss_disc_real_4: 0.21893727779388428 (0.2312217429280281) + | > loss_disc_real_5: 0.2759251296520233 (0.25729934871196747) + | > loss_0: 2.5438008308410645 (2.555506467819214) + | > loss_gen: 2.242720603942871 (2.290722608566284) + | > loss_kl: 2.0041229724884033 (2.3645882606506348) + | > loss_feat: 6.68140983581543 (6.206096410751343) + | > loss_mel: 17.735549926757812 (16.79362964630127) + | > loss_duration: 1.7053308486938477 (1.7318221926689148) + | > loss_1: 30.36913299560547 (29.386857986450195) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031442642211914062 (-0.0005996227264404297) + | > avg_loss_disc: 2.555506467819214 (+0.06652724742889404) + | > avg_loss_disc_real_0: 0.16246724128723145 (-0.009032920002937317) + | > avg_loss_disc_real_1: 0.2286050021648407 (-0.023144610226154327) + | > avg_loss_disc_real_2: 0.23155450075864792 (-0.04107492417097092) + | > avg_loss_disc_real_3: 0.2618979886174202 (+0.02784886211156845) + | > avg_loss_disc_real_4: 0.2312217429280281 (-0.040710508823394775) + | > avg_loss_disc_real_5: 0.25729934871196747 (+0.004292972385883331) + | > avg_loss_0: 2.555506467819214 (+0.06652724742889404) + | > avg_loss_gen: 2.290722608566284 (-0.4247157573699951) + | > avg_loss_kl: 2.3645882606506348 (-0.003208756446838379) + | > avg_loss_feat: 6.206096410751343 (-1.1364858150482178) + | > avg_loss_mel: 16.79362964630127 (-1.8303709030151367) + | > avg_loss_duration: 1.7318221926689148 (-0.006141722202301025) + | > avg_loss_1: 29.386857986450195 (-3.4009246826171875) + + + > EPOCH: 259/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:14:59)  + + --> TIME: 2025-04-19 03:15:05 -- STEP: 15/142 -- GLOBAL_STEP: 416075 + | > loss_disc: 2.3555827140808105 (2.4206573963165283) + | > loss_disc_real_0: 0.1321941465139389 (0.1120229572057724) + | > loss_disc_real_1: 0.20560786128044128 (0.19917061825593313) + | > loss_disc_real_2: 0.2494802325963974 (0.22547651429971058) + | > loss_disc_real_3: 0.20970585942268372 (0.2299460490544637) + | > loss_disc_real_4: 0.18252776563167572 (0.22735340694586437) + | > loss_disc_real_5: 0.2379571497440338 (0.24299490749835967) + | > loss_0: 2.3555827140808105 (2.4206573963165283) + | > grad_norm_0: tensor(6.4225, device='cuda:0') (tensor(7.7906, device='cuda:0')) + | > loss_gen: 2.4206628799438477 (2.410279401143392) + | > loss_kl: 1.8294520378112793 (1.8159079949061077) + | > loss_feat: 7.514561653137207 (7.4822038014729815) + | > loss_mel: 17.705228805541992 (18.30287806193034) + | > loss_duration: 1.428405523300171 (1.4190443595250448) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.898311614990234 (31.43031374613444) + | > grad_norm_1: tensor(253.0361, device='cuda:0') (tensor(211.7433, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.3353 (0.32801823616027825) + | > loader_time: 0.0046 (0.004222997029622396) + + + --> TIME: 2025-04-19 03:15:14 -- STEP: 40/142 -- GLOBAL_STEP: 416100 + | > loss_disc: 2.4984617233276367 (2.4442439496517183) + | > loss_disc_real_0: 0.07929743826389313 (0.1165512902662158) + | > loss_disc_real_1: 0.2150631695985794 (0.20419142581522465) + | > loss_disc_real_2: 0.21757455170154572 (0.22891962081193923) + | > loss_disc_real_3: 0.21438366174697876 (0.23027096427977084) + | > loss_disc_real_4: 0.2457059770822525 (0.22874965630471705) + | > loss_disc_real_5: 0.19983959197998047 (0.23335364907979966) + | > loss_0: 2.4984617233276367 (2.4442439496517183) + | > grad_norm_0: tensor(13.5496, device='cuda:0') (tensor(9.6678, device='cuda:0')) + | > loss_gen: 2.3063745498657227 (2.3943398773670195) + | > loss_kl: 1.9524691104888916 (1.813660329580307) + | > loss_feat: 5.9672160148620605 (7.301218044757843) + | > loss_mel: 16.074195861816406 (18.076449108123782) + | > loss_duration: 1.417844533920288 (1.4344164848327636) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.71809959411621 (31.020083713531495) + | > grad_norm_1: tensor(400.2317, device='cuda:0') (tensor(255.2132, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.36 (0.34070480465888975) + | > loader_time: 0.0054 (0.004776233434677124) + + + --> TIME: 2025-04-19 03:15:24 -- STEP: 65/142 -- GLOBAL_STEP: 416125 + | > loss_disc: 2.631169319152832 (2.4470085364121656) + | > loss_disc_real_0: 0.20908306539058685 (0.12056626517039079) + | > loss_disc_real_1: 0.1965009719133377 (0.20532271518157078) + | > loss_disc_real_2: 0.19634850323200226 (0.22644098767867454) + | > loss_disc_real_3: 0.20054370164871216 (0.22791226001886222) + | > loss_disc_real_4: 0.19363434612751007 (0.22820788805301373) + | > loss_disc_real_5: 0.20621870458126068 (0.2321576767242872) + | > loss_0: 2.631169319152832 (2.4470085364121656) + | > grad_norm_0: tensor(39.3091, device='cuda:0') (tensor(12.2498, device='cuda:0')) + | > loss_gen: 2.161926507949829 (2.406010679098276) + | > loss_kl: 1.8738363981246948 (1.8296247023802537) + | > loss_feat: 7.432254314422607 (7.295160256899321) + | > loss_mel: 18.56614875793457 (18.046766897348252) + | > loss_duration: 1.4513803720474243 (1.4469256511101356) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.485544204711914 (31.024488038283128) + | > grad_norm_1: tensor(349.6770, device='cuda:0') (tensor(277.9835, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.3899 (0.35321605388934796) + | > loader_time: 0.0065 (0.005159678825965296) + + + --> TIME: 2025-04-19 03:15:34 -- STEP: 90/142 -- GLOBAL_STEP: 416150 + | > loss_disc: 2.534608840942383 (2.4561550034417046) + | > loss_disc_real_0: 0.09562218189239502 (0.1224268480307526) + | > loss_disc_real_1: 0.19638586044311523 (0.206262692477968) + | > loss_disc_real_2: 0.22718340158462524 (0.2263891213470035) + | > loss_disc_real_3: 0.2511031925678253 (0.22752840485837725) + | > loss_disc_real_4: 0.2284543812274933 (0.22889471732907826) + | > loss_disc_real_5: 0.20605601370334625 (0.23240039116806455) + | > loss_0: 2.534608840942383 (2.4561550034417046) + | > grad_norm_0: tensor(9.2932, device='cuda:0') (tensor(14.1812, device='cuda:0')) + | > loss_gen: 2.30908203125 (2.4090550025304154) + | > loss_kl: 1.848218560218811 (1.8418307781219483) + | > loss_feat: 6.815267086029053 (7.220153003268772) + | > loss_mel: 17.201005935668945 (17.9628535164727) + | > loss_duration: 1.4907071590423584 (1.4579072263505717) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.66427993774414 (30.891799375745986) + | > grad_norm_1: tensor(247.5142, device='cuda:0') (tensor(291.1835, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.4346 (0.36645248995886903) + | > loader_time: 0.0067 (0.005497206581963436) + + + --> TIME: 2025-04-19 03:15:44 -- STEP: 115/142 -- GLOBAL_STEP: 416175 + | > loss_disc: 2.3598623275756836 (2.4593457512233554) + | > loss_disc_real_0: 0.10843636840581894 (0.12284863150638083) + | > loss_disc_real_1: 0.18841372430324554 (0.2059493750333786) + | > loss_disc_real_2: 0.2014472484588623 (0.22690926699534705) + | > loss_disc_real_3: 0.2155211865901947 (0.22848112116689268) + | > loss_disc_real_4: 0.24850742518901825 (0.22996261016182276) + | > loss_disc_real_5: 0.22922879457473755 (0.2326469135025273) + | > loss_0: 2.3598623275756836 (2.4593457512233554) + | > grad_norm_0: tensor(4.0644, device='cuda:0') (tensor(14.1690, device='cuda:0')) + | > loss_gen: 2.3791496753692627 (2.388049610801365) + | > loss_kl: 2.0439138412475586 (1.8539458513259888) + | > loss_feat: 7.529149532318115 (7.153335844952127) + | > loss_mel: 18.024879455566406 (17.86739506099535) + | > loss_duration: 1.5039695501327515 (1.464958995321522) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.481061935424805 (30.727685231747834) + | > grad_norm_1: tensor(37.0900, device='cuda:0') (tensor(291.4503, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.4253 (0.3771878988846488) + | > loader_time: 0.0075 (0.005865223511405615) + + + --> TIME: 2025-04-19 03:15:56 -- STEP: 140/142 -- GLOBAL_STEP: 416200 + | > loss_disc: 2.552015542984009 (2.467583377020699) + | > loss_disc_real_0: 0.12355392426252365 (0.12448714742703097) + | > loss_disc_real_1: 0.18320481479167938 (0.20633198033486094) + | > loss_disc_real_2: 0.2066284567117691 (0.22722356032047955) + | > loss_disc_real_3: 0.20698830485343933 (0.2280330166220665) + | > loss_disc_real_4: 0.21336407959461212 (0.23067190466182572) + | > loss_disc_real_5: 0.23985567688941956 (0.2343579888343811) + | > loss_0: 2.552015542984009 (2.467583377020699) + | > grad_norm_0: tensor(4.9479, device='cuda:0') (tensor(13.4796, device='cuda:0')) + | > loss_gen: 2.184866428375244 (2.371722870213645) + | > loss_kl: 1.9544627666473389 (1.8640428543090821) + | > loss_feat: 7.176895618438721 (7.114140568460737) + | > loss_mel: 19.262840270996094 (17.836524180003575) + | > loss_duration: 1.533860206604004 (1.4717448294162747) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.1129264831543 (30.658175209590368) + | > grad_norm_1: tensor(138.0081, device='cuda:0') (tensor(280.9370, device='cuda:0')) + | > current_lr_0: 0.00019362830021229025 + | > current_lr_1: 0.00019362830021229025 + | > step_time: 0.4584 (0.3889255353382655) + | > loader_time: 0.0077 (0.0061626298086983845) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.520087718963623 (2.520087718963623) + | > loss_disc_real_0: 0.15586724877357483 (0.15586724877357483) + | > loss_disc_real_1: 0.2263612300157547 (0.2263612300157547) + | > loss_disc_real_2: 0.251647412776947 (0.251647412776947) + | > loss_disc_real_3: 0.22378647327423096 (0.22378647327423096) + | > loss_disc_real_4: 0.24826017022132874 (0.24826017022132874) + | > loss_disc_real_5: 0.2031167596578598 (0.2031167596578598) + | > loss_0: 2.520087718963623 (2.520087718963623) + | > loss_gen: 2.2104475498199463 (2.2104475498199463) + | > loss_kl: 2.5593721866607666 (2.5593721866607666) + | > loss_feat: 6.60303258895874 (6.60303258895874) + | > loss_mel: 19.34771156311035 (19.34771156311035) + | > loss_duration: 1.6723787784576416 (1.6723787784576416) + | > loss_1: 32.3929443359375 (32.3929443359375) + + --> STEP: 1 + | > loss_disc: 2.4227025508880615 (2.4227025508880615) + | > loss_disc_real_0: 0.1459292769432068 (0.1459292769432068) + | > loss_disc_real_1: 0.27032652497291565 (0.27032652497291565) + | > loss_disc_real_2: 0.2713133692741394 (0.2713133692741394) + | > loss_disc_real_3: 0.22086460888385773 (0.22086460888385773) + | > loss_disc_real_4: 0.22545284032821655 (0.22545284032821655) + | > loss_disc_real_5: 0.1992388665676117 (0.1992388665676117) + | > loss_0: 2.4227025508880615 (2.4227025508880615) + | > loss_gen: 2.487122058868408 (2.487122058868408) + | > loss_kl: 2.6785805225372314 (2.6785805225372314) + | > loss_feat: 7.430216312408447 (7.430216312408447) + | > loss_mel: 19.07235336303711 (19.07235336303711) + | > loss_duration: 1.726517677307129 (1.726517677307129) + | > loss_1: 33.39479064941406 (33.39479064941406) + + --> STEP: 2 + | > loss_disc: 2.607393980026245 (2.5150482654571533) + | > loss_disc_real_0: 0.16127833724021912 (0.15360380709171295) + | > loss_disc_real_1: 0.2716645300388336 (0.27099552750587463) + | > loss_disc_real_2: 0.22801372408866882 (0.2496635466814041) + | > loss_disc_real_3: 0.21318092942237854 (0.21702276915311813) + | > loss_disc_real_4: 0.23370268940925598 (0.22957776486873627) + | > loss_disc_real_5: 0.19501450657844543 (0.19712668657302856) + | > loss_0: 2.607393980026245 (2.5150482654571533) + | > loss_gen: 2.1090400218963623 (2.2980810403823853) + | > loss_kl: 2.313508987426758 (2.4960447549819946) + | > loss_feat: 5.289143085479736 (6.359679698944092) + | > loss_mel: 15.338891983032227 (17.205622673034668) + | > loss_duration: 1.7072038650512695 (1.7168607711791992) + | > loss_1: 26.75778579711914 (30.0762882232666) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038132667541503906 (+0.0006690025329589844) + | > avg_loss_disc: 2.5150482654571533 (-0.04045820236206055) + | > avg_loss_disc_real_0: 0.15360380709171295 (-0.008863434195518494) + | > avg_loss_disc_real_1: 0.27099552750587463 (+0.042390525341033936) + | > avg_loss_disc_real_2: 0.2496635466814041 (+0.018109045922756195) + | > avg_loss_disc_real_3: 0.21702276915311813 (-0.04487521946430206) + | > avg_loss_disc_real_4: 0.22957776486873627 (-0.0016439780592918396) + | > avg_loss_disc_real_5: 0.19712668657302856 (-0.060172662138938904) + | > avg_loss_0: 2.5150482654571533 (-0.04045820236206055) + | > avg_loss_gen: 2.2980810403823853 (+0.007358431816101074) + | > avg_loss_kl: 2.4960447549819946 (+0.13145649433135986) + | > avg_loss_feat: 6.359679698944092 (+0.15358328819274902) + | > avg_loss_mel: 17.205622673034668 (+0.41199302673339844) + | > avg_loss_duration: 1.7168607711791992 (-0.014961421489715576) + | > avg_loss_1: 30.0762882232666 (+0.6894302368164062) + + + > EPOCH: 260/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:15:58)  + + --> TIME: 2025-04-19 03:16:07 -- STEP: 23/142 -- GLOBAL_STEP: 416225 + | > loss_disc: 2.4761033058166504 (2.4510062777477764) + | > loss_disc_real_0: 0.0804603099822998 (0.11978811738283737) + | > loss_disc_real_1: 0.2312224954366684 (0.21041873615721) + | > loss_disc_real_2: 0.25250688195228577 (0.22746188485104105) + | > loss_disc_real_3: 0.2176961749792099 (0.23008434280105258) + | > loss_disc_real_4: 0.2254800647497177 (0.22694309379743494) + | > loss_disc_real_5: 0.26080211997032166 (0.2388371028330015) + | > loss_0: 2.4761033058166504 (2.4510062777477764) + | > grad_norm_0: tensor(8.3654, device='cuda:0') (tensor(6.8749, device='cuda:0')) + | > loss_gen: 2.3113794326782227 (2.3521266501882794) + | > loss_kl: 1.8366888761520386 (1.8644404566806296) + | > loss_feat: 7.075176239013672 (7.132051737412162) + | > loss_mel: 17.890939712524414 (17.63093682994013) + | > loss_duration: 1.4529235363006592 (1.426281654316446) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.567108154296875 (30.405837432197902) + | > grad_norm_1: tensor(232.2139, device='cuda:0') (tensor(115.9203, device='cuda:0')) + | > current_lr_0: 0.0001936040966747637 + | > current_lr_1: 0.0001936040966747637 + | > step_time: 0.3435 (0.33356083994326385) + | > loader_time: 0.0051 (0.004745991333671237) + + + --> TIME: 2025-04-19 03:16:16 -- STEP: 48/142 -- GLOBAL_STEP: 416250 + | > loss_disc: 2.3311424255371094 (2.4760184884071355) + | > loss_disc_real_0: 0.12438280880451202 (0.12247620973115166) + | > loss_disc_real_1: 0.18101011216640472 (0.2073563744003574) + | > loss_disc_real_2: 0.2030107080936432 (0.22792203879604736) + | > loss_disc_real_3: 0.22943922877311707 (0.23286609631031752) + | > loss_disc_real_4: 0.20774823427200317 (0.23011155985295773) + | > loss_disc_real_5: 0.22708502411842346 (0.2359890000273784) + | > loss_0: 2.3311424255371094 (2.4760184884071355) + | > grad_norm_0: tensor(3.7729, device='cuda:0') (tensor(7.1646, device='cuda:0')) + | > loss_gen: 2.3395888805389404 (2.326749771833419) + | > loss_kl: 1.8189666271209717 (1.8638180444637935) + | > loss_feat: 6.869385242462158 (7.030941436688106) + | > loss_mel: 16.754674911499023 (17.571629643440254) + | > loss_duration: 1.4411449432373047 (1.4368012969692547) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.2237606048584 (30.22994029521942) + | > grad_norm_1: tensor(138.8228, device='cuda:0') (tensor(124.6482, device='cuda:0')) + | > current_lr_0: 0.0001936040966747637 + | > current_lr_1: 0.0001936040966747637 + | > step_time: 0.3635 (0.34633641441663104) + | > loader_time: 0.0057 (0.005088398853937784) + + + --> TIME: 2025-04-19 03:16:26 -- STEP: 73/142 -- GLOBAL_STEP: 416275 + | > loss_disc: 2.49434757232666 (2.4743137914840494) + | > loss_disc_real_0: 0.11075007915496826 (0.12475797768733272) + | > loss_disc_real_1: 0.24546831846237183 (0.2057890249033497) + | > loss_disc_real_2: 0.1870289146900177 (0.22691253480845935) + | > loss_disc_real_3: 0.1807381510734558 (0.2312693269285437) + | > loss_disc_real_4: 0.2164645791053772 (0.23115822693256483) + | > loss_disc_real_5: 0.2620020806789398 (0.23778957679663618) + | > loss_0: 2.49434757232666 (2.4743137914840494) + | > grad_norm_0: tensor(10.7034, device='cuda:0') (tensor(7.2012, device='cuda:0')) + | > loss_gen: 2.2937047481536865 (2.3391096232688584) + | > loss_kl: 1.8950657844543457 (1.873918649268477) + | > loss_feat: 7.665402412414551 (7.067245320098041) + | > loss_mel: 17.936309814453125 (17.64262723269529) + | > loss_duration: 1.505737066268921 (1.449416469221246) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.296220779418945 (30.372317510108424) + | > grad_norm_1: tensor(284.9597, device='cuda:0') (tensor(119.6853, device='cuda:0')) + | > current_lr_0: 0.0001936040966747637 + | > current_lr_1: 0.0001936040966747637 + | > step_time: 0.3923 (0.3580578026706226) + | > loader_time: 0.0064 (0.0054248032504565094) + + + --> TIME: 2025-04-19 03:16:36 -- STEP: 98/142 -- GLOBAL_STEP: 416300 + | > loss_disc: 2.460387706756592 (2.486740224215449) + | > loss_disc_real_0: 0.11663307249546051 (0.12725426439119839) + | > loss_disc_real_1: 0.21189998090267181 (0.2079886689173932) + | > loss_disc_real_2: 0.19872033596038818 (0.22798470909498175) + | > loss_disc_real_3: 0.2272048145532608 (0.23116403559640963) + | > loss_disc_real_4: 0.22810739278793335 (0.23022104343589472) + | > loss_disc_real_5: 0.23295660316944122 (0.23624312892860297) + | > loss_0: 2.460387706756592 (2.486740224215449) + | > grad_norm_0: tensor(11.2904, device='cuda:0') (tensor(7.3879, device='cuda:0')) + | > loss_gen: 2.21746826171875 (2.3240016479881427) + | > loss_kl: 2.0332698822021484 (1.8861083242357994) + | > loss_feat: 7.094545841217041 (6.984692680592439) + | > loss_mel: 16.77446937561035 (17.651661765818698) + | > loss_duration: 1.4570711851119995 (1.4599643568603358) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.576824188232422 (30.306428850913534) + | > grad_norm_1: tensor(254.1075, device='cuda:0') (tensor(130.4928, device='cuda:0')) + | > current_lr_0: 0.0001936040966747637 + | > current_lr_1: 0.0001936040966747637 + | > step_time: 0.4106 (0.36893402070415265) + | > loader_time: 0.0067 (0.005721513105898487) + + + --> TIME: 2025-04-19 03:16:47 -- STEP: 123/142 -- GLOBAL_STEP: 416325 + | > loss_disc: 2.502375364303589 (2.482810053398939) + | > loss_disc_real_0: 0.09425123780965805 (0.12826472169500042) + | > loss_disc_real_1: 0.21555279195308685 (0.20753228397873358) + | > loss_disc_real_2: 0.21812595427036285 (0.22773717137856211) + | > loss_disc_real_3: 0.2070663571357727 (0.23055211266851036) + | > loss_disc_real_4: 0.24090199172496796 (0.23002182225870893) + | > loss_disc_real_5: 0.2602493166923523 (0.2345944049397135) + | > loss_0: 2.502375364303589 (2.482810053398939) + | > grad_norm_0: tensor(13.3357, device='cuda:0') (tensor(7.5133, device='cuda:0')) + | > loss_gen: 2.3340210914611816 (2.324240508118296) + | > loss_kl: 1.8670802116394043 (1.896659528336874) + | > loss_feat: 7.250654220581055 (6.967460977352731) + | > loss_mel: 18.665334701538086 (17.61734120438739) + | > loss_duration: 1.487536072731018 (1.4675596971821976) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.604625701904297 (30.273261930884384) + | > grad_norm_1: tensor(282.0350, device='cuda:0') (tensor(134.3022, device='cuda:0')) + | > current_lr_0: 0.0001936040966747637 + | > current_lr_1: 0.0001936040966747637 + | > step_time: 0.4408 (0.3801002851346644) + | > loader_time: 0.0069 (0.00596650247651387) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4585022926330566 (2.4585022926330566) + | > loss_disc_real_0: 0.2062290757894516 (0.2062290757894516) + | > loss_disc_real_1: 0.23456530272960663 (0.23456530272960663) + | > loss_disc_real_2: 0.2606807053089142 (0.2606807053089142) + | > loss_disc_real_3: 0.2624843120574951 (0.2624843120574951) + | > loss_disc_real_4: 0.22390694916248322 (0.22390694916248322) + | > loss_disc_real_5: 0.22673559188842773 (0.22673559188842773) + | > loss_0: 2.4585022926330566 (2.4585022926330566) + | > loss_gen: 2.7638421058654785 (2.7638421058654785) + | > loss_kl: 2.4640283584594727 (2.4640283584594727) + | > loss_feat: 7.987015247344971 (7.987015247344971) + | > loss_mel: 19.325345993041992 (19.325345993041992) + | > loss_duration: 1.6853783130645752 (1.6853783130645752) + | > loss_1: 34.22561264038086 (34.22561264038086) + + --> STEP: 1 + | > loss_disc: 2.7012667655944824 (2.7012667655944824) + | > loss_disc_real_0: 0.26720213890075684 (0.26720213890075684) + | > loss_disc_real_1: 0.3049547076225281 (0.3049547076225281) + | > loss_disc_real_2: 0.303376168012619 (0.303376168012619) + | > loss_disc_real_3: 0.2863459885120392 (0.2863459885120392) + | > loss_disc_real_4: 0.28463420271873474 (0.28463420271873474) + | > loss_disc_real_5: 0.2822348475456238 (0.2822348475456238) + | > loss_0: 2.7012667655944824 (2.7012667655944824) + | > loss_gen: 2.698821783065796 (2.698821783065796) + | > loss_kl: 2.645815134048462 (2.645815134048462) + | > loss_feat: 5.956084251403809 (5.956084251403809) + | > loss_mel: 16.5496768951416 (16.5496768951416) + | > loss_duration: 1.7410234212875366 (1.7410234212875366) + | > loss_1: 29.59142303466797 (29.59142303466797) + + --> STEP: 2 + | > loss_disc: 2.4738574028015137 (2.587562084197998) + | > loss_disc_real_0: 0.171260267496109 (0.21923120319843292) + | > loss_disc_real_1: 0.2548202574253082 (0.27988748252391815) + | > loss_disc_real_2: 0.278174489736557 (0.290775328874588) + | > loss_disc_real_3: 0.2274092733860016 (0.2568776309490204) + | > loss_disc_real_4: 0.3000183701515198 (0.29232628643512726) + | > loss_disc_real_5: 0.21445123851299286 (0.24834304302930832) + | > loss_0: 2.4738574028015137 (2.587562084197998) + | > loss_gen: 2.5970475673675537 (2.647934675216675) + | > loss_kl: 2.249072790145874 (2.447443962097168) + | > loss_feat: 7.3738203048706055 (6.664952278137207) + | > loss_mel: 18.331562042236328 (17.440619468688965) + | > loss_duration: 1.7025235891342163 (1.7217735052108765) + | > loss_1: 32.2540283203125 (30.922725677490234) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033005475997924805 (-0.0005127191543579102) + | > avg_loss_disc: 2.587562084197998 (+0.07251381874084473) + | > avg_loss_disc_real_0: 0.21923120319843292 (+0.06562739610671997) + | > avg_loss_disc_real_1: 0.27988748252391815 (+0.008891955018043518) + | > avg_loss_disc_real_2: 0.290775328874588 (+0.0411117821931839) + | > avg_loss_disc_real_3: 0.2568776309490204 (+0.03985486179590225) + | > avg_loss_disc_real_4: 0.29232628643512726 (+0.06274852156639099) + | > avg_loss_disc_real_5: 0.24834304302930832 (+0.051216356456279755) + | > avg_loss_0: 2.587562084197998 (+0.07251381874084473) + | > avg_loss_gen: 2.647934675216675 (+0.34985363483428955) + | > avg_loss_kl: 2.447443962097168 (-0.04860079288482666) + | > avg_loss_feat: 6.664952278137207 (+0.30527257919311523) + | > avg_loss_mel: 17.440619468688965 (+0.23499679565429688) + | > avg_loss_duration: 1.7217735052108765 (+0.004912734031677246) + | > avg_loss_1: 30.922725677490234 (+0.8464374542236328) + + + > EPOCH: 261/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:16:57)  + + --> TIME: 2025-04-19 03:16:59 -- STEP: 6/142 -- GLOBAL_STEP: 416350 + | > loss_disc: 2.4425466060638428 (2.4726607402165732) + | > loss_disc_real_0: 0.1467805802822113 (0.12802681947747865) + | > loss_disc_real_1: 0.187030628323555 (0.20631622026364008) + | > loss_disc_real_2: 0.2033267319202423 (0.23304539918899536) + | > loss_disc_real_3: 0.20754176378250122 (0.24003137151400247) + | > loss_disc_real_4: 0.20372851192951202 (0.23734990259011587) + | > loss_disc_real_5: 0.24198392033576965 (0.2587093189358711) + | > loss_0: 2.4425466060638428 (2.4726607402165732) + | > grad_norm_0: tensor(13.0579, device='cuda:0') (tensor(11.6816, device='cuda:0')) + | > loss_gen: 2.18119740486145 (2.379095276196798) + | > loss_kl: 1.8424559831619263 (1.8304515480995178) + | > loss_feat: 7.4680914878845215 (7.495165983835856) + | > loss_mel: 17.36767578125 (17.909676869710285) + | > loss_duration: 1.4029245376586914 (1.4218199451764424) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.262344360351562 (31.036208788553875) + | > grad_norm_1: tensor(148.1025, device='cuda:0') (tensor(236.0334, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.3519 (0.3309447765350342) + | > loader_time: 0.0041 (0.0042877991994222) + + + --> TIME: 2025-04-19 03:17:08 -- STEP: 31/142 -- GLOBAL_STEP: 416375 + | > loss_disc: 2.479548931121826 (2.4404547137598835) + | > loss_disc_real_0: 0.1435905247926712 (0.11958976474500471) + | > loss_disc_real_1: 0.1835896372795105 (0.20607487809273503) + | > loss_disc_real_2: 0.22013939917087555 (0.22757563139161757) + | > loss_disc_real_3: 0.19844721257686615 (0.22998188339894818) + | > loss_disc_real_4: 0.20276358723640442 (0.22293946387306338) + | > loss_disc_real_5: 0.2798561155796051 (0.23515860976711397) + | > loss_0: 2.479548931121826 (2.4404547137598835) + | > grad_norm_0: tensor(13.5620, device='cuda:0') (tensor(9.0707, device='cuda:0')) + | > loss_gen: 2.4492340087890625 (2.388358854478406) + | > loss_kl: 1.8306864500045776 (1.8216751583160893) + | > loss_feat: 6.828323841094971 (7.374523362805767) + | > loss_mel: 16.588361740112305 (17.879354292346584) + | > loss_duration: 1.419844388961792 (1.4294885012411302) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.1164493560791 (30.893399884623864) + | > grad_norm_1: tensor(432.5797, device='cuda:0') (tensor(177.4344, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.3535 (0.3382685799752512) + | > loader_time: 0.0054 (0.004896848432479365) + + + --> TIME: 2025-04-19 03:17:17 -- STEP: 56/142 -- GLOBAL_STEP: 416400 + | > loss_disc: 2.3868091106414795 (2.4284145236015324) + | > loss_disc_real_0: 0.12697644531726837 (0.11933822171496493) + | > loss_disc_real_1: 0.2023945152759552 (0.20577592216432095) + | > loss_disc_real_2: 0.21840250492095947 (0.22525466047227383) + | > loss_disc_real_3: 0.2440408170223236 (0.22820757116590226) + | > loss_disc_real_4: 0.21695783734321594 (0.22632879896887712) + | > loss_disc_real_5: 0.23772668838500977 (0.2321533840149641) + | > loss_0: 2.3868091106414795 (2.4284145236015324) + | > grad_norm_0: tensor(10.8440, device='cuda:0') (tensor(9.7960, device='cuda:0')) + | > loss_gen: 2.3116002082824707 (2.3859316025461474) + | > loss_kl: 1.7919578552246094 (1.827985242009163) + | > loss_feat: 6.94696044921875 (7.290072815758841) + | > loss_mel: 18.064863204956055 (17.89688314710345) + | > loss_duration: 1.4681185483932495 (1.4405116055692946) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.583499908447266 (30.84138410431998) + | > grad_norm_1: tensor(231.6871, device='cuda:0') (tensor(218.1584, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.3724 (0.349081426858902) + | > loader_time: 0.0057 (0.005340005670275007) + + + --> TIME: 2025-04-19 03:17:27 -- STEP: 81/142 -- GLOBAL_STEP: 416425 + | > loss_disc: 2.5953640937805176 (2.440751172878125) + | > loss_disc_real_0: 0.06145496293902397 (0.11978282158573468) + | > loss_disc_real_1: 0.22801509499549866 (0.20569986896014508) + | > loss_disc_real_2: 0.22476200759410858 (0.22746248561658977) + | > loss_disc_real_3: 0.21342654526233673 (0.22846399633972733) + | > loss_disc_real_4: 0.2200792282819748 (0.2287020512201168) + | > loss_disc_real_5: 0.22433491051197052 (0.23201153914869568) + | > loss_0: 2.5953640937805176 (2.440751172878125) + | > grad_norm_0: tensor(31.3209, device='cuda:0') (tensor(11.0054, device='cuda:0')) + | > loss_gen: 2.140397310256958 (2.375655168368493) + | > loss_kl: 1.9646291732788086 (1.846977225056401) + | > loss_feat: 5.85927152633667 (7.209038416544597) + | > loss_mel: 16.09493637084961 (17.768120600853432) + | > loss_duration: 1.4538836479187012 (1.4519256794894182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.513118743896484 (30.651716915177712) + | > grad_norm_1: tensor(317.6720, device='cuda:0') (tensor(242.5926, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.3945 (0.3608866944725131) + | > loader_time: 0.0062 (0.005587510120721511) + + + --> TIME: 2025-04-19 03:17:38 -- STEP: 106/142 -- GLOBAL_STEP: 416450 + | > loss_disc: 2.440877676010132 (2.450296595411482) + | > loss_disc_real_0: 0.1318444460630417 (0.12261238690676554) + | > loss_disc_real_1: 0.22455281019210815 (0.20547855139340995) + | > loss_disc_real_2: 0.22092431783676147 (0.2278369759332459) + | > loss_disc_real_3: 0.223092719912529 (0.2289152152414592) + | > loss_disc_real_4: 0.21889793872833252 (0.2286197745575095) + | > loss_disc_real_5: 0.21457308530807495 (0.23281790020893206) + | > loss_0: 2.440877676010132 (2.450296595411482) + | > grad_norm_0: tensor(24.3523, device='cuda:0') (tensor(12.1329, device='cuda:0')) + | > loss_gen: 2.3095028400421143 (2.371237682846358) + | > loss_kl: 1.974199652671814 (1.8572002536845658) + | > loss_feat: 6.5081329345703125 (7.142741153825004) + | > loss_mel: 17.56928253173828 (17.735696441722357) + | > loss_duration: 1.510892629623413 (1.4624878932844918) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.872011184692383 (30.569363306153495) + | > grad_norm_1: tensor(394.2860, device='cuda:0') (tensor(251.0610, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.409 (0.3728262365988965) + | > loader_time: 0.0065 (0.005811484354846884) + + + --> TIME: 2025-04-19 03:17:49 -- STEP: 131/142 -- GLOBAL_STEP: 416475 + | > loss_disc: 2.433690309524536 (2.4556810091470047) + | > loss_disc_real_0: 0.1003984734416008 (0.12241509548239125) + | > loss_disc_real_1: 0.22123153507709503 (0.2057582078544238) + | > loss_disc_real_2: 0.22125090658664703 (0.22758530512565875) + | > loss_disc_real_3: 0.2355339229106903 (0.22879328846021463) + | > loss_disc_real_4: 0.21321459114551544 (0.22940580342107147) + | > loss_disc_real_5: 0.23960648477077484 (0.23383019113813647) + | > loss_0: 2.433690309524536 (2.4556810091470047) + | > grad_norm_0: tensor(4.5654, device='cuda:0') (tensor(12.5406, device='cuda:0')) + | > loss_gen: 2.345560073852539 (2.3660079268098793) + | > loss_kl: 1.8806482553482056 (1.863002257492706) + | > loss_feat: 6.8725457191467285 (7.091731344470541) + | > loss_mel: 17.035057067871094 (17.67433337582885) + | > loss_duration: 1.5015861988067627 (1.4684789089756158) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63539695739746 (30.463553741688038) + | > grad_norm_1: tensor(185.5671, device='cuda:0') (tensor(261.7057, device='cuda:0')) + | > current_lr_0: 0.00019357989616267934 + | > current_lr_1: 0.00019357989616267934 + | > step_time: 0.4497 (0.3853141788307947) + | > loader_time: 0.0078 (0.0061324570925181164) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.43022084236145 (2.43022084236145) + | > loss_disc_real_0: 0.1105566993355751 (0.1105566993355751) + | > loss_disc_real_1: 0.24208611249923706 (0.24208611249923706) + | > loss_disc_real_2: 0.2849899232387543 (0.2849899232387543) + | > loss_disc_real_3: 0.2620847225189209 (0.2620847225189209) + | > loss_disc_real_4: 0.2079959511756897 (0.2079959511756897) + | > loss_disc_real_5: 0.2387521117925644 (0.2387521117925644) + | > loss_0: 2.43022084236145 (2.43022084236145) + | > loss_gen: 2.442737102508545 (2.442737102508545) + | > loss_kl: 2.433072090148926 (2.433072090148926) + | > loss_feat: 6.8711371421813965 (6.8711371421813965) + | > loss_mel: 17.618061065673828 (17.618061065673828) + | > loss_duration: 1.6868048906326294 (1.6868048906326294) + | > loss_1: 31.05181312561035 (31.05181312561035) + + --> STEP: 1 + | > loss_disc: 2.4957540035247803 (2.4957540035247803) + | > loss_disc_real_0: 0.14588694274425507 (0.14588694274425507) + | > loss_disc_real_1: 0.21884043514728546 (0.21884043514728546) + | > loss_disc_real_2: 0.2587473392486572 (0.2587473392486572) + | > loss_disc_real_3: 0.23523889482021332 (0.23523889482021332) + | > loss_disc_real_4: 0.22726856172084808 (0.22726856172084808) + | > loss_disc_real_5: 0.20135684311389923 (0.20135684311389923) + | > loss_0: 2.4957540035247803 (2.4957540035247803) + | > loss_gen: 2.3539533615112305 (2.3539533615112305) + | > loss_kl: 2.6039109230041504 (2.6039109230041504) + | > loss_feat: 7.865865230560303 (7.865865230560303) + | > loss_mel: 19.158037185668945 (19.158037185668945) + | > loss_duration: 1.7361620664596558 (1.7361620664596558) + | > loss_1: 33.717926025390625 (33.717926025390625) + + --> STEP: 2 + | > loss_disc: 2.5303406715393066 (2.5130473375320435) + | > loss_disc_real_0: 0.151172935962677 (0.14852993935346603) + | > loss_disc_real_1: 0.19797350466251373 (0.2084069699048996) + | > loss_disc_real_2: 0.2748423218727112 (0.2667948305606842) + | > loss_disc_real_3: 0.2730766832828522 (0.25415778905153275) + | > loss_disc_real_4: 0.23958681523799896 (0.23342768847942352) + | > loss_disc_real_5: 0.22784142196178436 (0.2145991325378418) + | > loss_0: 2.5303406715393066 (2.5130473375320435) + | > loss_gen: 2.4233222007751465 (2.3886377811431885) + | > loss_kl: 2.3069145679473877 (2.455412745475769) + | > loss_feat: 6.650547027587891 (7.258206129074097) + | > loss_mel: 18.806726455688477 (18.98238182067871) + | > loss_duration: 1.7179477214813232 (1.7270548939704895) + | > loss_1: 31.90545654296875 (32.81169128417969) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003488898277282715 (+0.00018835067749023438) + | > avg_loss_disc: 2.5130473375320435 (-0.07451474666595459) + | > avg_loss_disc_real_0: 0.14852993935346603 (-0.07070126384496689) + | > avg_loss_disc_real_1: 0.2084069699048996 (-0.07148051261901855) + | > avg_loss_disc_real_2: 0.2667948305606842 (-0.02398049831390381) + | > avg_loss_disc_real_3: 0.25415778905153275 (-0.0027198418974876404) + | > avg_loss_disc_real_4: 0.23342768847942352 (-0.058898597955703735) + | > avg_loss_disc_real_5: 0.2145991325378418 (-0.03374391049146652) + | > avg_loss_0: 2.5130473375320435 (-0.07451474666595459) + | > avg_loss_gen: 2.3886377811431885 (-0.25929689407348633) + | > avg_loss_kl: 2.455412745475769 (+0.007968783378601074) + | > avg_loss_feat: 7.258206129074097 (+0.5932538509368896) + | > avg_loss_mel: 18.98238182067871 (+1.541762351989746) + | > avg_loss_duration: 1.7270548939704895 (+0.005281388759613037) + | > avg_loss_1: 32.81169128417969 (+1.8889656066894531) + + + > EPOCH: 262/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:17:56)  + + --> TIME: 2025-04-19 03:18:01 -- STEP: 14/142 -- GLOBAL_STEP: 416500 + | > loss_disc: 2.425309181213379 (2.4152464355741228) + | > loss_disc_real_0: 0.09008275717496872 (0.11920481388057981) + | > loss_disc_real_1: 0.177522674202919 (0.20729932827608927) + | > loss_disc_real_2: 0.20748071372509003 (0.22362490317651204) + | > loss_disc_real_3: 0.22285984456539154 (0.2277847837124552) + | > loss_disc_real_4: 0.2261483371257782 (0.22632804938725062) + | > loss_disc_real_5: 0.25607794523239136 (0.2251789335693632) + | > loss_0: 2.425309181213379 (2.4152464355741228) + | > grad_norm_0: tensor(4.8911, device='cuda:0') (tensor(8.0473, device='cuda:0')) + | > loss_gen: 2.1934027671813965 (2.3721203804016113) + | > loss_kl: 1.7773091793060303 (1.8107290863990784) + | > loss_feat: 7.323085784912109 (7.258591788155692) + | > loss_mel: 17.82521629333496 (17.28042479923793) + | > loss_duration: 1.4198479652404785 (1.4183406744684492) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.538862228393555 (30.140206473214285) + | > grad_norm_1: tensor(57.0785, device='cuda:0') (tensor(173.7273, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.3327 (0.3277413674763271) + | > loader_time: 0.0049 (0.004601802144731794) + + + --> TIME: 2025-04-19 03:18:10 -- STEP: 39/142 -- GLOBAL_STEP: 416525 + | > loss_disc: 2.4036693572998047 (2.4523298129057274) + | > loss_disc_real_0: 0.07760481536388397 (0.12054314941931994) + | > loss_disc_real_1: 0.23402221500873566 (0.20878288646539053) + | > loss_disc_real_2: 0.24612440168857574 (0.22819448052308497) + | > loss_disc_real_3: 0.23980045318603516 (0.226895177593598) + | > loss_disc_real_4: 0.2609753906726837 (0.2300576258164186) + | > loss_disc_real_5: 0.21733611822128296 (0.2302481730779012) + | > loss_0: 2.4036693572998047 (2.4523298129057274) + | > grad_norm_0: tensor(6.8055, device='cuda:0') (tensor(7.8075, device='cuda:0')) + | > loss_gen: 2.3442933559417725 (2.3535062288626647) + | > loss_kl: 1.8370654582977295 (1.8327603126183534) + | > loss_feat: 7.298762321472168 (7.2017574188036795) + | > loss_mel: 17.827831268310547 (17.629113955375477) + | > loss_duration: 1.463955283164978 (1.4308071931203206) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.771907806396484 (30.447944934551533) + | > grad_norm_1: tensor(54.1390, device='cuda:0') (tensor(146.0934, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.3548 (0.34073159022208965) + | > loader_time: 0.0056 (0.0052106197063739495) + + + --> TIME: 2025-04-19 03:18:19 -- STEP: 64/142 -- GLOBAL_STEP: 416550 + | > loss_disc: 2.484935760498047 (2.462169565260411) + | > loss_disc_real_0: 0.14382193982601166 (0.12197250861208886) + | > loss_disc_real_1: 0.22783318161964417 (0.2110852978657931) + | > loss_disc_real_2: 0.23597703874111176 (0.2266421434469521) + | > loss_disc_real_3: 0.23023365437984467 (0.22823916818015277) + | > loss_disc_real_4: 0.21604157984256744 (0.2302628019824624) + | > loss_disc_real_5: 0.2615530490875244 (0.23194355028681457) + | > loss_0: 2.484935760498047 (2.462169565260411) + | > grad_norm_0: tensor(5.0505, device='cuda:0') (tensor(7.3416, device='cuda:0')) + | > loss_gen: 2.4207072257995605 (2.3531785905361176) + | > loss_kl: 1.7710093259811401 (1.8431905303150415) + | > loss_feat: 7.2036309242248535 (7.1756072118878365) + | > loss_mel: 18.089624404907227 (17.677102655172355) + | > loss_duration: 1.4823815822601318 (1.4439920410513878) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.96735382080078 (30.493071109056473) + | > grad_norm_1: tensor(111.1299, device='cuda:0') (tensor(143.5983, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.3858 (0.3524368330836296) + | > loader_time: 0.0068 (0.005480270832777025) + + + --> TIME: 2025-04-19 03:18:29 -- STEP: 89/142 -- GLOBAL_STEP: 416575 + | > loss_disc: 2.490288496017456 (2.4699126254306756) + | > loss_disc_real_0: 0.12328272312879562 (0.12384290482555882) + | > loss_disc_real_1: 0.16991618275642395 (0.2101492423020052) + | > loss_disc_real_2: 0.210593119263649 (0.22735133984785402) + | > loss_disc_real_3: 0.22913268208503723 (0.2269778238253647) + | > loss_disc_real_4: 0.24426501989364624 (0.22936992916498292) + | > loss_disc_real_5: 0.23742426931858063 (0.2341446755977159) + | > loss_0: 2.490288496017456 (2.4699126254306756) + | > grad_norm_0: tensor(6.8140, device='cuda:0') (tensor(7.4186, device='cuda:0')) + | > loss_gen: 2.2862539291381836 (2.3454245556606343) + | > loss_kl: 1.9821441173553467 (1.8689993526158708) + | > loss_feat: 6.933722972869873 (7.088121382038245) + | > loss_mel: 17.850662231445312 (17.698323560564717) + | > loss_duration: 1.4680687189102173 (1.4553434366590514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.52085304260254 (30.456212268786484) + | > grad_norm_1: tensor(48.5298, device='cuda:0') (tensor(143.2713, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.4013 (0.3640346634254027) + | > loader_time: 0.0066 (0.005721220809422182) + + + --> TIME: 2025-04-19 03:18:40 -- STEP: 114/142 -- GLOBAL_STEP: 416600 + | > loss_disc: 2.4789557456970215 (2.4823341976132323) + | > loss_disc_real_0: 0.08778965473175049 (0.12579038713062024) + | > loss_disc_real_1: 0.19370459020137787 (0.21056449060377322) + | > loss_disc_real_2: 0.19959038496017456 (0.22733945961584123) + | > loss_disc_real_3: 0.2088111788034439 (0.2279379311622235) + | > loss_disc_real_4: 0.2125197947025299 (0.2308439800613805) + | > loss_disc_real_5: 0.23823192715644836 (0.23488658274474897) + | > loss_0: 2.4789557456970215 (2.4823341976132323) + | > grad_norm_0: tensor(15.5120, device='cuda:0') (tensor(7.9925, device='cuda:0')) + | > loss_gen: 2.3219664096832275 (2.337082208248607) + | > loss_kl: 1.9210600852966309 (1.8921067557836835) + | > loss_feat: 7.199161529541016 (7.053680281890066) + | > loss_mel: 17.417272567749023 (17.760063857362987) + | > loss_duration: 1.4858403205871582 (1.4626837906084558) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.345300674438477 (30.505616840563324) + | > grad_norm_1: tensor(330.4264, device='cuda:0') (tensor(155.8690, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.4172 (0.374518637071576) + | > loader_time: 0.007 (0.005958908482601768) + + + --> TIME: 2025-04-19 03:18:51 -- STEP: 139/142 -- GLOBAL_STEP: 416625 + | > loss_disc: 2.655982732772827 (2.48148181970171) + | > loss_disc_real_0: 0.07775788754224777 (0.1265005181483228) + | > loss_disc_real_1: 0.19269536435604095 (0.21073661412266517) + | > loss_disc_real_2: 0.22005455195903778 (0.22696055621980762) + | > loss_disc_real_3: 0.2204991728067398 (0.2282017757781118) + | > loss_disc_real_4: 0.22376158833503723 (0.2303401371129125) + | > loss_disc_real_5: 0.2543611228466034 (0.23478942862946353) + | > loss_0: 2.655982732772827 (2.48148181970171) + | > grad_norm_0: tensor(17.6085, device='cuda:0') (tensor(8.7602, device='cuda:0')) + | > loss_gen: 2.0702953338623047 (2.3401506444533093) + | > loss_kl: 1.9147106409072876 (1.8968941453549502) + | > loss_feat: 6.066925525665283 (7.0129923751885945) + | > loss_mel: 16.50568199157715 (17.706487195954914) + | > loss_duration: 1.4945924282073975 (1.469557251004006) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.05220603942871 (30.42608159566097) + | > grad_norm_1: tensor(214.9912, device='cuda:0') (tensor(176.1683, device='cuda:0')) + | > current_lr_0: 0.000193555698675659 + | > current_lr_1: 0.000193555698675659 + | > step_time: 0.4495 (0.38664072366069546) + | > loader_time: 0.0073 (0.006218767852234326) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.394960880279541 (2.394960880279541) + | > loss_disc_real_0: 0.14381535351276398 (0.14381535351276398) + | > loss_disc_real_1: 0.20499898493289948 (0.20499898493289948) + | > loss_disc_real_2: 0.22118408977985382 (0.22118408977985382) + | > loss_disc_real_3: 0.2022918164730072 (0.2022918164730072) + | > loss_disc_real_4: 0.20337997376918793 (0.20337997376918793) + | > loss_disc_real_5: 0.22137564420700073 (0.22137564420700073) + | > loss_0: 2.394960880279541 (2.394960880279541) + | > loss_gen: 2.320222854614258 (2.320222854614258) + | > loss_kl: 2.2930495738983154 (2.2930495738983154) + | > loss_feat: 6.130881309509277 (6.130881309509277) + | > loss_mel: 17.886079788208008 (17.886079788208008) + | > loss_duration: 1.6843981742858887 (1.6843981742858887) + | > loss_1: 30.314632415771484 (30.314632415771484) + + --> STEP: 1 + | > loss_disc: 2.5576252937316895 (2.5576252937316895) + | > loss_disc_real_0: 0.19996649026870728 (0.19996649026870728) + | > loss_disc_real_1: 0.24392345547676086 (0.24392345547676086) + | > loss_disc_real_2: 0.2466418296098709 (0.2466418296098709) + | > loss_disc_real_3: 0.2625306248664856 (0.2625306248664856) + | > loss_disc_real_4: 0.21780745685100555 (0.21780745685100555) + | > loss_disc_real_5: 0.23322021961212158 (0.23322021961212158) + | > loss_0: 2.5576252937316895 (2.5576252937316895) + | > loss_gen: 2.278085231781006 (2.278085231781006) + | > loss_kl: 2.4358859062194824 (2.4358859062194824) + | > loss_feat: 5.391341686248779 (5.391341686248779) + | > loss_mel: 16.232582092285156 (16.232582092285156) + | > loss_duration: 1.746220588684082 (1.746220588684082) + | > loss_1: 28.084117889404297 (28.084117889404297) + + --> STEP: 2 + | > loss_disc: 2.5406875610351562 (2.549156427383423) + | > loss_disc_real_0: 0.23343518376350403 (0.21670083701610565) + | > loss_disc_real_1: 0.21610042452812195 (0.2300119400024414) + | > loss_disc_real_2: 0.19897252321243286 (0.22280717641115189) + | > loss_disc_real_3: 0.24688607454299927 (0.25470834970474243) + | > loss_disc_real_4: 0.2275920808315277 (0.22269976884126663) + | > loss_disc_real_5: 0.21557597815990448 (0.22439809888601303) + | > loss_0: 2.5406875610351562 (2.549156427383423) + | > loss_gen: 2.3140738010406494 (2.2960795164108276) + | > loss_kl: 2.3476364612579346 (2.3917611837387085) + | > loss_feat: 6.634208679199219 (6.012775182723999) + | > loss_mel: 17.586877822875977 (16.909729957580566) + | > loss_duration: 1.6827208995819092 (1.7144707441329956) + | > loss_1: 30.565515518188477 (29.324816703796387) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035309791564941406 (+4.208087921142578e-05) + | > avg_loss_disc: 2.549156427383423 (+0.036109089851379395) + | > avg_loss_disc_real_0: 0.21670083701610565 (+0.06817089766263962) + | > avg_loss_disc_real_1: 0.2300119400024414 (+0.02160497009754181) + | > avg_loss_disc_real_2: 0.22280717641115189 (-0.04398765414953232) + | > avg_loss_disc_real_3: 0.25470834970474243 (+0.0005505606532096863) + | > avg_loss_disc_real_4: 0.22269976884126663 (-0.010727919638156891) + | > avg_loss_disc_real_5: 0.22439809888601303 (+0.009798966348171234) + | > avg_loss_0: 2.549156427383423 (+0.036109089851379395) + | > avg_loss_gen: 2.2960795164108276 (-0.09255826473236084) + | > avg_loss_kl: 2.3917611837387085 (-0.06365156173706055) + | > avg_loss_feat: 6.012775182723999 (-1.2454309463500977) + | > avg_loss_mel: 16.909729957580566 (-2.0726518630981445) + | > avg_loss_duration: 1.7144707441329956 (-0.012584149837493896) + | > avg_loss_1: 29.324816703796387 (-3.486874580383301) + + + > EPOCH: 263/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:18:54)  + + --> TIME: 2025-04-19 03:19:02 -- STEP: 22/142 -- GLOBAL_STEP: 416650 + | > loss_disc: 2.361428737640381 (2.442875385284424) + | > loss_disc_real_0: 0.08925070613622665 (0.11222103475169702) + | > loss_disc_real_1: 0.190246120095253 (0.2109795944257216) + | > loss_disc_real_2: 0.20296601951122284 (0.22815884988416324) + | > loss_disc_real_3: 0.19769497215747833 (0.23179033195430582) + | > loss_disc_real_4: 0.1809406876564026 (0.22514489699493756) + | > loss_disc_real_5: 0.21310576796531677 (0.2351225899024443) + | > loss_0: 2.361428737640381 (2.442875385284424) + | > grad_norm_0: tensor(6.2066, device='cuda:0') (tensor(14.7525, device='cuda:0')) + | > loss_gen: 2.2448019981384277 (2.4012886069037696) + | > loss_kl: 1.6912925243377686 (1.8022419972853227) + | > loss_feat: 7.084020137786865 (7.293533216823231) + | > loss_mel: 16.348146438598633 (17.924278172579676) + | > loss_duration: 1.4149014949798584 (1.4253085905855352) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.78316307067871 (30.846650296991523) + | > grad_norm_1: tensor(340.9794, device='cuda:0') (tensor(345.9170, device='cuda:0')) + | > current_lr_0: 0.00019353150421332454 + | > current_lr_1: 0.00019353150421332454 + | > step_time: 0.3508 (0.33423086729916657) + | > loader_time: 0.0057 (0.004944118586453524) + + + --> TIME: 2025-04-19 03:19:12 -- STEP: 47/142 -- GLOBAL_STEP: 416675 + | > loss_disc: 2.443453550338745 (2.4386918037495713) + | > loss_disc_real_0: 0.1448797583580017 (0.11346933943160037) + | > loss_disc_real_1: 0.2257918268442154 (0.20902891996059011) + | > loss_disc_real_2: 0.21448184549808502 (0.2251942408211688) + | > loss_disc_real_3: 0.24181871116161346 (0.2292978341909165) + | > loss_disc_real_4: 0.2408425658941269 (0.22640789030714237) + | > loss_disc_real_5: 0.21959850192070007 (0.23188901549958169) + | > loss_0: 2.443453550338745 (2.4386918037495713) + | > grad_norm_0: tensor(20.4816, device='cuda:0') (tensor(13.6436, device='cuda:0')) + | > loss_gen: 2.4617226123809814 (2.3811528378344597) + | > loss_kl: 1.814139723777771 (1.8157444862609213) + | > loss_feat: 7.2183403968811035 (7.1646923105767435) + | > loss_mel: 18.246816635131836 (17.84090285605573) + | > loss_duration: 1.4154481887817383 (1.4355759417757075) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.15646743774414 (30.638068462939973) + | > grad_norm_1: tensor(436.8394, device='cuda:0') (tensor(341.9051, device='cuda:0')) + | > current_lr_0: 0.00019353150421332454 + | > current_lr_1: 0.00019353150421332454 + | > step_time: 0.3632 (0.35207072724687294) + | > loader_time: 0.0061 (0.005342732084558365) + + + --> TIME: 2025-04-19 03:19:21 -- STEP: 72/142 -- GLOBAL_STEP: 416700 + | > loss_disc: 2.5354065895080566 (2.445345295800103) + | > loss_disc_real_0: 0.11682265251874924 (0.11739801398168008) + | > loss_disc_real_1: 0.24585580825805664 (0.20757130905985832) + | > loss_disc_real_2: 0.2113778293132782 (0.2237547675354613) + | > loss_disc_real_3: 0.21205034852027893 (0.2286479893243975) + | > loss_disc_real_4: 0.24042433500289917 (0.22767963632941246) + | > loss_disc_real_5: 0.26780927181243896 (0.2338705307079686) + | > loss_0: 2.5354065895080566 (2.445345295800103) + | > grad_norm_0: tensor(8.1778, device='cuda:0') (tensor(12.3627, device='cuda:0')) + | > loss_gen: 2.257190227508545 (2.3615307675467596) + | > loss_kl: 1.8148075342178345 (1.8414702481693692) + | > loss_feat: 6.950712203979492 (7.10290601518419) + | > loss_mel: 17.050146102905273 (17.774167776107788) + | > loss_duration: 1.4642887115478516 (1.4488876610994337) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.537145614624023 (30.5289626121521) + | > grad_norm_1: tensor(276.4329, device='cuda:0') (tensor(311.3159, device='cuda:0')) + | > current_lr_0: 0.00019353150421332454 + | > current_lr_1: 0.00019353150421332454 + | > step_time: 0.3937 (0.3622787594795227) + | > loader_time: 0.0057 (0.005580561028586494) + + + --> TIME: 2025-04-19 03:19:32 -- STEP: 97/142 -- GLOBAL_STEP: 416725 + | > loss_disc: 2.519505500793457 (2.452729119467981) + | > loss_disc_real_0: 0.22626538574695587 (0.12164455882667266) + | > loss_disc_real_1: 0.2020043283700943 (0.20871972377152787) + | > loss_disc_real_2: 0.20716503262519836 (0.22497831392534) + | > loss_disc_real_3: 0.23853562772274017 (0.22794115451193347) + | > loss_disc_real_4: 0.18647274374961853 (0.22618062993914811) + | > loss_disc_real_5: 0.20820166170597076 (0.23278159778757193) + | > loss_0: 2.519505500793457 (2.452729119467981) + | > grad_norm_0: tensor(30.5114, device='cuda:0') (tensor(12.9121, device='cuda:0')) + | > loss_gen: 2.3934195041656494 (2.3593016064044137) + | > loss_kl: 1.9877912998199463 (1.86267124992056) + | > loss_feat: 7.215541839599609 (7.0636103915185044) + | > loss_mel: 18.413339614868164 (17.798991704724493) + | > loss_duration: 1.4867384433746338 (1.460968849585228) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.496829986572266 (30.54554384762479) + | > grad_norm_1: tensor(97.6461, device='cuda:0') (tensor(305.0272, device='cuda:0')) + | > current_lr_0: 0.00019353150421332454 + | > current_lr_1: 0.00019353150421332454 + | > step_time: 0.4106 (0.37194014824542815) + | > loader_time: 0.0071 (0.0058239071639542735) + + + --> TIME: 2025-04-19 03:19:42 -- STEP: 122/142 -- GLOBAL_STEP: 416750 + | > loss_disc: 2.4816675186157227 (2.4556560614069953) + | > loss_disc_real_0: 0.15037091076374054 (0.12384863912326391) + | > loss_disc_real_1: 0.2461177408695221 (0.2075047660313669) + | > loss_disc_real_2: 0.23023054003715515 (0.22540544913929017) + | > loss_disc_real_3: 0.23546689748764038 (0.22907619447004599) + | > loss_disc_real_4: 0.24573729932308197 (0.22694618047260848) + | > loss_disc_real_5: 0.23782701790332794 (0.23282020038268605) + | > loss_0: 2.4816675186157227 (2.4556560614069953) + | > grad_norm_0: tensor(14.3366, device='cuda:0') (tensor(14.1479, device='cuda:0')) + | > loss_gen: 2.3956987857818604 (2.3681417723171054) + | > loss_kl: 1.973552942276001 (1.876651331049497) + | > loss_feat: 7.443536758422852 (7.072383993961772) + | > loss_mel: 18.74546241760254 (17.81002519951492) + | > loss_duration: 1.494141936302185 (1.468227310258834) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.052391052246094 (30.595429623713258) + | > grad_norm_1: tensor(413.2967, device='cuda:0') (tensor(310.6278, device='cuda:0')) + | > current_lr_0: 0.00019353150421332454 + | > current_lr_1: 0.00019353150421332454 + | > step_time: 0.4546 (0.3826026173888661) + | > loader_time: 0.0074 (0.006063299100907123) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.562208414077759 (2.562208414077759) + | > loss_disc_real_0: 0.23629049956798553 (0.23629049956798553) + | > loss_disc_real_1: 0.27273011207580566 (0.27273011207580566) + | > loss_disc_real_2: 0.2610544264316559 (0.2610544264316559) + | > loss_disc_real_3: 0.2602002024650574 (0.2602002024650574) + | > loss_disc_real_4: 0.28844791650772095 (0.28844791650772095) + | > loss_disc_real_5: 0.24856334924697876 (0.24856334924697876) + | > loss_0: 2.562208414077759 (2.562208414077759) + | > loss_gen: 2.7334768772125244 (2.7334768772125244) + | > loss_kl: 2.6424460411071777 (2.6424460411071777) + | > loss_feat: 6.378000259399414 (6.378000259399414) + | > loss_mel: 16.871490478515625 (16.871490478515625) + | > loss_duration: 1.7199668884277344 (1.7199668884277344) + | > loss_1: 30.345380783081055 (30.345380783081055) + + --> STEP: 1 + | > loss_disc: 2.5057642459869385 (2.5057642459869385) + | > loss_disc_real_0: 0.2033928632736206 (0.2033928632736206) + | > loss_disc_real_1: 0.28987449407577515 (0.28987449407577515) + | > loss_disc_real_2: 0.2252788245677948 (0.2252788245677948) + | > loss_disc_real_3: 0.2209518402814865 (0.2209518402814865) + | > loss_disc_real_4: 0.25799739360809326 (0.25799739360809326) + | > loss_disc_real_5: 0.24411599338054657 (0.24411599338054657) + | > loss_0: 2.5057642459869385 (2.5057642459869385) + | > loss_gen: 2.59253191947937 (2.59253191947937) + | > loss_kl: 2.6713058948516846 (2.6713058948516846) + | > loss_feat: 7.651124477386475 (7.651124477386475) + | > loss_mel: 18.916160583496094 (18.916160583496094) + | > loss_duration: 1.736954689025879 (1.736954689025879) + | > loss_1: 33.568077087402344 (33.568077087402344) + + --> STEP: 2 + | > loss_disc: 2.519620180130005 (2.5126922130584717) + | > loss_disc_real_0: 0.25651705265045166 (0.22995495796203613) + | > loss_disc_real_1: 0.3000536859035492 (0.29496408998966217) + | > loss_disc_real_2: 0.21449299156665802 (0.2198859080672264) + | > loss_disc_real_3: 0.21646800637245178 (0.21870992332696915) + | > loss_disc_real_4: 0.26614901423454285 (0.26207320392131805) + | > loss_disc_real_5: 0.20392878353595734 (0.22402238845825195) + | > loss_0: 2.519620180130005 (2.5126922130584717) + | > loss_gen: 2.596848964691162 (2.594690442085266) + | > loss_kl: 2.2301838397979736 (2.450744867324829) + | > loss_feat: 6.774868488311768 (7.212996482849121) + | > loss_mel: 14.940485954284668 (16.92832326889038) + | > loss_duration: 1.7282848358154297 (1.7326197624206543) + | > loss_1: 28.270671844482422 (30.919374465942383) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035865306854248047 (+5.555152893066406e-05) + | > avg_loss_disc: 2.5126922130584717 (-0.03646421432495117) + | > avg_loss_disc_real_0: 0.22995495796203613 (+0.013254120945930481) + | > avg_loss_disc_real_1: 0.29496408998966217 (+0.06495214998722076) + | > avg_loss_disc_real_2: 0.2198859080672264 (-0.002921268343925476) + | > avg_loss_disc_real_3: 0.21870992332696915 (-0.035998426377773285) + | > avg_loss_disc_real_4: 0.26207320392131805 (+0.03937343508005142) + | > avg_loss_disc_real_5: 0.22402238845825195 (-0.0003757104277610779) + | > avg_loss_0: 2.5126922130584717 (-0.03646421432495117) + | > avg_loss_gen: 2.594690442085266 (+0.2986109256744385) + | > avg_loss_kl: 2.450744867324829 (+0.058983683586120605) + | > avg_loss_feat: 7.212996482849121 (+1.200221300125122) + | > avg_loss_mel: 16.92832326889038 (+0.018593311309814453) + | > avg_loss_duration: 1.7326197624206543 (+0.01814901828765869) + | > avg_loss_1: 30.919374465942383 (+1.594557762145996) + + + > EPOCH: 264/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:19:53)  + + --> TIME: 2025-04-19 03:19:55 -- STEP: 5/142 -- GLOBAL_STEP: 416775 + | > loss_disc: 2.518075942993164 (2.4569085121154783) + | > loss_disc_real_0: 0.1299608200788498 (0.11345085799694062) + | > loss_disc_real_1: 0.24099589884281158 (0.19795649647712707) + | > loss_disc_real_2: 0.20404979586601257 (0.2242271214723587) + | > loss_disc_real_3: 0.21532084047794342 (0.22135671377182006) + | > loss_disc_real_4: 0.21686378121376038 (0.2543948471546173) + | > loss_disc_real_5: 0.2354777455329895 (0.24296228289604188) + | > loss_0: 2.518075942993164 (2.4569085121154783) + | > grad_norm_0: tensor(6.8448, device='cuda:0') (tensor(9.5608, device='cuda:0')) + | > loss_gen: 2.3463973999023438 (2.363779973983765) + | > loss_kl: 1.8176965713500977 (1.8153793573379517) + | > loss_feat: 7.208646297454834 (7.4380451202392575) + | > loss_mel: 18.232311248779297 (17.890672302246095) + | > loss_duration: 1.4359676837921143 (1.4227309226989746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.041019439697266 (30.930607986450195) + | > grad_norm_1: tensor(189.8581, device='cuda:0') (tensor(210.8509, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.3274 (0.32378277778625486) + | > loader_time: 0.0046 (0.0044291019439697266) + + + --> TIME: 2025-04-19 03:20:04 -- STEP: 30/142 -- GLOBAL_STEP: 416800 + | > loss_disc: 2.420571804046631 (2.445944325129191) + | > loss_disc_real_0: 0.11734256893396378 (0.12041631415486335) + | > loss_disc_real_1: 0.19614803791046143 (0.2076807975769043) + | > loss_disc_real_2: 0.20046231150627136 (0.22237075467904407) + | > loss_disc_real_3: 0.22165878117084503 (0.2255840505162875) + | > loss_disc_real_4: 0.23205527663230896 (0.2320047338803609) + | > loss_disc_real_5: 0.23140929639339447 (0.2396782343586286) + | > loss_0: 2.420571804046631 (2.445944325129191) + | > grad_norm_0: tensor(5.1489, device='cuda:0') (tensor(8.6411, device='cuda:0')) + | > loss_gen: 2.2536470890045166 (2.351501194636027) + | > loss_kl: 1.8633638620376587 (1.8577227632204691) + | > loss_feat: 7.2639946937561035 (7.258546320597331) + | > loss_mel: 17.416305541992188 (17.935855293273924) + | > loss_duration: 1.4453864097595215 (1.4297987461090087) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.24269676208496 (30.83342399597168) + | > grad_norm_1: tensor(128.2218, device='cuda:0') (tensor(193.6409, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.3494 (0.3380348761876425) + | > loader_time: 0.0055 (0.0050376971562703455) + + + --> TIME: 2025-04-19 03:20:13 -- STEP: 55/142 -- GLOBAL_STEP: 416825 + | > loss_disc: 2.3589868545532227 (2.4440509709444913) + | > loss_disc_real_0: 0.07467593997716904 (0.12095673287456686) + | > loss_disc_real_1: 0.16787897050380707 (0.2046695113182068) + | > loss_disc_real_2: 0.218878373503685 (0.22425394572994925) + | > loss_disc_real_3: 0.2062469720840454 (0.22654702798886733) + | > loss_disc_real_4: 0.23071333765983582 (0.22880922664295544) + | > loss_disc_real_5: 0.2341213971376419 (0.23590281659906562) + | > loss_0: 2.3589868545532227 (2.4440509709444913) + | > grad_norm_0: tensor(9.3630, device='cuda:0') (tensor(8.6875, device='cuda:0')) + | > loss_gen: 2.329847574234009 (2.3637802254069937) + | > loss_kl: 1.9999028444290161 (1.8642638704993508) + | > loss_feat: 8.405777931213379 (7.2712761358781295) + | > loss_mel: 19.4279727935791 (17.8628773429177) + | > loss_duration: 1.488107442855835 (1.440416193008423) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.651607513427734 (30.802613587812942) + | > grad_norm_1: tensor(300.7869, device='cuda:0') (tensor(187.6427, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.3698 (0.34893260435624557) + | > loader_time: 0.0059 (0.005368328094482422) + + + --> TIME: 2025-04-19 03:20:23 -- STEP: 80/142 -- GLOBAL_STEP: 416850 + | > loss_disc: 2.5185651779174805 (2.453463053703308) + | > loss_disc_real_0: 0.11063513904809952 (0.1221462451852858) + | > loss_disc_real_1: 0.213826984167099 (0.20499320905655627) + | > loss_disc_real_2: 0.21645814180374146 (0.2257316041737795) + | > loss_disc_real_3: 0.23487426340579987 (0.22761656381189824) + | > loss_disc_real_4: 0.2402726411819458 (0.22917224001139402) + | > loss_disc_real_5: 0.20135053992271423 (0.23436196316033603) + | > loss_0: 2.5185651779174805 (2.453463053703308) + | > grad_norm_0: tensor(11.7502, device='cuda:0') (tensor(8.5166, device='cuda:0')) + | > loss_gen: 2.1215600967407227 (2.3535156935453414) + | > loss_kl: 1.8972734212875366 (1.86865191757679) + | > loss_feat: 6.193636417388916 (7.196268743276596) + | > loss_mel: 16.83262062072754 (17.83125336170196) + | > loss_duration: 1.4755167961120605 (1.451912985742092) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.520606994628906 (30.70160241127014) + | > grad_norm_1: tensor(63.5438, device='cuda:0') (tensor(179.8591, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.4089 (0.36074627041816715) + | > loader_time: 0.006 (0.005638101696968078) + + + --> TIME: 2025-04-19 03:20:34 -- STEP: 105/142 -- GLOBAL_STEP: 416875 + | > loss_disc: 2.5151596069335938 (2.458763526734852) + | > loss_disc_real_0: 0.09585794061422348 (0.12391199256692614) + | > loss_disc_real_1: 0.19107820093631744 (0.20513617367971515) + | > loss_disc_real_2: 0.18148958683013916 (0.22630444509642464) + | > loss_disc_real_3: 0.22421380877494812 (0.22787534722260067) + | > loss_disc_real_4: 0.21696588397026062 (0.22932920427549452) + | > loss_disc_real_5: 0.23380634188652039 (0.23482411617324467) + | > loss_0: 2.5151596069335938 (2.458763526734852) + | > grad_norm_0: tensor(9.9000, device='cuda:0') (tensor(8.1480, device='cuda:0')) + | > loss_gen: 2.1313953399658203 (2.3400695755368184) + | > loss_kl: 2.042710304260254 (1.8759074915023077) + | > loss_feat: 7.159140586853027 (7.121229367029099) + | > loss_mel: 17.065343856811523 (17.796944590977265) + | > loss_duration: 1.5231575965881348 (1.4629472562244954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.9217472076416 (30.597098068963913) + | > grad_norm_1: tensor(127.8012, device='cuda:0') (tensor(163.7495, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.4208 (0.3712288811093285) + | > loader_time: 0.0076 (0.005889517920357841) + + + --> TIME: 2025-04-19 03:20:45 -- STEP: 130/142 -- GLOBAL_STEP: 416900 + | > loss_disc: 2.421882390975952 (2.4645464456998387) + | > loss_disc_real_0: 0.11384644359350204 (0.1252377483707208) + | > loss_disc_real_1: 0.20783399045467377 (0.2057122960686684) + | > loss_disc_real_2: 0.20969459414482117 (0.22653924352847613) + | > loss_disc_real_3: 0.24707120656967163 (0.2281312277683845) + | > loss_disc_real_4: 0.21584008634090424 (0.22925172872268237) + | > loss_disc_real_5: 0.18891499936580658 (0.23426044984505726) + | > loss_0: 2.421882390975952 (2.4645464456998387) + | > grad_norm_0: tensor(4.6249, device='cuda:0') (tensor(7.9448, device='cuda:0')) + | > loss_gen: 2.3256473541259766 (2.3327199459075922) + | > loss_kl: 1.7969448566436768 (1.8812978139290444) + | > loss_feat: 7.423743724822998 (7.074033238337591) + | > loss_mel: 18.223533630371094 (17.736941095498896) + | > loss_duration: 1.4885966777801514 (1.4698998221984272) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.258464813232422 (30.494891621516302) + | > grad_norm_1: tensor(87.6892, device='cuda:0') (tensor(160.5311, device='cuda:0')) + | > current_lr_0: 0.00019350731277529787 + | > current_lr_1: 0.00019350731277529787 + | > step_time: 0.4443 (0.3827976318506094) + | > loader_time: 0.0076 (0.006153321266174317) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3794755935668945 (2.3794755935668945) + | > loss_disc_real_0: 0.11829876899719238 (0.11829876899719238) + | > loss_disc_real_1: 0.2009563446044922 (0.2009563446044922) + | > loss_disc_real_2: 0.25843971967697144 (0.25843971967697144) + | > loss_disc_real_3: 0.19833582639694214 (0.19833582639694214) + | > loss_disc_real_4: 0.2096046358346939 (0.2096046358346939) + | > loss_disc_real_5: 0.24899381399154663 (0.24899381399154663) + | > loss_0: 2.3794755935668945 (2.3794755935668945) + | > loss_gen: 2.441061019897461 (2.441061019897461) + | > loss_kl: 2.4873783588409424 (2.4873783588409424) + | > loss_feat: 7.180211544036865 (7.180211544036865) + | > loss_mel: 18.26772689819336 (18.26772689819336) + | > loss_duration: 1.696319580078125 (1.696319580078125) + | > loss_1: 32.07270050048828 (32.07270050048828) + + --> STEP: 1 + | > loss_disc: 2.399223804473877 (2.399223804473877) + | > loss_disc_real_0: 0.14276985824108124 (0.14276985824108124) + | > loss_disc_real_1: 0.24277277290821075 (0.24277277290821075) + | > loss_disc_real_2: 0.26566144824028015 (0.26566144824028015) + | > loss_disc_real_3: 0.1961715817451477 (0.1961715817451477) + | > loss_disc_real_4: 0.2405100166797638 (0.2405100166797638) + | > loss_disc_real_5: 0.19118942320346832 (0.19118942320346832) + | > loss_0: 2.399223804473877 (2.399223804473877) + | > loss_gen: 2.4429988861083984 (2.4429988861083984) + | > loss_kl: 2.64319109916687 (2.64319109916687) + | > loss_feat: 6.597352027893066 (6.597352027893066) + | > loss_mel: 18.638999938964844 (18.638999938964844) + | > loss_duration: 1.7458165884017944 (1.7458165884017944) + | > loss_1: 32.068359375 (32.068359375) + + --> STEP: 2 + | > loss_disc: 2.710890769958496 (2.5550572872161865) + | > loss_disc_real_0: 0.20453613996505737 (0.1736529991030693) + | > loss_disc_real_1: 0.20635052025318146 (0.2245616465806961) + | > loss_disc_real_2: 0.2724531292915344 (0.2690572887659073) + | > loss_disc_real_3: 0.2361268699169159 (0.2161492258310318) + | > loss_disc_real_4: 0.1950477659702301 (0.21777889132499695) + | > loss_disc_real_5: 0.29192352294921875 (0.24155647307634354) + | > loss_0: 2.710890769958496 (2.5550572872161865) + | > loss_gen: 2.090308904647827 (2.266653895378113) + | > loss_kl: 2.252103805541992 (2.447647452354431) + | > loss_feat: 6.663661956787109 (6.630506992340088) + | > loss_mel: 16.53615951538086 (17.58757972717285) + | > loss_duration: 1.700688123703003 (1.7232523560523987) + | > loss_1: 29.242921829223633 (30.655640602111816) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003780841827392578 (+0.00019431114196777344) + | > avg_loss_disc: 2.5550572872161865 (+0.042365074157714844) + | > avg_loss_disc_real_0: 0.1736529991030693 (-0.05630195885896683) + | > avg_loss_disc_real_1: 0.2245616465806961 (-0.07040244340896606) + | > avg_loss_disc_real_2: 0.2690572887659073 (+0.04917138069868088) + | > avg_loss_disc_real_3: 0.2161492258310318 (-0.0025606974959373474) + | > avg_loss_disc_real_4: 0.21777889132499695 (-0.044294312596321106) + | > avg_loss_disc_real_5: 0.24155647307634354 (+0.017534084618091583) + | > avg_loss_0: 2.5550572872161865 (+0.042365074157714844) + | > avg_loss_gen: 2.266653895378113 (-0.3280365467071533) + | > avg_loss_kl: 2.447647452354431 (-0.0030974149703979492) + | > avg_loss_feat: 6.630506992340088 (-0.5824894905090332) + | > avg_loss_mel: 17.58757972717285 (+0.6592564582824707) + | > avg_loss_duration: 1.7232523560523987 (-0.009367406368255615) + | > avg_loss_1: 30.655640602111816 (-0.2637338638305664) + + + > EPOCH: 265/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:20:52)  + + --> TIME: 2025-04-19 03:20:57 -- STEP: 13/142 -- GLOBAL_STEP: 416925 + | > loss_disc: 2.4989089965820312 (2.489035826462966) + | > loss_disc_real_0: 0.09763985872268677 (0.12443114014772269) + | > loss_disc_real_1: 0.18530908226966858 (0.20749820883457476) + | > loss_disc_real_2: 0.2095196694135666 (0.2326203745145064) + | > loss_disc_real_3: 0.23434582352638245 (0.23293457122949454) + | > loss_disc_real_4: 0.21290619671344757 (0.22644003194112045) + | > loss_disc_real_5: 0.22002992033958435 (0.2352192413348418) + | > loss_0: 2.4989089965820312 (2.489035826462966) + | > grad_norm_0: tensor(5.1850, device='cuda:0') (tensor(11.0479, device='cuda:0')) + | > loss_gen: 2.13667368888855 (2.335593682069045) + | > loss_kl: 1.7398228645324707 (1.8149133645571196) + | > loss_feat: 6.883227825164795 (7.057771462660569) + | > loss_mel: 17.92026138305664 (17.676955003004807) + | > loss_duration: 1.4296419620513916 (1.4250511114413922) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.10962677001953 (30.3102843944843) + | > grad_norm_1: tensor(99.2960, device='cuda:0') (tensor(203.7797, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.3271 (0.32670917877784145) + | > loader_time: 0.0049 (0.004598159056443434) + + + --> TIME: 2025-04-19 03:21:06 -- STEP: 38/142 -- GLOBAL_STEP: 416950 + | > loss_disc: 2.535693645477295 (2.440317153930664) + | > loss_disc_real_0: 0.09602969139814377 (0.11817454546689987) + | > loss_disc_real_1: 0.19990479946136475 (0.20512612281661283) + | > loss_disc_real_2: 0.25805869698524475 (0.22660316997452787) + | > loss_disc_real_3: 0.24854399263858795 (0.22815567725583127) + | > loss_disc_real_4: 0.231724351644516 (0.2245139158085773) + | > loss_disc_real_5: 0.2354332059621811 (0.2310935231415849) + | > loss_0: 2.535693645477295 (2.440317153930664) + | > grad_norm_0: tensor(13.1348, device='cuda:0') (tensor(10.3425, device='cuda:0')) + | > loss_gen: 2.2394509315490723 (2.3734880497581075) + | > loss_kl: 1.791853427886963 (1.820039080946069) + | > loss_feat: 6.591522216796875 (7.133566116031847) + | > loss_mel: 16.652252197265625 (17.661883454573776) + | > loss_duration: 1.4602866172790527 (1.4372299721366482) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.73536491394043 (30.426206588745117) + | > grad_norm_1: tensor(314.6453, device='cuda:0') (tensor(275.6854, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.3515 (0.34074873673288447) + | > loader_time: 0.0055 (0.005097615091424239) + + + --> TIME: 2025-04-19 03:21:15 -- STEP: 63/142 -- GLOBAL_STEP: 416975 + | > loss_disc: 2.427924633026123 (2.451864586936103) + | > loss_disc_real_0: 0.10973236709833145 (0.1181709376119432) + | > loss_disc_real_1: 0.21559807658195496 (0.20640648215536087) + | > loss_disc_real_2: 0.18871906399726868 (0.22683064142862955) + | > loss_disc_real_3: 0.20194509625434875 (0.22973232207790253) + | > loss_disc_real_4: 0.2566451132297516 (0.2264474637451626) + | > loss_disc_real_5: 0.26557672023773193 (0.23457222656598167) + | > loss_0: 2.427924633026123 (2.451864586936103) + | > grad_norm_0: tensor(4.2117, device='cuda:0') (tensor(10.0315, device='cuda:0')) + | > loss_gen: 2.3984756469726562 (2.3704821949913373) + | > loss_kl: 2.1009011268615723 (1.8531602110181535) + | > loss_feat: 7.53011417388916 (7.092900488111708) + | > loss_mel: 19.36293601989746 (17.71457510145883) + | > loss_duration: 1.5044219493865967 (1.4476538177520506) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.8968505859375 (30.478771906050426) + | > grad_norm_1: tensor(52.8936, device='cuda:0') (tensor(268.0763, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.3911 (0.35261820989941794) + | > loader_time: 0.0059 (0.005414429165068126) + + + --> TIME: 2025-04-19 03:21:25 -- STEP: 88/142 -- GLOBAL_STEP: 417000 + | > loss_disc: 2.555199146270752 (2.456704719500108) + | > loss_disc_real_0: 0.09953569620847702 (0.12041458741507748) + | > loss_disc_real_1: 0.1980658918619156 (0.20620468749918722) + | > loss_disc_real_2: 0.22875253856182098 (0.22780707190659913) + | > loss_disc_real_3: 0.24872425198554993 (0.22901890782470052) + | > loss_disc_real_4: 0.253735214471817 (0.22733208181505854) + | > loss_disc_real_5: 0.21938127279281616 (0.23360475152730942) + | > loss_0: 2.555199146270752 (2.456704719500108) + | > grad_norm_0: tensor(9.8993, device='cuda:0') (tensor(10.3976, device='cuda:0')) + | > loss_gen: 2.231048583984375 (2.364384794777088) + | > loss_kl: 1.96308434009552 (1.875442610545592) + | > loss_feat: 6.636995792388916 (7.0549281076951464) + | > loss_mel: 17.754169464111328 (17.648678714578796) + | > loss_duration: 1.5189449787139893 (1.4589548327706074) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.104244232177734 (30.402389092878863) + | > grad_norm_1: tensor(296.4822, device='cuda:0') (tensor(270.1133, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.3934 (0.36427425796335394) + | > loader_time: 0.0068 (0.0057251643050800684) + + + --> TIME: 2025-04-19 03:21:36 -- STEP: 113/142 -- GLOBAL_STEP: 417025 + | > loss_disc: 2.518143653869629 (2.460796022837141) + | > loss_disc_real_0: 0.11918538808822632 (0.12307325592346952) + | > loss_disc_real_1: 0.22232022881507874 (0.20550667514315749) + | > loss_disc_real_2: 0.23641765117645264 (0.22714031138251314) + | > loss_disc_real_3: 0.20964454114437103 (0.22936533549718097) + | > loss_disc_real_4: 0.1949775069952011 (0.2277879654306226) + | > loss_disc_real_5: 0.24363292753696442 (0.2341197267570327) + | > loss_0: 2.518143653869629 (2.460796022837141) + | > grad_norm_0: tensor(29.9873, device='cuda:0') (tensor(11.4397, device='cuda:0')) + | > loss_gen: 2.467733860015869 (2.361184282640438) + | > loss_kl: 1.9282629489898682 (1.8843162218026355) + | > loss_feat: 7.151541709899902 (7.056264370943593) + | > loss_mel: 18.447551727294922 (17.7208827533553) + | > loss_duration: 1.503422737121582 (1.4655045808944025) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.498512268066406 (30.48815224233982) + | > grad_norm_1: tensor(496.1713, device='cuda:0') (tensor(287.6005, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.4259 (0.3748571239741503) + | > loader_time: 0.0073 (0.006006008755844252) + + + --> TIME: 2025-04-19 03:21:47 -- STEP: 138/142 -- GLOBAL_STEP: 417050 + | > loss_disc: 2.4549660682678223 (2.4720823229223057) + | > loss_disc_real_0: 0.10780811309814453 (0.12515335503047792) + | > loss_disc_real_1: 0.18804636597633362 (0.2068616903346518) + | > loss_disc_real_2: 0.23466897010803223 (0.22775766339854914) + | > loss_disc_real_3: 0.22293466329574585 (0.23032385866711105) + | > loss_disc_real_4: 0.2143421620130539 (0.22845916244862735) + | > loss_disc_real_5: 0.24835285544395447 (0.2365452100833257) + | > loss_0: 2.4549660682678223 (2.4720823229223057) + | > grad_norm_0: tensor(20.9593, device='cuda:0') (tensor(12.8042, device='cuda:0')) + | > loss_gen: 2.297356128692627 (2.361268819242282) + | > loss_kl: 1.889096975326538 (1.8855834629224695) + | > loss_feat: 6.847286224365234 (6.98058197118234) + | > loss_mel: 18.516469955444336 (17.616706689198804) + | > loss_duration: 1.540283441543579 (1.4720559025156326) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.090492248535156 (30.316196911576867) + | > grad_norm_1: tensor(289.5877, device='cuda:0') (tensor(293.0254, device='cuda:0')) + | > current_lr_0: 0.00019348312436120093 + | > current_lr_1: 0.00019348312436120093 + | > step_time: 0.4518 (0.38670697246772656) + | > loader_time: 0.0081 (0.006257426911506101) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4591007232666016 (2.4591007232666016) + | > loss_disc_real_0: 0.1404084712266922 (0.1404084712266922) + | > loss_disc_real_1: 0.1998908370733261 (0.1998908370733261) + | > loss_disc_real_2: 0.2150183767080307 (0.2150183767080307) + | > loss_disc_real_3: 0.23530706763267517 (0.23530706763267517) + | > loss_disc_real_4: 0.1674278974533081 (0.1674278974533081) + | > loss_disc_real_5: 0.23465920984745026 (0.23465920984745026) + | > loss_0: 2.4591007232666016 (2.4591007232666016) + | > loss_gen: 2.212251901626587 (2.212251901626587) + | > loss_kl: 2.5675933361053467 (2.5675933361053467) + | > loss_feat: 8.028538703918457 (8.028538703918457) + | > loss_mel: 19.51131820678711 (19.51131820678711) + | > loss_duration: 1.7131835222244263 (1.7131835222244263) + | > loss_1: 34.03288650512695 (34.03288650512695) + + --> STEP: 1 + | > loss_disc: 2.502225399017334 (2.502225399017334) + | > loss_disc_real_0: 0.13093061745166779 (0.13093061745166779) + | > loss_disc_real_1: 0.22160004079341888 (0.22160004079341888) + | > loss_disc_real_2: 0.24916329979896545 (0.24916329979896545) + | > loss_disc_real_3: 0.23012128472328186 (0.23012128472328186) + | > loss_disc_real_4: 0.21598780155181885 (0.21598780155181885) + | > loss_disc_real_5: 0.19043242931365967 (0.19043242931365967) + | > loss_0: 2.502225399017334 (2.502225399017334) + | > loss_gen: 2.035759925842285 (2.035759925842285) + | > loss_kl: 2.5638926029205322 (2.5638926029205322) + | > loss_feat: 5.822513103485107 (5.822513103485107) + | > loss_mel: 16.899139404296875 (16.899139404296875) + | > loss_duration: 1.7365047931671143 (1.7365047931671143) + | > loss_1: 29.057809829711914 (29.057809829711914) + + --> STEP: 2 + | > loss_disc: 2.424506664276123 (2.4633660316467285) + | > loss_disc_real_0: 0.14719606935977936 (0.13906334340572357) + | > loss_disc_real_1: 0.21150776743888855 (0.21655390411615372) + | > loss_disc_real_2: 0.23836687207221985 (0.24376508593559265) + | > loss_disc_real_3: 0.2508547604084015 (0.24048802256584167) + | > loss_disc_real_4: 0.18811340630054474 (0.2020506039261818) + | > loss_disc_real_5: 0.17936250567436218 (0.18489746749401093) + | > loss_0: 2.424506664276123 (2.4633660316467285) + | > loss_gen: 2.1981515884399414 (2.1169557571411133) + | > loss_kl: 2.5560736656188965 (2.5599831342697144) + | > loss_feat: 7.009112358093262 (6.415812730789185) + | > loss_mel: 16.835285186767578 (16.867212295532227) + | > loss_duration: 1.7114641666412354 (1.7239844799041748) + | > loss_1: 30.31008529663086 (29.683947563171387) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003971576690673828 (+0.00019073486328125) + | > avg_loss_disc: 2.4633660316467285 (-0.09169125556945801) + | > avg_loss_disc_real_0: 0.13906334340572357 (-0.034589655697345734) + | > avg_loss_disc_real_1: 0.21655390411615372 (-0.008007742464542389) + | > avg_loss_disc_real_2: 0.24376508593559265 (-0.025292202830314636) + | > avg_loss_disc_real_3: 0.24048802256584167 (+0.024338796734809875) + | > avg_loss_disc_real_4: 0.2020506039261818 (-0.015728287398815155) + | > avg_loss_disc_real_5: 0.18489746749401093 (-0.05665900558233261) + | > avg_loss_0: 2.4633660316467285 (-0.09169125556945801) + | > avg_loss_gen: 2.1169557571411133 (-0.1496981382369995) + | > avg_loss_kl: 2.5599831342697144 (+0.1123356819152832) + | > avg_loss_feat: 6.415812730789185 (-0.21469426155090332) + | > avg_loss_mel: 16.867212295532227 (-0.720367431640625) + | > avg_loss_duration: 1.7239844799041748 (+0.000732123851776123) + | > avg_loss_1: 29.683947563171387 (-0.9716930389404297) + + + > EPOCH: 266/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:21:50)  + + --> TIME: 2025-04-19 03:21:58 -- STEP: 21/142 -- GLOBAL_STEP: 417075 + | > loss_disc: 2.497481107711792 (2.448071979341053) + | > loss_disc_real_0: 0.15461282432079315 (0.1207506504087221) + | > loss_disc_real_1: 0.2505914866924286 (0.21108293817156837) + | > loss_disc_real_2: 0.2467142939567566 (0.2235544068472726) + | > loss_disc_real_3: 0.2587447762489319 (0.22965047331083388) + | > loss_disc_real_4: 0.2521430253982544 (0.2298576377687) + | > loss_disc_real_5: 0.2797364592552185 (0.22814602653185526) + | > loss_0: 2.497481107711792 (2.448071979341053) + | > grad_norm_0: tensor(18.2346, device='cuda:0') (tensor(14.5755, device='cuda:0')) + | > loss_gen: 2.605339288711548 (2.3866768337431408) + | > loss_kl: 1.9136780500411987 (1.8027999684924172) + | > loss_feat: 7.2383012771606445 (7.130227520352318) + | > loss_mel: 18.942583084106445 (17.68445146651495) + | > loss_duration: 1.4375038146972656 (1.419430937085833) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.13740539550781 (30.42358652750651) + | > grad_norm_1: tensor(308.0465, device='cuda:0') (tensor(297.4070, device='cuda:0')) + | > current_lr_0: 0.0001934589389706558 + | > current_lr_1: 0.0001934589389706558 + | > step_time: 0.3492 (0.3354294981275286) + | > loader_time: 0.0069 (0.005011410940261115) + + + --> TIME: 2025-04-19 03:22:07 -- STEP: 46/142 -- GLOBAL_STEP: 417100 + | > loss_disc: 2.3645377159118652 (2.42946601950604) + | > loss_disc_real_0: 0.11513552814722061 (0.11733173854325128) + | > loss_disc_real_1: 0.18097208440303802 (0.2069128155708313) + | > loss_disc_real_2: 0.20562724769115448 (0.2230723357718924) + | > loss_disc_real_3: 0.21564766764640808 (0.22557871788740158) + | > loss_disc_real_4: 0.20969520509243011 (0.22632119901802228) + | > loss_disc_real_5: 0.1793801635503769 (0.2292437297494515) + | > loss_0: 2.3645377159118652 (2.42946601950604) + | > grad_norm_0: tensor(7.6696, device='cuda:0') (tensor(12.4845, device='cuda:0')) + | > loss_gen: 2.4757680892944336 (2.3718435712482617) + | > loss_kl: 1.8378885984420776 (1.8221253312152366) + | > loss_feat: 7.677875995635986 (7.143194944962211) + | > loss_mel: 17.810991287231445 (17.818206144415807) + | > loss_duration: 1.4343570470809937 (1.4321978169938792) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.236881256103516 (30.587567702583645) + | > grad_norm_1: tensor(119.7315, device='cuda:0') (tensor(294.8359, device='cuda:0')) + | > current_lr_0: 0.0001934589389706558 + | > current_lr_1: 0.0001934589389706558 + | > step_time: 0.3654 (0.34641440536664875) + | > loader_time: 0.0062 (0.005422524783922278) + + + --> TIME: 2025-04-19 03:22:17 -- STEP: 71/142 -- GLOBAL_STEP: 417125 + | > loss_disc: 2.4575459957122803 (2.4369422415612454) + | > loss_disc_real_0: 0.08677851408720016 (0.11923795403309272) + | > loss_disc_real_1: 0.1769436150789261 (0.2082639300907162) + | > loss_disc_real_2: 0.2592840790748596 (0.2247785273152338) + | > loss_disc_real_3: 0.26716551184654236 (0.22573521901184404) + | > loss_disc_real_4: 0.2524029612541199 (0.22783968041480426) + | > loss_disc_real_5: 0.21848206222057343 (0.22834963886670664) + | > loss_0: 2.4575459957122803 (2.4369422415612454) + | > grad_norm_0: tensor(11.2337, device='cuda:0') (tensor(10.9890, device='cuda:0')) + | > loss_gen: 2.434877395629883 (2.3740349151718783) + | > loss_kl: 1.833314299583435 (1.8450308131500028) + | > loss_feat: 7.4738264083862305 (7.144716585186166) + | > loss_mel: 18.010730743408203 (17.70571383624009) + | > loss_duration: 1.478421688079834 (1.4438444913273127) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.231170654296875 (30.513340479891067) + | > grad_norm_1: tensor(283.2725, device='cuda:0') (tensor(268.3756, device='cuda:0')) + | > current_lr_0: 0.0001934589389706558 + | > current_lr_1: 0.0001934589389706558 + | > step_time: 0.3852 (0.3577567993755072) + | > loader_time: 0.0063 (0.0057080564364581044) + + + --> TIME: 2025-04-19 03:22:27 -- STEP: 96/142 -- GLOBAL_STEP: 417150 + | > loss_disc: 2.4873828887939453 (2.443233442803223) + | > loss_disc_real_0: 0.10780701786279678 (0.121648284761856) + | > loss_disc_real_1: 0.16863122582435608 (0.2076473847652475) + | > loss_disc_real_2: 0.204290509223938 (0.2253261422738433) + | > loss_disc_real_3: 0.24295562505722046 (0.22648365935310721) + | > loss_disc_real_4: 0.23432162404060364 (0.22908857961495718) + | > loss_disc_real_5: 0.22481416165828705 (0.22774440764139095) + | > loss_0: 2.4873828887939453 (2.443233442803223) + | > grad_norm_0: tensor(14.0696, device='cuda:0') (tensor(10.7487, device='cuda:0')) + | > loss_gen: 2.3375964164733887 (2.368421514829) + | > loss_kl: 1.879921555519104 (1.8513777690629165) + | > loss_feat: 6.734025001525879 (7.157846122980118) + | > loss_mel: 17.7314395904541 (17.794777790705357) + | > loss_duration: 1.48240065574646 (1.4557993573447066) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.16538429260254 (30.628222445646923) + | > grad_norm_1: tensor(337.4160, device='cuda:0') (tensor(262.0298, device='cuda:0')) + | > current_lr_0: 0.0001934589389706558 + | > current_lr_1: 0.0001934589389706558 + | > step_time: 0.4113 (0.3691776593526204) + | > loader_time: 0.0065 (0.005930225054423013) + + + --> TIME: 2025-04-19 03:22:38 -- STEP: 121/142 -- GLOBAL_STEP: 417175 + | > loss_disc: 2.4441707134246826 (2.443216704140024) + | > loss_disc_real_0: 0.12136662006378174 (0.1227718517800008) + | > loss_disc_real_1: 0.18951530754566193 (0.20656005102248232) + | > loss_disc_real_2: 0.2300603985786438 (0.22401803584138225) + | > loss_disc_real_3: 0.23117832839488983 (0.2273365525174732) + | > loss_disc_real_4: 0.25063440203666687 (0.22876889277095638) + | > loss_disc_real_5: 0.26406556367874146 (0.227861812661502) + | > loss_0: 2.4441707134246826 (2.443216704140024) + | > grad_norm_0: tensor(8.5561, device='cuda:0') (tensor(11.2013, device='cuda:0')) + | > loss_gen: 2.507567882537842 (2.3717329088321404) + | > loss_kl: 1.917325496673584 (1.8599901652533162) + | > loss_feat: 7.244720458984375 (7.153236006902269) + | > loss_mel: 17.807605743408203 (17.77161039399706) + | > loss_duration: 1.526050329208374 (1.4631643019431877) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.003271102905273 (30.619733637029473) + | > grad_norm_1: tensor(178.5367, device='cuda:0') (tensor(269.4326, device='cuda:0')) + | > current_lr_0: 0.0001934589389706558 + | > current_lr_1: 0.0001934589389706558 + | > step_time: 0.4427 (0.38091928505700484) + | > loader_time: 0.0079 (0.006167403922593297) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5896360874176025 (2.5896360874176025) + | > loss_disc_real_0: 0.22623465955257416 (0.22623465955257416) + | > loss_disc_real_1: 0.24917779862880707 (0.24917779862880707) + | > loss_disc_real_2: 0.2278422862291336 (0.2278422862291336) + | > loss_disc_real_3: 0.22293926775455475 (0.22293926775455475) + | > loss_disc_real_4: 0.2040228396654129 (0.2040228396654129) + | > loss_disc_real_5: 0.21923212707042694 (0.21923212707042694) + | > loss_0: 2.5896360874176025 (2.5896360874176025) + | > loss_gen: 2.2991182804107666 (2.2991182804107666) + | > loss_kl: 2.4472849369049072 (2.4472849369049072) + | > loss_feat: 6.349332809448242 (6.349332809448242) + | > loss_mel: 17.869314193725586 (17.869314193725586) + | > loss_duration: 1.685086965560913 (1.685086965560913) + | > loss_1: 30.650136947631836 (30.650136947631836) + + --> STEP: 1 + | > loss_disc: 2.566538095474243 (2.566538095474243) + | > loss_disc_real_0: 0.18022353947162628 (0.18022353947162628) + | > loss_disc_real_1: 0.24664005637168884 (0.24664005637168884) + | > loss_disc_real_2: 0.20932798087596893 (0.20932798087596893) + | > loss_disc_real_3: 0.20292146503925323 (0.20292146503925323) + | > loss_disc_real_4: 0.21362031996250153 (0.21362031996250153) + | > loss_disc_real_5: 0.2738143801689148 (0.2738143801689148) + | > loss_0: 2.566538095474243 (2.566538095474243) + | > loss_gen: 2.3126044273376465 (2.3126044273376465) + | > loss_kl: 2.6570208072662354 (2.6570208072662354) + | > loss_feat: 6.679731845855713 (6.679731845855713) + | > loss_mel: 17.43992805480957 (17.43992805480957) + | > loss_duration: 1.7286072969436646 (1.7286072969436646) + | > loss_1: 30.81789207458496 (30.81789207458496) + + --> STEP: 2 + | > loss_disc: 2.4284451007843018 (2.4974915981292725) + | > loss_disc_real_0: 0.1914767026901245 (0.1858501210808754) + | > loss_disc_real_1: 0.21100157499313354 (0.2288208156824112) + | > loss_disc_real_2: 0.2367212325334549 (0.22302460670471191) + | > loss_disc_real_3: 0.2077234387397766 (0.20532245188951492) + | > loss_disc_real_4: 0.23796826601028442 (0.22579429298639297) + | > loss_disc_real_5: 0.22779609262943268 (0.25080523639917374) + | > loss_0: 2.4284451007843018 (2.4974915981292725) + | > loss_gen: 2.66176176071167 (2.487183094024658) + | > loss_kl: 2.3473970890045166 (2.502208948135376) + | > loss_feat: 7.896410942077637 (7.288071393966675) + | > loss_mel: 18.808753967285156 (18.124341011047363) + | > loss_duration: 1.69179368019104 (1.7102004885673523) + | > loss_1: 33.40612030029297 (32.112006187438965) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0040323734283447266 (+6.079673767089844e-05) + | > avg_loss_disc: 2.4974915981292725 (+0.034125566482543945) + | > avg_loss_disc_real_0: 0.1858501210808754 (+0.046786777675151825) + | > avg_loss_disc_real_1: 0.2288208156824112 (+0.012266911566257477) + | > avg_loss_disc_real_2: 0.22302460670471191 (-0.020740479230880737) + | > avg_loss_disc_real_3: 0.20532245188951492 (-0.03516557067632675) + | > avg_loss_disc_real_4: 0.22579429298639297 (+0.02374368906021118) + | > avg_loss_disc_real_5: 0.25080523639917374 (+0.06590776890516281) + | > avg_loss_0: 2.4974915981292725 (+0.034125566482543945) + | > avg_loss_gen: 2.487183094024658 (+0.3702273368835449) + | > avg_loss_kl: 2.502208948135376 (-0.05777418613433838) + | > avg_loss_feat: 7.288071393966675 (+0.8722586631774902) + | > avg_loss_mel: 18.124341011047363 (+1.2571287155151367) + | > avg_loss_duration: 1.7102004885673523 (-0.01378399133682251) + | > avg_loss_1: 32.112006187438965 (+2.428058624267578) + + + > EPOCH: 267/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:22:49)  + + --> TIME: 2025-04-19 03:22:51 -- STEP: 4/142 -- GLOBAL_STEP: 417200 + | > loss_disc: 2.3789215087890625 (2.4060808420181274) + | > loss_disc_real_0: 0.09501194953918457 (0.0949457660317421) + | > loss_disc_real_1: 0.1777191013097763 (0.20565742999315262) + | > loss_disc_real_2: 0.23896339535713196 (0.2291680984199047) + | > loss_disc_real_3: 0.22288024425506592 (0.21624846756458282) + | > loss_disc_real_4: 0.1901107281446457 (0.20810541883111) + | > loss_disc_real_5: 0.17989729344844818 (0.20947520807385445) + | > loss_0: 2.3789215087890625 (2.4060808420181274) + | > grad_norm_0: tensor(16.6003, device='cuda:0') (tensor(13.8274, device='cuda:0')) + | > loss_gen: 2.526689291000366 (2.3640089631080627) + | > loss_kl: 1.9782071113586426 (1.8225396871566772) + | > loss_feat: 7.152050018310547 (6.9468525648117065) + | > loss_mel: 18.329599380493164 (17.62065887451172) + | > loss_duration: 1.4274693727493286 (1.418601632118225) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.41401481628418 (30.172661304473877) + | > grad_norm_1: tensor(333.2315, device='cuda:0') (tensor(263.4058, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.3225 (0.3264484405517578) + | > loader_time: 0.0049 (0.004320740699768066) + + + --> TIME: 2025-04-19 03:23:00 -- STEP: 29/142 -- GLOBAL_STEP: 417225 + | > loss_disc: 2.4547009468078613 (2.4346600647630363) + | > loss_disc_real_0: 0.09453883022069931 (0.11355700703530476) + | > loss_disc_real_1: 0.2097264677286148 (0.2036268608323459) + | > loss_disc_real_2: 0.2244294285774231 (0.23015542431124325) + | > loss_disc_real_3: 0.2521395683288574 (0.22837212219320494) + | > loss_disc_real_4: 0.2585488259792328 (0.2259895935140807) + | > loss_disc_real_5: 0.23722440004348755 (0.22786978144070197) + | > loss_0: 2.4547009468078613 (2.4346600647630363) + | > grad_norm_0: tensor(12.7903, device='cuda:0') (tensor(11.5850, device='cuda:0')) + | > loss_gen: 2.4598982334136963 (2.3747643848945357) + | > loss_kl: 1.858168125152588 (1.8064999498169998) + | > loss_feat: 6.613611221313477 (7.052295701257114) + | > loss_mel: 17.00663948059082 (17.586018529431577) + | > loss_duration: 1.42949640750885 (1.4278807187902516) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.367813110351562 (30.24745928007981) + | > grad_norm_1: tensor(420.5472, device='cuda:0') (tensor(290.7995, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.3458 (0.3355611357195624) + | > loader_time: 0.0053 (0.004891206478250438) + + + --> TIME: 2025-04-19 03:23:09 -- STEP: 54/142 -- GLOBAL_STEP: 417250 + | > loss_disc: 2.4008982181549072 (2.42411031987932) + | > loss_disc_real_0: 0.13078640401363373 (0.11407915295826064) + | > loss_disc_real_1: 0.17347870767116547 (0.20294022780877571) + | > loss_disc_real_2: 0.1862010657787323 (0.22374142071715108) + | > loss_disc_real_3: 0.18854723870754242 (0.2270768185456594) + | > loss_disc_real_4: 0.17824916541576385 (0.2252642294323003) + | > loss_disc_real_5: 0.21987661719322205 (0.23019324767368812) + | > loss_0: 2.4008982181549072 (2.42411031987932) + | > grad_norm_0: tensor(13.8997, device='cuda:0') (tensor(11.1001, device='cuda:0')) + | > loss_gen: 2.406369209289551 (2.3709768145172685) + | > loss_kl: 1.796420931816101 (1.8204787219012226) + | > loss_feat: 7.8371148109436035 (7.1122097880752) + | > loss_mel: 18.616365432739258 (17.68032301796808) + | > loss_duration: 1.4785513877868652 (1.4380455326150965) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.13481903076172 (30.422033698470504) + | > grad_norm_1: tensor(187.6183, device='cuda:0') (tensor(283.3530, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.3687 (0.34752161856050845) + | > loader_time: 0.0063 (0.005309899648030599) + + + --> TIME: 2025-04-19 03:23:19 -- STEP: 79/142 -- GLOBAL_STEP: 417275 + | > loss_disc: 2.443855047225952 (2.431269657762744) + | > loss_disc_real_0: 0.09920956939458847 (0.1161022873807557) + | > loss_disc_real_1: 0.2405083030462265 (0.20498133686524403) + | > loss_disc_real_2: 0.24899084866046906 (0.2262969680979282) + | > loss_disc_real_3: 0.22524066269397736 (0.22775550016874) + | > loss_disc_real_4: 0.22542482614517212 (0.22449301371846017) + | > loss_disc_real_5: 0.22098660469055176 (0.23043917666507674) + | > loss_0: 2.443855047225952 (2.431269657762744) + | > grad_norm_0: tensor(5.7827, device='cuda:0') (tensor(10.5247, device='cuda:0')) + | > loss_gen: 2.3911919593811035 (2.3648253452928762) + | > loss_kl: 1.8568540811538696 (1.8493859435938582) + | > loss_feat: 6.611103057861328 (7.1031543212600905) + | > loss_mel: 17.973703384399414 (17.70792457725429) + | > loss_duration: 1.5069122314453125 (1.4507822235928305) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.339765548706055 (30.476072383832328) + | > grad_norm_1: tensor(87.2073, device='cuda:0') (tensor(267.8534, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.3857 (0.359552727469915) + | > loader_time: 0.0063 (0.005604061899305899) + + + --> TIME: 2025-04-19 03:23:29 -- STEP: 104/142 -- GLOBAL_STEP: 417300 + | > loss_disc: 2.5616281032562256 (2.4352729068352614) + | > loss_disc_real_0: 0.13928277790546417 (0.11839750323158044) + | > loss_disc_real_1: 0.2270345538854599 (0.20507317781448364) + | > loss_disc_real_2: 0.20963150262832642 (0.2252227715574778) + | > loss_disc_real_3: 0.23338650166988373 (0.22830859084541982) + | > loss_disc_real_4: 0.23994630575180054 (0.2259964236559776) + | > loss_disc_real_5: 0.2846421003341675 (0.2316344307305721) + | > loss_0: 2.5616281032562256 (2.4352729068352614) + | > grad_norm_0: tensor(7.3989, device='cuda:0') (tensor(10.6662, device='cuda:0')) + | > loss_gen: 2.2318620681762695 (2.365933642937587) + | > loss_kl: 2.0492517948150635 (1.8641314334594286) + | > loss_feat: 6.043179512023926 (7.10267894084637) + | > loss_mel: 15.51435661315918 (17.728993039864772) + | > loss_duration: 1.508275032043457 (1.4609037775259748) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.346923828125 (30.522640815147987) + | > grad_norm_1: tensor(167.6830, device='cuda:0') (tensor(273.3647, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.4095 (0.3702343152119563) + | > loader_time: 0.0066 (0.005882584131681002) + + + --> TIME: 2025-04-19 03:23:40 -- STEP: 129/142 -- GLOBAL_STEP: 417325 + | > loss_disc: 2.5651841163635254 (2.4462499710940566) + | > loss_disc_real_0: 0.12195383757352829 (0.12232556124759275) + | > loss_disc_real_1: 0.1902119666337967 (0.20577238407707954) + | > loss_disc_real_2: 0.17804771661758423 (0.22523869008056877) + | > loss_disc_real_3: 0.19088739156723022 (0.228818002712819) + | > loss_disc_real_4: 0.23392631113529205 (0.22724753664445507) + | > loss_disc_real_5: 0.20516517758369446 (0.23169871263725814) + | > loss_0: 2.5651841163635254 (2.4462499710940566) + | > grad_norm_0: tensor(19.5509, device='cuda:0') (tensor(11.4736, device='cuda:0')) + | > loss_gen: 2.093963146209717 (2.3572006077729455) + | > loss_kl: 2.015623092651367 (1.8709201877431352) + | > loss_feat: 7.232386112213135 (7.034994188205216) + | > loss_mel: 18.131351470947266 (17.677911462709893) + | > loss_duration: 1.4893059730529785 (1.4680548961772473) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.962629318237305 (30.409081289010455) + | > grad_norm_1: tensor(476.0611, device='cuda:0') (tensor(277.7599, device='cuda:0')) + | > current_lr_0: 0.00019343475660328445 + | > current_lr_1: 0.00019343475660328445 + | > step_time: 0.4434 (0.38187790286633394) + | > loader_time: 0.0075 (0.006161673124446426) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5371551513671875 (2.5371551513671875) + | > loss_disc_real_0: 0.14825789630413055 (0.14825789630413055) + | > loss_disc_real_1: 0.2832804024219513 (0.2832804024219513) + | > loss_disc_real_2: 0.2272508591413498 (0.2272508591413498) + | > loss_disc_real_3: 0.22335369884967804 (0.22335369884967804) + | > loss_disc_real_4: 0.2636493146419525 (0.2636493146419525) + | > loss_disc_real_5: 0.23417292535305023 (0.23417292535305023) + | > loss_0: 2.5371551513671875 (2.5371551513671875) + | > loss_gen: 2.2706212997436523 (2.2706212997436523) + | > loss_kl: 2.3874332904815674 (2.3874332904815674) + | > loss_feat: 5.769123554229736 (5.769123554229736) + | > loss_mel: 15.90147876739502 (15.90147876739502) + | > loss_duration: 1.6941897869110107 (1.6941897869110107) + | > loss_1: 28.02284812927246 (28.02284812927246) + + --> STEP: 1 + | > loss_disc: 2.532442092895508 (2.532442092895508) + | > loss_disc_real_0: 0.11972309648990631 (0.11972309648990631) + | > loss_disc_real_1: 0.34506770968437195 (0.34506770968437195) + | > loss_disc_real_2: 0.2440073937177658 (0.2440073937177658) + | > loss_disc_real_3: 0.21687506139278412 (0.21687506139278412) + | > loss_disc_real_4: 0.21340489387512207 (0.21340489387512207) + | > loss_disc_real_5: 0.23051249980926514 (0.23051249980926514) + | > loss_0: 2.532442092895508 (2.532442092895508) + | > loss_gen: 2.3711671829223633 (2.3711671829223633) + | > loss_kl: 2.5548949241638184 (2.5548949241638184) + | > loss_feat: 6.384706020355225 (6.384706020355225) + | > loss_mel: 16.954925537109375 (16.954925537109375) + | > loss_duration: 1.7394781112670898 (1.7394781112670898) + | > loss_1: 30.005172729492188 (30.005172729492188) + + --> STEP: 2 + | > loss_disc: 2.3691749572753906 (2.450808525085449) + | > loss_disc_real_0: 0.1466279923915863 (0.1331755444407463) + | > loss_disc_real_1: 0.2923787534236908 (0.31872323155403137) + | > loss_disc_real_2: 0.2039966583251953 (0.22400202602148056) + | > loss_disc_real_3: 0.2251235991716385 (0.2209993302822113) + | > loss_disc_real_4: 0.23909316956996918 (0.22624903172254562) + | > loss_disc_real_5: 0.2264782190322876 (0.22849535942077637) + | > loss_0: 2.3691749572753906 (2.450808525085449) + | > loss_gen: 2.636239528656006 (2.5037033557891846) + | > loss_kl: 2.1213979721069336 (2.338146448135376) + | > loss_feat: 8.007060050964355 (7.19588303565979) + | > loss_mel: 19.003387451171875 (17.979156494140625) + | > loss_duration: 1.6942803859710693 (1.7168792486190796) + | > loss_1: 33.462364196777344 (31.733768463134766) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038957595825195312 (-0.0001366138458251953) + | > avg_loss_disc: 2.450808525085449 (-0.04668307304382324) + | > avg_loss_disc_real_0: 0.1331755444407463 (-0.05267457664012909) + | > avg_loss_disc_real_1: 0.31872323155403137 (+0.08990241587162018) + | > avg_loss_disc_real_2: 0.22400202602148056 (+0.0009774193167686462) + | > avg_loss_disc_real_3: 0.2209993302822113 (+0.01567687839269638) + | > avg_loss_disc_real_4: 0.22624903172254562 (+0.0004547387361526489) + | > avg_loss_disc_real_5: 0.22849535942077637 (-0.02230987697839737) + | > avg_loss_0: 2.450808525085449 (-0.04668307304382324) + | > avg_loss_gen: 2.5037033557891846 (+0.016520261764526367) + | > avg_loss_kl: 2.338146448135376 (-0.1640625) + | > avg_loss_feat: 7.19588303565979 (-0.09218835830688477) + | > avg_loss_mel: 17.979156494140625 (-0.14518451690673828) + | > avg_loss_duration: 1.7168792486190796 (+0.006678760051727295) + | > avg_loss_1: 31.733768463134766 (-0.3782377243041992) + + + > EPOCH: 268/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:23:47)  + + --> TIME: 2025-04-19 03:23:52 -- STEP: 12/142 -- GLOBAL_STEP: 417350 + | > loss_disc: 2.475041389465332 (2.4561722675959268) + | > loss_disc_real_0: 0.1705264300107956 (0.12414263437191646) + | > loss_disc_real_1: 0.2103448361158371 (0.20438454921046892) + | > loss_disc_real_2: 0.21260865032672882 (0.2320856253306071) + | > loss_disc_real_3: 0.20208507776260376 (0.22620250408848128) + | > loss_disc_real_4: 0.21327871084213257 (0.23813061540325484) + | > loss_disc_real_5: 0.22471341490745544 (0.23222482576966286) + | > loss_0: 2.475041389465332 (2.4561722675959268) + | > grad_norm_0: tensor(11.1956, device='cuda:0') (tensor(9.3457, device='cuda:0')) + | > loss_gen: 2.3252713680267334 (2.385968585809072) + | > loss_kl: 1.7111274003982544 (1.7984190781911213) + | > loss_feat: 6.797487735748291 (7.135194460550944) + | > loss_mel: 17.42769432067871 (17.7307235399882) + | > loss_duration: 1.4254765510559082 (1.4174046615759532) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.687057495117188 (30.46771017710368) + | > grad_norm_1: tensor(264.8730, device='cuda:0') (tensor(249.5988, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.3294 (0.32641830046971637) + | > loader_time: 0.0052 (0.004751662413279216) + + + --> TIME: 2025-04-19 03:24:01 -- STEP: 37/142 -- GLOBAL_STEP: 417375 + | > loss_disc: 2.3700602054595947 (2.4199262954093315) + | > loss_disc_real_0: 0.13519641757011414 (0.11813535299655553) + | > loss_disc_real_1: 0.206581249833107 (0.20489936986485044) + | > loss_disc_real_2: 0.23868529498577118 (0.2264040329971829) + | > loss_disc_real_3: 0.20492850244045258 (0.22193052116278056) + | > loss_disc_real_4: 0.21177439391613007 (0.22504811512457357) + | > loss_disc_real_5: 0.23454758524894714 (0.22464690015122696) + | > loss_0: 2.3700602054595947 (2.4199262954093315) + | > grad_norm_0: tensor(5.1495, device='cuda:0') (tensor(7.4464, device='cuda:0')) + | > loss_gen: 2.5286664962768555 (2.389430522918701) + | > loss_kl: 1.7970515489578247 (1.8216432043024011) + | > loss_feat: 7.490705966949463 (7.271692714175662) + | > loss_mel: 18.105710983276367 (17.809974103360563) + | > loss_duration: 1.453059196472168 (1.4319230221413277) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.375194549560547 (30.7246635024612) + | > grad_norm_1: tensor(170.4655, device='cuda:0') (tensor(171.6457, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.3881 (0.34569552782419566) + | > loader_time: 0.0059 (0.005204838675421638) + + + --> TIME: 2025-04-19 03:24:11 -- STEP: 62/142 -- GLOBAL_STEP: 417400 + | > loss_disc: 2.5504415035247803 (2.448691583448842) + | > loss_disc_real_0: 0.10113973915576935 (0.12443771189258944) + | > loss_disc_real_1: 0.23013558983802795 (0.2084785589287358) + | > loss_disc_real_2: 0.24743667244911194 (0.22674373056619399) + | > loss_disc_real_3: 0.2145070880651474 (0.22516221769394412) + | > loss_disc_real_4: 0.202184796333313 (0.22942506786315672) + | > loss_disc_real_5: 0.24368198215961456 (0.22659235856225413) + | > loss_0: 2.5504415035247803 (2.448691583448842) + | > grad_norm_0: tensor(15.7747, device='cuda:0') (tensor(7.5465, device='cuda:0')) + | > loss_gen: 2.1318681240081787 (2.3589460196033603) + | > loss_kl: 1.9068254232406616 (1.8277385081014326) + | > loss_feat: 6.612507343292236 (7.183488899661649) + | > loss_mel: 17.166851043701172 (17.792112319700184) + | > loss_duration: 1.5028895139694214 (1.4436469789474242) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.320941925048828 (30.605932727936775) + | > grad_norm_1: tensor(101.0679, device='cuda:0') (tensor(161.4039, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.3974 (0.3625642676507273) + | > loader_time: 0.0065 (0.005567019985568139) + + + --> TIME: 2025-04-19 03:24:21 -- STEP: 87/142 -- GLOBAL_STEP: 417425 + | > loss_disc: 2.562894344329834 (2.45332321079298) + | > loss_disc_real_0: 0.12394392490386963 (0.12604349663202788) + | > loss_disc_real_1: 0.24913230538368225 (0.20828972962395897) + | > loss_disc_real_2: 0.24180522561073303 (0.22637331896814808) + | > loss_disc_real_3: 0.24629749357700348 (0.22688229741721316) + | > loss_disc_real_4: 0.2737652659416199 (0.23050116721926064) + | > loss_disc_real_5: 0.24236451089382172 (0.22946575934859528) + | > loss_0: 2.562894344329834 (2.45332321079298) + | > grad_norm_0: tensor(6.2863, device='cuda:0') (tensor(7.6624, device='cuda:0')) + | > loss_gen: 2.2216637134552 (2.3601667058878926) + | > loss_kl: 1.8881103992462158 (1.8443394652728378) + | > loss_feat: 6.421332836151123 (7.143462340037028) + | > loss_mel: 17.740703582763672 (17.839714795693588) + | > loss_duration: 1.501712679862976 (1.455134239690057) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.77352523803711 (30.64281759590938) + | > grad_norm_1: tensor(40.4539, device='cuda:0') (tensor(146.6519, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.4155 (0.37510512615072317) + | > loader_time: 0.0069 (0.005848722896356693) + + + --> TIME: 2025-04-19 03:24:32 -- STEP: 112/142 -- GLOBAL_STEP: 417450 + | > loss_disc: 2.427542209625244 (2.463942121182168) + | > loss_disc_real_0: 0.13156186044216156 (0.12765351084194002) + | > loss_disc_real_1: 0.20598450303077698 (0.21045347702290332) + | > loss_disc_real_2: 0.21620643138885498 (0.22653526732964174) + | > loss_disc_real_3: 0.20329484343528748 (0.22683230907257115) + | > loss_disc_real_4: 0.2253798395395279 (0.22951089311391115) + | > loss_disc_real_5: 0.23631392419338226 (0.22949065893356288) + | > loss_0: 2.427542209625244 (2.463942121182168) + | > grad_norm_0: tensor(8.3730, device='cuda:0') (tensor(7.6179, device='cuda:0')) + | > loss_gen: 2.2309494018554688 (2.344798537237304) + | > loss_kl: 1.923670768737793 (1.8572388429726874) + | > loss_feat: 6.4137864112854 (7.066155812570027) + | > loss_mel: 16.231164932250977 (17.767517728464956) + | > loss_duration: 1.460712194442749 (1.4618336377399308) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.260284423828125 (30.49754456111363) + | > grad_norm_1: tensor(167.8647, device='cuda:0') (tensor(144.7421, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.4268 (0.38522409541266306) + | > loader_time: 0.0071 (0.006064583148275103) + + + --> TIME: 2025-04-19 03:24:43 -- STEP: 137/142 -- GLOBAL_STEP: 417475 + | > loss_disc: 2.4282875061035156 (2.466121036640918) + | > loss_disc_real_0: 0.11378736793994904 (0.12894887716448208) + | > loss_disc_real_1: 0.19137375056743622 (0.20952057740549102) + | > loss_disc_real_2: 0.2222587615251541 (0.22627489377547355) + | > loss_disc_real_3: 0.18961553275585175 (0.22756992595909287) + | > loss_disc_real_4: 0.20390506088733673 (0.22934204655407117) + | > loss_disc_real_5: 0.21588648855686188 (0.22993376807574809) + | > loss_0: 2.4282875061035156 (2.466121036640918) + | > grad_norm_0: tensor(5.5561, device='cuda:0') (tensor(7.8989, device='cuda:0')) + | > loss_gen: 2.3851163387298584 (2.3400348224779117) + | > loss_kl: 1.9300965070724487 (1.8623622986522035) + | > loss_feat: 7.081252574920654 (7.045556357307156) + | > loss_mel: 17.805530548095703 (17.718554378426003) + | > loss_duration: 1.4951183795928955 (1.4688144904853653) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.697114944458008 (30.435322350829185) + | > grad_norm_1: tensor(94.2691, device='cuda:0') (tensor(148.5316, device='cuda:0')) + | > current_lr_0: 0.00019341057725870902 + | > current_lr_1: 0.00019341057725870902 + | > step_time: 0.4523 (0.3951227108057398) + | > loader_time: 0.0076 (0.006295522634130325) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5333974361419678 (2.5333974361419678) + | > loss_disc_real_0: 0.19568561017513275 (0.19568561017513275) + | > loss_disc_real_1: 0.24637240171432495 (0.24637240171432495) + | > loss_disc_real_2: 0.25800982117652893 (0.25800982117652893) + | > loss_disc_real_3: 0.24184192717075348 (0.24184192717075348) + | > loss_disc_real_4: 0.2280997782945633 (0.2280997782945633) + | > loss_disc_real_5: 0.2020566612482071 (0.2020566612482071) + | > loss_0: 2.5333974361419678 (2.5333974361419678) + | > loss_gen: 2.3865389823913574 (2.3865389823913574) + | > loss_kl: 2.389277696609497 (2.389277696609497) + | > loss_feat: 7.223570823669434 (7.223570823669434) + | > loss_mel: 17.46854019165039 (17.46854019165039) + | > loss_duration: 1.6848737001419067 (1.6848737001419067) + | > loss_1: 31.152801513671875 (31.152801513671875) + + --> STEP: 1 + | > loss_disc: 2.5155065059661865 (2.5155065059661865) + | > loss_disc_real_0: 0.21181564033031464 (0.21181564033031464) + | > loss_disc_real_1: 0.21505391597747803 (0.21505391597747803) + | > loss_disc_real_2: 0.2525964081287384 (0.2525964081287384) + | > loss_disc_real_3: 0.22388578951358795 (0.22388578951358795) + | > loss_disc_real_4: 0.21827079355716705 (0.21827079355716705) + | > loss_disc_real_5: 0.23663543164730072 (0.23663543164730072) + | > loss_0: 2.5155065059661865 (2.5155065059661865) + | > loss_gen: 2.3779640197753906 (2.3779640197753906) + | > loss_kl: 2.5929999351501465 (2.5929999351501465) + | > loss_feat: 6.71968936920166 (6.71968936920166) + | > loss_mel: 18.68119239807129 (18.68119239807129) + | > loss_duration: 1.7187997102737427 (1.7187997102737427) + | > loss_1: 32.09064483642578 (32.09064483642578) + + --> STEP: 2 + | > loss_disc: 2.6158206462860107 (2.5656635761260986) + | > loss_disc_real_0: 0.17232713103294373 (0.19207138568162918) + | > loss_disc_real_1: 0.2425767183303833 (0.22881531715393066) + | > loss_disc_real_2: 0.25337454676628113 (0.25298547744750977) + | > loss_disc_real_3: 0.24623627960681915 (0.23506103456020355) + | > loss_disc_real_4: 0.237046018242836 (0.22765840590000153) + | > loss_disc_real_5: 0.2164432853460312 (0.22653935849666595) + | > loss_0: 2.6158206462860107 (2.5656635761260986) + | > loss_gen: 2.331221580505371 (2.354592800140381) + | > loss_kl: 2.315584897994995 (2.454292416572571) + | > loss_feat: 7.819200038909912 (7.269444704055786) + | > loss_mel: 18.56233024597168 (18.621761322021484) + | > loss_duration: 1.6579201221466064 (1.6883599162101746) + | > loss_1: 32.686256408691406 (32.388450622558594) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034220218658447266 (-0.0004737377166748047) + | > avg_loss_disc: 2.5656635761260986 (+0.11485505104064941) + | > avg_loss_disc_real_0: 0.19207138568162918 (+0.058895841240882874) + | > avg_loss_disc_real_1: 0.22881531715393066 (-0.08990791440010071) + | > avg_loss_disc_real_2: 0.25298547744750977 (+0.028983451426029205) + | > avg_loss_disc_real_3: 0.23506103456020355 (+0.014061704277992249) + | > avg_loss_disc_real_4: 0.22765840590000153 (+0.001409374177455902) + | > avg_loss_disc_real_5: 0.22653935849666595 (-0.0019560009241104126) + | > avg_loss_0: 2.5656635761260986 (+0.11485505104064941) + | > avg_loss_gen: 2.354592800140381 (-0.1491105556488037) + | > avg_loss_kl: 2.454292416572571 (+0.11614596843719482) + | > avg_loss_feat: 7.269444704055786 (+0.0735616683959961) + | > avg_loss_mel: 18.621761322021484 (+0.6426048278808594) + | > avg_loss_duration: 1.6883599162101746 (-0.02851933240890503) + | > avg_loss_1: 32.388450622558594 (+0.6546821594238281) + + + > EPOCH: 269/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:24:47)  + + --> TIME: 2025-04-19 03:24:55 -- STEP: 20/142 -- GLOBAL_STEP: 417500 + | > loss_disc: 2.4898440837860107 (2.4368891596794127) + | > loss_disc_real_0: 0.18056605756282806 (0.11554716378450394) + | > loss_disc_real_1: 0.22851012647151947 (0.20646776184439658) + | > loss_disc_real_2: 0.2083037793636322 (0.2300212174654007) + | > loss_disc_real_3: 0.2228279560804367 (0.22689996436238288) + | > loss_disc_real_4: 0.20546592772006989 (0.22485261633992196) + | > loss_disc_real_5: 0.22885555028915405 (0.23049930185079576) + | > loss_0: 2.4898440837860107 (2.4368891596794127) + | > grad_norm_0: tensor(8.1105, device='cuda:0') (tensor(8.4524, device='cuda:0')) + | > loss_gen: 2.286759614944458 (2.399389553070068) + | > loss_kl: 1.8091288805007935 (1.8300856709480287) + | > loss_feat: 7.010490894317627 (7.387876415252686) + | > loss_mel: 16.403579711914062 (17.877742195129393) + | > loss_duration: 1.4752845764160156 (1.4262026309967042) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.985244750976562 (30.921296691894533) + | > grad_norm_1: tensor(235.6125, device='cuda:0') (tensor(218.8862, device='cuda:0')) + | > current_lr_0: 0.00019338640093655168 + | > current_lr_1: 0.00019338640093655168 + | > step_time: 0.3412 (0.3314194560050964) + | > loader_time: 0.0052 (0.004828107357025146) + + + --> TIME: 2025-04-19 03:25:04 -- STEP: 45/142 -- GLOBAL_STEP: 417525 + | > loss_disc: 2.482211112976074 (2.4494567500220397) + | > loss_disc_real_0: 0.15547001361846924 (0.11984406461318334) + | > loss_disc_real_1: 0.22226102650165558 (0.20577729377481674) + | > loss_disc_real_2: 0.2402675300836563 (0.2311123573117786) + | > loss_disc_real_3: 0.21964669227600098 (0.2285140292512046) + | > loss_disc_real_4: 0.2434547394514084 (0.22881220016214582) + | > loss_disc_real_5: 0.20630475878715515 (0.23081279231442345) + | > loss_0: 2.482211112976074 (2.4494567500220397) + | > grad_norm_0: tensor(5.6564, device='cuda:0') (tensor(7.7295, device='cuda:0')) + | > loss_gen: 2.328331708908081 (2.3661810715993252) + | > loss_kl: 1.9089293479919434 (1.8709414111243354) + | > loss_feat: 7.280698776245117 (7.257791042327881) + | > loss_mel: 18.169763565063477 (17.757350709703232) + | > loss_duration: 1.4817264080047607 (1.438414732615153) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.169448852539062 (30.69067908393012) + | > grad_norm_1: tensor(232.7035, device='cuda:0') (tensor(202.0939, device='cuda:0')) + | > current_lr_0: 0.00019338640093655168 + | > current_lr_1: 0.00019338640093655168 + | > step_time: 0.3658 (0.34400731722513844) + | > loader_time: 0.0056 (0.005186902152167426) + + + --> TIME: 2025-04-19 03:25:13 -- STEP: 70/142 -- GLOBAL_STEP: 417550 + | > loss_disc: 2.464583396911621 (2.454641389846801) + | > loss_disc_real_0: 0.09452249854803085 (0.11926514304109982) + | > loss_disc_real_1: 0.1912544220685959 (0.20713354583297458) + | > loss_disc_real_2: 0.2339525818824768 (0.22942527681589125) + | > loss_disc_real_3: 0.26008275151252747 (0.2282039614660399) + | > loss_disc_real_4: 0.2529166042804718 (0.22990984788962773) + | > loss_disc_real_5: 0.21334382891654968 (0.23125616290739603) + | > loss_0: 2.464583396911621 (2.454641389846801) + | > grad_norm_0: tensor(4.7105, device='cuda:0') (tensor(8.7784, device='cuda:0')) + | > loss_gen: 2.5420479774475098 (2.364267979349409) + | > loss_kl: 1.914804220199585 (1.883093866280147) + | > loss_feat: 7.157686710357666 (7.172512790134975) + | > loss_mel: 17.684537887573242 (17.71023817062378) + | > loss_duration: 1.4917235374450684 (1.4492969053132194) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.790800094604492 (30.57940995352609) + | > grad_norm_1: tensor(189.6395, device='cuda:0') (tensor(227.2862, device='cuda:0')) + | > current_lr_0: 0.00019338640093655168 + | > current_lr_1: 0.00019338640093655168 + | > step_time: 0.3889 (0.3557403155735561) + | > loader_time: 0.0059 (0.005487755366734096) + + + --> TIME: 2025-04-19 03:25:23 -- STEP: 95/142 -- GLOBAL_STEP: 417575 + | > loss_disc: 2.3579583168029785 (2.4556675860756325) + | > loss_disc_real_0: 0.07416184991598129 (0.12210054091717068) + | > loss_disc_real_1: 0.1907862424850464 (0.2060528121496502) + | > loss_disc_real_2: 0.21219776570796967 (0.22791455184158527) + | > loss_disc_real_3: 0.2215045541524887 (0.2280564629717877) + | > loss_disc_real_4: 0.2242514044046402 (0.230339042293398) + | > loss_disc_real_5: 0.1972547173500061 (0.23051647936042988) + | > loss_0: 2.3579583168029785 (2.4556675860756325) + | > grad_norm_0: tensor(8.4227, device='cuda:0') (tensor(11.6191, device='cuda:0')) + | > loss_gen: 2.306793212890625 (2.368580667596114) + | > loss_kl: 1.8802844285964966 (1.885879931951824) + | > loss_feat: 6.760098934173584 (7.110530170641447) + | > loss_mel: 16.78213119506836 (17.70679501985249) + | > loss_duration: 1.5067613124847412 (1.459301787928531) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.23607063293457 (30.531087734824734) + | > grad_norm_1: tensor(363.0433, device='cuda:0') (tensor(260.4898, device='cuda:0')) + | > current_lr_0: 0.00019338640093655168 + | > current_lr_1: 0.00019338640093655168 + | > step_time: 0.4051 (0.36681228185954856) + | > loader_time: 0.0065 (0.005719890092548571) + + + --> TIME: 2025-04-19 03:25:34 -- STEP: 120/142 -- GLOBAL_STEP: 417600 + | > loss_disc: 2.4098708629608154 (2.462010308106741) + | > loss_disc_real_0: 0.10256019979715347 (0.12318466032544771) + | > loss_disc_real_1: 0.20884695649147034 (0.20696329797307653) + | > loss_disc_real_2: 0.2245042473077774 (0.2284152928739786) + | > loss_disc_real_3: 0.22559882700443268 (0.22989997006952761) + | > loss_disc_real_4: 0.23847661912441254 (0.23031468379000822) + | > loss_disc_real_5: 0.2159537822008133 (0.23175483122467994) + | > loss_0: 2.4098708629608154 (2.462010308106741) + | > grad_norm_0: tensor(7.9127, device='cuda:0') (tensor(11.9800, device='cuda:0')) + | > loss_gen: 2.1908621788024902 (2.3693226406971606) + | > loss_kl: 1.9129846096038818 (1.8891604542732239) + | > loss_feat: 6.061352729797363 (7.071260420481364) + | > loss_mel: 16.101593017578125 (17.637937347094212) + | > loss_duration: 1.5044498443603516 (1.4660342901945114) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.771244049072266 (30.43371532758077) + | > grad_norm_1: tensor(266.5670, device='cuda:0') (tensor(273.4245, device='cuda:0')) + | > current_lr_0: 0.00019338640093655168 + | > current_lr_1: 0.00019338640093655168 + | > step_time: 0.4328 (0.37784354488054916) + | > loader_time: 0.007 (0.005978679656982422) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5763914585113525 (2.5763914585113525) + | > loss_disc_real_0: 0.11036484688520432 (0.11036484688520432) + | > loss_disc_real_1: 0.3127892017364502 (0.3127892017364502) + | > loss_disc_real_2: 0.26100820302963257 (0.26100820302963257) + | > loss_disc_real_3: 0.27093949913978577 (0.27093949913978577) + | > loss_disc_real_4: 0.25890660285949707 (0.25890660285949707) + | > loss_disc_real_5: 0.2987075448036194 (0.2987075448036194) + | > loss_0: 2.5763914585113525 (2.5763914585113525) + | > loss_gen: 2.4264283180236816 (2.4264283180236816) + | > loss_kl: 2.525970935821533 (2.525970935821533) + | > loss_feat: 6.690978050231934 (6.690978050231934) + | > loss_mel: 17.467927932739258 (17.467927932739258) + | > loss_duration: 1.7047683000564575 (1.7047683000564575) + | > loss_1: 30.81607437133789 (30.81607437133789) + + --> STEP: 1 + | > loss_disc: 2.542145013809204 (2.542145013809204) + | > loss_disc_real_0: 0.17396853864192963 (0.17396853864192963) + | > loss_disc_real_1: 0.23778565227985382 (0.23778565227985382) + | > loss_disc_real_2: 0.25474369525909424 (0.25474369525909424) + | > loss_disc_real_3: 0.2585495710372925 (0.2585495710372925) + | > loss_disc_real_4: 0.3045385777950287 (0.3045385777950287) + | > loss_disc_real_5: 0.29898908734321594 (0.29898908734321594) + | > loss_0: 2.542145013809204 (2.542145013809204) + | > loss_gen: 2.4716172218322754 (2.4716172218322754) + | > loss_kl: 2.4994003772735596 (2.4994003772735596) + | > loss_feat: 5.959202766418457 (5.959202766418457) + | > loss_mel: 18.00844955444336 (18.00844955444336) + | > loss_duration: 1.7554835081100464 (1.7554835081100464) + | > loss_1: 30.69415283203125 (30.69415283203125) + + --> STEP: 2 + | > loss_disc: 2.4709243774414062 (2.506534695625305) + | > loss_disc_real_0: 0.16188067197799683 (0.16792460530996323) + | > loss_disc_real_1: 0.23530389368534088 (0.23654477298259735) + | > loss_disc_real_2: 0.2635374963283539 (0.25914059579372406) + | > loss_disc_real_3: 0.2573137581348419 (0.2579316645860672) + | > loss_disc_real_4: 0.25301456451416016 (0.2787765711545944) + | > loss_disc_real_5: 0.249967560172081 (0.27447832375764847) + | > loss_0: 2.4709243774414062 (2.506534695625305) + | > loss_gen: 2.4966447353363037 (2.4841309785842896) + | > loss_kl: 2.17889666557312 (2.33914852142334) + | > loss_feat: 6.96441650390625 (6.4618096351623535) + | > loss_mel: 18.93585205078125 (18.472150802612305) + | > loss_duration: 1.713454008102417 (1.7344687581062317) + | > loss_1: 32.28926467895508 (31.491708755493164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037946701049804688 (+0.0003726482391357422) + | > avg_loss_disc: 2.506534695625305 (-0.05912888050079346) + | > avg_loss_disc_real_0: 0.16792460530996323 (-0.024146780371665955) + | > avg_loss_disc_real_1: 0.23654477298259735 (+0.007729455828666687) + | > avg_loss_disc_real_2: 0.25914059579372406 (+0.006155118346214294) + | > avg_loss_disc_real_3: 0.2579316645860672 (+0.022870630025863647) + | > avg_loss_disc_real_4: 0.2787765711545944 (+0.051118165254592896) + | > avg_loss_disc_real_5: 0.27447832375764847 (+0.04793896526098251) + | > avg_loss_0: 2.506534695625305 (-0.05912888050079346) + | > avg_loss_gen: 2.4841309785842896 (+0.1295381784439087) + | > avg_loss_kl: 2.33914852142334 (-0.11514389514923096) + | > avg_loss_feat: 6.4618096351623535 (-0.8076350688934326) + | > avg_loss_mel: 18.472150802612305 (-0.1496105194091797) + | > avg_loss_duration: 1.7344687581062317 (+0.04610884189605713) + | > avg_loss_1: 31.491708755493164 (-0.8967418670654297) + + + > EPOCH: 270/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:25:46)  + + --> TIME: 2025-04-19 03:25:47 -- STEP: 3/142 -- GLOBAL_STEP: 417625 + | > loss_disc: 2.4625821113586426 (2.389442523320516) + | > loss_disc_real_0: 0.12737613916397095 (0.1421865920225779) + | > loss_disc_real_1: 0.17433688044548035 (0.20858202874660492) + | > loss_disc_real_2: 0.24508284032344818 (0.2475423961877823) + | > loss_disc_real_3: 0.24770577251911163 (0.23184776802857718) + | > loss_disc_real_4: 0.21575669944286346 (0.23173796633879343) + | > loss_disc_real_5: 0.21859075129032135 (0.20734102527300516) + | > loss_0: 2.4625821113586426 (2.389442523320516) + | > grad_norm_0: tensor(20.7712, device='cuda:0') (tensor(16.9230, device='cuda:0')) + | > loss_gen: 2.1060361862182617 (2.2849347591400146) + | > loss_kl: 1.834417700767517 (1.7919589281082153) + | > loss_feat: 6.61230993270874 (6.976254781087239) + | > loss_mel: 17.807220458984375 (17.102013270060223) + | > loss_duration: 1.4220942258834839 (1.4205973148345947) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.782079696655273 (29.575759887695312) + | > grad_norm_1: tensor(324.7028, device='cuda:0') (tensor(285.5599, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.3231 (0.3250574270884196) + | > loader_time: 0.0038 (0.0038553078969319663) + + + --> TIME: 2025-04-19 03:25:56 -- STEP: 28/142 -- GLOBAL_STEP: 417650 + | > loss_disc: 2.460420846939087 (2.489913140024458) + | > loss_disc_real_0: 0.1290316879749298 (0.1273667228275112) + | > loss_disc_real_1: 0.23862123489379883 (0.20740244697247232) + | > loss_disc_real_2: 0.23301832377910614 (0.23229206779173442) + | > loss_disc_real_3: 0.26046594977378845 (0.23083162946360453) + | > loss_disc_real_4: 0.24111974239349365 (0.23268217114465578) + | > loss_disc_real_5: 0.2472543567419052 (0.23295128984110697) + | > loss_0: 2.460420846939087 (2.489913140024458) + | > grad_norm_0: tensor(8.8401, device='cuda:0') (tensor(14.6412, device='cuda:0')) + | > loss_gen: 2.441915512084961 (2.3333335007940015) + | > loss_kl: 1.6554147005081177 (1.7949469430106026) + | > loss_feat: 6.872379779815674 (7.025635174342564) + | > loss_mel: 17.952011108398438 (17.554697070802963) + | > loss_duration: 1.4623805284500122 (1.4307477814810616) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.38410186767578 (30.139360972813197) + | > grad_norm_1: tensor(169.5402, device='cuda:0') (tensor(262.9534, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.3449 (0.3421846202441626) + | > loader_time: 0.0051 (0.004691047327859059) + + + --> TIME: 2025-04-19 03:26:05 -- STEP: 53/142 -- GLOBAL_STEP: 417675 + | > loss_disc: 2.3594298362731934 (2.463931124165372) + | > loss_disc_real_0: 0.13706336915493011 (0.12253295890284034) + | > loss_disc_real_1: 0.20807190239429474 (0.20487070252310555) + | > loss_disc_real_2: 0.22162359952926636 (0.22849974204909126) + | > loss_disc_real_3: 0.20351344347000122 (0.22807953858150626) + | > loss_disc_real_4: 0.19645702838897705 (0.2306293431880339) + | > loss_disc_real_5: 0.19664150476455688 (0.23359243487412074) + | > loss_0: 2.3594298362731934 (2.463931124165372) + | > grad_norm_0: tensor(18.5716, device='cuda:0') (tensor(12.4866, device='cuda:0')) + | > loss_gen: 2.461909294128418 (2.3530294580279647) + | > loss_kl: 1.8098410367965698 (1.8072421370812182) + | > loss_feat: 7.961664199829102 (7.117289318228668) + | > loss_mel: 17.984928131103516 (17.68360798313932) + | > loss_duration: 1.4424383640289307 (1.4387124551917023) + | > amp_scaler: 256.0 (289.81132075471703) + | > loss_1: 31.66077995300293 (30.399881722792138) + | > grad_norm_1: 0 (tensor(255.9845, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.3587 (0.3516824335422156) + | > loader_time: 0.0055 (0.005029295975307248) + + + --> TIME: 2025-04-19 03:26:15 -- STEP: 78/142 -- GLOBAL_STEP: 417700 + | > loss_disc: 2.4526262283325195 (2.461209309406769) + | > loss_disc_real_0: 0.09730825573205948 (0.12335842757079846) + | > loss_disc_real_1: 0.21167002618312836 (0.20627922411912528) + | > loss_disc_real_2: 0.23837009072303772 (0.22836341651586387) + | > loss_disc_real_3: 0.20961566269397736 (0.22783228449332407) + | > loss_disc_real_4: 0.251292884349823 (0.2298610407190445) + | > loss_disc_real_5: 0.2065720409154892 (0.23064782604193076) + | > loss_0: 2.4526262283325195 (2.461209309406769) + | > grad_norm_0: tensor(4.9194, device='cuda:0') (tensor(11.9014, device='cuda:0')) + | > loss_gen: 2.295506000518799 (2.348948732400551) + | > loss_kl: 1.9049558639526367 (1.8360476707800841) + | > loss_feat: 6.841247081756592 (7.073706657458574) + | > loss_mel: 17.19883155822754 (17.605665047963452) + | > loss_duration: 1.5116112232208252 (1.450892604314364) + | > amp_scaler: 256.0 (278.97435897435906) + | > loss_1: 29.752151489257812 (30.315260960505558) + | > grad_norm_1: tensor(155.4634, device='cuda:0') (tensor(251.1957, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.3927 (0.3643859105232434) + | > loader_time: 0.006 (0.005411725777846114) + + + --> TIME: 2025-04-19 03:26:26 -- STEP: 103/142 -- GLOBAL_STEP: 417725 + | > loss_disc: 2.4343230724334717 (2.4736212549857712) + | > loss_disc_real_0: 0.10159899294376373 (0.12549378335909936) + | > loss_disc_real_1: 0.20512762665748596 (0.20702532932017614) + | > loss_disc_real_2: 0.20356886088848114 (0.228530503013759) + | > loss_disc_real_3: 0.2393905073404312 (0.22955176812931172) + | > loss_disc_real_4: 0.18043763935565948 (0.23079680557389862) + | > loss_disc_real_5: 0.24576954543590546 (0.23395114167801384) + | > loss_0: 2.4343230724334717 (2.4736212549857712) + | > grad_norm_0: tensor(4.7457, device='cuda:0') (tensor(10.6266, device='cuda:0')) + | > loss_gen: 2.3198115825653076 (2.3362490899354493) + | > loss_kl: 1.9375560283660889 (1.8493772015988248) + | > loss_feat: 7.701008319854736 (6.986861330791585) + | > loss_mel: 18.8397216796875 (17.564696061958387) + | > loss_duration: 1.525939702987671 (1.4610464561332774) + | > amp_scaler: 256.0 (273.3980582524272) + | > loss_1: 32.32403564453125 (30.198230280459505) + | > grad_norm_1: tensor(67.4307, device='cuda:0') (tensor(217.8065, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.4091 (0.3738717384708738) + | > loader_time: 0.0078 (0.005714793807094535) + + + --> TIME: 2025-04-19 03:26:37 -- STEP: 128/142 -- GLOBAL_STEP: 417750 + | > loss_disc: 2.5309600830078125 (2.473897347226738) + | > loss_disc_real_0: 0.11098722368478775 (0.12717446094029577) + | > loss_disc_real_1: 0.21746936440467834 (0.20693298894912004) + | > loss_disc_real_2: 0.20439669489860535 (0.2283955740276724) + | > loss_disc_real_3: 0.21830351650714874 (0.22881547373253852) + | > loss_disc_real_4: 0.2215525358915329 (0.23006081604398787) + | > loss_disc_real_5: 0.2814372777938843 (0.2328788215527311) + | > loss_0: 2.5309600830078125 (2.473897347226738) + | > grad_norm_0: tensor(17.9158, device='cuda:0') (tensor(10.6180, device='cuda:0')) + | > loss_gen: 2.150090217590332 (2.3326210640370832) + | > loss_kl: 1.8206400871276855 (1.8582321479916573) + | > loss_feat: 6.141700744628906 (6.961365655064583) + | > loss_mel: 17.503326416015625 (17.56935124844313) + | > loss_duration: 1.5065984725952148 (1.4677459849044678) + | > amp_scaler: 256.0 (270.0000000000001) + | > loss_1: 29.122356414794922 (30.189316272735596) + | > grad_norm_1: tensor(344.0627, device='cuda:0') (tensor(218.5420, device='cuda:0')) + | > current_lr_0: 0.00019336222763643461 + | > current_lr_1: 0.00019336222763643461 + | > step_time: 0.4368 (0.38458229787647724) + | > loader_time: 0.0076 (0.006000246852636336) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.635615825653076 (2.635615825653076) + | > loss_disc_real_0: 0.13974817097187042 (0.13974817097187042) + | > loss_disc_real_1: 0.2833562195301056 (0.2833562195301056) + | > loss_disc_real_2: 0.22033703327178955 (0.22033703327178955) + | > loss_disc_real_3: 0.18974508345127106 (0.18974508345127106) + | > loss_disc_real_4: 0.23509933054447174 (0.23509933054447174) + | > loss_disc_real_5: 0.2307317852973938 (0.2307317852973938) + | > loss_0: 2.635615825653076 (2.635615825653076) + | > loss_gen: 2.045088052749634 (2.045088052749634) + | > loss_kl: 2.260483980178833 (2.260483980178833) + | > loss_feat: 6.153660774230957 (6.153660774230957) + | > loss_mel: 17.845937728881836 (17.845937728881836) + | > loss_duration: 1.6861047744750977 (1.6861047744750977) + | > loss_1: 29.991275787353516 (29.991275787353516) + + --> STEP: 1 + | > loss_disc: 2.3961377143859863 (2.3961377143859863) + | > loss_disc_real_0: 0.10260245949029922 (0.10260245949029922) + | > loss_disc_real_1: 0.20828771591186523 (0.20828771591186523) + | > loss_disc_real_2: 0.21853378415107727 (0.21853378415107727) + | > loss_disc_real_3: 0.2015484869480133 (0.2015484869480133) + | > loss_disc_real_4: 0.21459273993968964 (0.21459273993968964) + | > loss_disc_real_5: 0.23129044473171234 (0.23129044473171234) + | > loss_0: 2.3961377143859863 (2.3961377143859863) + | > loss_gen: 2.2368364334106445 (2.2368364334106445) + | > loss_kl: 2.585655689239502 (2.585655689239502) + | > loss_feat: 7.443113803863525 (7.443113803863525) + | > loss_mel: 18.9915771484375 (18.9915771484375) + | > loss_duration: 1.7425751686096191 (1.7425751686096191) + | > loss_1: 32.999759674072266 (32.999759674072266) + + --> STEP: 2 + | > loss_disc: 2.4307262897491455 (2.413432002067566) + | > loss_disc_real_0: 0.11465907096862793 (0.10863076522946358) + | > loss_disc_real_1: 0.2619335651397705 (0.23511064052581787) + | > loss_disc_real_2: 0.23275630176067352 (0.2256450429558754) + | > loss_disc_real_3: 0.19186681509017944 (0.19670765101909637) + | > loss_disc_real_4: 0.2713659703731537 (0.24297935515642166) + | > loss_disc_real_5: 0.20840716361999512 (0.21984880417585373) + | > loss_0: 2.4307262897491455 (2.413432002067566) + | > loss_gen: 2.2930705547332764 (2.2649534940719604) + | > loss_kl: 2.2277379035949707 (2.4066967964172363) + | > loss_feat: 6.984941005706787 (7.214027404785156) + | > loss_mel: 18.463396072387695 (18.727486610412598) + | > loss_duration: 1.6684155464172363 (1.7054953575134277) + | > loss_1: 31.63755989074707 (32.31865978240967) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003819584846496582 (+2.491474151611328e-05) + | > avg_loss_disc: 2.413432002067566 (-0.09310269355773926) + | > avg_loss_disc_real_0: 0.10863076522946358 (-0.05929384008049965) + | > avg_loss_disc_real_1: 0.23511064052581787 (-0.00143413245677948) + | > avg_loss_disc_real_2: 0.2256450429558754 (-0.03349555283784866) + | > avg_loss_disc_real_3: 0.19670765101909637 (-0.061224013566970825) + | > avg_loss_disc_real_4: 0.24297935515642166 (-0.03579721599817276) + | > avg_loss_disc_real_5: 0.21984880417585373 (-0.05462951958179474) + | > avg_loss_0: 2.413432002067566 (-0.09310269355773926) + | > avg_loss_gen: 2.2649534940719604 (-0.2191774845123291) + | > avg_loss_kl: 2.4066967964172363 (+0.06754827499389648) + | > avg_loss_feat: 7.214027404785156 (+0.7522177696228027) + | > avg_loss_mel: 18.727486610412598 (+0.25533580780029297) + | > avg_loss_duration: 1.7054953575134277 (-0.028973400592803955) + | > avg_loss_1: 32.31865978240967 (+0.8269510269165039) + + + > EPOCH: 271/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:26:44)  + + --> TIME: 2025-04-19 03:26:49 -- STEP: 11/142 -- GLOBAL_STEP: 417775 + | > loss_disc: 2.5004186630249023 (2.454652504487471) + | > loss_disc_real_0: 0.09930155426263809 (0.11739417097785255) + | > loss_disc_real_1: 0.16442549228668213 (0.20749485628171402) + | > loss_disc_real_2: 0.21830926835536957 (0.2269463986158371) + | > loss_disc_real_3: 0.23292192816734314 (0.23531097851016305) + | > loss_disc_real_4: 0.25695255398750305 (0.23204410482536664) + | > loss_disc_real_5: 0.27266186475753784 (0.23952105776830154) + | > loss_0: 2.5004186630249023 (2.454652504487471) + | > grad_norm_0: tensor(14.6436, device='cuda:0') (tensor(12.0027, device='cuda:0')) + | > loss_gen: 2.404733657836914 (2.449619314887307) + | > loss_kl: 1.7025312185287476 (1.7722272331064397) + | > loss_feat: 7.18550968170166 (7.5647431720386855) + | > loss_mel: 17.432092666625977 (17.969542936845258) + | > loss_duration: 1.4078178405761719 (1.4149777130647139) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.1326847076416 (31.17110997980291) + | > grad_norm_1: tensor(456.6131, device='cuda:0') (tensor(372.9338, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.3293 (0.32529293407093396) + | > loader_time: 0.005 (0.00449503551829945) + + + --> TIME: 2025-04-19 03:26:58 -- STEP: 36/142 -- GLOBAL_STEP: 417800 + | > loss_disc: 2.476762294769287 (2.4375777045885716) + | > loss_disc_real_0: 0.11075442284345627 (0.12157129775732756) + | > loss_disc_real_1: 0.22814424335956573 (0.20482169464230537) + | > loss_disc_real_2: 0.2241639494895935 (0.22269941866397858) + | > loss_disc_real_3: 0.2125304639339447 (0.22645320163832772) + | > loss_disc_real_4: 0.2257879078388214 (0.22936917179160649) + | > loss_disc_real_5: 0.23114196956157684 (0.22784804552793503) + | > loss_0: 2.476762294769287 (2.4375777045885716) + | > grad_norm_0: tensor(17.6272, device='cuda:0') (tensor(16.1022, device='cuda:0')) + | > loss_gen: 2.253603458404541 (2.4255025850401983) + | > loss_kl: 1.800189733505249 (1.7879296110735998) + | > loss_feat: 6.99533748626709 (7.400211625629002) + | > loss_mel: 17.39328384399414 (17.72032197316487) + | > loss_duration: 1.4254703521728516 (1.4277014566792383) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.86788558959961 (30.761667251586914) + | > grad_norm_1: tensor(168.1407, device='cuda:0') (tensor(376.0225, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.3532 (0.339175177945031) + | > loader_time: 0.0052 (0.005101323127746582) + + + --> TIME: 2025-04-19 03:27:07 -- STEP: 61/142 -- GLOBAL_STEP: 417825 + | > loss_disc: 2.5435965061187744 (2.436262173730818) + | > loss_disc_real_0: 0.11438155174255371 (0.12205504375647326) + | > loss_disc_real_1: 0.23067744076251984 (0.20511555622835628) + | > loss_disc_real_2: 0.21155531704425812 (0.22202173021973157) + | > loss_disc_real_3: 0.2167547047138214 (0.2249771981942849) + | > loss_disc_real_4: 0.21717891097068787 (0.2270743180005277) + | > loss_disc_real_5: 0.23055286705493927 (0.22906343345759345) + | > loss_0: 2.5435965061187744 (2.436262173730818) + | > grad_norm_0: tensor(6.5068, device='cuda:0') (tensor(13.1965, device='cuda:0')) + | > loss_gen: 2.25618052482605 (2.3870519810035575) + | > loss_kl: 1.9123674631118774 (1.812276517758604) + | > loss_feat: 6.476071834564209 (7.1982841179019115) + | > loss_mel: 16.21219253540039 (17.682163066551333) + | > loss_duration: 1.4477888345718384 (1.4397396455045606) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.30459976196289 (30.51951536585073) + | > grad_norm_1: tensor(87.9228, device='cuda:0') (tensor(309.5338, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.3827 (0.3507574933474181) + | > loader_time: 0.0062 (0.005496779426199491) + + + --> TIME: 2025-04-19 03:27:17 -- STEP: 86/142 -- GLOBAL_STEP: 417850 + | > loss_disc: 2.5374882221221924 (2.456972050112347) + | > loss_disc_real_0: 0.08990645408630371 (0.12452069249783837) + | > loss_disc_real_1: 0.18119573593139648 (0.20743818425161895) + | > loss_disc_real_2: 0.22106443345546722 (0.22481003906144653) + | > loss_disc_real_3: 0.21161207556724548 (0.2271625297360642) + | > loss_disc_real_4: 0.2322617471218109 (0.22943234824857048) + | > loss_disc_real_5: 0.24392767250537872 (0.23130151594794074) + | > loss_0: 2.5374882221221924 (2.456972050112347) + | > grad_norm_0: tensor(6.6413, device='cuda:0') (tensor(11.4640, device='cuda:0')) + | > loss_gen: 2.196396827697754 (2.365685443545496) + | > loss_kl: 1.807969331741333 (1.8283781575602154) + | > loss_feat: 6.764509677886963 (7.141702080881873) + | > loss_mel: 16.543588638305664 (17.686191381410104) + | > loss_duration: 1.4964890480041504 (1.4536793647810469) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.8089542388916 (30.475636437881825) + | > grad_norm_1: tensor(209.9529, device='cuda:0') (tensor(259.3622, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.3977 (0.3626796811126) + | > loader_time: 0.0067 (0.005752871202868083) + + + --> TIME: 2025-04-19 03:27:27 -- STEP: 111/142 -- GLOBAL_STEP: 417875 + | > loss_disc: 2.4126479625701904 (2.4592210816907465) + | > loss_disc_real_0: 0.12917838990688324 (0.1259413633730498) + | > loss_disc_real_1: 0.21226292848587036 (0.20793146754170325) + | > loss_disc_real_2: 0.20223784446716309 (0.2253769775768658) + | > loss_disc_real_3: 0.24722827970981598 (0.2277678464715545) + | > loss_disc_real_4: 0.24524427950382233 (0.2289302679332527) + | > loss_disc_real_5: 0.2193434238433838 (0.23038238542036968) + | > loss_0: 2.4126479625701904 (2.4592210816907465) + | > grad_norm_0: tensor(6.3410, device='cuda:0') (tensor(11.6597, device='cuda:0')) + | > loss_gen: 2.408714771270752 (2.3603329057092055) + | > loss_kl: 1.805587649345398 (1.8467036485671997) + | > loss_feat: 6.982087135314941 (7.118039706805805) + | > loss_mel: 18.143930435180664 (17.721346090505783) + | > loss_duration: 1.5222406387329102 (1.4627047132801363) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.862560272216797 (30.50912704124107) + | > grad_norm_1: tensor(245.9061, device='cuda:0') (tensor(264.0003, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.4255 (0.3732001394838901) + | > loader_time: 0.0068 (0.005966242369230802) + + + --> TIME: 2025-04-19 03:27:39 -- STEP: 136/142 -- GLOBAL_STEP: 417900 + | > loss_disc: 2.382883071899414 (2.4650641662233026) + | > loss_disc_real_0: 0.11300110816955566 (0.126863798537456) + | > loss_disc_real_1: 0.2031233310699463 (0.2078584752538625) + | > loss_disc_real_2: 0.21720758080482483 (0.226602557937012) + | > loss_disc_real_3: 0.23402832448482513 (0.2274436904665302) + | > loss_disc_real_4: 0.207808718085289 (0.2294457390466157) + | > loss_disc_real_5: 0.21676364541053772 (0.23157906280282667) + | > loss_0: 2.382883071899414 (2.4650641662233026) + | > grad_norm_0: tensor(11.9141, device='cuda:0') (tensor(11.1508, device='cuda:0')) + | > loss_gen: 2.256133794784546 (2.354776789160335) + | > loss_kl: 1.7701928615570068 (1.8537131951135748) + | > loss_feat: 7.450502872467041 (7.114540527848636) + | > loss_mel: 17.830047607421875 (17.70606237299302) + | > loss_duration: 1.5086336135864258 (1.4687612942036459) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.815509796142578 (30.497854106566486) + | > grad_norm_1: tensor(286.2944, device='cuda:0') (tensor(247.3812, device='cuda:0')) + | > current_lr_0: 0.00019333805735798006 + | > current_lr_1: 0.00019333805735798006 + | > step_time: 0.4521 (0.38517037910573626) + | > loader_time: 0.0074 (0.006221329464631923) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4786694049835205 (2.4786694049835205) + | > loss_disc_real_0: 0.15463887155056 (0.15463887155056) + | > loss_disc_real_1: 0.17084810137748718 (0.17084810137748718) + | > loss_disc_real_2: 0.222536101937294 (0.222536101937294) + | > loss_disc_real_3: 0.2329777628183365 (0.2329777628183365) + | > loss_disc_real_4: 0.22666116058826447 (0.22666116058826447) + | > loss_disc_real_5: 0.257527619600296 (0.257527619600296) + | > loss_0: 2.4786694049835205 (2.4786694049835205) + | > loss_gen: 2.3050479888916016 (2.3050479888916016) + | > loss_kl: 2.3884661197662354 (2.3884661197662354) + | > loss_feat: 7.2436909675598145 (7.2436909675598145) + | > loss_mel: 19.049577713012695 (19.049577713012695) + | > loss_duration: 1.7176413536071777 (1.7176413536071777) + | > loss_1: 32.70442581176758 (32.70442581176758) + + --> STEP: 1 + | > loss_disc: 2.436037540435791 (2.436037540435791) + | > loss_disc_real_0: 0.12815363705158234 (0.12815363705158234) + | > loss_disc_real_1: 0.15404213964939117 (0.15404213964939117) + | > loss_disc_real_2: 0.2082403004169464 (0.2082403004169464) + | > loss_disc_real_3: 0.18865618109703064 (0.18865618109703064) + | > loss_disc_real_4: 0.23693981766700745 (0.23693981766700745) + | > loss_disc_real_5: 0.2221461683511734 (0.2221461683511734) + | > loss_0: 2.436037540435791 (2.436037540435791) + | > loss_gen: 2.140287160873413 (2.140287160873413) + | > loss_kl: 2.5961532592773438 (2.5961532592773438) + | > loss_feat: 6.187642574310303 (6.187642574310303) + | > loss_mel: 16.48897361755371 (16.48897361755371) + | > loss_duration: 1.7565714120864868 (1.7565714120864868) + | > loss_1: 29.169628143310547 (29.169628143310547) + + --> STEP: 2 + | > loss_disc: 2.340202808380127 (2.388120174407959) + | > loss_disc_real_0: 0.11174189299345016 (0.11994776502251625) + | > loss_disc_real_1: 0.1642414927482605 (0.15914181619882584) + | > loss_disc_real_2: 0.20052890479564667 (0.20438460260629654) + | > loss_disc_real_3: 0.19177298247814178 (0.1902145817875862) + | > loss_disc_real_4: 0.2107672095298767 (0.22385351359844208) + | > loss_disc_real_5: 0.21158166229724884 (0.21686391532421112) + | > loss_0: 2.340202808380127 (2.388120174407959) + | > loss_gen: 2.370177745819092 (2.2552324533462524) + | > loss_kl: 2.284731388092041 (2.4404423236846924) + | > loss_feat: 8.21102523803711 (7.199333906173706) + | > loss_mel: 18.31311798095703 (17.40104579925537) + | > loss_duration: 1.6988656520843506 (1.7277185320854187) + | > loss_1: 32.8779182434082 (31.023773193359375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003699779510498047 (-0.00011980533599853516) + | > avg_loss_disc: 2.388120174407959 (-0.025311827659606934) + | > avg_loss_disc_real_0: 0.11994776502251625 (+0.011316999793052673) + | > avg_loss_disc_real_1: 0.15914181619882584 (-0.07596882432699203) + | > avg_loss_disc_real_2: 0.20438460260629654 (-0.021260440349578857) + | > avg_loss_disc_real_3: 0.1902145817875862 (-0.006493069231510162) + | > avg_loss_disc_real_4: 0.22385351359844208 (-0.019125841557979584) + | > avg_loss_disc_real_5: 0.21686391532421112 (-0.0029848888516426086) + | > avg_loss_0: 2.388120174407959 (-0.025311827659606934) + | > avg_loss_gen: 2.2552324533462524 (-0.009721040725708008) + | > avg_loss_kl: 2.4404423236846924 (+0.033745527267456055) + | > avg_loss_feat: 7.199333906173706 (-0.014693498611450195) + | > avg_loss_mel: 17.40104579925537 (-1.3264408111572266) + | > avg_loss_duration: 1.7277185320854187 (+0.022223174571990967) + | > avg_loss_1: 31.023773193359375 (-1.294886589050293) + + + > EPOCH: 272/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:27:43)  + + --> TIME: 2025-04-19 03:27:50 -- STEP: 19/142 -- GLOBAL_STEP: 417925 + | > loss_disc: 2.456855058670044 (2.4758621642464087) + | > loss_disc_real_0: 0.1108148917555809 (0.11078228958343204) + | > loss_disc_real_1: 0.17534641921520233 (0.20801653595347153) + | > loss_disc_real_2: 0.24450881779193878 (0.2309616208076477) + | > loss_disc_real_3: 0.2344568818807602 (0.23218677702702975) + | > loss_disc_real_4: 0.2632349729537964 (0.23792008349769994) + | > loss_disc_real_5: 0.22687476873397827 (0.23658360933002673) + | > loss_0: 2.456855058670044 (2.4758621642464087) + | > grad_norm_0: tensor(12.3086, device='cuda:0') (tensor(11.8770, device='cuda:0')) + | > loss_gen: 2.4998669624328613 (2.4055531903317102) + | > loss_kl: 1.8507272005081177 (1.8432052072725797) + | > loss_feat: 7.590649604797363 (7.427422724272075) + | > loss_mel: 17.286088943481445 (17.97581532126979) + | > loss_duration: 1.4257256984710693 (1.4240705527757342) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.653059005737305 (31.076066870438424) + | > grad_norm_1: tensor(583.2303, device='cuda:0') (tensor(328.5590, device='cuda:0')) + | > current_lr_0: 0.0001933138901008103 + | > current_lr_1: 0.0001933138901008103 + | > step_time: 0.3379 (0.3302697884409051) + | > loader_time: 0.0055 (0.004755986364264237) + + + --> TIME: 2025-04-19 03:27:59 -- STEP: 44/142 -- GLOBAL_STEP: 417950 + | > loss_disc: 2.379272937774658 (2.4651578122919258) + | > loss_disc_real_0: 0.10234246402978897 (0.1157280895859003) + | > loss_disc_real_1: 0.18781037628650665 (0.20590268075466156) + | > loss_disc_real_2: 0.2011491060256958 (0.2280755757608197) + | > loss_disc_real_3: 0.19802841544151306 (0.22754316201264208) + | > loss_disc_real_4: 0.2229665070772171 (0.23164785721085288) + | > loss_disc_real_5: 0.21007448434829712 (0.23534591055729173) + | > loss_0: 2.379272937774658 (2.4651578122919258) + | > grad_norm_0: tensor(12.3195, device='cuda:0') (tensor(10.9694, device='cuda:0')) + | > loss_gen: 2.5243966579437256 (2.3800752596421684) + | > loss_kl: 1.9333124160766602 (1.858406270092184) + | > loss_feat: 7.719639778137207 (7.250685713507912) + | > loss_mel: 18.554475784301758 (17.852837584235456) + | > loss_duration: 1.4854438304901123 (1.4370944960550827) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.21726989746094 (30.77909937771884) + | > grad_norm_1: tensor(378.2413, device='cuda:0') (tensor(291.3512, device='cuda:0')) + | > current_lr_0: 0.0001933138901008103 + | > current_lr_1: 0.0001933138901008103 + | > step_time: 0.3571 (0.34254373745484784) + | > loader_time: 0.0056 (0.005215861580588601) + + + --> TIME: 2025-04-19 03:28:09 -- STEP: 69/142 -- GLOBAL_STEP: 417975 + | > loss_disc: 2.4197118282318115 (2.4627792558808257) + | > loss_disc_real_0: 0.11713812500238419 (0.11903178843035214) + | > loss_disc_real_1: 0.19635885953903198 (0.20733931444693302) + | > loss_disc_real_2: 0.2136726826429367 (0.22784608149010202) + | > loss_disc_real_3: 0.2262144535779953 (0.2278186799823374) + | > loss_disc_real_4: 0.23708714544773102 (0.22945527278858682) + | > loss_disc_real_5: 0.22910092771053314 (0.23501140915829202) + | > loss_0: 2.4197118282318115 (2.4627792558808257) + | > grad_norm_0: tensor(5.8758, device='cuda:0') (tensor(9.8296, device='cuda:0')) + | > loss_gen: 2.4146058559417725 (2.3655915985936713) + | > loss_kl: 1.8666222095489502 (1.866435498431109) + | > loss_feat: 6.619959831237793 (7.196402010710343) + | > loss_mel: 17.5430908203125 (17.77001791415007) + | > loss_duration: 1.5227631330490112 (1.4490337440933005) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.967041015625 (30.64748081262561) + | > grad_norm_1: tensor(198.9916, device='cuda:0') (tensor(270.5592, device='cuda:0')) + | > current_lr_0: 0.0001933138901008103 + | > current_lr_1: 0.0001933138901008103 + | > step_time: 0.3938 (0.3549712671749834) + | > loader_time: 0.0057 (0.00551594513050024) + + + --> TIME: 2025-04-19 03:28:19 -- STEP: 94/142 -- GLOBAL_STEP: 418000 + | > loss_disc: 2.486403703689575 (2.4645390155467584) + | > loss_disc_real_0: 0.17221134901046753 (0.12146333454454199) + | > loss_disc_real_1: 0.22756710648536682 (0.20662107730799537) + | > loss_disc_real_2: 0.19179846346378326 (0.22733965928250172) + | > loss_disc_real_3: 0.21353080868721008 (0.2284662235290446) + | > loss_disc_real_4: 0.21904906630516052 (0.22925088554620743) + | > loss_disc_real_5: 0.23586881160736084 (0.2352204514627761) + | > loss_0: 2.486403703689575 (2.4645390155467584) + | > grad_norm_0: tensor(13.1781, device='cuda:0') (tensor(10.0135, device='cuda:0')) + | > loss_gen: 2.3605945110321045 (2.355557913475849) + | > loss_kl: 1.8184338808059692 (1.8842628344576409) + | > loss_feat: 6.905663967132568 (7.128806575815728) + | > loss_mel: 18.445287704467773 (17.77464274142651) + | > loss_duration: 1.5098793506622314 (1.4596555321774585) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.039859771728516 (30.602925604962287) + | > grad_norm_1: tensor(484.5889, device='cuda:0') (tensor(263.4265, device='cuda:0')) + | > current_lr_0: 0.0001933138901008103 + | > current_lr_1: 0.0001933138901008103 + | > step_time: 0.3973 (0.36783210267411903) + | > loader_time: 0.0064 (0.0057872280161431495) + + + --> TIME: 2025-04-19 03:28:30 -- STEP: 119/142 -- GLOBAL_STEP: 418025 + | > loss_disc: 2.4468142986297607 (2.4650108473641543) + | > loss_disc_real_0: 0.0781441479921341 (0.12196411417812861) + | > loss_disc_real_1: 0.18259258568286896 (0.20615338227328134) + | > loss_disc_real_2: 0.1816272884607315 (0.22744330175283575) + | > loss_disc_real_3: 0.18414358794689178 (0.22838425135412135) + | > loss_disc_real_4: 0.22905279695987701 (0.22849337143056533) + | > loss_disc_real_5: 0.21940088272094727 (0.23510413084711348) + | > loss_0: 2.4468142986297607 (2.4650108473641543) + | > grad_norm_0: tensor(7.8054, device='cuda:0') (tensor(10.6824, device='cuda:0')) + | > loss_gen: 2.402290105819702 (2.352741445813861) + | > loss_kl: 1.8377236127853394 (1.8882532169838913) + | > loss_feat: 7.693261623382568 (7.11416242904022) + | > loss_mel: 18.718626022338867 (17.776647375411354) + | > loss_duration: 1.542238473892212 (1.466731962035684) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.19414138793945 (30.598536499408112) + | > grad_norm_1: tensor(311.1693, device='cuda:0') (tensor(273.4926, device='cuda:0')) + | > current_lr_0: 0.0001933138901008103 + | > current_lr_1: 0.0001933138901008103 + | > step_time: 0.4259 (0.378619560674459) + | > loader_time: 0.0073 (0.006024554997933011) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4403762817382812 (2.4403762817382812) + | > loss_disc_real_0: 0.17848598957061768 (0.17848598957061768) + | > loss_disc_real_1: 0.29150867462158203 (0.29150867462158203) + | > loss_disc_real_2: 0.2633017897605896 (0.2633017897605896) + | > loss_disc_real_3: 0.2581004798412323 (0.2581004798412323) + | > loss_disc_real_4: 0.20613381266593933 (0.20613381266593933) + | > loss_disc_real_5: 0.2395874261856079 (0.2395874261856079) + | > loss_0: 2.4403762817382812 (2.4403762817382812) + | > loss_gen: 2.815054416656494 (2.815054416656494) + | > loss_kl: 2.398125410079956 (2.398125410079956) + | > loss_feat: 7.575076580047607 (7.575076580047607) + | > loss_mel: 19.041086196899414 (19.041086196899414) + | > loss_duration: 1.7094485759735107 (1.7094485759735107) + | > loss_1: 33.53879165649414 (33.53879165649414) + + --> STEP: 1 + | > loss_disc: 2.561850070953369 (2.561850070953369) + | > loss_disc_real_0: 0.20727744698524475 (0.20727744698524475) + | > loss_disc_real_1: 0.2863307297229767 (0.2863307297229767) + | > loss_disc_real_2: 0.2898685038089752 (0.2898685038089752) + | > loss_disc_real_3: 0.24743343889713287 (0.24743343889713287) + | > loss_disc_real_4: 0.23966386914253235 (0.23966386914253235) + | > loss_disc_real_5: 0.23162800073623657 (0.23162800073623657) + | > loss_0: 2.561850070953369 (2.561850070953369) + | > loss_gen: 2.498812437057495 (2.498812437057495) + | > loss_kl: 2.761784315109253 (2.761784315109253) + | > loss_feat: 6.223058223724365 (6.223058223724365) + | > loss_mel: 16.774150848388672 (16.774150848388672) + | > loss_duration: 1.7249903678894043 (1.7249903678894043) + | > loss_1: 29.98279571533203 (29.98279571533203) + + --> STEP: 2 + | > loss_disc: 2.6174511909484863 (2.5896506309509277) + | > loss_disc_real_0: 0.23514766991138458 (0.22121255844831467) + | > loss_disc_real_1: 0.2799522578716278 (0.28314149379730225) + | > loss_disc_real_2: 0.2519620656967163 (0.27091528475284576) + | > loss_disc_real_3: 0.24798010289669037 (0.24770677089691162) + | > loss_disc_real_4: 0.24116775393486023 (0.2404158115386963) + | > loss_disc_real_5: 0.25249892473220825 (0.2420634627342224) + | > loss_0: 2.6174511909484863 (2.5896506309509277) + | > loss_gen: 2.4780781269073486 (2.488445281982422) + | > loss_kl: 2.1859943866729736 (2.4738893508911133) + | > loss_feat: 6.730240345001221 (6.476649284362793) + | > loss_mel: 16.61345863342285 (16.69380474090576) + | > loss_duration: 1.6820157766342163 (1.7035030722618103) + | > loss_1: 29.689788818359375 (29.836292266845703) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0039044618606567383 (+0.0002046823501586914) + | > avg_loss_disc: 2.5896506309509277 (+0.20153045654296875) + | > avg_loss_disc_real_0: 0.22121255844831467 (+0.10126479342579842) + | > avg_loss_disc_real_1: 0.28314149379730225 (+0.12399967759847641) + | > avg_loss_disc_real_2: 0.27091528475284576 (+0.06653068214654922) + | > avg_loss_disc_real_3: 0.24770677089691162 (+0.05749218910932541) + | > avg_loss_disc_real_4: 0.2404158115386963 (+0.01656229794025421) + | > avg_loss_disc_real_5: 0.2420634627342224 (+0.02519954741001129) + | > avg_loss_0: 2.5896506309509277 (+0.20153045654296875) + | > avg_loss_gen: 2.488445281982422 (+0.23321282863616943) + | > avg_loss_kl: 2.4738893508911133 (+0.0334470272064209) + | > avg_loss_feat: 6.476649284362793 (-0.7226846218109131) + | > avg_loss_mel: 16.69380474090576 (-0.7072410583496094) + | > avg_loss_duration: 1.7035030722618103 (-0.0242154598236084) + | > avg_loss_1: 29.836292266845703 (-1.1874809265136719) + + + > EPOCH: 273/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:28:41)  + + --> TIME: 2025-04-19 03:28:43 -- STEP: 2/142 -- GLOBAL_STEP: 418050 + | > loss_disc: 2.4684011936187744 (2.4503637552261353) + | > loss_disc_real_0: 0.1399257928133011 (0.18154136091470718) + | > loss_disc_real_1: 0.18614539504051208 (0.20230238884687424) + | > loss_disc_real_2: 0.2334863394498825 (0.24120578914880753) + | > loss_disc_real_3: 0.21503284573554993 (0.22494787722826004) + | > loss_disc_real_4: 0.23011070489883423 (0.23634111136198044) + | > loss_disc_real_5: 0.22115568816661835 (0.21425233036279678) + | > loss_0: 2.4684011936187744 (2.4503637552261353) + | > grad_norm_0: tensor(9.1151, device='cuda:0') (tensor(13.9723, device='cuda:0')) + | > loss_gen: 2.1409270763397217 (2.34368097782135) + | > loss_kl: 1.894981861114502 (1.8967911005020142) + | > loss_feat: 6.927370071411133 (7.088116407394409) + | > loss_mel: 17.34769630432129 (17.117836952209473) + | > loss_duration: 1.4503858089447021 (1.4197090864181519) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.76136016845703 (29.86613368988037) + | > grad_norm_1: tensor(263.6900, device='cuda:0') (tensor(313.7799, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.321 (0.3266395330429077) + | > loader_time: 0.0039 (0.0040847063064575195) + + + --> TIME: 2025-04-19 03:28:51 -- STEP: 27/142 -- GLOBAL_STEP: 418075 + | > loss_disc: 2.426262140274048 (2.424550427330865) + | > loss_disc_real_0: 0.11592153459787369 (0.11469906099416592) + | > loss_disc_real_1: 0.2068147212266922 (0.2040967063771354) + | > loss_disc_real_2: 0.22920560836791992 (0.22574416078903056) + | > loss_disc_real_3: 0.22565776109695435 (0.2241572665947455) + | > loss_disc_real_4: 0.24099047482013702 (0.2300396958986918) + | > loss_disc_real_5: 0.2096465826034546 (0.22911835489449678) + | > loss_0: 2.426262140274048 (2.424550427330865) + | > grad_norm_0: tensor(16.5117, device='cuda:0') (tensor(14.5594, device='cuda:0')) + | > loss_gen: 2.3872876167297363 (2.3909129566616483) + | > loss_kl: 1.9112194776535034 (1.8283107457337555) + | > loss_feat: 7.24419641494751 (7.260366051285355) + | > loss_mel: 17.169431686401367 (17.607418696085613) + | > loss_duration: 1.4444575309753418 (1.427451248522158) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.156593322753906 (30.514459892555518) + | > grad_norm_1: tensor(443.3348, device='cuda:0') (tensor(341.6859, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.3679 (0.3370059331258138) + | > loader_time: 0.0053 (0.004983124909577546) + + + --> TIME: 2025-04-19 03:29:01 -- STEP: 52/142 -- GLOBAL_STEP: 418100 + | > loss_disc: 2.463958740234375 (2.442444691291221) + | > loss_disc_real_0: 0.09162578731775284 (0.11658455445789374) + | > loss_disc_real_1: 0.18801940977573395 (0.20512588465443024) + | > loss_disc_real_2: 0.2620980441570282 (0.22767276374193338) + | > loss_disc_real_3: 0.22505198419094086 (0.22648435773757788) + | > loss_disc_real_4: 0.24678559601306915 (0.230637841213208) + | > loss_disc_real_5: 0.2789890766143799 (0.23357675109918302) + | > loss_0: 2.463958740234375 (2.442444691291221) + | > grad_norm_0: tensor(11.9724, device='cuda:0') (tensor(13.5758, device='cuda:0')) + | > loss_gen: 2.3862221240997314 (2.3713796643110423) + | > loss_kl: 1.807688593864441 (1.8431206253858714) + | > loss_feat: 6.745304107666016 (7.137306268398579) + | > loss_mel: 18.051027297973633 (17.521507483262283) + | > loss_duration: 1.4873297214508057 (1.4361231418756337) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.477571487426758 (30.309437311612644) + | > grad_norm_1: tensor(379.3860, device='cuda:0') (tensor(319.6247, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.3757 (0.35282405064656186) + | > loader_time: 0.0077 (0.005483604394472563) + + + --> TIME: 2025-04-19 03:29:11 -- STEP: 77/142 -- GLOBAL_STEP: 418125 + | > loss_disc: 2.4571309089660645 (2.446441464609913) + | > loss_disc_real_0: 0.12843012809753418 (0.11995693209109368) + | > loss_disc_real_1: 0.20392541587352753 (0.20459828206471029) + | > loss_disc_real_2: 0.22700797021389008 (0.22686860584593438) + | > loss_disc_real_3: 0.28892409801483154 (0.22747137097569256) + | > loss_disc_real_4: 0.2474692463874817 (0.22995882994168765) + | > loss_disc_real_5: 0.24174194037914276 (0.2328355529091575) + | > loss_0: 2.4571309089660645 (2.446441464609913) + | > grad_norm_0: tensor(11.5382, device='cuda:0') (tensor(12.9182, device='cuda:0')) + | > loss_gen: 2.468796491622925 (2.3652821918586633) + | > loss_kl: 1.8002763986587524 (1.858156571140537) + | > loss_feat: 7.834077835083008 (7.121981750835072) + | > loss_mel: 18.82128143310547 (17.53147206987653) + | > loss_duration: 1.4551239013671875 (1.448675505526654) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.379554748535156 (30.325568186772333) + | > grad_norm_1: tensor(315.5524, device='cuda:0') (tensor(299.2671, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.4021 (0.3669915756621917) + | > loader_time: 0.0069 (0.005794373425570401) + + + --> TIME: 2025-04-19 03:29:22 -- STEP: 102/142 -- GLOBAL_STEP: 418150 + | > loss_disc: 2.4675097465515137 (2.4521476825078317) + | > loss_disc_real_0: 0.14507804811000824 (0.12168231365435264) + | > loss_disc_real_1: 0.23588739335536957 (0.2051985231684703) + | > loss_disc_real_2: 0.24609264731407166 (0.22661559327560313) + | > loss_disc_real_3: 0.23274487257003784 (0.22692728904532453) + | > loss_disc_real_4: 0.23796753585338593 (0.23067613252822092) + | > loss_disc_real_5: 0.24679115414619446 (0.23281880438912148) + | > loss_0: 2.4675097465515137 (2.4521476825078317) + | > grad_norm_0: tensor(15.9659, device='cuda:0') (tensor(12.7503, device='cuda:0')) + | > loss_gen: 2.2907333374023438 (2.3575077781490235) + | > loss_kl: 1.8782403469085693 (1.8733850168246848) + | > loss_feat: 6.0380988121032715 (7.113460919436286) + | > loss_mel: 17.271760940551758 (17.693680660397398) + | > loss_duration: 1.463575839996338 (1.4598770457155563) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.942407608032227 (30.497911546744554) + | > grad_norm_1: tensor(248.0883, device='cuda:0') (tensor(290.9535, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.4344 (0.3788690122903562) + | > loader_time: 0.0071 (0.0060271655811983) + + + --> TIME: 2025-04-19 03:29:33 -- STEP: 127/142 -- GLOBAL_STEP: 418175 + | > loss_disc: 2.431720495223999 (2.4556687704221463) + | > loss_disc_real_0: 0.12823478877544403 (0.1229961155437109) + | > loss_disc_real_1: 0.2249894142150879 (0.20673793378308059) + | > loss_disc_real_2: 0.23606491088867188 (0.2263185521514397) + | > loss_disc_real_3: 0.23374831676483154 (0.2264489811940456) + | > loss_disc_real_4: 0.24273312091827393 (0.2302906063832636) + | > loss_disc_real_5: 0.23344476521015167 (0.23289016593159653) + | > loss_0: 2.431720495223999 (2.4556687704221463) + | > grad_norm_0: tensor(21.5690, device='cuda:0') (tensor(13.5045, device='cuda:0')) + | > loss_gen: 2.4334628582000732 (2.3609370002596384) + | > loss_kl: 1.8311450481414795 (1.8770094621838547) + | > loss_feat: 8.015931129455566 (7.076144714055099) + | > loss_mel: 18.334671020507812 (17.680985510818594) + | > loss_duration: 1.4978519678115845 (1.467390536323307) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.113059997558594 (30.462467298732967) + | > grad_norm_1: tensor(571.7372, device='cuda:0') (tensor(302.5323, device='cuda:0')) + | > current_lr_0: 0.0001932897258645477 + | > current_lr_1: 0.0001932897258645477 + | > step_time: 0.4381 (0.3914170828391248) + | > loader_time: 0.0073 (0.0062567436788964455) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5862479209899902 (2.5862479209899902) + | > loss_disc_real_0: 0.25154587626457214 (0.25154587626457214) + | > loss_disc_real_1: 0.21755127608776093 (0.21755127608776093) + | > loss_disc_real_2: 0.20885638892650604 (0.20885638892650604) + | > loss_disc_real_3: 0.26372480392456055 (0.26372480392456055) + | > loss_disc_real_4: 0.22790901362895966 (0.22790901362895966) + | > loss_disc_real_5: 0.25961360335350037 (0.25961360335350037) + | > loss_0: 2.5862479209899902 (2.5862479209899902) + | > loss_gen: 2.50312876701355 (2.50312876701355) + | > loss_kl: 2.7375571727752686 (2.7375571727752686) + | > loss_feat: 6.864099979400635 (6.864099979400635) + | > loss_mel: 19.48847198486328 (19.48847198486328) + | > loss_duration: 1.7053651809692383 (1.7053651809692383) + | > loss_1: 33.298622131347656 (33.298622131347656) + + --> STEP: 1 + | > loss_disc: 2.87092924118042 (2.87092924118042) + | > loss_disc_real_0: 0.2988836467266083 (0.2988836467266083) + | > loss_disc_real_1: 0.2263157069683075 (0.2263157069683075) + | > loss_disc_real_2: 0.23575329780578613 (0.23575329780578613) + | > loss_disc_real_3: 0.2415558099746704 (0.2415558099746704) + | > loss_disc_real_4: 0.2577976882457733 (0.2577976882457733) + | > loss_disc_real_5: 0.2288745939731598 (0.2288745939731598) + | > loss_0: 2.87092924118042 (2.87092924118042) + | > loss_gen: 1.9395445585250854 (1.9395445585250854) + | > loss_kl: 2.8017375469207764 (2.8017375469207764) + | > loss_feat: 4.213366985321045 (4.213366985321045) + | > loss_mel: 13.778179168701172 (13.778179168701172) + | > loss_duration: 1.7208609580993652 (1.7208609580993652) + | > loss_1: 24.453689575195312 (24.453689575195312) + + --> STEP: 2 + | > loss_disc: 2.6457486152648926 (2.7583389282226562) + | > loss_disc_real_0: 0.24174517393112183 (0.27031441032886505) + | > loss_disc_real_1: 0.22877274453639984 (0.22754422575235367) + | > loss_disc_real_2: 0.21272166073322296 (0.22423747926950455) + | > loss_disc_real_3: 0.2518918514251709 (0.24672383069992065) + | > loss_disc_real_4: 0.24865812063217163 (0.2532279044389725) + | > loss_disc_real_5: 0.24106156826019287 (0.23496808111667633) + | > loss_0: 2.6457486152648926 (2.7583389282226562) + | > loss_gen: 2.475834369659424 (2.2076894640922546) + | > loss_kl: 2.0988330841064453 (2.450285315513611) + | > loss_feat: 7.0477294921875 (5.6305482387542725) + | > loss_mel: 17.99338722229004 (15.885783195495605) + | > loss_duration: 1.707282543182373 (1.7140717506408691) + | > loss_1: 31.32306671142578 (27.888378143310547) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037909746170043945 (-0.00011348724365234375) + | > avg_loss_disc: 2.7583389282226562 (+0.16868829727172852) + | > avg_loss_disc_real_0: 0.27031441032886505 (+0.049101851880550385) + | > avg_loss_disc_real_1: 0.22754422575235367 (-0.05559726804494858) + | > avg_loss_disc_real_2: 0.22423747926950455 (-0.04667780548334122) + | > avg_loss_disc_real_3: 0.24672383069992065 (-0.0009829401969909668) + | > avg_loss_disc_real_4: 0.2532279044389725 (+0.012812092900276184) + | > avg_loss_disc_real_5: 0.23496808111667633 (-0.0070953816175460815) + | > avg_loss_0: 2.7583389282226562 (+0.16868829727172852) + | > avg_loss_gen: 2.2076894640922546 (-0.28075581789016724) + | > avg_loss_kl: 2.450285315513611 (-0.02360403537750244) + | > avg_loss_feat: 5.6305482387542725 (-0.8461010456085205) + | > avg_loss_mel: 15.885783195495605 (-0.8080215454101562) + | > avg_loss_duration: 1.7140717506408691 (+0.010568678379058838) + | > avg_loss_1: 27.888378143310547 (-1.9479141235351562) + + + > EPOCH: 274/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:29:41)  + + --> TIME: 2025-04-19 03:29:45 -- STEP: 10/142 -- GLOBAL_STEP: 418200 + | > loss_disc: 2.5140271186828613 (2.4664758443832397) + | > loss_disc_real_0: 0.16297774016857147 (0.12018721476197243) + | > loss_disc_real_1: 0.18349967896938324 (0.21960338801145554) + | > loss_disc_real_2: 0.20951734483242035 (0.22097109407186508) + | > loss_disc_real_3: 0.24197539687156677 (0.2278115227818489) + | > loss_disc_real_4: 0.2175552397966385 (0.22462153881788255) + | > loss_disc_real_5: 0.1950705200433731 (0.22838071137666702) + | > loss_0: 2.5140271186828613 (2.4664758443832397) + | > grad_norm_0: tensor(6.4949, device='cuda:0') (tensor(11.8080, device='cuda:0')) + | > loss_gen: 2.252894401550293 (2.304765510559082) + | > loss_kl: 1.9189879894256592 (1.8023658633232116) + | > loss_feat: 7.188150882720947 (7.1801862716674805) + | > loss_mel: 18.07501792907715 (17.749621391296387) + | > loss_duration: 1.412369966506958 (1.4211686611175538) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.847421646118164 (30.458108329772948) + | > grad_norm_1: tensor(118.8506, device='cuda:0') (tensor(239.6288, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.3362 (0.3251789569854736) + | > loader_time: 0.0051 (0.004419302940368653) + + + --> TIME: 2025-04-19 03:29:54 -- STEP: 35/142 -- GLOBAL_STEP: 418225 + | > loss_disc: 2.3477611541748047 (2.418750034059797) + | > loss_disc_real_0: 0.18174457550048828 (0.11695758019174848) + | > loss_disc_real_1: 0.2134142518043518 (0.20568439705031258) + | > loss_disc_real_2: 0.22777226567268372 (0.22157080812113625) + | > loss_disc_real_3: 0.22551532089710236 (0.22931220148290907) + | > loss_disc_real_4: 0.20832495391368866 (0.22372174305575235) + | > loss_disc_real_5: 0.21642495691776276 (0.2262471569435937) + | > loss_0: 2.3477611541748047 (2.418750034059797) + | > grad_norm_0: tensor(13.7594, device='cuda:0') (tensor(10.0696, device='cuda:0')) + | > loss_gen: 2.5237836837768555 (2.3731172016688755) + | > loss_kl: 1.755290150642395 (1.8046101433890207) + | > loss_feat: 7.029702663421631 (7.265908786228725) + | > loss_mel: 17.96453094482422 (17.72835578918457) + | > loss_duration: 1.4549143314361572 (1.4347163438796997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.728221893310547 (30.60670874459403) + | > grad_norm_1: tensor(273.6741, device='cuda:0') (tensor(236.3554, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.3561 (0.338751254762922) + | > loader_time: 0.0058 (0.0049786772046770365) + + + --> TIME: 2025-04-19 03:30:03 -- STEP: 60/142 -- GLOBAL_STEP: 418250 + | > loss_disc: 2.477665424346924 (2.435293102264404) + | > loss_disc_real_0: 0.11572986841201782 (0.11844790341953436) + | > loss_disc_real_1: 0.24543730914592743 (0.2070438399910927) + | > loss_disc_real_2: 0.21976129710674286 (0.22398025517662365) + | > loss_disc_real_3: 0.21215404570102692 (0.22809835250178973) + | > loss_disc_real_4: 0.21392348408699036 (0.2256389230489731) + | > loss_disc_real_5: 0.19857901334762573 (0.22866154884298642) + | > loss_0: 2.477665424346924 (2.435293102264404) + | > grad_norm_0: tensor(11.2559, device='cuda:0') (tensor(10.3481, device='cuda:0')) + | > loss_gen: 2.3650600910186768 (2.3754777868588763) + | > loss_kl: 1.9753382205963135 (1.8166697820027669) + | > loss_feat: 7.517634868621826 (7.198023446400961) + | > loss_mel: 18.015016555786133 (17.731511481602983) + | > loss_duration: 1.4908597469329834 (1.4455918510754904) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.363910675048828 (30.56727466583252) + | > grad_norm_1: tensor(155.5012, device='cuda:0') (tensor(260.8107, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.3745 (0.3503441214561463) + | > loader_time: 0.0063 (0.005323362350463866) + + + --> TIME: 2025-04-19 03:30:13 -- STEP: 85/142 -- GLOBAL_STEP: 418275 + | > loss_disc: 2.4945149421691895 (2.4533206098219926) + | > loss_disc_real_0: 0.1737716943025589 (0.12240118138930377) + | > loss_disc_real_1: 0.2108631134033203 (0.20751935173483455) + | > loss_disc_real_2: 0.23592527210712433 (0.2257211627329097) + | > loss_disc_real_3: 0.21446548402309418 (0.22969129471217883) + | > loss_disc_real_4: 0.2302873283624649 (0.22737993860945982) + | > loss_disc_real_5: 0.2358822375535965 (0.2306816562133677) + | > loss_0: 2.4945149421691895 (2.4533206098219926) + | > grad_norm_0: tensor(5.9825, device='cuda:0') (tensor(9.8178, device='cuda:0')) + | > loss_gen: 2.2647714614868164 (2.3582591281217686) + | > loss_kl: 1.8261544704437256 (1.83750219485339) + | > loss_feat: 6.690566539764404 (7.123466463649974) + | > loss_mel: 17.02934455871582 (17.773814650142892) + | > loss_duration: 1.4830451011657715 (1.457587814331055) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.293882369995117 (30.550630434821635) + | > grad_norm_1: tensor(77.5902, device='cuda:0') (tensor(233.0033, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.3978 (0.36243217131670785) + | > loader_time: 0.0077 (0.005630299624274758) + + + --> TIME: 2025-04-19 03:30:24 -- STEP: 110/142 -- GLOBAL_STEP: 418300 + | > loss_disc: 2.582042694091797 (2.460994989221746) + | > loss_disc_real_0: 0.07647257298231125 (0.12382330068133095) + | > loss_disc_real_1: 0.1811397820711136 (0.20731523524631154) + | > loss_disc_real_2: 0.2120307981967926 (0.22601748623631218) + | > loss_disc_real_3: 0.2284517139196396 (0.2298173328692263) + | > loss_disc_real_4: 0.234205424785614 (0.2282796557654034) + | > loss_disc_real_5: 0.2413271814584732 (0.23161477487195622) + | > loss_0: 2.582042694091797 (2.460994989221746) + | > grad_norm_0: tensor(17.2880, device='cuda:0') (tensor(9.8468, device='cuda:0')) + | > loss_gen: 2.170431613922119 (2.3479384487325494) + | > loss_kl: 1.9888622760772705 (1.8508413553237915) + | > loss_feat: 6.742431640625 (7.0755900209600275) + | > loss_mel: 17.063411712646484 (17.743408142436646) + | > loss_duration: 1.4921653270721436 (1.4657719189470464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.457300186157227 (30.483550002358175) + | > grad_norm_1: tensor(117.3819, device='cuda:0') (tensor(226.3746, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.4349 (0.37309747609225186) + | > loader_time: 0.008 (0.005909531766718083) + + + --> TIME: 2025-04-19 03:30:35 -- STEP: 135/142 -- GLOBAL_STEP: 418325 + | > loss_disc: 2.446244716644287 (2.461686031906693) + | > loss_disc_real_0: 0.14542284607887268 (0.1250414296984673) + | > loss_disc_real_1: 0.20925506949424744 (0.20680813888708752) + | > loss_disc_real_2: 0.21986058354377747 (0.22603668162116297) + | > loss_disc_real_3: 0.21471893787384033 (0.22899507129633867) + | > loss_disc_real_4: 0.22829589247703552 (0.22834017331953402) + | > loss_disc_real_5: 0.21546988189220428 (0.23201388495939748) + | > loss_0: 2.446244716644287 (2.461686031906693) + | > grad_norm_0: tensor(4.6631, device='cuda:0') (tensor(9.9891, device='cuda:0')) + | > loss_gen: 2.2833099365234375 (2.339686734588058) + | > loss_kl: 1.9484286308288574 (1.8658577424508551) + | > loss_feat: 6.8047637939453125 (7.016332576892994) + | > loss_mel: 18.64571189880371 (17.660230678982213) + | > loss_duration: 1.4782779216766357 (1.4713897457829228) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.160491943359375 (30.353497554637766) + | > grad_norm_1: tensor(249.4407, device='cuda:0') (tensor(228.1688, device='cuda:0')) + | > current_lr_0: 0.00019326556464881463 + | > current_lr_1: 0.00019326556464881463 + | > step_time: 0.4588 (0.38569597138298883) + | > loader_time: 0.0076 (0.006192719494854962) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.49259090423584 (2.49259090423584) + | > loss_disc_real_0: 0.09493505209684372 (0.09493505209684372) + | > loss_disc_real_1: 0.2102949023246765 (0.2102949023246765) + | > loss_disc_real_2: 0.22921046614646912 (0.22921046614646912) + | > loss_disc_real_3: 0.22310474514961243 (0.22310474514961243) + | > loss_disc_real_4: 0.19618621468544006 (0.19618621468544006) + | > loss_disc_real_5: 0.23738303780555725 (0.23738303780555725) + | > loss_0: 2.49259090423584 (2.49259090423584) + | > loss_gen: 2.0783424377441406 (2.0783424377441406) + | > loss_kl: 2.545644998550415 (2.545644998550415) + | > loss_feat: 6.769240856170654 (6.769240856170654) + | > loss_mel: 16.922874450683594 (16.922874450683594) + | > loss_duration: 1.7115706205368042 (1.7115706205368042) + | > loss_1: 30.027673721313477 (30.027673721313477) + + --> STEP: 1 + | > loss_disc: 2.504024028778076 (2.504024028778076) + | > loss_disc_real_0: 0.11535874754190445 (0.11535874754190445) + | > loss_disc_real_1: 0.21986991167068481 (0.21986991167068481) + | > loss_disc_real_2: 0.17890748381614685 (0.17890748381614685) + | > loss_disc_real_3: 0.2312372922897339 (0.2312372922897339) + | > loss_disc_real_4: 0.2308785766363144 (0.2308785766363144) + | > loss_disc_real_5: 0.22605998814105988 (0.22605998814105988) + | > loss_0: 2.504024028778076 (2.504024028778076) + | > loss_gen: 2.078894853591919 (2.078894853591919) + | > loss_kl: 2.441532850265503 (2.441532850265503) + | > loss_feat: 7.217021465301514 (7.217021465301514) + | > loss_mel: 18.26953125 (18.26953125) + | > loss_duration: 1.7548933029174805 (1.7548933029174805) + | > loss_1: 31.76187515258789 (31.76187515258789) + + --> STEP: 2 + | > loss_disc: 2.515383243560791 (2.5097036361694336) + | > loss_disc_real_0: 0.10837157815694809 (0.11186516284942627) + | > loss_disc_real_1: 0.18759474158287048 (0.20373232662677765) + | > loss_disc_real_2: 0.20882795751094818 (0.19386772066354752) + | > loss_disc_real_3: 0.21530261635780334 (0.22326995432376862) + | > loss_disc_real_4: 0.20662857592105865 (0.21875357627868652) + | > loss_disc_real_5: 0.24824409186840057 (0.23715204000473022) + | > loss_0: 2.515383243560791 (2.5097036361694336) + | > loss_gen: 1.979770541191101 (2.02933269739151) + | > loss_kl: 2.2185142040252686 (2.3300235271453857) + | > loss_feat: 7.5038628578186035 (7.360442161560059) + | > loss_mel: 19.454723358154297 (18.86212730407715) + | > loss_duration: 1.6729097366333008 (1.7139015197753906) + | > loss_1: 32.82978057861328 (32.295827865600586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035436153411865234 (-0.0002473592758178711) + | > avg_loss_disc: 2.5097036361694336 (-0.24863529205322266) + | > avg_loss_disc_real_0: 0.11186516284942627 (-0.15844924747943878) + | > avg_loss_disc_real_1: 0.20373232662677765 (-0.02381189912557602) + | > avg_loss_disc_real_2: 0.19386772066354752 (-0.03036975860595703) + | > avg_loss_disc_real_3: 0.22326995432376862 (-0.02345387637615204) + | > avg_loss_disc_real_4: 0.21875357627868652 (-0.03447432816028595) + | > avg_loss_disc_real_5: 0.23715204000473022 (+0.002183958888053894) + | > avg_loss_0: 2.5097036361694336 (-0.24863529205322266) + | > avg_loss_gen: 2.02933269739151 (-0.17835676670074463) + | > avg_loss_kl: 2.3300235271453857 (-0.1202617883682251) + | > avg_loss_feat: 7.360442161560059 (+1.7298939228057861) + | > avg_loss_mel: 18.86212730407715 (+2.976344108581543) + | > avg_loss_duration: 1.7139015197753906 (-0.00017023086547851562) + | > avg_loss_1: 32.295827865600586 (+4.407449722290039) + + + > EPOCH: 275/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:30:40)  + + --> TIME: 2025-04-19 03:30:47 -- STEP: 18/142 -- GLOBAL_STEP: 418350 + | > loss_disc: 2.493300676345825 (2.456399533483717) + | > loss_disc_real_0: 0.13253667950630188 (0.12763014311591783) + | > loss_disc_real_1: 0.1894056797027588 (0.20202296475569406) + | > loss_disc_real_2: 0.22372111678123474 (0.22300163242552015) + | > loss_disc_real_3: 0.2376490831375122 (0.22729127440187666) + | > loss_disc_real_4: 0.21956346929073334 (0.22969329191578758) + | > loss_disc_real_5: 0.26696479320526123 (0.22956398046678966) + | > loss_0: 2.493300676345825 (2.456399533483717) + | > grad_norm_0: tensor(5.9139, device='cuda:0') (tensor(7.2004, device='cuda:0')) + | > loss_gen: 2.298433303833008 (2.3869667053222656) + | > loss_kl: 2.02799654006958 (1.8476971652772691) + | > loss_feat: 6.2991485595703125 (7.308733860651652) + | > loss_mel: 16.705535888671875 (17.772529708014595) + | > loss_duration: 1.4570590257644653 (1.4270343316925898) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.78817367553711 (30.74296145968967) + | > grad_norm_1: tensor(90.8689, device='cuda:0') (tensor(150.4764, device='cuda:0')) + | > current_lr_0: 0.0001932414064532335 + | > current_lr_1: 0.0001932414064532335 + | > step_time: 0.3387 (0.3298166063096788) + | > loader_time: 0.0049 (0.004597279760572646) + + + --> TIME: 2025-04-19 03:30:56 -- STEP: 43/142 -- GLOBAL_STEP: 418375 + | > loss_disc: 2.5426323413848877 (2.4645330406898673) + | > loss_disc_real_0: 0.12321493774652481 (0.12710782928868786) + | > loss_disc_real_1: 0.2094339281320572 (0.20645202384438627) + | > loss_disc_real_2: 0.21977004408836365 (0.22633500327897627) + | > loss_disc_real_3: 0.217767134308815 (0.22740260736886844) + | > loss_disc_real_4: 0.23267781734466553 (0.22731423412644586) + | > loss_disc_real_5: 0.23513680696487427 (0.23149970728297567) + | > loss_0: 2.5426323413848877 (2.4645330406898673) + | > grad_norm_0: tensor(11.1247, device='cuda:0') (tensor(7.7893, device='cuda:0')) + | > loss_gen: 2.1707308292388916 (2.3594765385916063) + | > loss_kl: 1.743640422821045 (1.8360809586768927) + | > loss_feat: 6.857371807098389 (7.162647291671398) + | > loss_mel: 18.28416633605957 (17.76769409623257) + | > loss_duration: 1.4516608715057373 (1.4381563746651935) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.507568359375 (30.564055243203807) + | > grad_norm_1: tensor(487.4237, device='cuda:0') (tensor(183.7085, device='cuda:0')) + | > current_lr_0: 0.0001932414064532335 + | > current_lr_1: 0.0001932414064532335 + | > step_time: 0.3629 (0.3425598754439243) + | > loader_time: 0.0057 (0.005153506301170173) + + + --> TIME: 2025-04-19 03:31:05 -- STEP: 68/142 -- GLOBAL_STEP: 418400 + | > loss_disc: 2.4659836292266846 (2.4655113430584183) + | > loss_disc_real_0: 0.12068688869476318 (0.126629503827323) + | > loss_disc_real_1: 0.20720769464969635 (0.20511302790221045) + | > loss_disc_real_2: 0.2239389419555664 (0.22623279142905683) + | > loss_disc_real_3: 0.22236225008964539 (0.22875765163232298) + | > loss_disc_real_4: 0.24518685042858124 (0.2291821690604967) + | > loss_disc_real_5: 0.2582964599132538 (0.23200553962413004) + | > loss_0: 2.4659836292266846 (2.4655113430584183) + | > grad_norm_0: tensor(17.5301, device='cuda:0') (tensor(9.1206, device='cuda:0')) + | > loss_gen: 2.4106574058532715 (2.3611898387179657) + | > loss_kl: 1.800554871559143 (1.8455072343349457) + | > loss_feat: 7.071486473083496 (7.156679104356205) + | > loss_mel: 18.064289093017578 (17.803552753785077) + | > loss_duration: 1.4550343751907349 (1.447089496780844) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.80202293395996 (30.614018384148093) + | > grad_norm_1: tensor(362.6706, device='cuda:0') (tensor(220.3356, device='cuda:0')) + | > current_lr_0: 0.0001932414064532335 + | > current_lr_1: 0.0001932414064532335 + | > step_time: 0.394 (0.354826338150922) + | > loader_time: 0.0062 (0.0054900646209716806) + + + --> TIME: 2025-04-19 03:31:16 -- STEP: 93/142 -- GLOBAL_STEP: 418425 + | > loss_disc: 2.497655153274536 (2.4613558912789957) + | > loss_disc_real_0: 0.15633682906627655 (0.12590648718578848) + | > loss_disc_real_1: 0.20333348214626312 (0.20512427437689998) + | > loss_disc_real_2: 0.2563716173171997 (0.2257234832291962) + | > loss_disc_real_3: 0.21727390587329865 (0.2285129874944687) + | > loss_disc_real_4: 0.22371764481067657 (0.22893305731716976) + | > loss_disc_real_5: 0.2318655252456665 (0.23304399735825035) + | > loss_0: 2.497655153274536 (2.4613558912789957) + | > grad_norm_0: tensor(10.6363, device='cuda:0') (tensor(10.9698, device='cuda:0')) + | > loss_gen: 2.44832181930542 (2.3784900788337953) + | > loss_kl: 1.7718580961227417 (1.8587088700263732) + | > loss_feat: 6.274648189544678 (7.17062621988276) + | > loss_mel: 15.992825508117676 (17.81059970650621) + | > loss_duration: 1.475956916809082 (1.4591231897313102) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.963611602783203 (30.677547988071236) + | > grad_norm_1: tensor(432.4679, device='cuda:0') (tensor(262.8728, device='cuda:0')) + | > current_lr_0: 0.0001932414064532335 + | > current_lr_1: 0.0001932414064532335 + | > step_time: 0.4044 (0.36750984191894515) + | > loader_time: 0.007 (0.005779250975578062) + + + --> TIME: 2025-04-19 03:31:26 -- STEP: 118/142 -- GLOBAL_STEP: 418450 + | > loss_disc: 2.453495740890503 (2.4629574003866175) + | > loss_disc_real_0: 0.15100286900997162 (0.12588109167576858) + | > loss_disc_real_1: 0.21667496860027313 (0.2049840588943433) + | > loss_disc_real_2: 0.2409265637397766 (0.22585040381399252) + | > loss_disc_real_3: 0.19970865547657013 (0.22892528115692784) + | > loss_disc_real_4: 0.2511953115463257 (0.22985188044228796) + | > loss_disc_real_5: 0.24568553268909454 (0.2327647137187295) + | > loss_0: 2.453495740890503 (2.4629574003866175) + | > grad_norm_0: tensor(29.7255, device='cuda:0') (tensor(12.5661, device='cuda:0')) + | > loss_gen: 2.415705680847168 (2.382733041957273) + | > loss_kl: 1.9012856483459473 (1.86731343350168) + | > loss_feat: 7.4541707038879395 (7.130779529021958) + | > loss_mel: 18.188749313354492 (17.747998609381206) + | > loss_duration: 1.4652104377746582 (1.466705272763462) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.425121307373047 (30.59552986338987) + | > grad_norm_1: tensor(439.9162, device='cuda:0') (tensor(285.9894, device='cuda:0')) + | > current_lr_0: 0.0001932414064532335 + | > current_lr_1: 0.0001932414064532335 + | > step_time: 0.4396 (0.37893073639627217) + | > loader_time: 0.0071 (0.0060479822805372336) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4670369625091553 (2.4670369625091553) + | > loss_disc_real_0: 0.09864594042301178 (0.09864594042301178) + | > loss_disc_real_1: 0.19254209101200104 (0.19254209101200104) + | > loss_disc_real_2: 0.23985391855239868 (0.23985391855239868) + | > loss_disc_real_3: 0.2260720580816269 (0.2260720580816269) + | > loss_disc_real_4: 0.2713232934474945 (0.2713232934474945) + | > loss_disc_real_5: 0.20489534735679626 (0.20489534735679626) + | > loss_0: 2.4670369625091553 (2.4670369625091553) + | > loss_gen: 2.2740321159362793 (2.2740321159362793) + | > loss_kl: 2.3767638206481934 (2.3767638206481934) + | > loss_feat: 6.368338108062744 (6.368338108062744) + | > loss_mel: 17.83088493347168 (17.83088493347168) + | > loss_duration: 1.707968831062317 (1.707968831062317) + | > loss_1: 30.5579891204834 (30.5579891204834) + + --> STEP: 1 + | > loss_disc: 2.410045623779297 (2.410045623779297) + | > loss_disc_real_0: 0.09340085834264755 (0.09340085834264755) + | > loss_disc_real_1: 0.2122134119272232 (0.2122134119272232) + | > loss_disc_real_2: 0.2711789906024933 (0.2711789906024933) + | > loss_disc_real_3: 0.22638173401355743 (0.22638173401355743) + | > loss_disc_real_4: 0.25718435645103455 (0.25718435645103455) + | > loss_disc_real_5: 0.23097455501556396 (0.23097455501556396) + | > loss_0: 2.410045623779297 (2.410045623779297) + | > loss_gen: 2.345512628555298 (2.345512628555298) + | > loss_kl: 2.5701022148132324 (2.5701022148132324) + | > loss_feat: 7.172473907470703 (7.172473907470703) + | > loss_mel: 19.574296951293945 (19.574296951293945) + | > loss_duration: 1.746178150177002 (1.746178150177002) + | > loss_1: 33.408565521240234 (33.408565521240234) + + --> STEP: 2 + | > loss_disc: 2.4445910453796387 (2.4273183345794678) + | > loss_disc_real_0: 0.09801585972309113 (0.09570835903286934) + | > loss_disc_real_1: 0.170441135764122 (0.1913272738456726) + | > loss_disc_real_2: 0.2354559451341629 (0.2533174678683281) + | > loss_disc_real_3: 0.211276113986969 (0.21882892400026321) + | > loss_disc_real_4: 0.23167924582958221 (0.24443180114030838) + | > loss_disc_real_5: 0.2088584452867508 (0.21991650015115738) + | > loss_0: 2.4445910453796387 (2.4273183345794678) + | > loss_gen: 2.1044259071350098 (2.224969267845154) + | > loss_kl: 2.371866464614868 (2.4709843397140503) + | > loss_feat: 7.656949043273926 (7.4147114753723145) + | > loss_mel: 18.281192779541016 (18.92774486541748) + | > loss_duration: 1.7105882167816162 (1.728383183479309) + | > loss_1: 32.125022888183594 (32.766794204711914) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036562681198120117 (+0.00011265277862548828) + | > avg_loss_disc: 2.4273183345794678 (-0.08238530158996582) + | > avg_loss_disc_real_0: 0.09570835903286934 (-0.01615680381655693) + | > avg_loss_disc_real_1: 0.1913272738456726 (-0.012405052781105042) + | > avg_loss_disc_real_2: 0.2533174678683281 (+0.05944974720478058) + | > avg_loss_disc_real_3: 0.21882892400026321 (-0.004441030323505402) + | > avg_loss_disc_real_4: 0.24443180114030838 (+0.025678224861621857) + | > avg_loss_disc_real_5: 0.21991650015115738 (-0.017235539853572845) + | > avg_loss_0: 2.4273183345794678 (-0.08238530158996582) + | > avg_loss_gen: 2.224969267845154 (+0.1956365704536438) + | > avg_loss_kl: 2.4709843397140503 (+0.14096081256866455) + | > avg_loss_feat: 7.4147114753723145 (+0.05426931381225586) + | > avg_loss_mel: 18.92774486541748 (+0.06561756134033203) + | > avg_loss_duration: 1.728383183479309 (+0.014481663703918457) + | > avg_loss_1: 32.766794204711914 (+0.4709663391113281) + + + > EPOCH: 276/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:31:39)  + + --> TIME: 2025-04-19 03:31:40 -- STEP: 1/142 -- GLOBAL_STEP: 418475 + | > loss_disc: 2.5543737411499023 (2.5543737411499023) + | > loss_disc_real_0: 0.11955767124891281 (0.11955767124891281) + | > loss_disc_real_1: 0.17856371402740479 (0.17856371402740479) + | > loss_disc_real_2: 0.24779027700424194 (0.24779027700424194) + | > loss_disc_real_3: 0.20512983202934265 (0.20512983202934265) + | > loss_disc_real_4: 0.251106321811676 (0.251106321811676) + | > loss_disc_real_5: 0.21913479268550873 (0.21913479268550873) + | > loss_0: 2.5543737411499023 (2.5543737411499023) + | > grad_norm_0: tensor(6.0961, device='cuda:0') (tensor(6.0961, device='cuda:0')) + | > loss_gen: 2.2825815677642822 (2.2825815677642822) + | > loss_kl: 1.7812854051589966 (1.7812854051589966) + | > loss_feat: 6.766921043395996 (6.766921043395996) + | > loss_mel: 16.71224594116211 (16.71224594116211) + | > loss_duration: 1.4079654216766357 (1.4079654216766357) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.950998306274414 (28.950998306274414) + | > grad_norm_1: tensor(228.5333, device='cuda:0') (tensor(228.5333, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.33 (0.32996463775634766) + | > loader_time: 0.005 (0.0049703121185302734) + + + --> TIME: 2025-04-19 03:31:48 -- STEP: 26/142 -- GLOBAL_STEP: 418500 + | > loss_disc: 2.4474151134490967 (2.473297330049368) + | > loss_disc_real_0: 0.12933631241321564 (0.11689532519533084) + | > loss_disc_real_1: 0.16510242223739624 (0.20528236776590347) + | > loss_disc_real_2: 0.21388748288154602 (0.23180837012254274) + | > loss_disc_real_3: 0.2302510291337967 (0.22960607420939666) + | > loss_disc_real_4: 0.22625815868377686 (0.23067608017187852) + | > loss_disc_real_5: 0.23486222326755524 (0.23567290088305107) + | > loss_0: 2.4474151134490967 (2.473297330049368) + | > grad_norm_0: tensor(7.3305, device='cuda:0') (tensor(7.0685, device='cuda:0')) + | > loss_gen: 2.2846627235412598 (2.330145624967722) + | > loss_kl: 1.8175172805786133 (1.8227179738191457) + | > loss_feat: 7.226680278778076 (6.973119588998648) + | > loss_mel: 17.905685424804688 (17.642167128049408) + | > loss_duration: 1.45003342628479 (1.4297542663720937) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.68457794189453 (30.19790451343243) + | > grad_norm_1: tensor(200.2170, device='cuda:0') (tensor(183.5632, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.3468 (0.33375293474930984) + | > loader_time: 0.0053 (0.004981407752403846) + + + --> TIME: 2025-04-19 03:31:57 -- STEP: 51/142 -- GLOBAL_STEP: 418525 + | > loss_disc: 2.423285722732544 (2.4510195956510654) + | > loss_disc_real_0: 0.11688697338104248 (0.12072309486421884) + | > loss_disc_real_1: 0.1916382908821106 (0.20220117972177615) + | > loss_disc_real_2: 0.23058217763900757 (0.22643203010746077) + | > loss_disc_real_3: 0.2120036482810974 (0.22681238224693373) + | > loss_disc_real_4: 0.22550813853740692 (0.22773656891841515) + | > loss_disc_real_5: 0.2208200991153717 (0.23464342396633298) + | > loss_0: 2.423285722732544 (2.4510195956510654) + | > grad_norm_0: tensor(7.6081, device='cuda:0') (tensor(7.8446, device='cuda:0')) + | > loss_gen: 2.3198318481445312 (2.3371522052615292) + | > loss_kl: 1.854972004890442 (1.8521792351030837) + | > loss_feat: 7.834897041320801 (7.009635485854804) + | > loss_mel: 18.865692138671875 (17.58235237645168) + | > loss_duration: 1.446545124053955 (1.4389110639983533) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.321937561035156 (30.220230289534026) + | > grad_norm_1: tensor(239.8200, device='cuda:0') (tensor(174.1852, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.3679 (0.34648135129143204) + | > loader_time: 0.0059 (0.005272360409007351) + + + --> TIME: 2025-04-19 03:32:07 -- STEP: 76/142 -- GLOBAL_STEP: 418550 + | > loss_disc: 2.458446741104126 (2.467174379449141) + | > loss_disc_real_0: 0.14486835896968842 (0.12447615203104521) + | > loss_disc_real_1: 0.19231286644935608 (0.2056111909056964) + | > loss_disc_real_2: 0.20008762180805206 (0.2263614700028771) + | > loss_disc_real_3: 0.23820428550243378 (0.22913438513090736) + | > loss_disc_real_4: 0.2293565571308136 (0.22936314932609858) + | > loss_disc_real_5: 0.2651713192462921 (0.23530611709544533) + | > loss_0: 2.458446741104126 (2.467174379449141) + | > grad_norm_0: tensor(8.9324, device='cuda:0') (tensor(8.3005, device='cuda:0')) + | > loss_gen: 2.344296932220459 (2.3342357089644983) + | > loss_kl: 1.9748879671096802 (1.876655051582738) + | > loss_feat: 6.911756992340088 (7.023934433334752) + | > loss_mel: 19.534679412841797 (17.660613925833456) + | > loss_duration: 1.489262342453003 (1.4506964573734686) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.254886627197266 (30.346135566109105) + | > grad_norm_1: tensor(181.8993, device='cuda:0') (tensor(181.8884, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.399 (0.35891903074164133) + | > loader_time: 0.0064 (0.005574436564194527) + + + --> TIME: 2025-04-19 03:32:18 -- STEP: 101/142 -- GLOBAL_STEP: 418575 + | > loss_disc: 2.4275543689727783 (2.4725992113056745) + | > loss_disc_real_0: 0.1445726603269577 (0.12677216802788255) + | > loss_disc_real_1: 0.18838657438755035 (0.20644238620701397) + | > loss_disc_real_2: 0.2289431244134903 (0.22728533910052612) + | > loss_disc_real_3: 0.2207341343164444 (0.22835230989621416) + | > loss_disc_real_4: 0.2394203543663025 (0.22911692004982787) + | > loss_disc_real_5: 0.229386568069458 (0.23479371377737215) + | > loss_0: 2.4275543689727783 (2.4725992113056745) + | > grad_norm_0: tensor(6.1324, device='cuda:0') (tensor(8.4713, device='cuda:0')) + | > loss_gen: 2.356816530227661 (2.3337773993463813) + | > loss_kl: 1.9685585498809814 (1.8951770492119364) + | > loss_feat: 7.024609088897705 (7.001074356607871) + | > loss_mel: 16.90829086303711 (17.671942002702473) + | > loss_duration: 1.4986190795898438 (1.461998724701381) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.756895065307617 (30.363969519586846) + | > grad_norm_1: tensor(84.9061, device='cuda:0') (tensor(189.5148, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.4191 (0.37018399663490825) + | > loader_time: 0.0068 (0.005840792514310025) + + + --> TIME: 2025-04-19 03:32:29 -- STEP: 126/142 -- GLOBAL_STEP: 418600 + | > loss_disc: 2.445524215698242 (2.475241799203176) + | > loss_disc_real_0: 0.12579850852489471 (0.12718148606400642) + | > loss_disc_real_1: 0.1974734514951706 (0.20678730072483179) + | > loss_disc_real_2: 0.20817914605140686 (0.22773053047675934) + | > loss_disc_real_3: 0.21536391973495483 (0.22868482472877655) + | > loss_disc_real_4: 0.2149607390165329 (0.22989681658763733) + | > loss_disc_real_5: 0.22880320250988007 (0.23448237173614048) + | > loss_0: 2.445524215698242 (2.475241799203176) + | > grad_norm_0: tensor(4.6015, device='cuda:0') (tensor(8.3529, device='cuda:0')) + | > loss_gen: 2.205143451690674 (2.3279652955040113) + | > loss_kl: 2.0744478702545166 (1.901365633994814) + | > loss_feat: 6.126524925231934 (6.972813946860177) + | > loss_mel: 15.786398887634277 (17.674850395747594) + | > loss_duration: 1.488144874572754 (1.4688484252445282) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.680660247802734 (30.34584370870439) + | > grad_norm_1: tensor(40.2342, device='cuda:0') (tensor(184.9445, device='cuda:0')) + | > current_lr_0: 0.00019321725127742685 + | > current_lr_1: 0.00019321725127742685 + | > step_time: 0.4424 (0.381947708508325) + | > loader_time: 0.0072 (0.006091167056371295) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4312589168548584 (2.4312589168548584) + | > loss_disc_real_0: 0.15038025379180908 (0.15038025379180908) + | > loss_disc_real_1: 0.18043531477451324 (0.18043531477451324) + | > loss_disc_real_2: 0.2220105528831482 (0.2220105528831482) + | > loss_disc_real_3: 0.21112492680549622 (0.21112492680549622) + | > loss_disc_real_4: 0.1854916214942932 (0.1854916214942932) + | > loss_disc_real_5: 0.22213760018348694 (0.22213760018348694) + | > loss_0: 2.4312589168548584 (2.4312589168548584) + | > loss_gen: 2.13126540184021 (2.13126540184021) + | > loss_kl: 2.5520834922790527 (2.5520834922790527) + | > loss_feat: 6.040066719055176 (6.040066719055176) + | > loss_mel: 16.323272705078125 (16.323272705078125) + | > loss_duration: 1.7154208421707153 (1.7154208421707153) + | > loss_1: 28.762109756469727 (28.762109756469727) + + --> STEP: 1 + | > loss_disc: 2.4475972652435303 (2.4475972652435303) + | > loss_disc_real_0: 0.12917058169841766 (0.12917058169841766) + | > loss_disc_real_1: 0.17252296209335327 (0.17252296209335327) + | > loss_disc_real_2: 0.2107812762260437 (0.2107812762260437) + | > loss_disc_real_3: 0.2028800994157791 (0.2028800994157791) + | > loss_disc_real_4: 0.17526403069496155 (0.17526403069496155) + | > loss_disc_real_5: 0.205373615026474 (0.205373615026474) + | > loss_0: 2.4475972652435303 (2.4475972652435303) + | > loss_gen: 2.054431438446045 (2.054431438446045) + | > loss_kl: 2.611818313598633 (2.611818313598633) + | > loss_feat: 7.073177814483643 (7.073177814483643) + | > loss_mel: 18.045591354370117 (18.045591354370117) + | > loss_duration: 1.7398245334625244 (1.7398245334625244) + | > loss_1: 31.524843215942383 (31.524843215942383) + + --> STEP: 2 + | > loss_disc: 2.43871808052063 (2.44315767288208) + | > loss_disc_real_0: 0.14116312563419342 (0.13516685366630554) + | > loss_disc_real_1: 0.19269798696041107 (0.18261047452688217) + | > loss_disc_real_2: 0.19534006714820862 (0.20306067168712616) + | > loss_disc_real_3: 0.2200784981250763 (0.2114792987704277) + | > loss_disc_real_4: 0.23885905742645264 (0.2070615440607071) + | > loss_disc_real_5: 0.22585132718086243 (0.2156124711036682) + | > loss_0: 2.43871808052063 (2.44315767288208) + | > loss_gen: 2.168694257736206 (2.1115628480911255) + | > loss_kl: 2.323659658432007 (2.46773898601532) + | > loss_feat: 6.566860198974609 (6.820019006729126) + | > loss_mel: 17.341279983520508 (17.693435668945312) + | > loss_duration: 1.7010947465896606 (1.7204596400260925) + | > loss_1: 30.10158920288086 (30.81321620941162) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032804012298583984 (-0.0003758668899536133) + | > avg_loss_disc: 2.44315767288208 (+0.015839338302612305) + | > avg_loss_disc_real_0: 0.13516685366630554 (+0.0394584946334362) + | > avg_loss_disc_real_1: 0.18261047452688217 (-0.008716799318790436) + | > avg_loss_disc_real_2: 0.20306067168712616 (-0.050256796181201935) + | > avg_loss_disc_real_3: 0.2114792987704277 (-0.00734962522983551) + | > avg_loss_disc_real_4: 0.2070615440607071 (-0.03737025707960129) + | > avg_loss_disc_real_5: 0.2156124711036682 (-0.004304029047489166) + | > avg_loss_0: 2.44315767288208 (+0.015839338302612305) + | > avg_loss_gen: 2.1115628480911255 (-0.11340641975402832) + | > avg_loss_kl: 2.46773898601532 (-0.0032453536987304688) + | > avg_loss_feat: 6.820019006729126 (-0.5946924686431885) + | > avg_loss_mel: 17.693435668945312 (-1.234309196472168) + | > avg_loss_duration: 1.7204596400260925 (-0.007923543453216553) + | > avg_loss_1: 30.81321620941162 (-1.953577995300293) + + + > EPOCH: 277/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:32:37)  + + --> TIME: 2025-04-19 03:32:41 -- STEP: 9/142 -- GLOBAL_STEP: 418625 + | > loss_disc: 2.4181180000305176 (2.389587958653768) + | > loss_disc_real_0: 0.13126665353775024 (0.1280306875705719) + | > loss_disc_real_1: 0.222612664103508 (0.19788719382550982) + | > loss_disc_real_2: 0.226558119058609 (0.22327772941854265) + | > loss_disc_real_3: 0.23924045264720917 (0.22757966816425323) + | > loss_disc_real_4: 0.2179202139377594 (0.21720868017938402) + | > loss_disc_real_5: 0.2477003037929535 (0.23314833144346872) + | > loss_0: 2.4181180000305176 (2.389587958653768) + | > grad_norm_0: tensor(8.0683, device='cuda:0') (tensor(7.1362, device='cuda:0')) + | > loss_gen: 2.4211597442626953 (2.4140247768825955) + | > loss_kl: 1.6949920654296875 (1.841714964972602) + | > loss_feat: 6.688870906829834 (7.106206787957086) + | > loss_mel: 16.1961669921875 (17.24502733018663) + | > loss_duration: 1.4124608039855957 (1.4213487837049696) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.413650512695312 (30.028322219848633) + | > grad_norm_1: tensor(270.9466, device='cuda:0') (tensor(142.4699, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.3259 (0.3256946669684516) + | > loader_time: 0.0045 (0.004617320166693794) + + + --> TIME: 2025-04-19 03:32:50 -- STEP: 34/142 -- GLOBAL_STEP: 418650 + | > loss_disc: 2.3816773891448975 (2.435470237451441) + | > loss_disc_real_0: 0.0848160907626152 (0.12381736758877249) + | > loss_disc_real_1: 0.197786346077919 (0.20127936452627182) + | > loss_disc_real_2: 0.22252081334590912 (0.22431026837405035) + | > loss_disc_real_3: 0.24315550923347473 (0.22684023792252822) + | > loss_disc_real_4: 0.18623697757720947 (0.22838965452769222) + | > loss_disc_real_5: 0.2397056668996811 (0.23009611666202545) + | > loss_0: 2.3816773891448975 (2.435470237451441) + | > grad_norm_0: tensor(18.3934, device='cuda:0') (tensor(9.4214, device='cuda:0')) + | > loss_gen: 2.396636962890625 (2.3682703060262345) + | > loss_kl: 2.0127720832824707 (1.8881488337236292) + | > loss_feat: 8.341115951538086 (7.087893822613885) + | > loss_mel: 18.33700180053711 (17.385995696572703) + | > loss_duration: 1.4446903467178345 (1.4294981605866377) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.5322151184082 (30.159806644215305) + | > grad_norm_1: tensor(463.0533, device='cuda:0') (tensor(221.6041, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.3576 (0.33924396599040313) + | > loader_time: 0.0053 (0.005101694780237535) + + + --> TIME: 2025-04-19 03:32:59 -- STEP: 59/142 -- GLOBAL_STEP: 418675 + | > loss_disc: 2.4345924854278564 (2.436727325795061) + | > loss_disc_real_0: 0.20054011046886444 (0.12368343379032815) + | > loss_disc_real_1: 0.2336515337228775 (0.20137405420764018) + | > loss_disc_real_2: 0.24064446985721588 (0.22497693865986193) + | > loss_disc_real_3: 0.2310250997543335 (0.22650182853310796) + | > loss_disc_real_4: 0.20445194840431213 (0.22698199117587783) + | > loss_disc_real_5: 0.25764816999435425 (0.2332990927211309) + | > loss_0: 2.4345924854278564 (2.436727325795061) + | > grad_norm_0: tensor(13.8195, device='cuda:0') (tensor(10.4941, device='cuda:0')) + | > loss_gen: 2.644972562789917 (2.39098051038839) + | > loss_kl: 1.7497698068618774 (1.8904537588863048) + | > loss_feat: 7.383484840393066 (7.175245907347081) + | > loss_mel: 18.54532241821289 (17.67426012330137) + | > loss_duration: 1.4633733034133911 (1.4426194530422405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.786922454833984 (30.57355958324368) + | > grad_norm_1: tensor(526.7489, device='cuda:0') (tensor(252.0988, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.3865 (0.35046488147670946) + | > loader_time: 0.0062 (0.005411208686182055) + + + --> TIME: 2025-04-19 03:33:09 -- STEP: 84/142 -- GLOBAL_STEP: 418700 + | > loss_disc: 2.4312808513641357 (2.444068959781102) + | > loss_disc_real_0: 0.08703340590000153 (0.1240078414834681) + | > loss_disc_real_1: 0.1988595724105835 (0.20261859769622484) + | > loss_disc_real_2: 0.26608169078826904 (0.22627350368670054) + | > loss_disc_real_3: 0.22881361842155457 (0.22575024231558755) + | > loss_disc_real_4: 0.23790831863880157 (0.2272998953149432) + | > loss_disc_real_5: 0.23705846071243286 (0.23321579911169552) + | > loss_0: 2.4312808513641357 (2.444068959781102) + | > grad_norm_0: tensor(6.3195, device='cuda:0') (tensor(10.7550, device='cuda:0')) + | > loss_gen: 2.572240114212036 (2.3805884945960276) + | > loss_kl: 1.9416710138320923 (1.896372173513685) + | > loss_feat: 6.763632297515869 (7.149915360269093) + | > loss_mel: 17.667972564697266 (17.7131170091175) + | > loss_duration: 1.5065131187438965 (1.4545514796461378) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.452030181884766 (30.59454431987944) + | > grad_norm_1: tensor(88.1118, device='cuda:0') (tensor(262.3283, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.3944 (0.36343144802820126) + | > loader_time: 0.0067 (0.005747133777255103) + + + --> TIME: 2025-04-19 03:33:20 -- STEP: 109/142 -- GLOBAL_STEP: 418725 + | > loss_disc: 2.4876692295074463 (2.4517383422326593) + | > loss_disc_real_0: 0.11665961891412735 (0.12582021418514602) + | > loss_disc_real_1: 0.26434779167175293 (0.20327707468916517) + | > loss_disc_real_2: 0.2474772334098816 (0.2262759181337619) + | > loss_disc_real_3: 0.2375858873128891 (0.22760199625557714) + | > loss_disc_real_4: 0.25081706047058105 (0.22869995906265503) + | > loss_disc_real_5: 0.24723364412784576 (0.2336487873978571) + | > loss_0: 2.4876692295074463 (2.4517383422326593) + | > grad_norm_0: tensor(12.8349, device='cuda:0') (tensor(11.5217, device='cuda:0')) + | > loss_gen: 2.308621883392334 (2.366242365005914) + | > loss_kl: 1.961391806602478 (1.8928594261134437) + | > loss_feat: 7.087416648864746 (7.064671997630268) + | > loss_mel: 19.305007934570312 (17.656056710339477) + | > loss_duration: 1.4859620332717896 (1.4626951742609706) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.14840316772461 (30.442525601168292) + | > grad_norm_1: tensor(268.5319, device='cuda:0') (tensor(272.4915, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.4127 (0.37346367442279793) + | > loader_time: 0.0069 (0.005982339929003234) + + + --> TIME: 2025-04-19 03:33:31 -- STEP: 134/142 -- GLOBAL_STEP: 418750 + | > loss_disc: 2.4643754959106445 (2.4560561962981713) + | > loss_disc_real_0: 0.12418796122074127 (0.12603068907759082) + | > loss_disc_real_1: 0.21916116774082184 (0.20279527099719688) + | > loss_disc_real_2: 0.22135154902935028 (0.22668932611817746) + | > loss_disc_real_3: 0.23676146566867828 (0.22794386577695164) + | > loss_disc_real_4: 0.2439928948879242 (0.22923763796909533) + | > loss_disc_real_5: 0.21809078752994537 (0.23337754851846554) + | > loss_0: 2.4643754959106445 (2.4560561962981713) + | > grad_norm_0: tensor(5.7492, device='cuda:0') (tensor(10.9296, device='cuda:0')) + | > loss_gen: 2.3704886436462402 (2.3570266623995204) + | > loss_kl: 1.928362250328064 (1.8896479375326811) + | > loss_feat: 6.698671340942383 (7.031659738341374) + | > loss_mel: 16.748632431030273 (17.628955798362625) + | > loss_duration: 1.525289535522461 (1.4693183587558234) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.27144432067871 (30.37660840731948) + | > grad_norm_1: tensor(102.4007, device='cuda:0') (tensor(253.3802, device='cuda:0')) + | > current_lr_0: 0.00019319309912101717 + | > current_lr_1: 0.00019319309912101717 + | > step_time: 0.4383 (0.38515885196514993) + | > loader_time: 0.0078 (0.0062710402616814) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.440993309020996 (2.440993309020996) + | > loss_disc_real_0: 0.14270390570163727 (0.14270390570163727) + | > loss_disc_real_1: 0.20803038775920868 (0.20803038775920868) + | > loss_disc_real_2: 0.25558316707611084 (0.25558316707611084) + | > loss_disc_real_3: 0.2378132939338684 (0.2378132939338684) + | > loss_disc_real_4: 0.24004684388637543 (0.24004684388637543) + | > loss_disc_real_5: 0.22280925512313843 (0.22280925512313843) + | > loss_0: 2.440993309020996 (2.440993309020996) + | > loss_gen: 2.425565004348755 (2.425565004348755) + | > loss_kl: 2.344804525375366 (2.344804525375366) + | > loss_feat: 6.9842963218688965 (6.9842963218688965) + | > loss_mel: 18.273202896118164 (18.273202896118164) + | > loss_duration: 1.7036962509155273 (1.7036962509155273) + | > loss_1: 31.731563568115234 (31.731563568115234) + + --> STEP: 1 + | > loss_disc: 2.5626516342163086 (2.5626516342163086) + | > loss_disc_real_0: 0.1654823124408722 (0.1654823124408722) + | > loss_disc_real_1: 0.20154796540737152 (0.20154796540737152) + | > loss_disc_real_2: 0.26981639862060547 (0.26981639862060547) + | > loss_disc_real_3: 0.23278459906578064 (0.23278459906578064) + | > loss_disc_real_4: 0.20813895761966705 (0.20813895761966705) + | > loss_disc_real_5: 0.19589345157146454 (0.19589345157146454) + | > loss_0: 2.5626516342163086 (2.5626516342163086) + | > loss_gen: 2.0879874229431152 (2.0879874229431152) + | > loss_kl: 2.6035687923431396 (2.6035687923431396) + | > loss_feat: 7.317505359649658 (7.317505359649658) + | > loss_mel: 19.163448333740234 (19.163448333740234) + | > loss_duration: 1.7242368459701538 (1.7242368459701538) + | > loss_1: 32.89674758911133 (32.89674758911133) + + --> STEP: 2 + | > loss_disc: 2.6585702896118164 (2.6106109619140625) + | > loss_disc_real_0: 0.18357102572917938 (0.1745266690850258) + | > loss_disc_real_1: 0.22918963432312012 (0.21536879986524582) + | > loss_disc_real_2: 0.28356969356536865 (0.27669304609298706) + | > loss_disc_real_3: 0.23433183133602142 (0.23355821520090103) + | > loss_disc_real_4: 0.2643730342388153 (0.23625599592924118) + | > loss_disc_real_5: 0.220703586935997 (0.20829851925373077) + | > loss_0: 2.6585702896118164 (2.6106109619140625) + | > loss_gen: 2.186899185180664 (2.1374433040618896) + | > loss_kl: 2.1687498092651367 (2.386159300804138) + | > loss_feat: 5.353743553161621 (6.33562445640564) + | > loss_mel: 14.561591148376465 (16.86251974105835) + | > loss_duration: 1.6619981527328491 (1.6931174993515015) + | > loss_1: 25.932981491088867 (29.414864540100098) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031903982162475586 (-9.000301361083984e-05) + | > avg_loss_disc: 2.6106109619140625 (+0.16745328903198242) + | > avg_loss_disc_real_0: 0.1745266690850258 (+0.039359815418720245) + | > avg_loss_disc_real_1: 0.21536879986524582 (+0.03275832533836365) + | > avg_loss_disc_real_2: 0.27669304609298706 (+0.0736323744058609) + | > avg_loss_disc_real_3: 0.23355821520090103 (+0.022078916430473328) + | > avg_loss_disc_real_4: 0.23625599592924118 (+0.029194451868534088) + | > avg_loss_disc_real_5: 0.20829851925373077 (-0.007313951849937439) + | > avg_loss_0: 2.6106109619140625 (+0.16745328903198242) + | > avg_loss_gen: 2.1374433040618896 (+0.02588045597076416) + | > avg_loss_kl: 2.386159300804138 (-0.08157968521118164) + | > avg_loss_feat: 6.33562445640564 (-0.48439455032348633) + | > avg_loss_mel: 16.86251974105835 (-0.8309159278869629) + | > avg_loss_duration: 1.6931174993515015 (-0.027342140674591064) + | > avg_loss_1: 29.414864540100098 (-1.3983516693115234) + + + > EPOCH: 278/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:33:36)  + + --> TIME: 2025-04-19 03:33:42 -- STEP: 17/142 -- GLOBAL_STEP: 418775 + | > loss_disc: 2.393935441970825 (2.469186376122867) + | > loss_disc_real_0: 0.11183171719312668 (0.12115324507741367) + | > loss_disc_real_1: 0.19060063362121582 (0.21019281885203192) + | > loss_disc_real_2: 0.22342686355113983 (0.23120173636604757) + | > loss_disc_real_3: 0.237394779920578 (0.23124082386493683) + | > loss_disc_real_4: 0.23549938201904297 (0.22801295098136454) + | > loss_disc_real_5: 0.2295108437538147 (0.22939411156317768) + | > loss_0: 2.393935441970825 (2.469186376122867) + | > grad_norm_0: tensor(9.3274, device='cuda:0') (tensor(6.1200, device='cuda:0')) + | > loss_gen: 2.4320414066314697 (2.371181277667775) + | > loss_kl: 1.724867582321167 (1.8356243161594166) + | > loss_feat: 7.261488914489746 (7.299497856813319) + | > loss_mel: 17.69782829284668 (17.93627738952637) + | > loss_duration: 1.4253450632095337 (1.4229767392663395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.54157066345215 (30.86555783888873) + | > grad_norm_1: tensor(145.7412, device='cuda:0') (tensor(118.4841, device='cuda:0')) + | > current_lr_0: 0.00019316894998362705 + | > current_lr_1: 0.00019316894998362705 + | > step_time: 0.3445 (0.330103313221651) + | > loader_time: 0.0051 (0.004682120154885685) + + + --> TIME: 2025-04-19 03:33:52 -- STEP: 42/142 -- GLOBAL_STEP: 418800 + | > loss_disc: 2.4821839332580566 (2.4182700713475547) + | > loss_disc_real_0: 0.11244640499353409 (0.11960417990173612) + | > loss_disc_real_1: 0.18492096662521362 (0.20240451785780134) + | > loss_disc_real_2: 0.23948192596435547 (0.22280461802369073) + | > loss_disc_real_3: 0.22604672610759735 (0.2248625588558969) + | > loss_disc_real_4: 0.1942361295223236 (0.22428376582406817) + | > loss_disc_real_5: 0.23098036646842957 (0.22740790638185682) + | > loss_0: 2.4821839332580566 (2.4182700713475547) + | > grad_norm_0: tensor(3.5538, device='cuda:0') (tensor(7.2870, device='cuda:0')) + | > loss_gen: 2.3846943378448486 (2.3904084875470124) + | > loss_kl: 1.854013442993164 (1.8472702162606376) + | > loss_feat: 6.772515296936035 (7.347903058642433) + | > loss_mel: 17.394485473632812 (17.868057795933314) + | > loss_duration: 1.4702129364013672 (1.4354532701628548) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.87592315673828 (30.889092944917223) + | > grad_norm_1: tensor(100.0691, device='cuda:0') (tensor(156.9175, device='cuda:0')) + | > current_lr_0: 0.00019316894998362705 + | > current_lr_1: 0.00019316894998362705 + | > step_time: 0.373 (0.34722355433872765) + | > loader_time: 0.0058 (0.005292268026442754) + + + --> TIME: 2025-04-19 03:34:01 -- STEP: 67/142 -- GLOBAL_STEP: 418825 + | > loss_disc: 2.487154722213745 (2.456178626017784) + | > loss_disc_real_0: 0.10656028240919113 (0.12183202903217344) + | > loss_disc_real_1: 0.19450262188911438 (0.20555597276829962) + | > loss_disc_real_2: 0.1972341388463974 (0.226201353233252) + | > loss_disc_real_3: 0.2079210877418518 (0.2285474209643122) + | > loss_disc_real_4: 0.19614063203334808 (0.22836003663824567) + | > loss_disc_real_5: 0.2418016791343689 (0.2332544019862787) + | > loss_0: 2.487154722213745 (2.456178626017784) + | > grad_norm_0: tensor(14.4401, device='cuda:0') (tensor(8.7402, device='cuda:0')) + | > loss_gen: 2.356793165206909 (2.372576450234029) + | > loss_kl: 1.941663146018982 (1.8589413112668849) + | > loss_feat: 7.374416828155518 (7.233935420192889) + | > loss_mel: 17.554134368896484 (17.80415431065346) + | > loss_duration: 1.3943395614624023 (1.444899934441296) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.62134552001953 (30.714507487282823) + | > grad_norm_1: tensor(72.9847, device='cuda:0') (tensor(181.2431, device='cuda:0')) + | > current_lr_0: 0.00019316894998362705 + | > current_lr_1: 0.00019316894998362705 + | > step_time: 0.3965 (0.36170377304304885) + | > loader_time: 0.0064 (0.00562596321105957) + + + --> TIME: 2025-04-19 03:34:12 -- STEP: 92/142 -- GLOBAL_STEP: 418850 + | > loss_disc: 2.397571563720703 (2.4585377781287483) + | > loss_disc_real_0: 0.10135302692651749 (0.12264286192215007) + | > loss_disc_real_1: 0.17522872984409332 (0.20621859525208888) + | > loss_disc_real_2: 0.25978365540504456 (0.22680721836893455) + | > loss_disc_real_3: 0.24564941227436066 (0.22932862749566202) + | > loss_disc_real_4: 0.20171532034873962 (0.22829502626605655) + | > loss_disc_real_5: 0.24892780184745789 (0.2319856801110765) + | > loss_0: 2.397571563720703 (2.4585377781287483) + | > grad_norm_0: tensor(18.4600, device='cuda:0') (tensor(9.6969, device='cuda:0')) + | > loss_gen: 2.529815435409546 (2.3680319449175955) + | > loss_kl: 1.8892964124679565 (1.8631692127041195) + | > loss_feat: 7.234258651733398 (7.15035633418871) + | > loss_mel: 18.02214813232422 (17.74011656512385) + | > loss_duration: 1.4537991285324097 (1.4569744040136752) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.129316329956055 (30.578648525735606) + | > grad_norm_1: tensor(362.0143, device='cuda:0') (tensor(206.5578, device='cuda:0')) + | > current_lr_0: 0.00019316894998362705 + | > current_lr_1: 0.00019316894998362705 + | > step_time: 0.4225 (0.3743190298909724) + | > loader_time: 0.007 (0.005950500135836394) + + + --> TIME: 2025-04-19 03:34:23 -- STEP: 117/142 -- GLOBAL_STEP: 418875 + | > loss_disc: 2.4216184616088867 (2.4679113310626426) + | > loss_disc_real_0: 0.07545019686222076 (0.1257261995576386) + | > loss_disc_real_1: 0.23656360805034637 (0.2081710087437915) + | > loss_disc_real_2: 0.2638842463493347 (0.2266625293299683) + | > loss_disc_real_3: 0.24322088062763214 (0.22951601522091108) + | > loss_disc_real_4: 0.20488178730010986 (0.2290822862305193) + | > loss_disc_real_5: 0.2573901414871216 (0.2316843529797008) + | > loss_0: 2.4216184616088867 (2.4679113310626426) + | > grad_norm_0: tensor(13.7651, device='cuda:0') (tensor(10.8576, device='cuda:0')) + | > loss_gen: 2.3144681453704834 (2.3568177162072588) + | > loss_kl: 1.8041467666625977 (1.8715658513908713) + | > loss_feat: 7.59138822555542 (7.074547612768972) + | > loss_mel: 19.089054107666016 (17.65408358614669) + | > loss_duration: 1.4920239448547363 (1.4636734622156518) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.291080474853516 (30.420688221597263) + | > grad_norm_1: tensor(208.3634, device='cuda:0') (tensor(219.7039, device='cuda:0')) + | > current_lr_0: 0.00019316894998362705 + | > current_lr_1: 0.00019316894998362705 + | > step_time: 0.4536 (0.3861472932701435) + | > loader_time: 0.0098 (0.006238425898755717) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4991562366485596 (2.4991562366485596) + | > loss_disc_real_0: 0.13241878151893616 (0.13241878151893616) + | > loss_disc_real_1: 0.1803995966911316 (0.1803995966911316) + | > loss_disc_real_2: 0.2661101222038269 (0.2661101222038269) + | > loss_disc_real_3: 0.22617971897125244 (0.22617971897125244) + | > loss_disc_real_4: 0.2405937761068344 (0.2405937761068344) + | > loss_disc_real_5: 0.23402027785778046 (0.23402027785778046) + | > loss_0: 2.4991562366485596 (2.4991562366485596) + | > loss_gen: 2.214210033416748 (2.214210033416748) + | > loss_kl: 2.49869704246521 (2.49869704246521) + | > loss_feat: 6.172278881072998 (6.172278881072998) + | > loss_mel: 15.917699813842773 (15.917699813842773) + | > loss_duration: 1.715139389038086 (1.715139389038086) + | > loss_1: 28.51802635192871 (28.51802635192871) + + --> STEP: 1 + | > loss_disc: 2.416968822479248 (2.416968822479248) + | > loss_disc_real_0: 0.08011377602815628 (0.08011377602815628) + | > loss_disc_real_1: 0.18620283901691437 (0.18620283901691437) + | > loss_disc_real_2: 0.25361573696136475 (0.25361573696136475) + | > loss_disc_real_3: 0.20896272361278534 (0.20896272361278534) + | > loss_disc_real_4: 0.20260363817214966 (0.20260363817214966) + | > loss_disc_real_5: 0.2416956126689911 (0.2416956126689911) + | > loss_0: 2.416968822479248 (2.416968822479248) + | > loss_gen: 2.1207399368286133 (2.1207399368286133) + | > loss_kl: 2.6163437366485596 (2.6163437366485596) + | > loss_feat: 6.653181552886963 (6.653181552886963) + | > loss_mel: 17.600536346435547 (17.600536346435547) + | > loss_duration: 1.7731306552886963 (1.7731306552886963) + | > loss_1: 30.763933181762695 (30.763933181762695) + + --> STEP: 2 + | > loss_disc: 2.6197381019592285 (2.5183534622192383) + | > loss_disc_real_0: 0.12054130434989929 (0.10032754018902779) + | > loss_disc_real_1: 0.2013876885175705 (0.19379526376724243) + | > loss_disc_real_2: 0.2586307227611542 (0.25612322986125946) + | > loss_disc_real_3: 0.26274263858795166 (0.2358526811003685) + | > loss_disc_real_4: 0.23801599442958832 (0.220309816300869) + | > loss_disc_real_5: 0.2255515307188034 (0.23362357169389725) + | > loss_0: 2.6197381019592285 (2.5183534622192383) + | > loss_gen: 2.0258564949035645 (2.073298215866089) + | > loss_kl: 2.312833309173584 (2.4645885229110718) + | > loss_feat: 6.179610729217529 (6.416396141052246) + | > loss_mel: 14.146631240844727 (15.873583793640137) + | > loss_duration: 1.6982147693634033 (1.7356727123260498) + | > loss_1: 26.363147735595703 (28.5635404586792) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003322005271911621 (+0.0001316070556640625) + | > avg_loss_disc: 2.5183534622192383 (-0.09225749969482422) + | > avg_loss_disc_real_0: 0.10032754018902779 (-0.074199128895998) + | > avg_loss_disc_real_1: 0.19379526376724243 (-0.021573536098003387) + | > avg_loss_disc_real_2: 0.25612322986125946 (-0.0205698162317276) + | > avg_loss_disc_real_3: 0.2358526811003685 (+0.0022944658994674683) + | > avg_loss_disc_real_4: 0.220309816300869 (-0.015946179628372192) + | > avg_loss_disc_real_5: 0.23362357169389725 (+0.025325052440166473) + | > avg_loss_0: 2.5183534622192383 (-0.09225749969482422) + | > avg_loss_gen: 2.073298215866089 (-0.06414508819580078) + | > avg_loss_kl: 2.4645885229110718 (+0.0784292221069336) + | > avg_loss_feat: 6.416396141052246 (+0.08077168464660645) + | > avg_loss_mel: 15.873583793640137 (-0.9889359474182129) + | > avg_loss_duration: 1.7356727123260498 (+0.04255521297454834) + | > avg_loss_1: 28.5635404586792 (-0.8513240814208984) + + + > EPOCH: 279/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:34:36)  + + --> TIME: 2025-04-19 03:34:37 -- STEP: 0/142 -- GLOBAL_STEP: 418900 + | > loss_disc: 2.4393017292022705 (2.4393017292022705) + | > loss_disc_real_0: 0.08987734466791153 (0.08987734466791153) + | > loss_disc_real_1: 0.20193931460380554 (0.20193931460380554) + | > loss_disc_real_2: 0.26375138759613037 (0.26375138759613037) + | > loss_disc_real_3: 0.22745701670646667 (0.22745701670646667) + | > loss_disc_real_4: 0.24636930227279663 (0.24636930227279663) + | > loss_disc_real_5: 0.23822659254074097 (0.23822659254074097) + | > loss_0: 2.4393017292022705 (2.4393017292022705) + | > grad_norm_0: tensor(8.9663, device='cuda:0') (tensor(8.9663, device='cuda:0')) + | > loss_gen: 2.7868096828460693 (2.7868096828460693) + | > loss_kl: 1.9245859384536743 (1.9245859384536743) + | > loss_feat: 7.382641792297363 (7.382641792297363) + | > loss_mel: 17.709447860717773 (17.709447860717773) + | > loss_duration: 1.433449387550354 (1.433449387550354) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.236934661865234 (31.236934661865234) + | > grad_norm_1: tensor(389.6357, device='cuda:0') (tensor(389.6357, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.3632 (0.36322021484375) + | > loader_time: 0.3245 (0.32452392578125) + + + --> TIME: 2025-04-19 03:34:45 -- STEP: 25/142 -- GLOBAL_STEP: 418925 + | > loss_disc: 2.339315176010132 (2.4885295677185066) + | > loss_disc_real_0: 0.07206536829471588 (0.13846623480319972) + | > loss_disc_real_1: 0.18305635452270508 (0.20508394241333008) + | > loss_disc_real_2: 0.21496209502220154 (0.22760107278823852) + | > loss_disc_real_3: 0.2273557037115097 (0.2316765606403351) + | > loss_disc_real_4: 0.23176920413970947 (0.23411345541477202) + | > loss_disc_real_5: 0.21017026901245117 (0.2352053540945053) + | > loss_0: 2.339315176010132 (2.4885295677185066) + | > grad_norm_0: tensor(17.2827, device='cuda:0') (tensor(20.5609, device='cuda:0')) + | > loss_gen: 2.3770906925201416 (2.370985059738159) + | > loss_kl: 1.8798422813415527 (1.815065679550171) + | > loss_feat: 7.779181003570557 (7.265273208618164) + | > loss_mel: 17.801795959472656 (17.824721755981447) + | > loss_duration: 1.4811718463897705 (1.428781614303589) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.319082260131836 (30.704827270507813) + | > grad_norm_1: tensor(63.6068, device='cuda:0') (tensor(325.4351, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.346 (0.3340472888946533) + | > loader_time: 0.0052 (0.00482508659362793) + + + --> TIME: 2025-04-19 03:34:54 -- STEP: 50/142 -- GLOBAL_STEP: 418950 + | > loss_disc: 2.557023763656616 (2.46895823955536) + | > loss_disc_real_0: 0.1121390089392662 (0.13011476337909694) + | > loss_disc_real_1: 0.19183434545993805 (0.20492031633853913) + | > loss_disc_real_2: 0.22335317730903625 (0.22621508330106735) + | > loss_disc_real_3: 0.2384580820798874 (0.23122507065534592) + | > loss_disc_real_4: 0.2513112723827362 (0.2283533549308777) + | > loss_disc_real_5: 0.2695951759815216 (0.23494184613227845) + | > loss_0: 2.557023763656616 (2.46895823955536) + | > grad_norm_0: tensor(10.4388, device='cuda:0') (tensor(16.0678, device='cuda:0')) + | > loss_gen: 2.180323362350464 (2.350313763618469) + | > loss_kl: 1.800614833831787 (1.8432797551155091) + | > loss_feat: 6.210282802581787 (7.101603546142578) + | > loss_mel: 16.01826286315918 (17.543948326110844) + | > loss_duration: 1.4716216325759888 (1.437368860244751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.68110466003418 (30.27651412963867) + | > grad_norm_1: tensor(302.6840, device='cuda:0') (tensor(308.5631, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.37 (0.3463502073287964) + | > loader_time: 0.0061 (0.005192618370056152) + + + --> TIME: 2025-04-19 03:35:04 -- STEP: 75/142 -- GLOBAL_STEP: 418975 + | > loss_disc: 2.4231672286987305 (2.471038958231609) + | > loss_disc_real_0: 0.12503938376903534 (0.12861371328433355) + | > loss_disc_real_1: 0.2061089128255844 (0.2066364953915278) + | > loss_disc_real_2: 0.20269349217414856 (0.22593045771121978) + | > loss_disc_real_3: 0.20836634933948517 (0.23179659247398376) + | > loss_disc_real_4: 0.21439532935619354 (0.2283542170127233) + | > loss_disc_real_5: 0.23701035976409912 (0.23598510444164275) + | > loss_0: 2.4231672286987305 (2.471038958231609) + | > grad_norm_0: tensor(5.5093, device='cuda:0') (tensor(14.2678, device='cuda:0')) + | > loss_gen: 2.304723024368286 (2.349340130488078) + | > loss_kl: 1.9322346448898315 (1.8738123973210652) + | > loss_feat: 6.970256328582764 (7.113550090789795) + | > loss_mel: 16.103750228881836 (17.61547955830892) + | > loss_duration: 1.444698691368103 (1.4492386436462403) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.75566291809082 (30.401420618693034) + | > grad_norm_1: tensor(71.1850, device='cuda:0') (tensor(289.4270, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.3993 (0.35842642148335785) + | > loader_time: 0.0067 (0.005503060022989909) + + + --> TIME: 2025-04-19 03:35:15 -- STEP: 100/142 -- GLOBAL_STEP: 419000 + | > loss_disc: 2.375192165374756 (2.475877997875214) + | > loss_disc_real_0: 0.10505077987909317 (0.12817347183823583) + | > loss_disc_real_1: 0.19652818143367767 (0.2072206601500511) + | > loss_disc_real_2: 0.21738740801811218 (0.22732525438070297) + | > loss_disc_real_3: 0.2153547704219818 (0.23064934238791465) + | > loss_disc_real_4: 0.21973109245300293 (0.22812831073999404) + | > loss_disc_real_5: 0.20709985494613647 (0.23273886770009994) + | > loss_0: 2.375192165374756 (2.475877997875214) + | > grad_norm_0: tensor(9.8803, device='cuda:0') (tensor(13.4236, device='cuda:0')) + | > loss_gen: 2.332692861557007 (2.3287161421775817) + | > loss_kl: 1.8598066568374634 (1.8847362661361695) + | > loss_feat: 7.088444232940674 (7.032372665405274) + | > loss_mel: 17.40214729309082 (17.593558616638187) + | > loss_duration: 1.4897394180297852 (1.4589730429649352) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.172832489013672 (30.298356742858886) + | > grad_norm_1: tensor(244.8406, device='cuda:0') (tensor(277.7775, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.4104 (0.3700266122817994) + | > loader_time: 0.007 (0.005806312561035156) + + + --> TIME: 2025-04-19 03:35:25 -- STEP: 125/142 -- GLOBAL_STEP: 419025 + | > loss_disc: 2.442319631576538 (2.475249639511109) + | > loss_disc_real_0: 0.09371522814035416 (0.12844603806734078) + | > loss_disc_real_1: 0.23703114688396454 (0.2075936198234558) + | > loss_disc_real_2: 0.27996206283569336 (0.22748120737075805) + | > loss_disc_real_3: 0.2653859853744507 (0.23041571986675263) + | > loss_disc_real_4: 0.2620481550693512 (0.22865894412994384) + | > loss_disc_real_5: 0.2715625464916229 (0.23389234364032746) + | > loss_0: 2.442319631576538 (2.475249639511109) + | > grad_norm_0: tensor(7.5252, device='cuda:0') (tensor(12.6735, device='cuda:0')) + | > loss_gen: 2.422126054763794 (2.327778251647949) + | > loss_kl: 1.9920116662979126 (1.8952538814544677) + | > loss_feat: 7.095134258270264 (6.980267303466797) + | > loss_mel: 18.7611141204834 (17.52224553680421) + | > loss_duration: 1.5346927642822266 (1.466623303413391) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.805078506469727 (30.192168258666992) + | > grad_norm_1: tensor(285.0609, device='cuda:0') (tensor(274.9605, device='cuda:0')) + | > current_lr_0: 0.0001931448038648791 + | > current_lr_1: 0.0001931448038648791 + | > step_time: 0.4369 (0.3812044582366944) + | > loader_time: 0.0078 (0.006092208862304688) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3650128841400146 (2.3650128841400146) + | > loss_disc_real_0: 0.15669187903404236 (0.15669187903404236) + | > loss_disc_real_1: 0.20501698553562164 (0.20501698553562164) + | > loss_disc_real_2: 0.19214436411857605 (0.19214436411857605) + | > loss_disc_real_3: 0.18819303810596466 (0.18819303810596466) + | > loss_disc_real_4: 0.23652099072933197 (0.23652099072933197) + | > loss_disc_real_5: 0.2126697599887848 (0.2126697599887848) + | > loss_0: 2.3650128841400146 (2.3650128841400146) + | > loss_gen: 2.3342885971069336 (2.3342885971069336) + | > loss_kl: 2.4898524284362793 (2.4898524284362793) + | > loss_feat: 6.758357048034668 (6.758357048034668) + | > loss_mel: 17.345077514648438 (17.345077514648438) + | > loss_duration: 1.701582670211792 (1.701582670211792) + | > loss_1: 30.6291561126709 (30.6291561126709) + + --> STEP: 1 + | > loss_disc: 2.5231597423553467 (2.5231597423553467) + | > loss_disc_real_0: 0.20662084221839905 (0.20662084221839905) + | > loss_disc_real_1: 0.2182886302471161 (0.2182886302471161) + | > loss_disc_real_2: 0.19491884112358093 (0.19491884112358093) + | > loss_disc_real_3: 0.20410457253456116 (0.20410457253456116) + | > loss_disc_real_4: 0.21703071892261505 (0.21703071892261505) + | > loss_disc_real_5: 0.19480645656585693 (0.19480645656585693) + | > loss_0: 2.5231597423553467 (2.5231597423553467) + | > loss_gen: 2.136268138885498 (2.136268138885498) + | > loss_kl: 2.787336826324463 (2.787336826324463) + | > loss_feat: 6.767052173614502 (6.767052173614502) + | > loss_mel: 17.53335189819336 (17.53335189819336) + | > loss_duration: 1.7458206415176392 (1.7458206415176392) + | > loss_1: 30.969831466674805 (30.969831466674805) + + --> STEP: 2 + | > loss_disc: 2.5885632038116455 (2.555861473083496) + | > loss_disc_real_0: 0.2217625379562378 (0.21419169008731842) + | > loss_disc_real_1: 0.24665579199790955 (0.23247221112251282) + | > loss_disc_real_2: 0.2231747955083847 (0.20904681831598282) + | > loss_disc_real_3: 0.19985681772232056 (0.20198069512844086) + | > loss_disc_real_4: 0.23782451450824738 (0.2274276167154312) + | > loss_disc_real_5: 0.2677405774593353 (0.23127351701259613) + | > loss_0: 2.5885632038116455 (2.555861473083496) + | > loss_gen: 2.3115384578704834 (2.2239032983779907) + | > loss_kl: 2.2998499870300293 (2.543593406677246) + | > loss_feat: 6.507603168487549 (6.637327671051025) + | > loss_mel: 18.901262283325195 (18.217307090759277) + | > loss_duration: 1.6923599243164062 (1.7190902829170227) + | > loss_1: 31.712614059448242 (31.341222763061523) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003265857696533203 (-5.614757537841797e-05) + | > avg_loss_disc: 2.555861473083496 (+0.03750801086425781) + | > avg_loss_disc_real_0: 0.21419169008731842 (+0.11386414989829063) + | > avg_loss_disc_real_1: 0.23247221112251282 (+0.038676947355270386) + | > avg_loss_disc_real_2: 0.20904681831598282 (-0.04707641154527664) + | > avg_loss_disc_real_3: 0.20198069512844086 (-0.03387198597192764) + | > avg_loss_disc_real_4: 0.2274276167154312 (+0.007117800414562225) + | > avg_loss_disc_real_5: 0.23127351701259613 (-0.002350054681301117) + | > avg_loss_0: 2.555861473083496 (+0.03750801086425781) + | > avg_loss_gen: 2.2239032983779907 (+0.15060508251190186) + | > avg_loss_kl: 2.543593406677246 (+0.07900488376617432) + | > avg_loss_feat: 6.637327671051025 (+0.2209315299987793) + | > avg_loss_mel: 18.217307090759277 (+2.3437232971191406) + | > avg_loss_duration: 1.7190902829170227 (-0.0165824294090271) + | > avg_loss_1: 31.341222763061523 (+2.777682304382324) + + + > EPOCH: 280/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:35:35)  + + --> TIME: 2025-04-19 03:35:38 -- STEP: 8/142 -- GLOBAL_STEP: 419050 + | > loss_disc: 2.4571645259857178 (2.452503502368927) + | > loss_disc_real_0: 0.08164453506469727 (0.11666251346468925) + | > loss_disc_real_1: 0.18789170682430267 (0.20918740704655647) + | > loss_disc_real_2: 0.17777588963508606 (0.21830003336071968) + | > loss_disc_real_3: 0.20250840485095978 (0.23019875027239323) + | > loss_disc_real_4: 0.17786017060279846 (0.22895438969135284) + | > loss_disc_real_5: 0.20493614673614502 (0.22953198477625847) + | > loss_0: 2.4571645259857178 (2.452503502368927) + | > grad_norm_0: tensor(8.8747, device='cuda:0') (tensor(13.9217, device='cuda:0')) + | > loss_gen: 2.202713966369629 (2.3342732191085815) + | > loss_kl: 1.8200743198394775 (1.872916340827942) + | > loss_feat: 7.310062408447266 (6.859281539916992) + | > loss_mel: 18.031139373779297 (16.962314248085022) + | > loss_duration: 1.430640459060669 (1.425110250711441) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.794631958007812 (29.453895568847656) + | > grad_norm_1: tensor(259.5969, device='cuda:0') (tensor(323.7028, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.323 (0.32411977648735046) + | > loader_time: 0.005 (0.004483431577682495) + + + --> TIME: 2025-04-19 03:35:47 -- STEP: 33/142 -- GLOBAL_STEP: 419075 + | > loss_disc: 2.477842092514038 (2.451000993902033) + | > loss_disc_real_0: 0.1682557314634323 (0.11862997642972252) + | > loss_disc_real_1: 0.18830719590187073 (0.20740843225609173) + | > loss_disc_real_2: 0.21042807400226593 (0.22452395070682873) + | > loss_disc_real_3: 0.23334312438964844 (0.2286267357342171) + | > loss_disc_real_4: 0.251980721950531 (0.22796921341708212) + | > loss_disc_real_5: 0.26548683643341064 (0.2341189863103809) + | > loss_0: 2.477842092514038 (2.451000993902033) + | > grad_norm_0: tensor(19.3223, device='cuda:0') (tensor(14.0996, device='cuda:0')) + | > loss_gen: 2.418386459350586 (2.3791900548067964) + | > loss_kl: 1.7771766185760498 (1.8364553632158223) + | > loss_feat: 6.90854024887085 (7.199943918170351) + | > loss_mel: 17.02056312561035 (17.65570724371708) + | > loss_duration: 1.436466932296753 (1.429029070969784) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.561132431030273 (30.50032546303489) + | > grad_norm_1: tensor(442.5726, device='cuda:0') (tensor(338.8032, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.3565 (0.3367272073572332) + | > loader_time: 0.0053 (0.005088488260904949) + + + --> TIME: 2025-04-19 03:35:56 -- STEP: 58/142 -- GLOBAL_STEP: 419100 + | > loss_disc: 2.4363508224487305 (2.442799679164229) + | > loss_disc_real_0: 0.12083609402179718 (0.11829010217354216) + | > loss_disc_real_1: 0.20464396476745605 (0.20711021253774906) + | > loss_disc_real_2: 0.22814780473709106 (0.22450239031479277) + | > loss_disc_real_3: 0.22233979403972626 (0.22808324699771815) + | > loss_disc_real_4: 0.2504555583000183 (0.22732657528129116) + | > loss_disc_real_5: 0.22562524676322937 (0.23228199959828935) + | > loss_0: 2.4363508224487305 (2.442799679164229) + | > grad_norm_0: tensor(12.0830, device='cuda:0') (tensor(12.1433, device='cuda:0')) + | > loss_gen: 2.444053888320923 (2.383874021727463) + | > loss_kl: 1.9927455186843872 (1.8638473112007667) + | > loss_feat: 7.032341480255127 (7.199613127215155) + | > loss_mel: 19.002883911132812 (17.64556116893375) + | > loss_duration: 1.477357029914856 (1.4406847666049827) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.949382781982422 (30.533580286749476) + | > grad_norm_1: tensor(100.9546, device='cuda:0') (tensor(304.3669, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.3735 (0.35203336025106496) + | > loader_time: 0.0055 (0.005437637197560279) + + + --> TIME: 2025-04-19 03:36:06 -- STEP: 83/142 -- GLOBAL_STEP: 419125 + | > loss_disc: 2.458228349685669 (2.4458069772605433) + | > loss_disc_real_0: 0.14898860454559326 (0.1204183776694608) + | > loss_disc_real_1: 0.19808264076709747 (0.20622343237859658) + | > loss_disc_real_2: 0.21816661953926086 (0.22437694201986474) + | > loss_disc_real_3: 0.207469180226326 (0.2284924443586763) + | > loss_disc_real_4: 0.18318523466587067 (0.22688004248831645) + | > loss_disc_real_5: 0.20268718898296356 (0.23082132404109082) + | > loss_0: 2.458228349685669 (2.4458069772605433) + | > grad_norm_0: tensor(5.2221, device='cuda:0') (tensor(10.9817, device='cuda:0')) + | > loss_gen: 2.391854763031006 (2.372091462813228) + | > loss_kl: 1.9208996295928955 (1.8713536679026592) + | > loss_feat: 6.899789810180664 (7.161027465958193) + | > loss_mel: 18.32638168334961 (17.7150499734534) + | > loss_duration: 1.543837308883667 (1.4523533151810428) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.082763671875 (30.57187585945589) + | > grad_norm_1: tensor(97.1376, device='cuda:0') (tensor(263.1908, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.3982 (0.363483021058232) + | > loader_time: 0.0065 (0.005689801939998766) + + + --> TIME: 2025-04-19 03:36:17 -- STEP: 108/142 -- GLOBAL_STEP: 419150 + | > loss_disc: 2.469740867614746 (2.4528066339316172) + | > loss_disc_real_0: 0.12767885625362396 (0.1228818063520723) + | > loss_disc_real_1: 0.19967928528785706 (0.20705932847879552) + | > loss_disc_real_2: 0.23792916536331177 (0.22482272347918264) + | > loss_disc_real_3: 0.25405481457710266 (0.22831630403244937) + | > loss_disc_real_4: 0.2472677081823349 (0.22783900783569724) + | > loss_disc_real_5: 0.26789456605911255 (0.23152548433453948) + | > loss_0: 2.469740867614746 (2.4528066339316172) + | > grad_norm_0: tensor(7.2228, device='cuda:0') (tensor(9.9124, device='cuda:0')) + | > loss_gen: 2.347386360168457 (2.357698888690383) + | > loss_kl: 1.8661495447158813 (1.8739215841999761) + | > loss_feat: 6.689577579498291 (7.081586484555845) + | > loss_mel: 18.324548721313477 (17.680410058410104) + | > loss_duration: 1.496185302734375 (1.4610854310018044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.723846435546875 (30.4547023596587) + | > grad_norm_1: tensor(155.7700, device='cuda:0') (tensor(234.3073, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.4225 (0.37366980755770646) + | > loader_time: 0.0079 (0.005942949542292843) + + + --> TIME: 2025-04-19 03:36:28 -- STEP: 133/142 -- GLOBAL_STEP: 419175 + | > loss_disc: 2.405141830444336 (2.4630800207754704) + | > loss_disc_real_0: 0.09044600278139114 (0.12568044281543644) + | > loss_disc_real_1: 0.19208204746246338 (0.207490421439472) + | > loss_disc_real_2: 0.2221488654613495 (0.2247541732806012) + | > loss_disc_real_3: 0.23103567957878113 (0.22831427121072784) + | > loss_disc_real_4: 0.21414567530155182 (0.22837428009151517) + | > loss_disc_real_5: 0.21157386898994446 (0.23135513661051155) + | > loss_0: 2.405141830444336 (2.4630800207754704) + | > grad_norm_0: tensor(4.4142, device='cuda:0') (tensor(10.4768, device='cuda:0')) + | > loss_gen: 2.51344633102417 (2.3449222163150183) + | > loss_kl: 2.0401744842529297 (1.8820218663466604) + | > loss_feat: 7.48796272277832 (7.013733799296214) + | > loss_mel: 18.934223175048828 (17.617598547971365) + | > loss_duration: 1.542862892150879 (1.468099506277787) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.51866912841797 (30.32637586091694) + | > grad_norm_1: tensor(298.9099, device='cuda:0') (tensor(235.1910, device='cuda:0')) + | > current_lr_0: 0.00019312066076439597 + | > current_lr_1: 0.00019312066076439597 + | > step_time: 0.4377 (0.3853780165650791) + | > loader_time: 0.007 (0.006183948731960211) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.525752544403076 (2.525752544403076) + | > loss_disc_real_0: 0.1256732940673828 (0.1256732940673828) + | > loss_disc_real_1: 0.2640599310398102 (0.2640599310398102) + | > loss_disc_real_2: 0.27431946992874146 (0.27431946992874146) + | > loss_disc_real_3: 0.25373631715774536 (0.25373631715774536) + | > loss_disc_real_4: 0.21807856857776642 (0.21807856857776642) + | > loss_disc_real_5: 0.2843072712421417 (0.2843072712421417) + | > loss_0: 2.525752544403076 (2.525752544403076) + | > loss_gen: 2.505270004272461 (2.505270004272461) + | > loss_kl: 2.445622444152832 (2.445622444152832) + | > loss_feat: 7.456470966339111 (7.456470966339111) + | > loss_mel: 18.963603973388672 (18.963603973388672) + | > loss_duration: 1.709275484085083 (1.709275484085083) + | > loss_1: 33.08024215698242 (33.08024215698242) + + --> STEP: 1 + | > loss_disc: 2.522082567214966 (2.522082567214966) + | > loss_disc_real_0: 0.13632652163505554 (0.13632652163505554) + | > loss_disc_real_1: 0.2555124759674072 (0.2555124759674072) + | > loss_disc_real_2: 0.23914672434329987 (0.23914672434329987) + | > loss_disc_real_3: 0.23260650038719177 (0.23260650038719177) + | > loss_disc_real_4: 0.1843375563621521 (0.1843375563621521) + | > loss_disc_real_5: 0.24180090427398682 (0.24180090427398682) + | > loss_0: 2.522082567214966 (2.522082567214966) + | > loss_gen: 2.2098116874694824 (2.2098116874694824) + | > loss_kl: 2.6532199382781982 (2.6532199382781982) + | > loss_feat: 6.530216217041016 (6.530216217041016) + | > loss_mel: 17.68087387084961 (17.68087387084961) + | > loss_duration: 1.7305727005004883 (1.7305727005004883) + | > loss_1: 30.80469512939453 (30.80469512939453) + + --> STEP: 2 + | > loss_disc: 2.403498411178589 (2.4627904891967773) + | > loss_disc_real_0: 0.12515072524547577 (0.13073862344026566) + | > loss_disc_real_1: 0.21226836740970612 (0.23389042168855667) + | > loss_disc_real_2: 0.21506407856941223 (0.22710540145635605) + | > loss_disc_real_3: 0.21715384721755981 (0.2248801738023758) + | > loss_disc_real_4: 0.23379437625408173 (0.2090659663081169) + | > loss_disc_real_5: 0.272684782743454 (0.2572428435087204) + | > loss_0: 2.403498411178589 (2.4627904891967773) + | > loss_gen: 2.3681700229644775 (2.28899085521698) + | > loss_kl: 2.2084567546844482 (2.4308383464813232) + | > loss_feat: 8.429450035095215 (7.479833126068115) + | > loss_mel: 19.177610397338867 (18.42924213409424) + | > loss_duration: 1.68336820602417 (1.706970453262329) + | > loss_1: 33.86705780029297 (32.33587646484375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003921985626220703 (+0.0006561279296875) + | > avg_loss_disc: 2.4627904891967773 (-0.09307098388671875) + | > avg_loss_disc_real_0: 0.13073862344026566 (-0.08345306664705276) + | > avg_loss_disc_real_1: 0.23389042168855667 (+0.0014182105660438538) + | > avg_loss_disc_real_2: 0.22710540145635605 (+0.01805858314037323) + | > avg_loss_disc_real_3: 0.2248801738023758 (+0.022899478673934937) + | > avg_loss_disc_real_4: 0.2090659663081169 (-0.0183616504073143) + | > avg_loss_disc_real_5: 0.2572428435087204 (+0.025969326496124268) + | > avg_loss_0: 2.4627904891967773 (-0.09307098388671875) + | > avg_loss_gen: 2.28899085521698 (+0.06508755683898926) + | > avg_loss_kl: 2.4308383464813232 (-0.11275506019592285) + | > avg_loss_feat: 7.479833126068115 (+0.8425054550170898) + | > avg_loss_mel: 18.42924213409424 (+0.21193504333496094) + | > avg_loss_duration: 1.706970453262329 (-0.012119829654693604) + | > avg_loss_1: 32.33587646484375 (+0.9946537017822266) + + + > EPOCH: 281/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:36:33)  + + --> TIME: 2025-04-19 03:36:39 -- STEP: 16/142 -- GLOBAL_STEP: 419200 + | > loss_disc: 2.3872146606445312 (2.396432861685753) + | > loss_disc_real_0: 0.12306606769561768 (0.1203485750593245) + | > loss_disc_real_1: 0.1865522265434265 (0.19970146007835865) + | > loss_disc_real_2: 0.22526197135448456 (0.2198148164898157) + | > loss_disc_real_3: 0.19790662825107574 (0.2174080451950431) + | > loss_disc_real_4: 0.2714168429374695 (0.2258390672504902) + | > loss_disc_real_5: 0.22841677069664001 (0.22624309174716473) + | > loss_0: 2.3872146606445312 (2.396432861685753) + | > grad_norm_0: tensor(7.3364, device='cuda:0') (tensor(11.8974, device='cuda:0')) + | > loss_gen: 2.4351673126220703 (2.387524664402008) + | > loss_kl: 1.9043432474136353 (1.800004668533802) + | > loss_feat: 7.047933578491211 (7.229822903871536) + | > loss_mel: 17.552743911743164 (17.703746676445007) + | > loss_duration: 1.4118064641952515 (1.420289620757103) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.351993560791016 (30.541388154029846) + | > grad_norm_1: tensor(306.1316, device='cuda:0') (tensor(264.8500, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.3406 (0.32867099344730377) + | > loader_time: 0.0051 (0.004667550325393677) + + + --> TIME: 2025-04-19 03:36:48 -- STEP: 41/142 -- GLOBAL_STEP: 419225 + | > loss_disc: 2.4270641803741455 (2.4332980353657785) + | > loss_disc_real_0: 0.14190007746219635 (0.12124580690046637) + | > loss_disc_real_1: 0.19846415519714355 (0.20484297849783084) + | > loss_disc_real_2: 0.20498983561992645 (0.22434119553100773) + | > loss_disc_real_3: 0.23485049605369568 (0.22652420110818816) + | > loss_disc_real_4: 0.21475264430046082 (0.22706314194493177) + | > loss_disc_real_5: 0.22775501012802124 (0.2292453392249782) + | > loss_0: 2.4270641803741455 (2.4332980353657785) + | > grad_norm_0: tensor(6.0038, device='cuda:0') (tensor(10.4817, device='cuda:0')) + | > loss_gen: 2.2178850173950195 (2.3633161637841207) + | > loss_kl: 1.7554978132247925 (1.8249566962079304) + | > loss_feat: 7.075537204742432 (7.137903201870802) + | > loss_mel: 16.528905868530273 (17.616306700357576) + | > loss_duration: 1.4350022077560425 (1.4316448641986383) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.01282501220703 (30.37412731821944) + | > grad_norm_1: tensor(48.5689, device='cuda:0') (tensor(233.7233, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.3611 (0.34134653137951365) + | > loader_time: 0.0056 (0.005163867299149676) + + + --> TIME: 2025-04-19 03:36:58 -- STEP: 66/142 -- GLOBAL_STEP: 419250 + | > loss_disc: 2.4606709480285645 (2.442851503690084) + | > loss_disc_real_0: 0.14205734431743622 (0.1227473843504082) + | > loss_disc_real_1: 0.21863269805908203 (0.20667080942428473) + | > loss_disc_real_2: 0.22564947605133057 (0.22545099506775537) + | > loss_disc_real_3: 0.19697727262973785 (0.22518418368064996) + | > loss_disc_real_4: 0.19125011563301086 (0.22712371701543982) + | > loss_disc_real_5: 0.2330256849527359 (0.22957306942253403) + | > loss_0: 2.4606709480285645 (2.442851503690084) + | > grad_norm_0: tensor(5.0908, device='cuda:0') (tensor(9.5972, device='cuda:0')) + | > loss_gen: 2.324788808822632 (2.3622492335059433) + | > loss_kl: 1.787636399269104 (1.840186565211325) + | > loss_feat: 7.194904804229736 (7.116998325694691) + | > loss_mel: 18.39410972595215 (17.5989102305788) + | > loss_duration: 1.4474103450775146 (1.445458786054091) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.14885139465332 (30.363802909851074) + | > grad_norm_1: tensor(126.5020, device='cuda:0') (tensor(220.5130, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.3843 (0.3538046714031335) + | > loader_time: 0.0074 (0.00549117001620206) + + + --> TIME: 2025-04-19 03:37:08 -- STEP: 91/142 -- GLOBAL_STEP: 419275 + | > loss_disc: 2.3694396018981934 (2.4632611091320333) + | > loss_disc_real_0: 0.09023502469062805 (0.1262690840022905) + | > loss_disc_real_1: 0.2050122618675232 (0.20691632765990037) + | > loss_disc_real_2: 0.2122740000486374 (0.22645017144444224) + | > loss_disc_real_3: 0.20398852229118347 (0.22725483955262782) + | > loss_disc_real_4: 0.2093905359506607 (0.22873307764530182) + | > loss_disc_real_5: 0.19116146862506866 (0.2313562954192633) + | > loss_0: 2.3694396018981934 (2.4632611091320333) + | > grad_norm_0: tensor(19.0566, device='cuda:0') (tensor(9.1978, device='cuda:0')) + | > loss_gen: 2.323725700378418 (2.3519233714093217) + | > loss_kl: 1.912253737449646 (1.859546260519342) + | > loss_feat: 7.1034722328186035 (7.060360484070831) + | > loss_mel: 18.319477081298828 (17.695607206323636) + | > loss_duration: 1.494985580444336 (1.4564425119986895) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.153913497924805 (30.42387968629271) + | > grad_norm_1: tensor(444.8227, device='cuda:0') (tensor(200.9423, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.4007 (0.3649652161441007) + | > loader_time: 0.0063 (0.005783230393797487) + + + --> TIME: 2025-04-19 03:37:19 -- STEP: 116/142 -- GLOBAL_STEP: 419300 + | > loss_disc: 2.4402849674224854 (2.4720383142602853) + | > loss_disc_real_0: 0.20577998459339142 (0.12843687453403563) + | > loss_disc_real_1: 0.20304881036281586 (0.20743516922510905) + | > loss_disc_real_2: 0.24942991137504578 (0.22688656891214437) + | > loss_disc_real_3: 0.22291791439056396 (0.22748602322977166) + | > loss_disc_real_4: 0.24886515736579895 (0.2291288230696629) + | > loss_disc_real_5: 0.21684284508228302 (0.23110361448649702) + | > loss_0: 2.4402849674224854 (2.4720383142602853) + | > grad_norm_0: tensor(10.7264, device='cuda:0') (tensor(9.7070, device='cuda:0')) + | > loss_gen: 2.514174461364746 (2.3403441166055625) + | > loss_kl: 1.9009363651275635 (1.8669112119181404) + | > loss_feat: 7.37692928314209 (6.988831166563363) + | > loss_mel: 18.010223388671875 (17.65727032464126) + | > loss_duration: 1.4772615432739258 (1.4631072971327546) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.279525756835938 (30.316464045952106) + | > grad_norm_1: tensor(489.2854, device='cuda:0') (tensor(221.0849, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.426 (0.3764787460195608) + | > loader_time: 0.0069 (0.006029932663358491) + + + --> TIME: 2025-04-19 03:37:30 -- STEP: 141/142 -- GLOBAL_STEP: 419325 + | > loss_disc: 2.504246234893799 (2.477280395250794) + | > loss_disc_real_0: 0.11299414932727814 (0.1279511695846599) + | > loss_disc_real_1: 0.22738787531852722 (0.20759947386616512) + | > loss_disc_real_2: 0.23539604246616364 (0.22656239495209768) + | > loss_disc_real_3: 0.23457220196723938 (0.22854914341835267) + | > loss_disc_real_4: 0.22031937539577484 (0.23000820594053742) + | > loss_disc_real_5: 0.26307666301727295 (0.23385726306455354) + | > loss_0: 2.504246234893799 (2.477280395250794) + | > grad_norm_0: tensor(17.8135, device='cuda:0') (tensor(10.2506, device='cuda:0')) + | > loss_gen: 2.274373769760132 (2.3390299479166674) + | > loss_kl: 1.6040713787078857 (1.8713133013840262) + | > loss_feat: 5.846871852874756 (6.983996658460468) + | > loss_mel: 15.218475341796875 (17.66832175153367) + | > loss_duration: 1.3462026119232178 (1.4688550242295497) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.289995193481445 (30.331516576996933) + | > grad_norm_1: tensor(430.7971, device='cuda:0') (tensor(233.9854, device='cuda:0')) + | > current_lr_0: 0.0001930965206818004 + | > current_lr_1: 0.0001930965206818004 + | > step_time: 0.2003 (0.3866290640323721) + | > loader_time: 0.0027 (0.006274192891222365) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.641596794128418 (2.641596794128418) + | > loss_disc_real_0: 0.10443203896284103 (0.10443203896284103) + | > loss_disc_real_1: 0.2545110285282135 (0.2545110285282135) + | > loss_disc_real_2: 0.26111650466918945 (0.26111650466918945) + | > loss_disc_real_3: 0.24621649086475372 (0.24621649086475372) + | > loss_disc_real_4: 0.22037936747074127 (0.22037936747074127) + | > loss_disc_real_5: 0.20043806731700897 (0.20043806731700897) + | > loss_0: 2.641596794128418 (2.641596794128418) + | > loss_gen: 2.0619959831237793 (2.0619959831237793) + | > loss_kl: 2.4721755981445312 (2.4721755981445312) + | > loss_feat: 6.668830871582031 (6.668830871582031) + | > loss_mel: 18.664047241210938 (18.664047241210938) + | > loss_duration: 1.6948124170303345 (1.6948124170303345) + | > loss_1: 31.56186294555664 (31.56186294555664) + + --> STEP: 1 + | > loss_disc: 2.5894923210144043 (2.5894923210144043) + | > loss_disc_real_0: 0.13556142151355743 (0.13556142151355743) + | > loss_disc_real_1: 0.24607063829898834 (0.24607063829898834) + | > loss_disc_real_2: 0.2459690272808075 (0.2459690272808075) + | > loss_disc_real_3: 0.24118171632289886 (0.24118171632289886) + | > loss_disc_real_4: 0.22818796336650848 (0.22818796336650848) + | > loss_disc_real_5: 0.24036063253879547 (0.24036063253879547) + | > loss_0: 2.5894923210144043 (2.5894923210144043) + | > loss_gen: 2.1993234157562256 (2.1993234157562256) + | > loss_kl: 2.6812644004821777 (2.6812644004821777) + | > loss_feat: 6.446295261383057 (6.446295261383057) + | > loss_mel: 17.37353515625 (17.37353515625) + | > loss_duration: 1.7332613468170166 (1.7332613468170166) + | > loss_1: 30.433679580688477 (30.433679580688477) + + --> STEP: 2 + | > loss_disc: 2.6104483604431152 (2.5999703407287598) + | > loss_disc_real_0: 0.102754145860672 (0.11915778368711472) + | > loss_disc_real_1: 0.2154061198234558 (0.23073837906122208) + | > loss_disc_real_2: 0.23418673872947693 (0.2400778830051422) + | > loss_disc_real_3: 0.24862204492092133 (0.2449018806219101) + | > loss_disc_real_4: 0.22223909199237823 (0.22521352767944336) + | > loss_disc_real_5: 0.20104622840881348 (0.22070343047380447) + | > loss_0: 2.6104483604431152 (2.5999703407287598) + | > loss_gen: 2.0246243476867676 (2.1119738817214966) + | > loss_kl: 2.2315785884857178 (2.4564214944839478) + | > loss_feat: 5.892609119415283 (6.16945219039917) + | > loss_mel: 16.692459106445312 (17.032997131347656) + | > loss_duration: 1.6815693378448486 (1.7074153423309326) + | > loss_1: 28.52284049987793 (29.478260040283203) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036368370056152344 (-0.00028514862060546875) + | > avg_loss_disc: 2.5999703407287598 (+0.13717985153198242) + | > avg_loss_disc_real_0: 0.11915778368711472 (-0.01158083975315094) + | > avg_loss_disc_real_1: 0.23073837906122208 (-0.0031520426273345947) + | > avg_loss_disc_real_2: 0.2400778830051422 (+0.012972481548786163) + | > avg_loss_disc_real_3: 0.2449018806219101 (+0.020021706819534302) + | > avg_loss_disc_real_4: 0.22521352767944336 (+0.016147561371326447) + | > avg_loss_disc_real_5: 0.22070343047380447 (-0.036539413034915924) + | > avg_loss_0: 2.5999703407287598 (+0.13717985153198242) + | > avg_loss_gen: 2.1119738817214966 (-0.1770169734954834) + | > avg_loss_kl: 2.4564214944839478 (+0.02558314800262451) + | > avg_loss_feat: 6.16945219039917 (-1.3103809356689453) + | > avg_loss_mel: 17.032997131347656 (-1.396245002746582) + | > avg_loss_duration: 1.7074153423309326 (+0.0004448890686035156) + | > avg_loss_1: 29.478260040283203 (-2.857616424560547) + + + > EPOCH: 282/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:37:32)  + + --> TIME: 2025-04-19 03:37:41 -- STEP: 24/142 -- GLOBAL_STEP: 419350 + | > loss_disc: 2.377410411834717 (2.4238055845101676) + | > loss_disc_real_0: 0.108647421002388 (0.11976055366297562) + | > loss_disc_real_1: 0.19299761950969696 (0.2042514725277821) + | > loss_disc_real_2: 0.2598595917224884 (0.2254134068886439) + | > loss_disc_real_3: 0.22899846732616425 (0.22557923942804337) + | > loss_disc_real_4: 0.25026455521583557 (0.2229465370376905) + | > loss_disc_real_5: 0.25158044695854187 (0.22946631846328577) + | > loss_0: 2.377410411834717 (2.4238055845101676) + | > grad_norm_0: tensor(10.7060, device='cuda:0') (tensor(9.0860, device='cuda:0')) + | > loss_gen: 2.4384920597076416 (2.371917168299357) + | > loss_kl: 1.8092851638793945 (1.8149222483237584) + | > loss_feat: 6.912237167358398 (7.095768253008525) + | > loss_mel: 18.88880157470703 (17.61608517169952) + | > loss_duration: 1.4649006128311157 (1.42813245455424) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.513717651367188 (30.326825459798176) + | > grad_norm_1: tensor(232.7684, device='cuda:0') (tensor(209.3326, device='cuda:0')) + | > current_lr_0: 0.00019307238361671516 + | > current_lr_1: 0.00019307238361671516 + | > step_time: 0.3432 (0.3364023168881734) + | > loader_time: 0.0053 (0.0049663782119750985) + + + --> TIME: 2025-04-19 03:37:50 -- STEP: 49/142 -- GLOBAL_STEP: 419375 + | > loss_disc: 2.3552422523498535 (2.426258403427747) + | > loss_disc_real_0: 0.12160217016935349 (0.11908710033309702) + | > loss_disc_real_1: 0.19066114723682404 (0.20409187005490673) + | > loss_disc_real_2: 0.2398299276828766 (0.2246064093648171) + | > loss_disc_real_3: 0.23489445447921753 (0.22550054076983003) + | > loss_disc_real_4: 0.23599888384342194 (0.22827407626473173) + | > loss_disc_real_5: 0.2624199688434601 (0.23202625434009397) + | > loss_0: 2.3552422523498535 (2.426258403427747) + | > grad_norm_0: tensor(14.8861, device='cuda:0') (tensor(11.0806, device='cuda:0')) + | > loss_gen: 2.361062526702881 (2.3982854570661267) + | > loss_kl: 1.8983420133590698 (1.849451833841752) + | > loss_feat: 7.734092712402344 (7.25924893787929) + | > loss_mel: 19.13327407836914 (17.929718367907462) + | > loss_duration: 1.4436038732528687 (1.438106539298077) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.570377349853516 (30.874811211410833) + | > grad_norm_1: tensor(517.4458, device='cuda:0') (tensor(268.2453, device='cuda:0')) + | > current_lr_0: 0.00019307238361671516 + | > current_lr_1: 0.00019307238361671516 + | > step_time: 0.3704 (0.34703821065474527) + | > loader_time: 0.0055 (0.0053069007639982265) + + + --> TIME: 2025-04-19 03:38:00 -- STEP: 74/142 -- GLOBAL_STEP: 419400 + | > loss_disc: 2.408937454223633 (2.4388485889177067) + | > loss_disc_real_0: 0.1750832498073578 (0.12196793860277615) + | > loss_disc_real_1: 0.20585761964321136 (0.20333424069591471) + | > loss_disc_real_2: 0.20327192544937134 (0.22398935278525223) + | > loss_disc_real_3: 0.2095889449119568 (0.2264088283519487) + | > loss_disc_real_4: 0.23247377574443817 (0.22796157103132558) + | > loss_disc_real_5: 0.22517627477645874 (0.23292636931748004) + | > loss_0: 2.408937454223633 (2.4388485889177067) + | > grad_norm_0: tensor(16.1599, device='cuda:0') (tensor(12.4491, device='cuda:0')) + | > loss_gen: 2.305706024169922 (2.3926021730577616) + | > loss_kl: 1.9286221265792847 (1.8547382709142324) + | > loss_feat: 7.056826591491699 (7.147729081076545) + | > loss_mel: 18.51432991027832 (17.774202759201454) + | > loss_duration: 1.4721968173980713 (1.4502195345388877) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.277681350708008 (30.61949180912327) + | > grad_norm_1: tensor(197.9451, device='cuda:0') (tensor(285.8090, device='cuda:0')) + | > current_lr_0: 0.00019307238361671516 + | > current_lr_1: 0.00019307238361671516 + | > step_time: 0.3943 (0.35931181907653814) + | > loader_time: 0.0061 (0.005654157818974675) + + + --> TIME: 2025-04-19 03:38:10 -- STEP: 99/142 -- GLOBAL_STEP: 419425 + | > loss_disc: 2.6492130756378174 (2.45338601536221) + | > loss_disc_real_0: 0.13389591872692108 (0.12378565365015859) + | > loss_disc_real_1: 0.2175142914056778 (0.20482575456903437) + | > loss_disc_real_2: 0.2825911343097687 (0.22683218121528625) + | > loss_disc_real_3: 0.29119470715522766 (0.2262846976518631) + | > loss_disc_real_4: 0.294511079788208 (0.22843602615775485) + | > loss_disc_real_5: 0.26325473189353943 (0.23270749609277705) + | > loss_0: 2.6492130756378174 (2.45338601536221) + | > grad_norm_0: tensor(28.3369, device='cuda:0') (tensor(13.8457, device='cuda:0')) + | > loss_gen: 2.3359341621398926 (2.386813758599637) + | > loss_kl: 1.6574673652648926 (1.8729498422507085) + | > loss_feat: 6.83651876449585 (7.102361770591351) + | > loss_mel: 15.947639465332031 (17.770188986653025) + | > loss_duration: 1.5082182884216309 (1.4603068431218464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.285778045654297 (30.59262133126307) + | > grad_norm_1: tensor(496.5336, device='cuda:0') (tensor(292.8795, device='cuda:0')) + | > current_lr_0: 0.00019307238361671516 + | > current_lr_1: 0.00019307238361671516 + | > step_time: 0.4075 (0.37060734719941113) + | > loader_time: 0.0067 (0.005892907730256667) + + + --> TIME: 2025-04-19 03:38:21 -- STEP: 124/142 -- GLOBAL_STEP: 419450 + | > loss_disc: 2.472546339035034 (2.4566886982610154) + | > loss_disc_real_0: 0.12058525532484055 (0.12405890767132083) + | > loss_disc_real_1: 0.1817885786294937 (0.20480726647280878) + | > loss_disc_real_2: 0.22210483253002167 (0.22607784898531053) + | > loss_disc_real_3: 0.22155429422855377 (0.22626613392945258) + | > loss_disc_real_4: 0.19440999627113342 (0.22833686201803147) + | > loss_disc_real_5: 0.23682616651058197 (0.23361954044911168) + | > loss_0: 2.472546339035034 (2.4566886982610154) + | > grad_norm_0: tensor(10.5255, device='cuda:0') (tensor(13.8119, device='cuda:0')) + | > loss_gen: 2.3162622451782227 (2.373802037008346) + | > loss_kl: 1.8801435232162476 (1.8843486001414638) + | > loss_feat: 7.456418037414551 (7.039410468070738) + | > loss_mel: 18.76718521118164 (17.688882004830138) + | > loss_duration: 1.511652946472168 (1.4666993368056513) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.931663513183594 (30.453142581447477) + | > grad_norm_1: tensor(335.4393, device='cuda:0') (tensor(297.1047, device='cuda:0')) + | > current_lr_0: 0.00019307238361671516 + | > current_lr_1: 0.00019307238361671516 + | > step_time: 0.441 (0.3825125002091931) + | > loader_time: 0.007 (0.006135040713894751) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6784372329711914 (2.6784372329711914) + | > loss_disc_real_0: 0.21849167346954346 (0.21849167346954346) + | > loss_disc_real_1: 0.27429673075675964 (0.27429673075675964) + | > loss_disc_real_2: 0.2742389738559723 (0.2742389738559723) + | > loss_disc_real_3: 0.26888230443000793 (0.26888230443000793) + | > loss_disc_real_4: 0.25148987770080566 (0.25148987770080566) + | > loss_disc_real_5: 0.2483810931444168 (0.2483810931444168) + | > loss_0: 2.6784372329711914 (2.6784372329711914) + | > loss_gen: 2.3448526859283447 (2.3448526859283447) + | > loss_kl: 2.5905532836914062 (2.5905532836914062) + | > loss_feat: 5.44957971572876 (5.44957971572876) + | > loss_mel: 15.83353042602539 (15.83353042602539) + | > loss_duration: 1.7051804065704346 (1.7051804065704346) + | > loss_1: 27.923696517944336 (27.923696517944336) + + --> STEP: 1 + | > loss_disc: 2.506640672683716 (2.506640672683716) + | > loss_disc_real_0: 0.181069478392601 (0.181069478392601) + | > loss_disc_real_1: 0.26095181703567505 (0.26095181703567505) + | > loss_disc_real_2: 0.24047425389289856 (0.24047425389289856) + | > loss_disc_real_3: 0.2559773325920105 (0.2559773325920105) + | > loss_disc_real_4: 0.2548476457595825 (0.2548476457595825) + | > loss_disc_real_5: 0.2042466551065445 (0.2042466551065445) + | > loss_0: 2.506640672683716 (2.506640672683716) + | > loss_gen: 2.4053633213043213 (2.4053633213043213) + | > loss_kl: 2.695267915725708 (2.695267915725708) + | > loss_feat: 6.882645130157471 (6.882645130157471) + | > loss_mel: 19.378673553466797 (19.378673553466797) + | > loss_duration: 1.7268635034561157 (1.7268635034561157) + | > loss_1: 33.08881378173828 (33.08881378173828) + + --> STEP: 2 + | > loss_disc: 2.4322142601013184 (2.469427466392517) + | > loss_disc_real_0: 0.2160184681415558 (0.1985439732670784) + | > loss_disc_real_1: 0.2267008125782013 (0.24382631480693817) + | > loss_disc_real_2: 0.23009087145328522 (0.2352825626730919) + | > loss_disc_real_3: 0.2150665521621704 (0.23552194237709045) + | > loss_disc_real_4: 0.227997288107872 (0.24142246693372726) + | > loss_disc_real_5: 0.22859013080596924 (0.21641839295625687) + | > loss_0: 2.4322142601013184 (2.469427466392517) + | > loss_gen: 2.4975380897521973 (2.4514507055282593) + | > loss_kl: 2.2692997455596924 (2.4822838306427) + | > loss_feat: 7.9103474617004395 (7.396496295928955) + | > loss_mel: 19.4248104095459 (19.401741981506348) + | > loss_duration: 1.688561201095581 (1.7077123522758484) + | > loss_1: 33.790557861328125 (33.4396858215332) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003262042999267578 (-0.00037479400634765625) + | > avg_loss_disc: 2.469427466392517 (-0.13054287433624268) + | > avg_loss_disc_real_0: 0.1985439732670784 (+0.07938618957996368) + | > avg_loss_disc_real_1: 0.24382631480693817 (+0.013087935745716095) + | > avg_loss_disc_real_2: 0.2352825626730919 (-0.0047953203320503235) + | > avg_loss_disc_real_3: 0.23552194237709045 (-0.009379938244819641) + | > avg_loss_disc_real_4: 0.24142246693372726 (+0.016208939254283905) + | > avg_loss_disc_real_5: 0.21641839295625687 (-0.004285037517547607) + | > avg_loss_0: 2.469427466392517 (-0.13054287433624268) + | > avg_loss_gen: 2.4514507055282593 (+0.3394768238067627) + | > avg_loss_kl: 2.4822838306427 (+0.02586233615875244) + | > avg_loss_feat: 7.396496295928955 (+1.2270441055297852) + | > avg_loss_mel: 19.401741981506348 (+2.3687448501586914) + | > avg_loss_duration: 1.7077123522758484 (+0.0002970099449157715) + | > avg_loss_1: 33.4396858215332 (+3.96142578125) + + + > EPOCH: 283/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:38:31)  + + --> TIME: 2025-04-19 03:38:34 -- STEP: 7/142 -- GLOBAL_STEP: 419475 + | > loss_disc: 2.3102123737335205 (2.4464055810655867) + | > loss_disc_real_0: 0.12448885291814804 (0.12589328842503683) + | > loss_disc_real_1: 0.23045200109481812 (0.20865909116608755) + | > loss_disc_real_2: 0.2134082168340683 (0.2262063537325178) + | > loss_disc_real_3: 0.18907058238983154 (0.22744672851903097) + | > loss_disc_real_4: 0.18589994311332703 (0.22651306646210806) + | > loss_disc_real_5: 0.24538485705852509 (0.2397403780903135) + | > loss_0: 2.3102123737335205 (2.4464055810655867) + | > grad_norm_0: tensor(6.6936, device='cuda:0') (tensor(8.1438, device='cuda:0')) + | > loss_gen: 2.4677319526672363 (2.3391221932002475) + | > loss_kl: 1.729272723197937 (1.802695666040693) + | > loss_feat: 7.941706657409668 (7.137031555175781) + | > loss_mel: 19.544960021972656 (18.04088374546596) + | > loss_duration: 1.406224250793457 (1.417123453957694) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.08989334106445 (30.736856733049667) + | > grad_norm_1: tensor(143.6196, device='cuda:0') (tensor(185.1097, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.3214 (0.32465924535478863) + | > loader_time: 0.0042 (0.00428077152797154) + + + --> TIME: 2025-04-19 03:38:42 -- STEP: 32/142 -- GLOBAL_STEP: 419500 + | > loss_disc: 2.4445278644561768 (2.438823446631431) + | > loss_disc_real_0: 0.11524056643247604 (0.11899352329783142) + | > loss_disc_real_1: 0.21483556926250458 (0.20644385740160942) + | > loss_disc_real_2: 0.24046768248081207 (0.2264940799213946) + | > loss_disc_real_3: 0.2580917477607727 (0.22747469414025545) + | > loss_disc_real_4: 0.1944013237953186 (0.2257490549236536) + | > loss_disc_real_5: 0.2447795271873474 (0.23287799814715981) + | > loss_0: 2.4445278644561768 (2.438823446631431) + | > grad_norm_0: tensor(9.4185, device='cuda:0') (tensor(7.6534, device='cuda:0')) + | > loss_gen: 2.4121854305267334 (2.3574534729123116) + | > loss_kl: 1.7790518999099731 (1.8265108987689018) + | > loss_feat: 7.009843349456787 (7.2447828352451324) + | > loss_mel: 17.748044967651367 (18.02376037836075) + | > loss_duration: 1.4315881729125977 (1.4303583800792694) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.380714416503906 (30.882865846157074) + | > grad_norm_1: tensor(69.4818, device='cuda:0') (tensor(155.6773, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.3538 (0.33725055307149887) + | > loader_time: 0.0055 (0.005186058580875396) + + + --> TIME: 2025-04-19 03:38:52 -- STEP: 57/142 -- GLOBAL_STEP: 419525 + | > loss_disc: 2.546522617340088 (2.4417627276035767) + | > loss_disc_real_0: 0.11175964027643204 (0.12188983630192907) + | > loss_disc_real_1: 0.24018891155719757 (0.20684799264397538) + | > loss_disc_real_2: 0.24936813116073608 (0.2251459263395845) + | > loss_disc_real_3: 0.2573816776275635 (0.22619769500012984) + | > loss_disc_real_4: 0.24617111682891846 (0.22564681161913955) + | > loss_disc_real_5: 0.21815674006938934 (0.2315417334698794) + | > loss_0: 2.546522617340088 (2.4417627276035767) + | > grad_norm_0: tensor(8.0613, device='cuda:0') (tensor(7.9557, device='cuda:0')) + | > loss_gen: 2.2150909900665283 (2.3549559618297384) + | > loss_kl: 1.8378841876983643 (1.83519769133183) + | > loss_feat: 7.046528339385986 (7.178857677861264) + | > loss_mel: 16.494380950927734 (17.87048122339082) + | > loss_duration: 1.4848370552062988 (1.4407032079863968) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.07872200012207 (30.680195691292745) + | > grad_norm_1: tensor(261.2194, device='cuda:0') (tensor(169.8711, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.3723 (0.3487016820071036) + | > loader_time: 0.0059 (0.00552711988750257) + + + --> TIME: 2025-04-19 03:39:02 -- STEP: 82/142 -- GLOBAL_STEP: 419550 + | > loss_disc: 2.562849998474121 (2.449576462187417) + | > loss_disc_real_0: 0.183428093791008 (0.122656998292702) + | > loss_disc_real_1: 0.21769246459007263 (0.2065680116051581) + | > loss_disc_real_2: 0.23389650881290436 (0.2261880527182323) + | > loss_disc_real_3: 0.22982026636600494 (0.22654105050534737) + | > loss_disc_real_4: 0.23966434597969055 (0.22592360653528354) + | > loss_disc_real_5: 0.27229657769203186 (0.2325501282040666) + | > loss_0: 2.562849998474121 (2.449576462187417) + | > grad_norm_0: tensor(4.0403, device='cuda:0') (tensor(8.4205, device='cuda:0')) + | > loss_gen: 2.313300132751465 (2.345854564410885) + | > loss_kl: 2.0371882915496826 (1.8465120588860862) + | > loss_feat: 5.951976299285889 (7.073660879600339) + | > loss_mel: 16.31315040588379 (17.768448922692283) + | > loss_duration: 1.5429757833480835 (1.4518112961838885) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.158592224121094 (30.486287628732075) + | > grad_norm_1: tensor(239.8954, device='cuda:0') (tensor(186.9127, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.3919 (0.36133834792346486) + | > loader_time: 0.0062 (0.005796455755466369) + + + --> TIME: 2025-04-19 03:39:12 -- STEP: 107/142 -- GLOBAL_STEP: 419575 + | > loss_disc: 2.5261993408203125 (2.460442761394464) + | > loss_disc_real_0: 0.14726482331752777 (0.1242489738163547) + | > loss_disc_real_1: 0.22400589287281036 (0.20754662114326086) + | > loss_disc_real_2: 0.2629425823688507 (0.22647457312200672) + | > loss_disc_real_3: 0.23876634240150452 (0.2271261375362628) + | > loss_disc_real_4: 0.257183313369751 (0.22700950489422986) + | > loss_disc_real_5: 0.25140076875686646 (0.23382120210433674) + | > loss_0: 2.5261993408203125 (2.460442761394464) + | > grad_norm_0: tensor(30.7510, device='cuda:0') (tensor(10.3493, device='cuda:0')) + | > loss_gen: 2.5514936447143555 (2.3526336300038855) + | > loss_kl: 1.7234069108963013 (1.8515031259750652) + | > loss_feat: 7.219029426574707 (7.06674980003143) + | > loss_mel: 18.60239028930664 (17.76597841209341) + | > loss_duration: 1.471876859664917 (1.4615690986686778) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.56819725036621 (30.498434048946773) + | > grad_norm_1: tensor(231.8400, device='cuda:0') (tensor(213.1123, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.4149 (0.37424785845747616) + | > loader_time: 0.0069 (0.006017582438816534) + + + --> TIME: 2025-04-19 03:39:23 -- STEP: 132/142 -- GLOBAL_STEP: 419600 + | > loss_disc: 2.431079149246216 (2.468356475685581) + | > loss_disc_real_0: 0.16839878261089325 (0.12797678166040863) + | > loss_disc_real_1: 0.21686121821403503 (0.20795007366122623) + | > loss_disc_real_2: 0.27061203122138977 (0.22664210155154718) + | > loss_disc_real_3: 0.24625825881958008 (0.22675674882802097) + | > loss_disc_real_4: 0.24232806265354156 (0.2272940981342937) + | > loss_disc_real_5: 0.2372073382139206 (0.2323171988581166) + | > loss_0: 2.431079149246216 (2.468356475685581) + | > grad_norm_0: tensor(22.4980, device='cuda:0') (tensor(11.7627, device='cuda:0')) + | > loss_gen: 2.424548625946045 (2.341483226328186) + | > loss_kl: 1.925050973892212 (1.8617244868567495) + | > loss_feat: 6.768229007720947 (7.000625375545386) + | > loss_mel: 17.761024475097656 (17.69500318440524) + | > loss_duration: 1.4753692150115967 (1.4678304195404053) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35422134399414 (30.366666663776744) + | > grad_norm_1: tensor(379.0911, device='cuda:0') (tensor(223.6102, device='cuda:0')) + | > current_lr_0: 0.00019304824956876306 + | > current_lr_1: 0.00019304824956876306 + | > step_time: 0.463 (0.38624167081081506) + | > loader_time: 0.0079 (0.00625846241459702) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5043129920959473 (2.5043129920959473) + | > loss_disc_real_0: 0.17559432983398438 (0.17559432983398438) + | > loss_disc_real_1: 0.27572372555732727 (0.27572372555732727) + | > loss_disc_real_2: 0.2041969895362854 (0.2041969895362854) + | > loss_disc_real_3: 0.3204801380634308 (0.3204801380634308) + | > loss_disc_real_4: 0.22049351036548615 (0.22049351036548615) + | > loss_disc_real_5: 0.2781538963317871 (0.2781538963317871) + | > loss_0: 2.5043129920959473 (2.5043129920959473) + | > loss_gen: 2.651160478591919 (2.651160478591919) + | > loss_kl: 2.2997384071350098 (2.2997384071350098) + | > loss_feat: 7.261651515960693 (7.261651515960693) + | > loss_mel: 19.547204971313477 (19.547204971313477) + | > loss_duration: 1.6907073259353638 (1.6907073259353638) + | > loss_1: 33.450462341308594 (33.450462341308594) + + --> STEP: 1 + | > loss_disc: 2.359111785888672 (2.359111785888672) + | > loss_disc_real_0: 0.14354458451271057 (0.14354458451271057) + | > loss_disc_real_1: 0.24432915449142456 (0.24432915449142456) + | > loss_disc_real_2: 0.20304535329341888 (0.20304535329341888) + | > loss_disc_real_3: 0.25094592571258545 (0.25094592571258545) + | > loss_disc_real_4: 0.2027798742055893 (0.2027798742055893) + | > loss_disc_real_5: 0.1960318386554718 (0.1960318386554718) + | > loss_0: 2.359111785888672 (2.359111785888672) + | > loss_gen: 2.477506399154663 (2.477506399154663) + | > loss_kl: 2.738935708999634 (2.738935708999634) + | > loss_feat: 7.774611473083496 (7.774611473083496) + | > loss_mel: 18.264907836914062 (18.264907836914062) + | > loss_duration: 1.7463668584823608 (1.7463668584823608) + | > loss_1: 33.00232696533203 (33.00232696533203) + + --> STEP: 2 + | > loss_disc: 2.511315107345581 (2.4352134466171265) + | > loss_disc_real_0: 0.21994978189468384 (0.1817471832036972) + | > loss_disc_real_1: 0.3050200343132019 (0.27467459440231323) + | > loss_disc_real_2: 0.22455136477947235 (0.21379835903644562) + | > loss_disc_real_3: 0.2850858271121979 (0.26801587641239166) + | > loss_disc_real_4: 0.2274334728717804 (0.21510667353868484) + | > loss_disc_real_5: 0.27350914478302 (0.2347704917192459) + | > loss_0: 2.511315107345581 (2.4352134466171265) + | > loss_gen: 2.7081964015960693 (2.592851400375366) + | > loss_kl: 2.1236820220947266 (2.43130886554718) + | > loss_feat: 7.3669610023498535 (7.570786237716675) + | > loss_mel: 17.884489059448242 (18.074698448181152) + | > loss_duration: 1.6987117528915405 (1.7225393056869507) + | > loss_1: 31.782039642333984 (32.39218330383301) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003503084182739258 (+0.0002410411834716797) + | > avg_loss_disc: 2.4352134466171265 (-0.034214019775390625) + | > avg_loss_disc_real_0: 0.1817471832036972 (-0.016796790063381195) + | > avg_loss_disc_real_1: 0.27467459440231323 (+0.03084827959537506) + | > avg_loss_disc_real_2: 0.21379835903644562 (-0.02148420363664627) + | > avg_loss_disc_real_3: 0.26801587641239166 (+0.03249393403530121) + | > avg_loss_disc_real_4: 0.21510667353868484 (-0.02631579339504242) + | > avg_loss_disc_real_5: 0.2347704917192459 (+0.018352098762989044) + | > avg_loss_0: 2.4352134466171265 (-0.034214019775390625) + | > avg_loss_gen: 2.592851400375366 (+0.14140069484710693) + | > avg_loss_kl: 2.43130886554718 (-0.05097496509552002) + | > avg_loss_feat: 7.570786237716675 (+0.17428994178771973) + | > avg_loss_mel: 18.074698448181152 (-1.3270435333251953) + | > avg_loss_duration: 1.7225393056869507 (+0.014826953411102295) + | > avg_loss_1: 32.39218330383301 (-1.0475025177001953) + + + > EPOCH: 284/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:39:30)  + + --> TIME: 2025-04-19 03:39:35 -- STEP: 15/142 -- GLOBAL_STEP: 419625 + | > loss_disc: 2.4549951553344727 (2.460409100850423) + | > loss_disc_real_0: 0.1469300091266632 (0.11747880975405375) + | > loss_disc_real_1: 0.17252056300640106 (0.1964891622463862) + | > loss_disc_real_2: 0.2436976432800293 (0.23106541037559508) + | > loss_disc_real_3: 0.20926935970783234 (0.23331703344980875) + | > loss_disc_real_4: 0.22188551723957062 (0.24785406788190206) + | > loss_disc_real_5: 0.22807666659355164 (0.236823433637619) + | > loss_0: 2.4549951553344727 (2.460409100850423) + | > grad_norm_0: tensor(7.7438, device='cuda:0') (tensor(7.6781, device='cuda:0')) + | > loss_gen: 2.3650293350219727 (2.3346537748972573) + | > loss_kl: 1.7652579545974731 (1.810415252049764) + | > loss_feat: 7.41825532913208 (7.119116719563802) + | > loss_mel: 16.864009857177734 (17.452031517028807) + | > loss_duration: 1.4255752563476562 (1.4210164705912272) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.83812713623047 (30.137233861287434) + | > grad_norm_1: tensor(79.6909, device='cuda:0') (tensor(112.6028, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.3341 (0.32799544334411623) + | > loader_time: 0.0057 (0.0047721385955810545) + + + --> TIME: 2025-04-19 03:39:44 -- STEP: 40/142 -- GLOBAL_STEP: 419650 + | > loss_disc: 2.468269109725952 (2.4612197935581204) + | > loss_disc_real_0: 0.07489629089832306 (0.12132683172821998) + | > loss_disc_real_1: 0.16293007135391235 (0.20790084302425385) + | > loss_disc_real_2: 0.21753644943237305 (0.22787635512650012) + | > loss_disc_real_3: 0.22760291397571564 (0.22895557358860968) + | > loss_disc_real_4: 0.21640560030937195 (0.2313681960105896) + | > loss_disc_real_5: 0.21408557891845703 (0.23487799018621444) + | > loss_0: 2.468269109725952 (2.4612197935581204) + | > grad_norm_0: tensor(11.3904, device='cuda:0') (tensor(7.6150, device='cuda:0')) + | > loss_gen: 2.280151844024658 (2.33191785812378) + | > loss_kl: 1.7877788543701172 (1.8134079188108445) + | > loss_feat: 6.952399730682373 (7.034254491329193) + | > loss_mel: 16.939613342285156 (17.49909927845001) + | > loss_duration: 1.4294369220733643 (1.434632456302643) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.389381408691406 (30.113312196731567) + | > grad_norm_1: tensor(76.3442, device='cuda:0') (tensor(121.6219, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.3591 (0.3409563541412353) + | > loader_time: 0.0061 (0.00525856614112854) + + + --> TIME: 2025-04-19 03:39:54 -- STEP: 65/142 -- GLOBAL_STEP: 419675 + | > loss_disc: 2.464369773864746 (2.463489836912889) + | > loss_disc_real_0: 0.15198688209056854 (0.12254260090681222) + | > loss_disc_real_1: 0.193451926112175 (0.20751607509759756) + | > loss_disc_real_2: 0.20423609018325806 (0.22778571064655598) + | > loss_disc_real_3: 0.2260241061449051 (0.22812748780617348) + | > loss_disc_real_4: 0.21567124128341675 (0.23201335691488706) + | > loss_disc_real_5: 0.25941306352615356 (0.2344450042797969) + | > loss_0: 2.464369773864746 (2.463489836912889) + | > grad_norm_0: tensor(5.6568, device='cuda:0') (tensor(7.7354, device='cuda:0')) + | > loss_gen: 2.3127293586730957 (2.333300843605629) + | > loss_kl: 1.9023107290267944 (1.8351405382156372) + | > loss_feat: 7.839541912078857 (7.069167892749493) + | > loss_mel: 17.62547492980957 (17.68462073986347) + | > loss_duration: 1.4594923257827759 (1.4469514791782085) + | > amp_scaler: 512.0 (259.9384615384615) + | > loss_1: 31.139549255371094 (30.369181735698994) + | > grad_norm_1: tensor(120.9137, device='cuda:0') (tensor(141.2376, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.3897 (0.35272400929377623) + | > loader_time: 0.0067 (0.005533284407395583) + + + --> TIME: 2025-04-19 03:40:04 -- STEP: 90/142 -- GLOBAL_STEP: 419700 + | > loss_disc: 2.3451919555664062 (2.4569524923960366) + | > loss_disc_real_0: 0.13554219901561737 (0.12357121258974076) + | > loss_disc_real_1: 0.1868692934513092 (0.20845206429560978) + | > loss_disc_real_2: 0.20893864333629608 (0.226376164621777) + | > loss_disc_real_3: 0.22633948922157288 (0.2274963163667255) + | > loss_disc_real_4: 0.2272999882698059 (0.23034410327672958) + | > loss_disc_real_5: 0.20273084938526154 (0.23291855686240726) + | > loss_0: 2.3451919555664062 (2.4569524923960366) + | > grad_norm_0: tensor(9.7390, device='cuda:0') (tensor(7.4584, device='cuda:0')) + | > loss_gen: 2.440826416015625 (2.3385856442981297) + | > loss_kl: 1.8435310125350952 (1.8450000431802538) + | > loss_feat: 7.8914923667907715 (7.094930744171142) + | > loss_mel: 19.116374969482422 (17.76454879972669) + | > loss_duration: 1.5110230445861816 (1.457866811752319) + | > amp_scaler: 512.0 (329.9555555555554) + | > loss_1: 32.80324935913086 (30.500932269626194) + | > grad_norm_1: tensor(352.8056, device='cuda:0') (tensor(144.1932, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.4063 (0.36427597204844175) + | > loader_time: 0.0066 (0.005794575479295519) + + + --> TIME: 2025-04-19 03:40:14 -- STEP: 115/142 -- GLOBAL_STEP: 419725 + | > loss_disc: 2.4123458862304688 (2.459994641594265) + | > loss_disc_real_0: 0.16831529140472412 (0.12549262131037917) + | > loss_disc_real_1: 0.18596290051937103 (0.20754117900910585) + | > loss_disc_real_2: 0.25272440910339355 (0.22583826508211052) + | > loss_disc_real_3: 0.22637347877025604 (0.22747669440248738) + | > loss_disc_real_4: 0.23029907047748566 (0.2292679795752401) + | > loss_disc_real_5: 0.21354533731937408 (0.23321220745211063) + | > loss_0: 2.4123458862304688 (2.459994641594265) + | > grad_norm_0: tensor(6.4751, device='cuda:0') (tensor(7.3490, device='cuda:0')) + | > loss_gen: 2.528458595275879 (2.3363022990848705) + | > loss_kl: 1.8578137159347534 (1.8589072300040204) + | > loss_feat: 7.158064842224121 (7.04996162497479) + | > loss_mel: 17.690980911254883 (17.752666208018425) + | > loss_duration: 1.4991962909698486 (1.4645244453264317) + | > amp_scaler: 512.0 (369.53043478260855) + | > loss_1: 30.734514236450195 (30.462361957715906) + | > grad_norm_1: tensor(211.1284, device='cuda:0') (tensor(144.4127, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.4247 (0.3749757663063382) + | > loader_time: 0.0071 (0.006036507565042247) + + + --> TIME: 2025-04-19 03:40:26 -- STEP: 140/142 -- GLOBAL_STEP: 419750 + | > loss_disc: 2.439878225326538 (2.463785905497415) + | > loss_disc_real_0: 0.13821782171726227 (0.12663230938570838) + | > loss_disc_real_1: 0.20088134706020355 (0.20787665460790908) + | > loss_disc_real_2: 0.23850621283054352 (0.22584133467503956) + | > loss_disc_real_3: 0.22921401262283325 (0.22746039990867886) + | > loss_disc_real_4: 0.2328292727470398 (0.2301737622490951) + | > loss_disc_real_5: 0.22535958886146545 (0.23239466537322317) + | > loss_0: 2.439878225326538 (2.463785905497415) + | > grad_norm_0: tensor(11.2106, device='cuda:0') (tensor(7.6187, device='cuda:0')) + | > loss_gen: 2.503293037414551 (2.331637992177691) + | > loss_kl: 1.9718867540359497 (1.8691930796418872) + | > loss_feat: 7.356430530548096 (7.03323141166142) + | > loss_mel: 17.918521881103516 (17.71321157046727) + | > loss_duration: 1.5223214626312256 (1.4707376335348399) + | > amp_scaler: 512.0 (394.9714285714285) + | > loss_1: 31.2724552154541 (30.418011801583425) + | > grad_norm_1: tensor(357.8804, device='cuda:0') (tensor(158.0925, device='cuda:0')) + | > current_lr_0: 0.00019302411853756697 + | > current_lr_1: 0.00019302411853756697 + | > step_time: 0.4592 (0.38717706203460706) + | > loader_time: 0.0078 (0.0062886289187840064) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4677915573120117 (2.4677915573120117) + | > loss_disc_real_0: 0.15628844499588013 (0.15628844499588013) + | > loss_disc_real_1: 0.24472802877426147 (0.24472802877426147) + | > loss_disc_real_2: 0.19417837262153625 (0.19417837262153625) + | > loss_disc_real_3: 0.22635339200496674 (0.22635339200496674) + | > loss_disc_real_4: 0.24542991816997528 (0.24542991816997528) + | > loss_disc_real_5: 0.210748091340065 (0.210748091340065) + | > loss_0: 2.4677915573120117 (2.4677915573120117) + | > loss_gen: 2.320793390274048 (2.320793390274048) + | > loss_kl: 2.4877970218658447 (2.4877970218658447) + | > loss_feat: 6.837565898895264 (6.837565898895264) + | > loss_mel: 18.74127197265625 (18.74127197265625) + | > loss_duration: 1.6879857778549194 (1.6879857778549194) + | > loss_1: 32.07541275024414 (32.07541275024414) + + --> STEP: 1 + | > loss_disc: 2.2586593627929688 (2.2586593627929688) + | > loss_disc_real_0: 0.14184996485710144 (0.14184996485710144) + | > loss_disc_real_1: 0.23862764239311218 (0.23862764239311218) + | > loss_disc_real_2: 0.1847996860742569 (0.1847996860742569) + | > loss_disc_real_3: 0.1871458739042282 (0.1871458739042282) + | > loss_disc_real_4: 0.26209840178489685 (0.26209840178489685) + | > loss_disc_real_5: 0.22046974301338196 (0.22046974301338196) + | > loss_0: 2.2586593627929688 (2.2586593627929688) + | > loss_gen: 2.5642499923706055 (2.5642499923706055) + | > loss_kl: 2.549921989440918 (2.549921989440918) + | > loss_feat: 7.989008903503418 (7.989008903503418) + | > loss_mel: 19.108762741088867 (19.108762741088867) + | > loss_duration: 1.7477673292160034 (1.7477673292160034) + | > loss_1: 33.959712982177734 (33.959712982177734) + + --> STEP: 2 + | > loss_disc: 2.4314658641815186 (2.3450626134872437) + | > loss_disc_real_0: 0.16613401472568512 (0.15399198979139328) + | > loss_disc_real_1: 0.21044376492500305 (0.22453570365905762) + | > loss_disc_real_2: 0.2019300013780594 (0.19336484372615814) + | > loss_disc_real_3: 0.2211669236421585 (0.20415639877319336) + | > loss_disc_real_4: 0.27919521927833557 (0.2706468105316162) + | > loss_disc_real_5: 0.26315709948539734 (0.24181342124938965) + | > loss_0: 2.4314658641815186 (2.3450626134872437) + | > loss_gen: 2.534708261489868 (2.549479126930237) + | > loss_kl: 2.32485032081604 (2.437386155128479) + | > loss_feat: 7.8220720291137695 (7.905540466308594) + | > loss_mel: 18.8649845123291 (18.986873626708984) + | > loss_duration: 1.7029484510421753 (1.7253578901290894) + | > loss_1: 33.24956130981445 (33.604637145996094) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003458738327026367 (-4.4345855712890625e-05) + | > avg_loss_disc: 2.3450626134872437 (-0.09015083312988281) + | > avg_loss_disc_real_0: 0.15399198979139328 (-0.027755193412303925) + | > avg_loss_disc_real_1: 0.22453570365905762 (-0.050138890743255615) + | > avg_loss_disc_real_2: 0.19336484372615814 (-0.020433515310287476) + | > avg_loss_disc_real_3: 0.20415639877319336 (-0.0638594776391983) + | > avg_loss_disc_real_4: 0.2706468105316162 (+0.055540136992931366) + | > avg_loss_disc_real_5: 0.24181342124938965 (+0.007042929530143738) + | > avg_loss_0: 2.3450626134872437 (-0.09015083312988281) + | > avg_loss_gen: 2.549479126930237 (-0.043372273445129395) + | > avg_loss_kl: 2.437386155128479 (+0.006077289581298828) + | > avg_loss_feat: 7.905540466308594 (+0.33475422859191895) + | > avg_loss_mel: 18.986873626708984 (+0.912175178527832) + | > avg_loss_duration: 1.7253578901290894 (+0.002818584442138672) + | > avg_loss_1: 33.604637145996094 (+1.212453842163086) + + + > EPOCH: 285/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:40:28)  + + --> TIME: 2025-04-19 03:40:37 -- STEP: 23/142 -- GLOBAL_STEP: 419775 + | > loss_disc: 2.4015097618103027 (2.4132514518240225) + | > loss_disc_real_0: 0.12596525251865387 (0.11747769557911417) + | > loss_disc_real_1: 0.22419270873069763 (0.20345182328120523) + | > loss_disc_real_2: 0.21607919037342072 (0.22338878395764725) + | > loss_disc_real_3: 0.252569317817688 (0.22761289516220923) + | > loss_disc_real_4: 0.2231978178024292 (0.23325551787148352) + | > loss_disc_real_5: 0.2261517494916916 (0.23073004380516385) + | > loss_0: 2.4015097618103027 (2.4132514518240225) + | > grad_norm_0: tensor(10.6781, device='cuda:0') (tensor(11.2020, device='cuda:0')) + | > loss_gen: 2.4355576038360596 (2.4441468197366465) + | > loss_kl: 1.9204742908477783 (1.8270503904508508) + | > loss_feat: 7.103573799133301 (7.577203833538553) + | > loss_mel: 18.252681732177734 (18.335066505100418) + | > loss_duration: 1.4398560523986816 (1.423275315243265) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.152143478393555 (31.606742610102113) + | > grad_norm_1: tensor(255.3898, device='cuda:0') (tensor(302.4724, device='cuda:0')) + | > current_lr_0: 0.00019299999052274977 + | > current_lr_1: 0.00019299999052274977 + | > step_time: 0.3497 (0.34198627264603326) + | > loader_time: 0.006 (0.0049845757691756535) + + + --> TIME: 2025-04-19 03:40:46 -- STEP: 48/142 -- GLOBAL_STEP: 419800 + | > loss_disc: 2.3733034133911133 (2.4369173149267835) + | > loss_disc_real_0: 0.13459420204162598 (0.11845166825999816) + | > loss_disc_real_1: 0.20934513211250305 (0.20804071333259344) + | > loss_disc_real_2: 0.24793630838394165 (0.22614455471436182) + | > loss_disc_real_3: 0.21608267724514008 (0.22529744170606136) + | > loss_disc_real_4: 0.21624137461185455 (0.23009304081400236) + | > loss_disc_real_5: 0.2106744349002838 (0.229105768725276) + | > loss_0: 2.3733034133911133 (2.4369173149267835) + | > grad_norm_0: tensor(8.5840, device='cuda:0') (tensor(12.0216, device='cuda:0')) + | > loss_gen: 2.5173473358154297 (2.3992797533671055) + | > loss_kl: 1.9479836225509644 (1.8455379803975422) + | > loss_feat: 7.527491569519043 (7.315081973870595) + | > loss_mel: 18.48427963256836 (17.98461983601253) + | > loss_duration: 1.4464302062988281 (1.4364262769619625) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.923532485961914 (30.980945706367493) + | > grad_norm_1: tensor(347.3157, device='cuda:0') (tensor(306.0854, device='cuda:0')) + | > current_lr_0: 0.00019299999052274977 + | > current_lr_1: 0.00019299999052274977 + | > step_time: 0.3646 (0.3492875049511592) + | > loader_time: 0.0054 (0.0053325444459915135) + + + --> TIME: 2025-04-19 03:40:55 -- STEP: 73/142 -- GLOBAL_STEP: 419825 + | > loss_disc: 2.466284990310669 (2.459990814940572) + | > loss_disc_real_0: 0.12016239017248154 (0.12249564450897582) + | > loss_disc_real_1: 0.2085641771554947 (0.20842170531619086) + | > loss_disc_real_2: 0.21594493091106415 (0.2270990685649114) + | > loss_disc_real_3: 0.256546288728714 (0.22935866227705184) + | > loss_disc_real_4: 0.2571263015270233 (0.23182259397963956) + | > loss_disc_real_5: 0.23193767666816711 (0.2325360450434358) + | > loss_0: 2.466284990310669 (2.459990814940572) + | > grad_norm_0: tensor(10.7565, device='cuda:0') (tensor(11.0192, device='cuda:0')) + | > loss_gen: 2.3050036430358887 (2.3723499089071196) + | > loss_kl: 1.9294586181640625 (1.8702370548901492) + | > loss_feat: 6.419022560119629 (7.189136446338811) + | > loss_mel: 16.811656951904297 (17.84296939797598) + | > loss_duration: 1.4993937015533447 (1.4486251481591839) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.964534759521484 (30.723317943207206) + | > grad_norm_1: tensor(252.8592, device='cuda:0') (tensor(267.1082, device='cuda:0')) + | > current_lr_0: 0.00019299999052274977 + | > current_lr_1: 0.00019299999052274977 + | > step_time: 0.3927 (0.35982931476749785) + | > loader_time: 0.0064 (0.005639200341211605) + + + --> TIME: 2025-04-19 03:41:06 -- STEP: 98/142 -- GLOBAL_STEP: 419850 + | > loss_disc: 2.534639358520508 (2.4630905049187812) + | > loss_disc_real_0: 0.11063700169324875 (0.12434510483729597) + | > loss_disc_real_1: 0.19593173265457153 (0.20713083841362778) + | > loss_disc_real_2: 0.1987067312002182 (0.22577973820117056) + | > loss_disc_real_3: 0.24596241116523743 (0.2284482109303377) + | > loss_disc_real_4: 0.23079469799995422 (0.23161320342701308) + | > loss_disc_real_5: 0.19406461715698242 (0.23233849768127715) + | > loss_0: 2.534639358520508 (2.4630905049187812) + | > grad_norm_0: tensor(4.9177, device='cuda:0') (tensor(10.7627, device='cuda:0')) + | > loss_gen: 2.2485225200653076 (2.3572570718064605) + | > loss_kl: 1.9414597749710083 (1.8758315212872563) + | > loss_feat: 6.814605236053467 (7.114270930387536) + | > loss_mel: 17.2689208984375 (17.748066220964706) + | > loss_duration: 1.4586765766143799 (1.4587938797717195) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.73218536376953 (30.554219635165467) + | > grad_norm_1: tensor(50.9014, device='cuda:0') (tensor(248.9217, device='cuda:0')) + | > current_lr_0: 0.00019299999052274977 + | > current_lr_1: 0.00019299999052274977 + | > step_time: 0.4086 (0.36979196509536433) + | > loader_time: 0.0067 (0.005906474833585776) + + + --> TIME: 2025-04-19 03:41:16 -- STEP: 123/142 -- GLOBAL_STEP: 419875 + | > loss_disc: 2.518280506134033 (2.4679041936145576) + | > loss_disc_real_0: 0.14150190353393555 (0.1257631252330493) + | > loss_disc_real_1: 0.20905625820159912 (0.2079834384404547) + | > loss_disc_real_2: 0.23529945313930511 (0.22664150074730074) + | > loss_disc_real_3: 0.21972161531448364 (0.2286864838949064) + | > loss_disc_real_4: 0.21946190297603607 (0.2313943667625024) + | > loss_disc_real_5: 0.1903291791677475 (0.23163331475684312) + | > loss_0: 2.518280506134033 (2.4679041936145576) + | > grad_norm_0: tensor(10.7854, device='cuda:0') (tensor(11.3122, device='cuda:0')) + | > loss_gen: 2.4167187213897705 (2.3527013790316706) + | > loss_kl: 1.8950591087341309 (1.8773364138797046) + | > loss_feat: 7.452349662780762 (7.087185165746425) + | > loss_mel: 18.608732223510742 (17.751699788783633) + | > loss_duration: 1.484510064125061 (1.4658890293865672) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.857370376586914 (30.53481187277693) + | > grad_norm_1: tensor(415.6552, device='cuda:0') (tensor(259.8478, device='cuda:0')) + | > current_lr_0: 0.00019299999052274977 + | > current_lr_1: 0.00019299999052274977 + | > step_time: 0.4383 (0.3808049961803405) + | > loader_time: 0.0085 (0.0061629361253443754) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.555695056915283 (2.555695056915283) + | > loss_disc_real_0: 0.16805614531040192 (0.16805614531040192) + | > loss_disc_real_1: 0.26175954937934875 (0.26175954937934875) + | > loss_disc_real_2: 0.254182904958725 (0.254182904958725) + | > loss_disc_real_3: 0.24257515370845795 (0.24257515370845795) + | > loss_disc_real_4: 0.20538468658924103 (0.20538468658924103) + | > loss_disc_real_5: 0.2449948638677597 (0.2449948638677597) + | > loss_0: 2.555695056915283 (2.555695056915283) + | > loss_gen: 2.39867901802063 (2.39867901802063) + | > loss_kl: 2.521695613861084 (2.521695613861084) + | > loss_feat: 6.924037933349609 (6.924037933349609) + | > loss_mel: 17.449161529541016 (17.449161529541016) + | > loss_duration: 1.6943472623825073 (1.6943472623825073) + | > loss_1: 30.98792266845703 (30.98792266845703) + + --> STEP: 1 + | > loss_disc: 2.3040199279785156 (2.3040199279785156) + | > loss_disc_real_0: 0.162369966506958 (0.162369966506958) + | > loss_disc_real_1: 0.26820626854896545 (0.26820626854896545) + | > loss_disc_real_2: 0.2208048403263092 (0.2208048403263092) + | > loss_disc_real_3: 0.20654964447021484 (0.20654964447021484) + | > loss_disc_real_4: 0.18818354606628418 (0.18818354606628418) + | > loss_disc_real_5: 0.2263367772102356 (0.2263367772102356) + | > loss_0: 2.3040199279785156 (2.3040199279785156) + | > loss_gen: 2.7174792289733887 (2.7174792289733887) + | > loss_kl: 2.7588531970977783 (2.7588531970977783) + | > loss_feat: 7.498997688293457 (7.498997688293457) + | > loss_mel: 18.6655216217041 (18.6655216217041) + | > loss_duration: 1.7499700784683228 (1.7499700784683228) + | > loss_1: 33.39082336425781 (33.39082336425781) + + --> STEP: 2 + | > loss_disc: 2.5837135314941406 (2.443866729736328) + | > loss_disc_real_0: 0.27336275577545166 (0.21786636114120483) + | > loss_disc_real_1: 0.24453990161418915 (0.2563730850815773) + | > loss_disc_real_2: 0.2795368432998657 (0.25017084181308746) + | > loss_disc_real_3: 0.2508331835269928 (0.22869141399860382) + | > loss_disc_real_4: 0.2428605705499649 (0.21552205830812454) + | > loss_disc_real_5: 0.2607136368751526 (0.2435252070426941) + | > loss_0: 2.5837135314941406 (2.443866729736328) + | > loss_gen: 2.6300764083862305 (2.6737778186798096) + | > loss_kl: 2.237898588180542 (2.49837589263916) + | > loss_feat: 6.2083611488342285 (6.853679418563843) + | > loss_mel: 18.446453094482422 (18.55598735809326) + | > loss_duration: 1.6976521015167236 (1.7238110899925232) + | > loss_1: 31.220441818237305 (32.30563259124756) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038635730743408203 (+0.0004048347473144531) + | > avg_loss_disc: 2.443866729736328 (+0.09880411624908447) + | > avg_loss_disc_real_0: 0.21786636114120483 (+0.06387437134981155) + | > avg_loss_disc_real_1: 0.2563730850815773 (+0.031837381422519684) + | > avg_loss_disc_real_2: 0.25017084181308746 (+0.05680599808692932) + | > avg_loss_disc_real_3: 0.22869141399860382 (+0.02453501522541046) + | > avg_loss_disc_real_4: 0.21552205830812454 (-0.05512475222349167) + | > avg_loss_disc_real_5: 0.2435252070426941 (+0.0017117857933044434) + | > avg_loss_0: 2.443866729736328 (+0.09880411624908447) + | > avg_loss_gen: 2.6737778186798096 (+0.12429869174957275) + | > avg_loss_kl: 2.49837589263916 (+0.06098973751068115) + | > avg_loss_feat: 6.853679418563843 (-1.051861047744751) + | > avg_loss_mel: 18.55598735809326 (-0.43088626861572266) + | > avg_loss_duration: 1.7238110899925232 (-0.0015468001365661621) + | > avg_loss_1: 32.30563259124756 (-1.2990045547485352) + + + > EPOCH: 286/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:41:27)  + + --> TIME: 2025-04-19 03:41:29 -- STEP: 6/142 -- GLOBAL_STEP: 419900 + | > loss_disc: 2.5215377807617188 (2.4413904349009194) + | > loss_disc_real_0: 0.13725994527339935 (0.12110709026455879) + | > loss_disc_real_1: 0.23001264035701752 (0.19966534028450647) + | > loss_disc_real_2: 0.21173004806041718 (0.22326061129570007) + | > loss_disc_real_3: 0.2400353103876114 (0.23074242224295935) + | > loss_disc_real_4: 0.22368380427360535 (0.24462568511565527) + | > loss_disc_real_5: 0.23152615129947662 (0.23110807687044144) + | > loss_0: 2.5215377807617188 (2.4413904349009194) + | > grad_norm_0: tensor(13.1417, device='cuda:0') (tensor(17.3953, device='cuda:0')) + | > loss_gen: 2.5110907554626465 (2.4625250895818076) + | > loss_kl: 1.7797865867614746 (1.8039940396944683) + | > loss_feat: 6.647003650665283 (7.123712778091431) + | > loss_mel: 17.092288970947266 (17.26048755645752) + | > loss_duration: 1.4138062000274658 (1.426635166009267) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.44397735595703 (30.077354431152344) + | > grad_norm_1: tensor(543.7430, device='cuda:0') (tensor(394.1826, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.3521 (0.3359563748041789) + | > loader_time: 0.0053 (0.004918932914733887) + + + --> TIME: 2025-04-19 03:41:38 -- STEP: 31/142 -- GLOBAL_STEP: 419925 + | > loss_disc: 2.4901537895202637 (2.432128506322061) + | > loss_disc_real_0: 0.1903272122144699 (0.12343471785706855) + | > loss_disc_real_1: 0.1956336349248886 (0.20341642393219855) + | > loss_disc_real_2: 0.23722104728221893 (0.22418407807427068) + | > loss_disc_real_3: 0.21103709936141968 (0.22487355384134478) + | > loss_disc_real_4: 0.23106491565704346 (0.2243750758709446) + | > loss_disc_real_5: 0.21452227234840393 (0.22968803201952287) + | > loss_0: 2.4901537895202637 (2.432128506322061) + | > grad_norm_0: tensor(33.1813, device='cuda:0') (tensor(20.7267, device='cuda:0')) + | > loss_gen: 2.3332321643829346 (2.4338057194986655) + | > loss_kl: 1.9086904525756836 (1.790540833627024) + | > loss_feat: 7.158563613891602 (7.230567039981965) + | > loss_mel: 17.18826675415039 (17.764700428132084) + | > loss_duration: 1.4204673767089844 (1.4285561192420222) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.009220123291016 (30.648170102027155) + | > grad_norm_1: tensor(505.1603, device='cuda:0') (tensor(394.5465, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.3515 (0.34547320488960515) + | > loader_time: 0.006 (0.00523486445027013) + + + --> TIME: 2025-04-19 03:41:48 -- STEP: 56/142 -- GLOBAL_STEP: 419950 + | > loss_disc: 2.4500949382781982 (2.4506664872169504) + | > loss_disc_real_0: 0.12203454971313477 (0.12567647313699118) + | > loss_disc_real_1: 0.20585668087005615 (0.20573362920965468) + | > loss_disc_real_2: 0.2260148972272873 (0.22680250661713736) + | > loss_disc_real_3: 0.2344183623790741 (0.22759057315332548) + | > loss_disc_real_4: 0.24529819190502167 (0.22649600596300193) + | > loss_disc_real_5: 0.2317884862422943 (0.22909491562417575) + | > loss_0: 2.4500949382781982 (2.4506664872169504) + | > grad_norm_0: tensor(9.0145, device='cuda:0') (tensor(18.8636, device='cuda:0')) + | > loss_gen: 2.3742380142211914 (2.3936033632074083) + | > loss_kl: 1.8889697790145874 (1.8146702221461706) + | > loss_feat: 7.661928653717041 (7.131373303277152) + | > loss_mel: 18.153303146362305 (17.624455009187965) + | > loss_duration: 1.4670441150665283 (1.4391104876995082) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.545482635498047 (30.40321217264448) + | > grad_norm_1: tensor(234.6802, device='cuda:0') (tensor(361.1328, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.3875 (0.35375772203717915) + | > loader_time: 0.006 (0.005520616258893693) + + + --> TIME: 2025-04-19 03:41:57 -- STEP: 81/142 -- GLOBAL_STEP: 419975 + | > loss_disc: 2.540736198425293 (2.461735021920852) + | > loss_disc_real_0: 0.1034509688615799 (0.12577150417146846) + | > loss_disc_real_1: 0.19063922762870789 (0.20690219067497018) + | > loss_disc_real_2: 0.2404342144727707 (0.2268530038403876) + | > loss_disc_real_3: 0.21488042175769806 (0.2283853202322383) + | > loss_disc_real_4: 0.24621181190013885 (0.2279604485741368) + | > loss_disc_real_5: 0.2325262725353241 (0.23082464859809404) + | > loss_0: 2.540736198425293 (2.461735021920852) + | > grad_norm_0: tensor(15.4201, device='cuda:0') (tensor(16.2638, device='cuda:0')) + | > loss_gen: 2.207357883453369 (2.3710576546045) + | > loss_kl: 1.9832133054733276 (1.8322230180104573) + | > loss_feat: 6.606801986694336 (7.081579667550546) + | > loss_mel: 17.690670013427734 (17.589587859165515) + | > loss_duration: 1.471232295036316 (1.4500480301586194) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.959274291992188 (30.32449602197718) + | > grad_norm_1: tensor(370.8359, device='cuda:0') (tensor(321.5099, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.393 (0.3640877435236801) + | > loader_time: 0.006 (0.005822414233360762) + + + --> TIME: 2025-04-19 03:42:08 -- STEP: 106/142 -- GLOBAL_STEP: 420000 + | > loss_disc: 2.381476640701294 (2.466794580783484) + | > loss_disc_real_0: 0.08408316224813461 (0.12739470014172896) + | > loss_disc_real_1: 0.14256204664707184 (0.2070094071469217) + | > loss_disc_real_2: 0.18198123574256897 (0.22653500862279027) + | > loss_disc_real_3: 0.22777678072452545 (0.22851278675052356) + | > loss_disc_real_4: 0.19147871434688568 (0.2281004544698967) + | > loss_disc_real_5: 0.2442537397146225 (0.23155223044022075) + | > loss_0: 2.381476640701294 (2.466794580783484) + | > grad_norm_0: tensor(6.1239, device='cuda:0') (tensor(15.3588, device='cuda:0')) + | > loss_gen: 2.480201244354248 (2.355395712942448) + | > loss_kl: 1.9449232816696167 (1.8427582844248358) + | > loss_feat: 7.9116692543029785 (6.993671021371518) + | > loss_mel: 18.256650924682617 (17.518407263845763) + | > loss_duration: 1.473327398300171 (1.4606107158480945) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.0667724609375 (30.170842854481823) + | > grad_norm_1: tensor(168.9865, device='cuda:0') (tensor(301.9417, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.4085 (0.37372995547528537) + | > loader_time: 0.0065 (0.006056985765133265) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_420000.pth + + --> TIME: 2025-04-19 03:42:20 -- STEP: 131/142 -- GLOBAL_STEP: 420025 + | > loss_disc: 2.5053508281707764 (2.4763735068663384) + | > loss_disc_real_0: 0.09387949109077454 (0.12917810357140216) + | > loss_disc_real_1: 0.18992303311824799 (0.20792860861952978) + | > loss_disc_real_2: 0.24259649217128754 (0.22737985599131985) + | > loss_disc_real_3: 0.22520294785499573 (0.2287033853185086) + | > loss_disc_real_4: 0.24408888816833496 (0.229559869825385) + | > loss_disc_real_5: 0.22168542444705963 (0.23215325722712596) + | > loss_0: 2.5053508281707764 (2.4763735068663384) + | > grad_norm_0: tensor(8.2250, device='cuda:0') (tensor(14.2089, device='cuda:0')) + | > loss_gen: 2.357261896133423 (2.3458700343852743) + | > loss_kl: 1.962873101234436 (1.8480557394391708) + | > loss_feat: 7.277458190917969 (6.972821323016218) + | > loss_mel: 18.086471557617188 (17.496355624599303) + | > loss_duration: 1.5041834115982056 (1.4670807414382465) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.188247680664062 (30.13018338734867) + | > grad_norm_1: tensor(173.6244, device='cuda:0') (tensor(277.9256, device='cuda:0')) + | > current_lr_0: 0.0001929758655239344 + | > current_lr_1: 0.0001929758655239344 + | > step_time: 0.4479 (0.3856079632999333) + | > loader_time: 0.0073 (0.006305325122279976) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.2483127117156982 (2.2483127117156982) + | > loss_disc_real_0: 0.18021129071712494 (0.18021129071712494) + | > loss_disc_real_1: 0.2212788462638855 (0.2212788462638855) + | > loss_disc_real_2: 0.20803947746753693 (0.20803947746753693) + | > loss_disc_real_3: 0.23200762271881104 (0.23200762271881104) + | > loss_disc_real_4: 0.17055292427539825 (0.17055292427539825) + | > loss_disc_real_5: 0.19415174424648285 (0.19415174424648285) + | > loss_0: 2.2483127117156982 (2.2483127117156982) + | > loss_gen: 2.558593988418579 (2.558593988418579) + | > loss_kl: 2.530794143676758 (2.530794143676758) + | > loss_feat: 8.02064037322998 (8.02064037322998) + | > loss_mel: 19.68131446838379 (19.68131446838379) + | > loss_duration: 1.6869887113571167 (1.6869887113571167) + | > loss_1: 34.47833251953125 (34.47833251953125) + + --> STEP: 1 + | > loss_disc: 2.577030658721924 (2.577030658721924) + | > loss_disc_real_0: 0.1916082352399826 (0.1916082352399826) + | > loss_disc_real_1: 0.2579413950443268 (0.2579413950443268) + | > loss_disc_real_2: 0.23743315041065216 (0.23743315041065216) + | > loss_disc_real_3: 0.2386929988861084 (0.2386929988861084) + | > loss_disc_real_4: 0.21188220381736755 (0.21188220381736755) + | > loss_disc_real_5: 0.2023290991783142 (0.2023290991783142) + | > loss_0: 2.577030658721924 (2.577030658721924) + | > loss_gen: 2.201629638671875 (2.201629638671875) + | > loss_kl: 2.7722527980804443 (2.7722527980804443) + | > loss_feat: 6.652521133422852 (6.652521133422852) + | > loss_mel: 17.266042709350586 (17.266042709350586) + | > loss_duration: 1.7324517965316772 (1.7324517965316772) + | > loss_1: 30.62489891052246 (30.62489891052246) + + --> STEP: 2 + | > loss_disc: 2.512179374694824 (2.544605016708374) + | > loss_disc_real_0: 0.22143039107322693 (0.20651931315660477) + | > loss_disc_real_1: 0.23269321024417877 (0.24531730264425278) + | > loss_disc_real_2: 0.2281101793050766 (0.23277166485786438) + | > loss_disc_real_3: 0.23985880613327026 (0.23927590250968933) + | > loss_disc_real_4: 0.21617744863033295 (0.21402982622385025) + | > loss_disc_real_5: 0.3100215494632721 (0.25617532432079315) + | > loss_0: 2.512179374694824 (2.544605016708374) + | > loss_gen: 2.5292704105377197 (2.3654500246047974) + | > loss_kl: 2.0854156017303467 (2.4288341999053955) + | > loss_feat: 6.167396545410156 (6.409958839416504) + | > loss_mel: 17.41158103942871 (17.33881187438965) + | > loss_duration: 1.6833384037017822 (1.7078951001167297) + | > loss_1: 29.877002716064453 (30.250950813293457) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033261775970458984 (-0.0005373954772949219) + | > avg_loss_disc: 2.544605016708374 (+0.1007382869720459) + | > avg_loss_disc_real_0: 0.20651931315660477 (-0.011347047984600067) + | > avg_loss_disc_real_1: 0.24531730264425278 (-0.011055782437324524) + | > avg_loss_disc_real_2: 0.23277166485786438 (-0.017399176955223083) + | > avg_loss_disc_real_3: 0.23927590250968933 (+0.01058448851108551) + | > avg_loss_disc_real_4: 0.21402982622385025 (-0.001492232084274292) + | > avg_loss_disc_real_5: 0.25617532432079315 (+0.01265011727809906) + | > avg_loss_0: 2.544605016708374 (+0.1007382869720459) + | > avg_loss_gen: 2.3654500246047974 (-0.3083277940750122) + | > avg_loss_kl: 2.4288341999053955 (-0.06954169273376465) + | > avg_loss_feat: 6.409958839416504 (-0.44372057914733887) + | > avg_loss_mel: 17.33881187438965 (-1.2171754837036133) + | > avg_loss_duration: 1.7078951001167297 (-0.015915989875793457) + | > avg_loss_1: 30.250950813293457 (-2.0546817779541016) + + + > EPOCH: 287/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:42:27)  + + --> TIME: 2025-04-19 03:42:32 -- STEP: 14/142 -- GLOBAL_STEP: 420050 + | > loss_disc: 2.4171035289764404 (2.3672295127596175) + | > loss_disc_real_0: 0.12942154705524445 (0.11090947581189019) + | > loss_disc_real_1: 0.2003178596496582 (0.19892288212265288) + | > loss_disc_real_2: 0.20277747511863708 (0.21407347811119898) + | > loss_disc_real_3: 0.19963926076889038 (0.22010056035859243) + | > loss_disc_real_4: 0.2157849222421646 (0.22531692045075552) + | > loss_disc_real_5: 0.22495797276496887 (0.21608514338731766) + | > loss_0: 2.4171035289764404 (2.3672295127596175) + | > grad_norm_0: tensor(5.0932, device='cuda:0') (tensor(7.9604, device='cuda:0')) + | > loss_gen: 2.459083318710327 (2.4014653308050975) + | > loss_kl: 1.7562559843063354 (1.8013250402041845) + | > loss_feat: 7.798182964324951 (7.5087067399706156) + | > loss_mel: 18.66531753540039 (17.522263254438126) + | > loss_duration: 1.4230471849441528 (1.4151658756392342) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.10188674926758 (30.64892578125) + | > grad_norm_1: tensor(110.1463, device='cuda:0') (tensor(131.3121, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.3313 (0.3267308473587036) + | > loader_time: 0.0051 (0.004738824708121163) + + + --> TIME: 2025-04-19 03:42:41 -- STEP: 39/142 -- GLOBAL_STEP: 420075 + | > loss_disc: 2.4393441677093506 (2.4123495114155302) + | > loss_disc_real_0: 0.10689367353916168 (0.11621909416638888) + | > loss_disc_real_1: 0.1786045879125595 (0.20537088200067863) + | > loss_disc_real_2: 0.2229665219783783 (0.22467153729536596) + | > loss_disc_real_3: 0.230552077293396 (0.22805038132728675) + | > loss_disc_real_4: 0.21855753660202026 (0.22774239495778695) + | > loss_disc_real_5: 0.24781444668769836 (0.2241174265360221) + | > loss_0: 2.4393441677093506 (2.4123495114155302) + | > grad_norm_0: tensor(8.1726, device='cuda:0') (tensor(7.0145, device='cuda:0')) + | > loss_gen: 2.3668928146362305 (2.4014001259436975) + | > loss_kl: 1.8479740619659424 (1.8143354990543463) + | > loss_feat: 7.369693756103516 (7.474171565129207) + | > loss_mel: 17.83599281311035 (17.950568076891777) + | > loss_duration: 1.4693604707717896 (1.432873805363973) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.889915466308594 (31.073348852304314) + | > grad_norm_1: tensor(113.5710, device='cuda:0') (tensor(117.7836, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.3582 (0.3401377873542982) + | > loader_time: 0.0068 (0.0052942373813726976) + + + --> TIME: 2025-04-19 03:42:51 -- STEP: 64/142 -- GLOBAL_STEP: 420100 + | > loss_disc: 2.427281618118286 (2.4228840544819827) + | > loss_disc_real_0: 0.1691097468137741 (0.12130128720309585) + | > loss_disc_real_1: 0.22291246056556702 (0.2063127316068858) + | > loss_disc_real_2: 0.2774359881877899 (0.22445398615673184) + | > loss_disc_real_3: 0.23937201499938965 (0.22663946030661464) + | > loss_disc_real_4: 0.2651817798614502 (0.22718544187955558) + | > loss_disc_real_5: 0.2328338623046875 (0.2237149579450488) + | > loss_0: 2.427281618118286 (2.4228840544819827) + | > grad_norm_0: tensor(8.1738, device='cuda:0') (tensor(7.4330, device='cuda:0')) + | > loss_gen: 2.5347554683685303 (2.3858833387494087) + | > loss_kl: 1.8681812286376953 (1.8315413184463978) + | > loss_feat: 7.471289157867432 (7.323636516928673) + | > loss_mel: 17.864242553710938 (17.80044695734978) + | > loss_duration: 1.4925987720489502 (1.445498863235116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.231067657470703 (30.787006855010986) + | > grad_norm_1: tensor(280.6947, device='cuda:0') (tensor(126.1248, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.3826 (0.3518369197845459) + | > loader_time: 0.0066 (0.005558256059885026) + + + --> TIME: 2025-04-19 03:43:01 -- STEP: 89/142 -- GLOBAL_STEP: 420125 + | > loss_disc: 2.346717357635498 (2.4395201875922368) + | > loss_disc_real_0: 0.11140811443328857 (0.12248126182998165) + | > loss_disc_real_1: 0.21216203272342682 (0.2069659860951177) + | > loss_disc_real_2: 0.21579238772392273 (0.22490370826105052) + | > loss_disc_real_3: 0.2111760526895523 (0.227764244662242) + | > loss_disc_real_4: 0.22614839673042297 (0.23016395026378417) + | > loss_disc_real_5: 0.2224845290184021 (0.22696974940514297) + | > loss_0: 2.346717357635498 (2.4395201875922368) + | > grad_norm_0: tensor(5.1981, device='cuda:0') (tensor(7.5753, device='cuda:0')) + | > loss_gen: 2.4671571254730225 (2.370274222298955) + | > loss_kl: 1.9454734325408936 (1.8413587955946333) + | > loss_feat: 7.540369510650635 (7.254246550999331) + | > loss_mel: 18.868253707885742 (17.84593487857433) + | > loss_duration: 1.4612433910369873 (1.4558769102846638) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.28249740600586 (30.76769134435761) + | > grad_norm_1: tensor(41.4206, device='cuda:0') (tensor(131.8345, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.3999 (0.3639516160729227) + | > loader_time: 0.0064 (0.005828865458456317) + + + --> TIME: 2025-04-19 03:43:11 -- STEP: 114/142 -- GLOBAL_STEP: 420150 + | > loss_disc: 2.424257278442383 (2.4445082702134764) + | > loss_disc_real_0: 0.12159749120473862 (0.12479079253318016) + | > loss_disc_real_1: 0.16604185104370117 (0.20667741879036552) + | > loss_disc_real_2: 0.1999090611934662 (0.2254032981500291) + | > loss_disc_real_3: 0.2015557736158371 (0.22730103095895365) + | > loss_disc_real_4: 0.1950342059135437 (0.22870495510205888) + | > loss_disc_real_5: 0.2555447816848755 (0.2275024998867721) + | > loss_0: 2.424257278442383 (2.4445082702134764) + | > grad_norm_0: tensor(3.8394, device='cuda:0') (tensor(7.4568, device='cuda:0')) + | > loss_gen: 2.4619483947753906 (2.3639566835604224) + | > loss_kl: 1.827783465385437 (1.8475914597511292) + | > loss_feat: 7.32150936126709 (7.207835385673924) + | > loss_mel: 17.245954513549805 (17.853853643986216) + | > loss_duration: 1.4994938373565674 (1.463288690960198) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.356691360473633 (30.736525769819295) + | > grad_norm_1: tensor(81.7845, device='cuda:0') (tensor(133.9288, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.42 (0.37484023445530945) + | > loader_time: 0.0073 (0.0060771034474958446) + + + --> TIME: 2025-04-19 03:43:23 -- STEP: 139/142 -- GLOBAL_STEP: 420175 + | > loss_disc: 2.5097315311431885 (2.447422161376732) + | > loss_disc_real_0: 0.1467968225479126 (0.12556561195164276) + | > loss_disc_real_1: 0.22797410190105438 (0.2081247428767115) + | > loss_disc_real_2: 0.22842645645141602 (0.22615152736790745) + | > loss_disc_real_3: 0.23092544078826904 (0.22791807203412914) + | > loss_disc_real_4: 0.27212902903556824 (0.22884330657317484) + | > loss_disc_real_5: 0.23770035803318024 (0.2265507319419504) + | > loss_0: 2.5097315311431885 (2.447422161376732) + | > grad_norm_0: tensor(22.0270, device='cuda:0') (tensor(8.7267, device='cuda:0')) + | > loss_gen: 2.4545795917510986 (2.366640229876951) + | > loss_kl: 1.9462740421295166 (1.8543965893683674) + | > loss_feat: 6.687396049499512 (7.169811543800848) + | > loss_mel: 18.27845573425293 (17.84085098788035) + | > loss_duration: 1.5040894746780396 (1.470043389059657) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.87079429626465 (30.701742707396583) + | > grad_norm_1: tensor(350.8364, device='cuda:0') (tensor(163.8571, device='cuda:0')) + | > current_lr_0: 0.0001929517435407439 + | > current_lr_1: 0.0001929517435407439 + | > step_time: 0.4511 (0.38714568048930004) + | > loader_time: 0.0074 (0.00631939592978937) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5831823348999023 (2.5831823348999023) + | > loss_disc_real_0: 0.12393543869256973 (0.12393543869256973) + | > loss_disc_real_1: 0.26084280014038086 (0.26084280014038086) + | > loss_disc_real_2: 0.24664117395877838 (0.24664117395877838) + | > loss_disc_real_3: 0.23905257880687714 (0.23905257880687714) + | > loss_disc_real_4: 0.22483724355697632 (0.22483724355697632) + | > loss_disc_real_5: 0.2215580940246582 (0.2215580940246582) + | > loss_0: 2.5831823348999023 (2.5831823348999023) + | > loss_gen: 2.1118431091308594 (2.1118431091308594) + | > loss_kl: 2.2143421173095703 (2.2143421173095703) + | > loss_feat: 6.632380962371826 (6.632380962371826) + | > loss_mel: 18.239398956298828 (18.239398956298828) + | > loss_duration: 1.7001460790634155 (1.7001460790634155) + | > loss_1: 30.89811134338379 (30.89811134338379) + + --> STEP: 1 + | > loss_disc: 2.6482949256896973 (2.6482949256896973) + | > loss_disc_real_0: 0.1447569727897644 (0.1447569727897644) + | > loss_disc_real_1: 0.2625921368598938 (0.2625921368598938) + | > loss_disc_real_2: 0.26805347204208374 (0.26805347204208374) + | > loss_disc_real_3: 0.22958269715309143 (0.22958269715309143) + | > loss_disc_real_4: 0.21767649054527283 (0.21767649054527283) + | > loss_disc_real_5: 0.21567368507385254 (0.21567368507385254) + | > loss_0: 2.6482949256896973 (2.6482949256896973) + | > loss_gen: 2.1309919357299805 (2.1309919357299805) + | > loss_kl: 2.854149341583252 (2.854149341583252) + | > loss_feat: 7.608092308044434 (7.608092308044434) + | > loss_mel: 18.095808029174805 (18.095808029174805) + | > loss_duration: 1.7205069065093994 (1.7205069065093994) + | > loss_1: 32.409549713134766 (32.409549713134766) + + --> STEP: 2 + | > loss_disc: 2.3542981147766113 (2.5012965202331543) + | > loss_disc_real_0: 0.09721268713474274 (0.12098482996225357) + | > loss_disc_real_1: 0.23986047506332397 (0.2512263059616089) + | > loss_disc_real_2: 0.24060776829719543 (0.2543306201696396) + | > loss_disc_real_3: 0.21115811169147491 (0.22037040442228317) + | > loss_disc_real_4: 0.2093517929315567 (0.21351414173841476) + | > loss_disc_real_5: 0.17184433341026306 (0.1937590092420578) + | > loss_0: 2.3542981147766113 (2.5012965202331543) + | > loss_gen: 2.2888405323028564 (2.2099162340164185) + | > loss_kl: 2.4301812648773193 (2.6421653032302856) + | > loss_feat: 8.529061317443848 (8.06857681274414) + | > loss_mel: 18.364898681640625 (18.230353355407715) + | > loss_duration: 1.673136591911316 (1.6968217492103577) + | > loss_1: 33.2861213684082 (32.847835540771484) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032796859741210938 (-4.649162292480469e-05) + | > avg_loss_disc: 2.5012965202331543 (-0.04330849647521973) + | > avg_loss_disc_real_0: 0.12098482996225357 (-0.0855344831943512) + | > avg_loss_disc_real_1: 0.2512263059616089 (+0.00590900331735611) + | > avg_loss_disc_real_2: 0.2543306201696396 (+0.021558955311775208) + | > avg_loss_disc_real_3: 0.22037040442228317 (-0.01890549808740616) + | > avg_loss_disc_real_4: 0.21351414173841476 (-0.0005156844854354858) + | > avg_loss_disc_real_5: 0.1937590092420578 (-0.06241631507873535) + | > avg_loss_0: 2.5012965202331543 (-0.04330849647521973) + | > avg_loss_gen: 2.2099162340164185 (-0.1555337905883789) + | > avg_loss_kl: 2.6421653032302856 (+0.21333110332489014) + | > avg_loss_feat: 8.06857681274414 (+1.6586179733276367) + | > avg_loss_mel: 18.230353355407715 (+0.8915414810180664) + | > avg_loss_duration: 1.6968217492103577 (-0.01107335090637207) + | > avg_loss_1: 32.847835540771484 (+2.5968847274780273) + + + > EPOCH: 288/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:43:25)  + + --> TIME: 2025-04-19 03:43:34 -- STEP: 22/142 -- GLOBAL_STEP: 420200 + | > loss_disc: 2.414980888366699 (2.4079162532633003) + | > loss_disc_real_0: 0.12485835701227188 (0.11811126734722745) + | > loss_disc_real_1: 0.21058641374111176 (0.20803470706397836) + | > loss_disc_real_2: 0.2163209766149521 (0.22014786980368875) + | > loss_disc_real_3: 0.22734719514846802 (0.22888220643455331) + | > loss_disc_real_4: 0.2407672107219696 (0.224007593637163) + | > loss_disc_real_5: 0.24825473129749298 (0.22473674877123398) + | > loss_0: 2.414980888366699 (2.4079162532633003) + | > grad_norm_0: tensor(16.9612, device='cuda:0') (tensor(11.6719, device='cuda:0')) + | > loss_gen: 2.52711820602417 (2.4192149205641313) + | > loss_kl: 1.8643012046813965 (1.832089131528681) + | > loss_feat: 7.91829776763916 (7.35424128445712) + | > loss_mel: 18.646730422973633 (17.722964503548365) + | > loss_duration: 1.4028794765472412 (1.42388362234289) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.35932540893555 (30.75239302895286) + | > grad_norm_1: tensor(476.1501, device='cuda:0') (tensor(272.5992, device='cuda:0')) + | > current_lr_0: 0.00019292762457280132 + | > current_lr_1: 0.00019292762457280132 + | > step_time: 0.3679 (0.3363535729321567) + | > loader_time: 0.0051 (0.005046042529019442) + + + --> TIME: 2025-04-19 03:43:43 -- STEP: 47/142 -- GLOBAL_STEP: 420225 + | > loss_disc: 2.2793099880218506 (2.4071213387428445) + | > loss_disc_real_0: 0.08789873868227005 (0.11514171276320802) + | > loss_disc_real_1: 0.18547502160072327 (0.20572932064533234) + | > loss_disc_real_2: 0.2192823886871338 (0.22372639337752728) + | > loss_disc_real_3: 0.1894056349992752 (0.22572049625376436) + | > loss_disc_real_4: 0.18295079469680786 (0.22352827895194927) + | > loss_disc_real_5: 0.20342077314853668 (0.22593362217253826) + | > loss_0: 2.2793099880218506 (2.4071213387428445) + | > grad_norm_0: tensor(13.1860, device='cuda:0') (tensor(9.7691, device='cuda:0')) + | > loss_gen: 2.5048818588256836 (2.399032278263823) + | > loss_kl: 1.805969476699829 (1.8435546235835298) + | > loss_feat: 8.495657920837402 (7.3425486442890575) + | > loss_mel: 19.463281631469727 (17.75777350080774) + | > loss_duration: 1.4200102090835571 (1.4379775727048836) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.68980026245117 (30.78088638630319) + | > grad_norm_1: tensor(399.4228, device='cuda:0') (tensor(233.6568, device='cuda:0')) + | > current_lr_0: 0.00019292762457280132 + | > current_lr_1: 0.00019292762457280132 + | > step_time: 0.3655 (0.3501212901257454) + | > loader_time: 0.006 (0.005490095057386033) + + + --> TIME: 2025-04-19 03:43:53 -- STEP: 72/142 -- GLOBAL_STEP: 420250 + | > loss_disc: 2.488861083984375 (2.421186841196484) + | > loss_disc_real_0: 0.14702551066875458 (0.11802272001902263) + | > loss_disc_real_1: 0.25217756628990173 (0.20681935569478407) + | > loss_disc_real_2: 0.22961397469043732 (0.22443886825607884) + | > loss_disc_real_3: 0.22335399687290192 (0.22771102521154615) + | > loss_disc_real_4: 0.24205385148525238 (0.2259811040841871) + | > loss_disc_real_5: 0.2307170033454895 (0.22641469227770963) + | > loss_0: 2.488861083984375 (2.421186841196484) + | > grad_norm_0: tensor(12.5723, device='cuda:0') (tensor(8.8896, device='cuda:0')) + | > loss_gen: 2.2968218326568604 (2.4001260134908895) + | > loss_kl: 1.8366936445236206 (1.8516777339908812) + | > loss_feat: 7.329769134521484 (7.316353109147814) + | > loss_mel: 18.087575912475586 (17.821146488189697) + | > loss_duration: 1.446050763130188 (1.4487429774469802) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.996912002563477 (30.838046232859295) + | > grad_norm_1: tensor(326.3341, device='cuda:0') (tensor(210.8338, device='cuda:0')) + | > current_lr_0: 0.00019292762457280132 + | > current_lr_1: 0.00019292762457280132 + | > step_time: 0.3937 (0.36121493246820235) + | > loader_time: 0.0065 (0.005780617396036784) + + + --> TIME: 2025-04-19 03:44:03 -- STEP: 97/142 -- GLOBAL_STEP: 420275 + | > loss_disc: 2.3988146781921387 (2.434962543015627) + | > loss_disc_real_0: 0.14157243072986603 (0.12135956874213268) + | > loss_disc_real_1: 0.19177746772766113 (0.20703748897793367) + | > loss_disc_real_2: 0.2050715833902359 (0.22407016004483724) + | > loss_disc_real_3: 0.21268831193447113 (0.2265153322944936) + | > loss_disc_real_4: 0.21246649324893951 (0.2273695335252998) + | > loss_disc_real_5: 0.2190537303686142 (0.22738950492180499) + | > loss_0: 2.3988146781921387 (2.434962543015627) + | > grad_norm_0: tensor(6.0243, device='cuda:0') (tensor(9.5512, device='cuda:0')) + | > loss_gen: 2.3873848915100098 (2.3802703532975986) + | > loss_kl: 1.8635164499282837 (1.8628317351193773) + | > loss_feat: 6.5592041015625 (7.208100736755686) + | > loss_mel: 16.04556655883789 (17.753012234402686) + | > loss_duration: 1.4769591093063354 (1.4597422130329087) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.332632064819336 (30.663957281210987) + | > grad_norm_1: tensor(278.4632, device='cuda:0') (tensor(234.0774, device='cuda:0')) + | > current_lr_0: 0.00019292762457280132 + | > current_lr_1: 0.00019292762457280132 + | > step_time: 0.4133 (0.3722510214933416) + | > loader_time: 0.0072 (0.006050355655630839) + + + --> TIME: 2025-04-19 03:44:14 -- STEP: 122/142 -- GLOBAL_STEP: 420300 + | > loss_disc: 2.462252378463745 (2.4440518226779866) + | > loss_disc_real_0: 0.17516322433948517 (0.12336615931059493) + | > loss_disc_real_1: 0.22028827667236328 (0.2075482654522677) + | > loss_disc_real_2: 0.1794443130493164 (0.22430912026616392) + | > loss_disc_real_3: 0.20841707289218903 (0.22789915638868927) + | > loss_disc_real_4: 0.2319873869419098 (0.22729709768881562) + | > loss_disc_real_5: 0.22346454858779907 (0.2287534510747331) + | > loss_0: 2.462252378463745 (2.4440518226779866) + | > grad_norm_0: tensor(25.2930, device='cuda:0') (tensor(9.8565, device='cuda:0')) + | > loss_gen: 2.284475564956665 (2.3702689350628474) + | > loss_kl: 1.868310809135437 (1.8776121979854146) + | > loss_feat: 6.845536708831787 (7.1526863965831815) + | > loss_mel: 17.258363723754883 (17.71031351558498) + | > loss_duration: 1.4660413265228271 (1.4662255107379345) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.722726821899414 (30.577106522732095) + | > grad_norm_1: tensor(336.8518, device='cuda:0') (tensor(235.5246, device='cuda:0')) + | > current_lr_0: 0.00019292762457280132 + | > current_lr_1: 0.00019292762457280132 + | > step_time: 0.4517 (0.3840653349141606) + | > loader_time: 0.0076 (0.006285180811022148) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.508391857147217 (2.508391857147217) + | > loss_disc_real_0: 0.24571876227855682 (0.24571876227855682) + | > loss_disc_real_1: 0.20956188440322876 (0.20956188440322876) + | > loss_disc_real_2: 0.2378862202167511 (0.2378862202167511) + | > loss_disc_real_3: 0.2527274489402771 (0.2527274489402771) + | > loss_disc_real_4: 0.2534187436103821 (0.2534187436103821) + | > loss_disc_real_5: 0.22212782502174377 (0.22212782502174377) + | > loss_0: 2.508391857147217 (2.508391857147217) + | > loss_gen: 2.625242233276367 (2.625242233276367) + | > loss_kl: 2.3009700775146484 (2.3009700775146484) + | > loss_feat: 6.380263328552246 (6.380263328552246) + | > loss_mel: 18.690778732299805 (18.690778732299805) + | > loss_duration: 1.6984035968780518 (1.6984035968780518) + | > loss_1: 31.695659637451172 (31.695659637451172) + + --> STEP: 1 + | > loss_disc: 2.589755058288574 (2.589755058288574) + | > loss_disc_real_0: 0.265726238489151 (0.265726238489151) + | > loss_disc_real_1: 0.2540039122104645 (0.2540039122104645) + | > loss_disc_real_2: 0.23689095675945282 (0.23689095675945282) + | > loss_disc_real_3: 0.23614759743213654 (0.23614759743213654) + | > loss_disc_real_4: 0.2311962991952896 (0.2311962991952896) + | > loss_disc_real_5: 0.23969623446464539 (0.23969623446464539) + | > loss_0: 2.589755058288574 (2.589755058288574) + | > loss_gen: 2.4832165241241455 (2.4832165241241455) + | > loss_kl: 2.5404911041259766 (2.5404911041259766) + | > loss_feat: 5.9027419090271 (5.9027419090271) + | > loss_mel: 17.811054229736328 (17.811054229736328) + | > loss_duration: 1.7455244064331055 (1.7455244064331055) + | > loss_1: 30.483028411865234 (30.483028411865234) + + --> STEP: 2 + | > loss_disc: 2.5839712619781494 (2.586863160133362) + | > loss_disc_real_0: 0.3304484784603119 (0.29808735847473145) + | > loss_disc_real_1: 0.2121129184961319 (0.2330584153532982) + | > loss_disc_real_2: 0.2386513203382492 (0.237771138548851) + | > loss_disc_real_3: 0.2389766424894333 (0.2375621199607849) + | > loss_disc_real_4: 0.22856810688972473 (0.22988220304250717) + | > loss_disc_real_5: 0.24105069041252136 (0.24037346243858337) + | > loss_0: 2.5839712619781494 (2.586863160133362) + | > loss_gen: 2.632420539855957 (2.5578185319900513) + | > loss_kl: 2.3746984004974365 (2.4575947523117065) + | > loss_feat: 5.972719669342041 (5.93773078918457) + | > loss_mel: 17.097326278686523 (17.454190254211426) + | > loss_duration: 1.680267095565796 (1.7128957509994507) + | > loss_1: 29.757431030273438 (30.120229721069336) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003669142723083496 (+0.00038945674896240234) + | > avg_loss_disc: 2.586863160133362 (+0.08556663990020752) + | > avg_loss_disc_real_0: 0.29808735847473145 (+0.17710252851247787) + | > avg_loss_disc_real_1: 0.2330584153532982 (-0.0181678906083107) + | > avg_loss_disc_real_2: 0.237771138548851 (-0.016559481620788574) + | > avg_loss_disc_real_3: 0.2375621199607849 (+0.01719171553850174) + | > avg_loss_disc_real_4: 0.22988220304250717 (+0.016368061304092407) + | > avg_loss_disc_real_5: 0.24037346243858337 (+0.046614453196525574) + | > avg_loss_0: 2.586863160133362 (+0.08556663990020752) + | > avg_loss_gen: 2.5578185319900513 (+0.3479022979736328) + | > avg_loss_kl: 2.4575947523117065 (-0.1845705509185791) + | > avg_loss_feat: 5.93773078918457 (-2.1308460235595703) + | > avg_loss_mel: 17.454190254211426 (-0.7761631011962891) + | > avg_loss_duration: 1.7128957509994507 (+0.016074001789093018) + | > avg_loss_1: 30.120229721069336 (-2.7276058197021484) + + + > EPOCH: 289/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:44:25)  + + --> TIME: 2025-04-19 03:44:27 -- STEP: 5/142 -- GLOBAL_STEP: 420325 + | > loss_disc: 2.488010883331299 (2.4474853038787843) + | > loss_disc_real_0: 0.19049929082393646 (0.13117255121469498) + | > loss_disc_real_1: 0.23436588048934937 (0.2209498554468155) + | > loss_disc_real_2: 0.28168725967407227 (0.23667224645614623) + | > loss_disc_real_3: 0.24158646166324615 (0.22955255210399628) + | > loss_disc_real_4: 0.256841242313385 (0.23880124986171722) + | > loss_disc_real_5: 0.23045609891414642 (0.2437213808298111) + | > loss_0: 2.488010883331299 (2.4474853038787843) + | > grad_norm_0: tensor(11.3619, device='cuda:0') (tensor(8.7204, device='cuda:0')) + | > loss_gen: 2.4340856075286865 (2.4389758586883543) + | > loss_kl: 1.7658655643463135 (1.8174988508224488) + | > loss_feat: 7.056266784667969 (7.173021221160889) + | > loss_mel: 16.98516082763672 (17.680429649353027) + | > loss_duration: 1.4459059238433838 (1.428170394897461) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.687284469604492 (30.53809509277344) + | > grad_norm_1: tensor(171.5688, device='cuda:0') (tensor(166.9382, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.3359 (0.32596664428710936) + | > loader_time: 0.0044 (0.004593658447265625) + + + --> TIME: 2025-04-19 03:44:36 -- STEP: 30/142 -- GLOBAL_STEP: 420350 + | > loss_disc: 2.3827648162841797 (2.457446885108948) + | > loss_disc_real_0: 0.10284014046192169 (0.12312652344504992) + | > loss_disc_real_1: 0.2074246108531952 (0.21171305974324545) + | > loss_disc_real_2: 0.216547891497612 (0.2279447555541992) + | > loss_disc_real_3: 0.2133890837430954 (0.22893928835789362) + | > loss_disc_real_4: 0.22265301644802094 (0.2289286623398463) + | > loss_disc_real_5: 0.2157781422138214 (0.2363258292277654) + | > loss_0: 2.3827648162841797 (2.457446885108948) + | > grad_norm_0: tensor(4.8365, device='cuda:0') (tensor(8.0120, device='cuda:0')) + | > loss_gen: 2.4342641830444336 (2.3674017985661826) + | > loss_kl: 1.9595175981521606 (1.8558399438858033) + | > loss_feat: 7.455838680267334 (7.23135240872701) + | > loss_mel: 18.143903732299805 (17.919555346171062) + | > loss_duration: 1.4569237232208252 (1.4306315223375956) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.450450897216797 (30.804780960083008) + | > grad_norm_1: tensor(58.0909, device='cuda:0') (tensor(125.7119, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.3516 (0.3372624079386393) + | > loader_time: 0.0057 (0.005103246370951334) + + + --> TIME: 2025-04-19 03:44:45 -- STEP: 55/142 -- GLOBAL_STEP: 420375 + | > loss_disc: 2.435547351837158 (2.4389708909121426) + | > loss_disc_real_0: 0.14191552996635437 (0.12410880300131712) + | > loss_disc_real_1: 0.21090397238731384 (0.2064157339659604) + | > loss_disc_real_2: 0.21770745515823364 (0.22580394040454518) + | > loss_disc_real_3: 0.2418115735054016 (0.22745173167098653) + | > loss_disc_real_4: 0.2349115014076233 (0.22925227853384886) + | > loss_disc_real_5: 0.23263594508171082 (0.232126354358413) + | > loss_0: 2.435547351837158 (2.4389708909121426) + | > grad_norm_0: tensor(12.6256, device='cuda:0') (tensor(8.0989, device='cuda:0')) + | > loss_gen: 2.2483205795288086 (2.375468687577682) + | > loss_kl: 1.908636212348938 (1.8752458398992364) + | > loss_feat: 6.7415289878845215 (7.283533781225031) + | > loss_mel: 17.180301666259766 (17.97439731251109) + | > loss_duration: 1.4986337423324585 (1.4396013173190032) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.577423095703125 (30.94824700789018) + | > grad_norm_1: tensor(100.7730, device='cuda:0') (tensor(138.9977, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.3719 (0.3489373077045787) + | > loader_time: 0.0058 (0.005481919375332919) + + + --> TIME: 2025-04-19 03:44:55 -- STEP: 80/142 -- GLOBAL_STEP: 420400 + | > loss_disc: 2.5552237033843994 (2.4528485387563705) + | > loss_disc_real_0: 0.11508617550134659 (0.12562451707199215) + | > loss_disc_real_1: 0.1809549182653427 (0.2065073721110821) + | > loss_disc_real_2: 0.21335722506046295 (0.2260630175471306) + | > loss_disc_real_3: 0.23791059851646423 (0.22723767813295126) + | > loss_disc_real_4: 0.24268782138824463 (0.22788999695330858) + | > loss_disc_real_5: 0.2519747316837311 (0.2349605428054929) + | > loss_0: 2.5552237033843994 (2.4528485387563705) + | > grad_norm_0: tensor(11.2040, device='cuda:0') (tensor(8.1540, device='cuda:0')) + | > loss_gen: 2.186882257461548 (2.3636154681444177) + | > loss_kl: 1.9236772060394287 (1.8843596637248994) + | > loss_feat: 6.624044418334961 (7.179911404848099) + | > loss_mel: 17.700302124023438 (17.829410016536706) + | > loss_duration: 1.4797117710113525 (1.449987538158894) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.91461753845215 (30.707283973693848) + | > grad_norm_1: tensor(251.4522, device='cuda:0') (tensor(144.2076, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.408 (0.36123761832714085) + | > loader_time: 0.0062 (0.005738371610641482) + + + --> TIME: 2025-04-19 03:45:05 -- STEP: 105/142 -- GLOBAL_STEP: 420425 + | > loss_disc: 2.5005500316619873 (2.457747702371507) + | > loss_disc_real_0: 0.10708525031805038 (0.12602530327581224) + | > loss_disc_real_1: 0.20514880120754242 (0.2075908950396947) + | > loss_disc_real_2: 0.20950831472873688 (0.22626985481807163) + | > loss_disc_real_3: 0.24473324418067932 (0.22733378396147774) + | > loss_disc_real_4: 0.22104768455028534 (0.22942916850248973) + | > loss_disc_real_5: 0.1896430253982544 (0.23276025099413736) + | > loss_0: 2.5005500316619873 (2.457747702371507) + | > grad_norm_0: tensor(5.2713, device='cuda:0') (tensor(8.6288, device='cuda:0')) + | > loss_gen: 2.3257243633270264 (2.3549557367960614) + | > loss_kl: 1.9592695236206055 (1.8997802632195608) + | > loss_feat: 7.613679885864258 (7.107165254865374) + | > loss_mel: 19.039628982543945 (17.75757803235734) + | > loss_duration: 1.5049619674682617 (1.4604327133723671) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.44326400756836 (30.579911895025344) + | > grad_norm_1: tensor(100.4884, device='cuda:0') (tensor(162.2512, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.4257 (0.3723855245681036) + | > loader_time: 0.007 (0.005995816276186991) + + + --> TIME: 2025-04-19 03:45:16 -- STEP: 130/142 -- GLOBAL_STEP: 420450 + | > loss_disc: 2.383432388305664 (2.4589165082344646) + | > loss_disc_real_0: 0.12858746945858002 (0.12675775971550218) + | > loss_disc_real_1: 0.19193035364151 (0.20711443836872398) + | > loss_disc_real_2: 0.20380626618862152 (0.22601718776501142) + | > loss_disc_real_3: 0.1736910492181778 (0.22698180835980636) + | > loss_disc_real_4: 0.2314530462026596 (0.22948941657176383) + | > loss_disc_real_5: 0.20546075701713562 (0.23268258296526395) + | > loss_0: 2.383432388305664 (2.4589165082344646) + | > grad_norm_0: tensor(10.0047, device='cuda:0') (tensor(8.9565, device='cuda:0')) + | > loss_gen: 2.413740873336792 (2.3471930045347946) + | > loss_kl: 1.9816282987594604 (1.9053056900317853) + | > loss_feat: 7.090097904205322 (7.064257926207322) + | > loss_mel: 17.67460823059082 (17.72455467810996) + | > loss_duration: 1.482053279876709 (1.4667420589006865) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.642127990722656 (30.50805326608511) + | > grad_norm_1: tensor(371.6465, device='cuda:0') (tensor(175.9702, device='cuda:0')) + | > current_lr_0: 0.0001929035086197297 + | > current_lr_1: 0.0001929035086197297 + | > step_time: 0.4517 (0.3843386099888728) + | > loader_time: 0.0071 (0.006254742695735053) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3736233711242676 (2.3736233711242676) + | > loss_disc_real_0: 0.1752910017967224 (0.1752910017967224) + | > loss_disc_real_1: 0.19682376086711884 (0.19682376086711884) + | > loss_disc_real_2: 0.2129671424627304 (0.2129671424627304) + | > loss_disc_real_3: 0.2581639885902405 (0.2581639885902405) + | > loss_disc_real_4: 0.2504400908946991 (0.2504400908946991) + | > loss_disc_real_5: 0.21521514654159546 (0.21521514654159546) + | > loss_0: 2.3736233711242676 (2.3736233711242676) + | > loss_gen: 2.57255482673645 (2.57255482673645) + | > loss_kl: 2.5053539276123047 (2.5053539276123047) + | > loss_feat: 7.107875823974609 (7.107875823974609) + | > loss_mel: 17.746912002563477 (17.746912002563477) + | > loss_duration: 1.7078183889389038 (1.7078183889389038) + | > loss_1: 31.64051628112793 (31.64051628112793) + + --> STEP: 1 + | > loss_disc: 2.428476572036743 (2.428476572036743) + | > loss_disc_real_0: 0.180160790681839 (0.180160790681839) + | > loss_disc_real_1: 0.2459065020084381 (0.2459065020084381) + | > loss_disc_real_2: 0.2215421050786972 (0.2215421050786972) + | > loss_disc_real_3: 0.19400843977928162 (0.19400843977928162) + | > loss_disc_real_4: 0.23366400599479675 (0.23366400599479675) + | > loss_disc_real_5: 0.22836816310882568 (0.22836816310882568) + | > loss_0: 2.428476572036743 (2.428476572036743) + | > loss_gen: 2.5205438137054443 (2.5205438137054443) + | > loss_kl: 2.75954532623291 (2.75954532623291) + | > loss_feat: 7.698427200317383 (7.698427200317383) + | > loss_mel: 18.78875160217285 (18.78875160217285) + | > loss_duration: 1.7491230964660645 (1.7491230964660645) + | > loss_1: 33.51639175415039 (33.51639175415039) + + --> STEP: 2 + | > loss_disc: 2.6781845092773438 (2.5533305406570435) + | > loss_disc_real_0: 0.19610054790973663 (0.1881306692957878) + | > loss_disc_real_1: 0.21683749556541443 (0.23137199878692627) + | > loss_disc_real_2: 0.24145862460136414 (0.23150036484003067) + | > loss_disc_real_3: 0.27945488691329956 (0.2367316633462906) + | > loss_disc_real_4: 0.2670813202857971 (0.25037266314029694) + | > loss_disc_real_5: 0.2385946363210678 (0.23348139971494675) + | > loss_0: 2.6781845092773438 (2.5533305406570435) + | > loss_gen: 2.602987289428711 (2.5617655515670776) + | > loss_kl: 2.1800572872161865 (2.4698013067245483) + | > loss_feat: 7.372668743133545 (7.535547971725464) + | > loss_mel: 17.958526611328125 (18.37363910675049) + | > loss_duration: 1.6795010566711426 (1.7143120765686035) + | > loss_1: 31.79374122619629 (32.65506649017334) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00400388240814209 (+0.00033473968505859375) + | > avg_loss_disc: 2.5533305406570435 (-0.03353261947631836) + | > avg_loss_disc_real_0: 0.1881306692957878 (-0.10995668917894363) + | > avg_loss_disc_real_1: 0.23137199878692627 (-0.0016864165663719177) + | > avg_loss_disc_real_2: 0.23150036484003067 (-0.006270773708820343) + | > avg_loss_disc_real_3: 0.2367316633462906 (-0.0008304566144943237) + | > avg_loss_disc_real_4: 0.25037266314029694 (+0.020490460097789764) + | > avg_loss_disc_real_5: 0.23348139971494675 (-0.006892062723636627) + | > avg_loss_0: 2.5533305406570435 (-0.03353261947631836) + | > avg_loss_gen: 2.5617655515670776 (+0.003947019577026367) + | > avg_loss_kl: 2.4698013067245483 (+0.012206554412841797) + | > avg_loss_feat: 7.535547971725464 (+1.5978171825408936) + | > avg_loss_mel: 18.37363910675049 (+0.9194488525390625) + | > avg_loss_duration: 1.7143120765686035 (+0.001416325569152832) + | > avg_loss_1: 32.65506649017334 (+2.534836769104004) + + + > EPOCH: 290/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:45:23)  + + --> TIME: 2025-04-19 03:45:28 -- STEP: 13/142 -- GLOBAL_STEP: 420475 + | > loss_disc: 2.5345399379730225 (2.4412772838885965) + | > loss_disc_real_0: 0.12221512943506241 (0.11425014069447151) + | > loss_disc_real_1: 0.18616345524787903 (0.20077122174776518) + | > loss_disc_real_2: 0.23338355123996735 (0.23267750442028046) + | > loss_disc_real_3: 0.24184784293174744 (0.23121088857834154) + | > loss_disc_real_4: 0.24831824004650116 (0.2312399561588581) + | > loss_disc_real_5: 0.2404497265815735 (0.23398191195267898) + | > loss_0: 2.5345399379730225 (2.4412772838885965) + | > grad_norm_0: tensor(9.2420, device='cuda:0') (tensor(6.5716, device='cuda:0')) + | > loss_gen: 2.2384846210479736 (2.4568569843585673) + | > loss_kl: 1.5722897052764893 (1.8288763761520386) + | > loss_feat: 6.719566822052002 (7.609687878535344) + | > loss_mel: 16.792224884033203 (18.11125007042518) + | > loss_duration: 1.4315509796142578 (1.4194545929248517) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.754117965698242 (31.426125306349533) + | > grad_norm_1: tensor(216.3126, device='cuda:0') (tensor(174.1326, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.3298 (0.33215838212233323) + | > loader_time: 0.0052 (0.004658038799579327) + + + --> TIME: 2025-04-19 03:45:37 -- STEP: 38/142 -- GLOBAL_STEP: 420500 + | > loss_disc: 2.5159096717834473 (2.4374120800118697) + | > loss_disc_real_0: 0.14214888215065002 (0.11613006203582413) + | > loss_disc_real_1: 0.23313826322555542 (0.20505032139389137) + | > loss_disc_real_2: 0.24614126980304718 (0.2273544947567739) + | > loss_disc_real_3: 0.2121398001909256 (0.22886159427856145) + | > loss_disc_real_4: 0.2192343920469284 (0.22901822391309237) + | > loss_disc_real_5: 0.238551527261734 (0.23169655274403722) + | > loss_0: 2.5159096717834473 (2.4374120800118697) + | > grad_norm_0: tensor(6.4299, device='cuda:0') (tensor(7.6339, device='cuda:0')) + | > loss_gen: 2.2792344093322754 (2.436213593733938) + | > loss_kl: 1.948843002319336 (1.8283246473262185) + | > loss_feat: 7.077137470245361 (7.502898417021099) + | > loss_mel: 17.21827507019043 (18.138759261683408) + | > loss_duration: 1.46193265914917 (1.4322625461377594) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.985422134399414 (31.338458563152113) + | > grad_norm_1: tensor(256.5293, device='cuda:0') (tensor(202.3800, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.3563 (0.3420655162710893) + | > loader_time: 0.0057 (0.005159810969704076) + + + --> TIME: 2025-04-19 03:45:47 -- STEP: 63/142 -- GLOBAL_STEP: 420525 + | > loss_disc: 2.4152822494506836 (2.450141675888546) + | > loss_disc_real_0: 0.12408695369958878 (0.11828755492728854) + | > loss_disc_real_1: 0.21374662220478058 (0.20453472009726933) + | > loss_disc_real_2: 0.22499747574329376 (0.2266828351550632) + | > loss_disc_real_3: 0.24713599681854248 (0.23022993665838998) + | > loss_disc_real_4: 0.22421899437904358 (0.23279112648396266) + | > loss_disc_real_5: 0.20241643488407135 (0.22899722674536327) + | > loss_0: 2.4152822494506836 (2.450141675888546) + | > grad_norm_0: tensor(12.9079, device='cuda:0') (tensor(8.1701, device='cuda:0')) + | > loss_gen: 2.4781875610351562 (2.4019091961875794) + | > loss_kl: 2.0190327167510986 (1.8526678501613556) + | > loss_feat: 7.628834247589111 (7.308936701880561) + | > loss_mel: 19.4971981048584 (17.935854094369066) + | > loss_duration: 1.500345230102539 (1.443233679211329) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.12359619140625 (30.94260161263602) + | > grad_norm_1: tensor(493.6915, device='cuda:0') (tensor(216.6484, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.3982 (0.35573896150740364) + | > loader_time: 0.006 (0.005558282609969851) + + + --> TIME: 2025-04-19 03:45:57 -- STEP: 88/142 -- GLOBAL_STEP: 420550 + | > loss_disc: 2.4008255004882812 (2.4574637819420215) + | > loss_disc_real_0: 0.07355626672506332 (0.12047290624204007) + | > loss_disc_real_1: 0.19982874393463135 (0.20598920993506908) + | > loss_disc_real_2: 0.1931782215833664 (0.22651175748218189) + | > loss_disc_real_3: 0.24419982731342316 (0.2299623287875544) + | > loss_disc_real_4: 0.22098703682422638 (0.2323728764260357) + | > loss_disc_real_5: 0.19094446301460266 (0.2308223152702505) + | > loss_0: 2.4008255004882812 (2.4574637819420215) + | > grad_norm_0: tensor(18.9773, device='cuda:0') (tensor(8.9933, device='cuda:0')) + | > loss_gen: 2.3256568908691406 (2.3848721899769525) + | > loss_kl: 1.8968383073806763 (1.8687711710279638) + | > loss_feat: 7.251532554626465 (7.258302027528936) + | > loss_mel: 18.458877563476562 (17.995043613693927) + | > loss_duration: 1.522672176361084 (1.4557950618592175) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.455577850341797 (30.962784160267223) + | > grad_norm_1: tensor(241.0417, device='cuda:0') (tensor(240.7978, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.3981 (0.3665499064055356) + | > loader_time: 0.0073 (0.005841512571681631) + + + --> TIME: 2025-04-19 03:46:08 -- STEP: 113/142 -- GLOBAL_STEP: 420575 + | > loss_disc: 2.433708429336548 (2.4554742437548356) + | > loss_disc_real_0: 0.05851968750357628 (0.12367881850990574) + | > loss_disc_real_1: 0.2331448793411255 (0.2058587989448446) + | > loss_disc_real_2: 0.2128158062696457 (0.22621249717421235) + | > loss_disc_real_3: 0.21290956437587738 (0.22930106388784088) + | > loss_disc_real_4: 0.24466486275196075 (0.23167718353524672) + | > loss_disc_real_5: 0.19509710371494293 (0.2300758949950733) + | > loss_0: 2.433708429336548 (2.4554742437548356) + | > grad_norm_0: tensor(8.5592, device='cuda:0') (tensor(11.7684, device='cuda:0')) + | > loss_gen: 2.5086216926574707 (2.3970260240335395) + | > loss_kl: 1.9380035400390625 (1.8769188866151119) + | > loss_feat: 7.634256839752197 (7.224415534365494) + | > loss_mel: 18.400665283203125 (17.93539943526277) + | > loss_duration: 1.494637131690979 (1.4629599531139947) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.97618293762207 (30.896719890358174) + | > grad_norm_1: tensor(205.1688, device='cuda:0') (tensor(264.4276, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.4282 (0.37662187930756963) + | > loader_time: 0.0063 (0.006048101239499793) + + + --> TIME: 2025-04-19 03:46:19 -- STEP: 138/142 -- GLOBAL_STEP: 420600 + | > loss_disc: 2.5267562866210938 (2.460176892902541) + | > loss_disc_real_0: 0.11132309585809708 (0.12375312430811101) + | > loss_disc_real_1: 0.23139077425003052 (0.20716853256242862) + | > loss_disc_real_2: 0.25475797057151794 (0.22601302335227746) + | > loss_disc_real_3: 0.2553582191467285 (0.22960476523292236) + | > loss_disc_real_4: 0.2645164132118225 (0.2314131376535996) + | > loss_disc_real_5: 0.24986839294433594 (0.23010950362768726) + | > loss_0: 2.5267562866210938 (2.460176892902541) + | > grad_norm_0: tensor(11.6103, device='cuda:0') (tensor(12.0523, device='cuda:0')) + | > loss_gen: 2.3541359901428223 (2.3966496215350404) + | > loss_kl: 1.968156337738037 (1.8837952207827913) + | > loss_feat: 6.4801926612854 (7.1935412883758545) + | > loss_mel: 16.626184463500977 (17.916075982909277) + | > loss_duration: 1.5345168113708496 (1.4699294869450554) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.963186264038086 (30.859991612641707) + | > grad_norm_1: tensor(296.0903, device='cuda:0') (tensor(265.1331, device='cuda:0')) + | > current_lr_0: 0.00019287939568115225 + | > current_lr_1: 0.00019287939568115225 + | > step_time: 0.4486 (0.38844646578249725) + | > loader_time: 0.0077 (0.006299937980762427) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3934648036956787 (2.3934648036956787) + | > loss_disc_real_0: 0.10065430402755737 (0.10065430402755737) + | > loss_disc_real_1: 0.20636580884456635 (0.20636580884456635) + | > loss_disc_real_2: 0.22398000955581665 (0.22398000955581665) + | > loss_disc_real_3: 0.2562684118747711 (0.2562684118747711) + | > loss_disc_real_4: 0.23516087234020233 (0.23516087234020233) + | > loss_disc_real_5: 0.28706493973731995 (0.28706493973731995) + | > loss_0: 2.3934648036956787 (2.3934648036956787) + | > loss_gen: 2.4654617309570312 (2.4654617309570312) + | > loss_kl: 2.3231611251831055 (2.3231611251831055) + | > loss_feat: 7.2472710609436035 (7.2472710609436035) + | > loss_mel: 19.633384704589844 (19.633384704589844) + | > loss_duration: 1.730647087097168 (1.730647087097168) + | > loss_1: 33.399925231933594 (33.399925231933594) + + --> STEP: 1 + | > loss_disc: 2.592498302459717 (2.592498302459717) + | > loss_disc_real_0: 0.14986564218997955 (0.14986564218997955) + | > loss_disc_real_1: 0.23183484375476837 (0.23183484375476837) + | > loss_disc_real_2: 0.2243792563676834 (0.2243792563676834) + | > loss_disc_real_3: 0.24155084788799286 (0.24155084788799286) + | > loss_disc_real_4: 0.25777116417884827 (0.25777116417884827) + | > loss_disc_real_5: 0.28172188997268677 (0.28172188997268677) + | > loss_0: 2.592498302459717 (2.592498302459717) + | > loss_gen: 2.190880298614502 (2.190880298614502) + | > loss_kl: 2.8006398677825928 (2.8006398677825928) + | > loss_feat: 6.751217842102051 (6.751217842102051) + | > loss_mel: 17.172670364379883 (17.172670364379883) + | > loss_duration: 1.7511346340179443 (1.7511346340179443) + | > loss_1: 30.66654396057129 (30.66654396057129) + + --> STEP: 2 + | > loss_disc: 2.5435009002685547 (2.5679996013641357) + | > loss_disc_real_0: 0.15177632868289948 (0.15082098543643951) + | > loss_disc_real_1: 0.21810173988342285 (0.2249682918190956) + | > loss_disc_real_2: 0.22093717753887177 (0.2226582169532776) + | > loss_disc_real_3: 0.23033323884010315 (0.235942043364048) + | > loss_disc_real_4: 0.25441810488700867 (0.25609463453292847) + | > loss_disc_real_5: 0.22533518075942993 (0.25352853536605835) + | > loss_0: 2.5435009002685547 (2.5679996013641357) + | > loss_gen: 2.14648175239563 (2.168681025505066) + | > loss_kl: 2.293306827545166 (2.5469733476638794) + | > loss_feat: 6.975835800170898 (6.863526821136475) + | > loss_mel: 17.426509857177734 (17.29959011077881) + | > loss_duration: 1.6637414693832397 (1.707438051700592) + | > loss_1: 30.505876541137695 (30.586210250854492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003536224365234375 (-0.00046765804290771484) + | > avg_loss_disc: 2.5679996013641357 (+0.014669060707092285) + | > avg_loss_disc_real_0: 0.15082098543643951 (-0.0373096838593483) + | > avg_loss_disc_real_1: 0.2249682918190956 (-0.006403706967830658) + | > avg_loss_disc_real_2: 0.2226582169532776 (-0.008842147886753082) + | > avg_loss_disc_real_3: 0.235942043364048 (-0.0007896199822425842) + | > avg_loss_disc_real_4: 0.25609463453292847 (+0.005721971392631531) + | > avg_loss_disc_real_5: 0.25352853536605835 (+0.020047135651111603) + | > avg_loss_0: 2.5679996013641357 (+0.014669060707092285) + | > avg_loss_gen: 2.168681025505066 (-0.3930845260620117) + | > avg_loss_kl: 2.5469733476638794 (+0.07717204093933105) + | > avg_loss_feat: 6.863526821136475 (-0.6720211505889893) + | > avg_loss_mel: 17.29959011077881 (-1.0740489959716797) + | > avg_loss_duration: 1.707438051700592 (-0.006874024868011475) + | > avg_loss_1: 30.586210250854492 (-2.0688562393188477) + + + > EPOCH: 291/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:46:22)  + + --> TIME: 2025-04-19 03:46:30 -- STEP: 21/142 -- GLOBAL_STEP: 420625 + | > loss_disc: 2.4812896251678467 (2.4134750820341564) + | > loss_disc_real_0: 0.11636435985565186 (0.11459636581795556) + | > loss_disc_real_1: 0.22102892398834229 (0.20150616339274816) + | > loss_disc_real_2: 0.23089700937271118 (0.2258028792483466) + | > loss_disc_real_3: 0.2311563491821289 (0.22324146827061972) + | > loss_disc_real_4: 0.2201692759990692 (0.22641602158546448) + | > loss_disc_real_5: 0.22308088839054108 (0.2288246098018828) + | > loss_0: 2.4812896251678467 (2.4134750820341564) + | > grad_norm_0: tensor(3.3482, device='cuda:0') (tensor(7.0587, device='cuda:0')) + | > loss_gen: 2.379805326461792 (2.397950921739851) + | > loss_kl: 1.7907991409301758 (1.770505354517982) + | > loss_feat: 6.642828464508057 (7.328343754722958) + | > loss_mel: 16.533702850341797 (17.420386450631277) + | > loss_duration: 1.4343605041503906 (1.4233796142396473) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.781496047973633 (30.340566271827335) + | > grad_norm_1: tensor(215.9614, device='cuda:0') (tensor(130.3644, device='cuda:0')) + | > current_lr_0: 0.0001928552857566921 + | > current_lr_1: 0.0001928552857566921 + | > step_time: 0.3478 (0.3327408177512033) + | > loader_time: 0.0056 (0.004856813521612258) + + + --> TIME: 2025-04-19 03:46:39 -- STEP: 46/142 -- GLOBAL_STEP: 420650 + | > loss_disc: 2.4283080101013184 (2.43841719109079) + | > loss_disc_real_0: 0.1209576353430748 (0.1190932998838632) + | > loss_disc_real_1: 0.194157674908638 (0.2034581514156383) + | > loss_disc_real_2: 0.21438901126384735 (0.22888646112835925) + | > loss_disc_real_3: 0.20203618705272675 (0.2254708788317183) + | > loss_disc_real_4: 0.22340376675128937 (0.22868156983800556) + | > loss_disc_real_5: 0.22643235325813293 (0.23016587903966074) + | > loss_0: 2.4283080101013184 (2.43841719109079) + | > grad_norm_0: tensor(5.5271, device='cuda:0') (tensor(6.6932, device='cuda:0')) + | > loss_gen: 2.2055253982543945 (2.370169613672339) + | > loss_kl: 1.7626656293869019 (1.7839652947757556) + | > loss_feat: 6.259080410003662 (7.196504706921785) + | > loss_mel: 16.259029388427734 (17.640008262965985) + | > loss_duration: 1.4260571002960205 (1.4356704017390376) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.9123592376709 (30.42631837596064) + | > grad_norm_1: tensor(41.2633, device='cuda:0') (tensor(119.0546, device='cuda:0')) + | > current_lr_0: 0.0001928552857566921 + | > current_lr_1: 0.0001928552857566921 + | > step_time: 0.3766 (0.34947430050891376) + | > loader_time: 0.006 (0.005319061486617379) + + + --> TIME: 2025-04-19 03:46:49 -- STEP: 71/142 -- GLOBAL_STEP: 420675 + | > loss_disc: 2.4810688495635986 (2.444482846998832) + | > loss_disc_real_0: 0.15168297290802002 (0.12336237539707774) + | > loss_disc_real_1: 0.2078525871038437 (0.20460538755000476) + | > loss_disc_real_2: 0.21864913403987885 (0.22756561259148825) + | > loss_disc_real_3: 0.2946167290210724 (0.22698809479323911) + | > loss_disc_real_4: 0.258774995803833 (0.22787342692764706) + | > loss_disc_real_5: 0.2693319022655487 (0.2299072154810731) + | > loss_0: 2.4810688495635986 (2.444482846998832) + | > grad_norm_0: tensor(5.4609, device='cuda:0') (tensor(6.4872, device='cuda:0')) + | > loss_gen: 2.3566370010375977 (2.365148295818919) + | > loss_kl: 1.8414080142974854 (1.807987841082291) + | > loss_feat: 6.304137229919434 (7.110795833695103) + | > loss_mel: 16.467979431152344 (17.587608700067225) + | > loss_duration: 1.4823224544525146 (1.4471071421260564) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.452484130859375 (30.318647841332663) + | > grad_norm_1: tensor(82.3494, device='cuda:0') (tensor(110.5967, device='cuda:0')) + | > current_lr_0: 0.0001928552857566921 + | > current_lr_1: 0.0001928552857566921 + | > step_time: 0.3984 (0.3629720244609134) + | > loader_time: 0.0059 (0.005713754976299447) + + + --> TIME: 2025-04-19 03:47:00 -- STEP: 96/142 -- GLOBAL_STEP: 420700 + | > loss_disc: 2.4032750129699707 (2.449779128034909) + | > loss_disc_real_0: 0.14484678208827972 (0.12540124434356886) + | > loss_disc_real_1: 0.18298397958278656 (0.2049271447273592) + | > loss_disc_real_2: 0.19930298626422882 (0.22613712881381312) + | > loss_disc_real_3: 0.2143012285232544 (0.22736165253445506) + | > loss_disc_real_4: 0.20692196488380432 (0.22831858694553372) + | > loss_disc_real_5: 0.20746499300003052 (0.22901339332262674) + | > loss_0: 2.4032750129699707 (2.449779128034909) + | > grad_norm_0: tensor(6.1427, device='cuda:0') (tensor(6.6359, device='cuda:0')) + | > loss_gen: 2.27179217338562 (2.354104042053222) + | > loss_kl: 1.8179404735565186 (1.8249349296092987) + | > loss_feat: 7.015345573425293 (7.063915774226189) + | > loss_mel: 16.714567184448242 (17.556665241718285) + | > loss_duration: 1.4865570068359375 (1.4576083719730377) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.306201934814453 (30.257228314876556) + | > grad_norm_1: tensor(166.1068, device='cuda:0') (tensor(112.9531, device='cuda:0')) + | > current_lr_0: 0.0001928552857566921 + | > current_lr_1: 0.0001928552857566921 + | > step_time: 0.4069 (0.3756258984406789) + | > loader_time: 0.0066 (0.005966216325759888) + + + --> TIME: 2025-04-19 03:47:10 -- STEP: 121/142 -- GLOBAL_STEP: 420725 + | > loss_disc: 2.462801694869995 (2.4572959064452116) + | > loss_disc_real_0: 0.11685767024755478 (0.1267861408019853) + | > loss_disc_real_1: 0.22963593900203705 (0.20523700549090204) + | > loss_disc_real_2: 0.2333843857049942 (0.22691639174114575) + | > loss_disc_real_3: 0.23891109228134155 (0.22858337897899722) + | > loss_disc_real_4: 0.2484646439552307 (0.22937721851443454) + | > loss_disc_real_5: 0.23190636932849884 (0.22978691473480098) + | > loss_0: 2.462801694869995 (2.4572959064452116) + | > grad_norm_0: tensor(4.5091, device='cuda:0') (tensor(7.2891, device='cuda:0')) + | > loss_gen: 2.452116012573242 (2.3500529163139916) + | > loss_kl: 1.9601407051086426 (1.8357591638880328) + | > loss_feat: 6.73970365524292 (7.025271675803444) + | > loss_mel: 17.949832916259766 (17.556807139688278) + | > loss_duration: 1.5165297985076904 (1.4651678396650583) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.618322372436523 (30.233058661468757) + | > grad_norm_1: tensor(98.9394, device='cuda:0') (tensor(132.8218, device='cuda:0')) + | > current_lr_0: 0.0001928552857566921 + | > current_lr_1: 0.0001928552857566921 + | > step_time: 0.4428 (0.3852452955955316) + | > loader_time: 0.0073 (0.006214496517969557) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6751790046691895 (2.6751790046691895) + | > loss_disc_real_0: 0.16697533428668976 (0.16697533428668976) + | > loss_disc_real_1: 0.2896391451358795 (0.2896391451358795) + | > loss_disc_real_2: 0.2654639780521393 (0.2654639780521393) + | > loss_disc_real_3: 0.25900164246559143 (0.25900164246559143) + | > loss_disc_real_4: 0.25465303659439087 (0.25465303659439087) + | > loss_disc_real_5: 0.3505653440952301 (0.3505653440952301) + | > loss_0: 2.6751790046691895 (2.6751790046691895) + | > loss_gen: 2.426182985305786 (2.426182985305786) + | > loss_kl: 2.2965822219848633 (2.2965822219848633) + | > loss_feat: 6.448976993560791 (6.448976993560791) + | > loss_mel: 16.60419273376465 (16.60419273376465) + | > loss_duration: 1.692411184310913 (1.692411184310913) + | > loss_1: 29.468347549438477 (29.468347549438477) + + --> STEP: 1 + | > loss_disc: 2.671041250228882 (2.671041250228882) + | > loss_disc_real_0: 0.22419218719005585 (0.22419218719005585) + | > loss_disc_real_1: 0.3117506802082062 (0.3117506802082062) + | > loss_disc_real_2: 0.2380092591047287 (0.2380092591047287) + | > loss_disc_real_3: 0.24822846055030823 (0.24822846055030823) + | > loss_disc_real_4: 0.25671321153640747 (0.25671321153640747) + | > loss_disc_real_5: 0.2602740526199341 (0.2602740526199341) + | > loss_0: 2.671041250228882 (2.671041250228882) + | > loss_gen: 2.361443519592285 (2.361443519592285) + | > loss_kl: 2.84401535987854 (2.84401535987854) + | > loss_feat: 5.648149490356445 (5.648149490356445) + | > loss_mel: 16.076261520385742 (16.076261520385742) + | > loss_duration: 1.7317371368408203 (1.7317371368408203) + | > loss_1: 28.66160774230957 (28.66160774230957) + + --> STEP: 2 + | > loss_disc: 2.5579817295074463 (2.614511489868164) + | > loss_disc_real_0: 0.15528079867362976 (0.1897364929318428) + | > loss_disc_real_1: 0.29481828212738037 (0.3032844811677933) + | > loss_disc_real_2: 0.2780482769012451 (0.2580287680029869) + | > loss_disc_real_3: 0.27303680777549744 (0.26063263416290283) + | > loss_disc_real_4: 0.2767239511013031 (0.2667185813188553) + | > loss_disc_real_5: 0.25243714451789856 (0.2563555985689163) + | > loss_0: 2.5579817295074463 (2.614511489868164) + | > loss_gen: 2.4934632778167725 (2.427453398704529) + | > loss_kl: 2.1074182987213135 (2.4757168292999268) + | > loss_feat: 7.144847393035889 (6.396498441696167) + | > loss_mel: 16.912317276000977 (16.49428939819336) + | > loss_duration: 1.719264268875122 (1.7255007028579712) + | > loss_1: 30.37731170654297 (29.51945972442627) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035293102264404297 (-6.9141387939453125e-06) + | > avg_loss_disc: 2.614511489868164 (+0.04651188850402832) + | > avg_loss_disc_real_0: 0.1897364929318428 (+0.03891550749540329) + | > avg_loss_disc_real_1: 0.3032844811677933 (+0.07831618934869766) + | > avg_loss_disc_real_2: 0.2580287680029869 (+0.03537055104970932) + | > avg_loss_disc_real_3: 0.26063263416290283 (+0.024690590798854828) + | > avg_loss_disc_real_4: 0.2667185813188553 (+0.010623946785926819) + | > avg_loss_disc_real_5: 0.2563555985689163 (+0.002827063202857971) + | > avg_loss_0: 2.614511489868164 (+0.04651188850402832) + | > avg_loss_gen: 2.427453398704529 (+0.2587723731994629) + | > avg_loss_kl: 2.4757168292999268 (-0.07125651836395264) + | > avg_loss_feat: 6.396498441696167 (-0.4670283794403076) + | > avg_loss_mel: 16.49428939819336 (-0.8053007125854492) + | > avg_loss_duration: 1.7255007028579712 (+0.01806265115737915) + | > avg_loss_1: 29.51945972442627 (-1.0667505264282227) + + + > EPOCH: 292/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:47:21)  + + --> TIME: 2025-04-19 03:47:23 -- STEP: 4/142 -- GLOBAL_STEP: 420750 + | > loss_disc: 2.5270402431488037 (2.4605469703674316) + | > loss_disc_real_0: 0.10137584060430527 (0.13480987399816513) + | > loss_disc_real_1: 0.2282124012708664 (0.20974114164710045) + | > loss_disc_real_2: 0.20521776378154755 (0.22870900481939316) + | > loss_disc_real_3: 0.22880513966083527 (0.25048041716217995) + | > loss_disc_real_4: 0.21748606860637665 (0.2184634506702423) + | > loss_disc_real_5: 0.2094167023897171 (0.2476930357515812) + | > loss_0: 2.5270402431488037 (2.4605469703674316) + | > grad_norm_0: tensor(12.3103, device='cuda:0') (tensor(13.8966, device='cuda:0')) + | > loss_gen: 2.2306628227233887 (2.336629092693329) + | > loss_kl: 1.7822582721710205 (1.7533091008663177) + | > loss_feat: 6.830271244049072 (7.0731542110443115) + | > loss_mel: 16.8486385345459 (17.596835613250732) + | > loss_duration: 1.4286315441131592 (1.4251448512077332) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.12046241760254 (30.185072422027588) + | > grad_norm_1: tensor(333.6850, device='cuda:0') (tensor(348.8889, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.3223 (0.3226471543312073) + | > loader_time: 0.0046 (0.004294991493225098) + + + --> TIME: 2025-04-19 03:47:32 -- STEP: 29/142 -- GLOBAL_STEP: 420775 + | > loss_disc: 2.4260854721069336 (2.4391170452381004) + | > loss_disc_real_0: 0.1289840191602707 (0.1300719129885065) + | > loss_disc_real_1: 0.22313246130943298 (0.20377345588700524) + | > loss_disc_real_2: 0.23513609170913696 (0.22423503522215218) + | > loss_disc_real_3: 0.22507692873477936 (0.2272589304323854) + | > loss_disc_real_4: 0.2425684779882431 (0.2244694140450708) + | > loss_disc_real_5: 0.21744607388973236 (0.22871717296797653) + | > loss_0: 2.4260854721069336 (2.4391170452381004) + | > grad_norm_0: tensor(4.7692, device='cuda:0') (tensor(14.9936, device='cuda:0')) + | > loss_gen: 2.392299175262451 (2.3957465928176354) + | > loss_kl: 1.836506724357605 (1.79796223393802) + | > loss_feat: 7.574251174926758 (7.251827733270053) + | > loss_mel: 17.78416633605957 (17.782354815252894) + | > loss_duration: 1.44022798538208 (1.4277439980671323) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.027450561523438 (30.655635307575093) + | > grad_norm_1: tensor(271.3611, device='cuda:0') (tensor(298.7962, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.345 (0.33488838425998035) + | > loader_time: 0.0052 (0.005110987301530509) + + + --> TIME: 2025-04-19 03:47:41 -- STEP: 54/142 -- GLOBAL_STEP: 420800 + | > loss_disc: 2.254349946975708 (2.414199705477114) + | > loss_disc_real_0: 0.14161737263202667 (0.12306297936097337) + | > loss_disc_real_1: 0.1705079972743988 (0.2016567830686216) + | > loss_disc_real_2: 0.1983485370874405 (0.22170119732618332) + | > loss_disc_real_3: 0.21421752870082855 (0.22609831282386073) + | > loss_disc_real_4: 0.21266959607601166 (0.2221474669597767) + | > loss_disc_real_5: 0.23163817822933197 (0.22763276431295607) + | > loss_0: 2.254349946975708 (2.414199705477114) + | > grad_norm_0: tensor(26.3693, device='cuda:0') (tensor(13.4435, device='cuda:0')) + | > loss_gen: 2.6486589908599854 (2.4049767476541017) + | > loss_kl: 1.850793480873108 (1.8095036016570196) + | > loss_feat: 8.431160926818848 (7.289610491858588) + | > loss_mel: 18.040287017822266 (17.694703596609607) + | > loss_duration: 1.4712426662445068 (1.436475274739442) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.442142486572266 (30.635269836143213) + | > grad_norm_1: tensor(475.5066, device='cuda:0') (tensor(303.5588, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.3741 (0.3479914444464223) + | > loader_time: 0.0061 (0.005456716926009565) + + + --> TIME: 2025-04-19 03:47:51 -- STEP: 79/142 -- GLOBAL_STEP: 420825 + | > loss_disc: 2.353593111038208 (2.424260821523545) + | > loss_disc_real_0: 0.12887956202030182 (0.12338347879203057) + | > loss_disc_real_1: 0.20261800289154053 (0.2046445845803128) + | > loss_disc_real_2: 0.24310505390167236 (0.22201515564435645) + | > loss_disc_real_3: 0.2332654595375061 (0.22557857198805747) + | > loss_disc_real_4: 0.27820196747779846 (0.22692182263996027) + | > loss_disc_real_5: 0.22973105311393738 (0.22700973138024536) + | > loss_0: 2.353593111038208 (2.424260821523545) + | > grad_norm_0: tensor(12.9797, device='cuda:0') (tensor(13.0365, device='cuda:0')) + | > loss_gen: 2.507124900817871 (2.3917062735255756) + | > loss_kl: 1.8447144031524658 (1.8279602603067326) + | > loss_feat: 6.941178321838379 (7.214944121203845) + | > loss_mel: 17.497934341430664 (17.713628189473212) + | > loss_duration: 1.5070056915283203 (1.4490338729906687) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.297956466674805 (30.597272824637496) + | > grad_norm_1: tensor(340.1013, device='cuda:0') (tensor(291.3065, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.3849 (0.36131125462206093) + | > loader_time: 0.0062 (0.005709198456776291) + + + --> TIME: 2025-04-19 03:48:02 -- STEP: 104/142 -- GLOBAL_STEP: 420850 + | > loss_disc: 2.4040274620056152 (2.4304659343682795) + | > loss_disc_real_0: 0.1106407418847084 (0.12374180207888666) + | > loss_disc_real_1: 0.19235709309577942 (0.20473028447192448) + | > loss_disc_real_2: 0.17438459396362305 (0.2225180216706716) + | > loss_disc_real_3: 0.20766951143741608 (0.2264905354151359) + | > loss_disc_real_4: 0.20615658164024353 (0.22755368455098227) + | > loss_disc_real_5: 0.23873473703861237 (0.22937773411663678) + | > loss_0: 2.4040274620056152 (2.4304659343682795) + | > grad_norm_0: tensor(8.6309, device='cuda:0') (tensor(11.7791, device='cuda:0')) + | > loss_gen: 2.207669496536255 (2.382267997815058) + | > loss_kl: 1.909456729888916 (1.8322412130924373) + | > loss_feat: 7.0981926918029785 (7.187003135681152) + | > loss_mel: 17.264240264892578 (17.75128021607032) + | > loss_duration: 1.506382703781128 (1.4597875487345915) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.98594093322754 (30.612580189338097) + | > grad_norm_1: tensor(82.2205, device='cuda:0') (tensor(258.9240, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.4092 (0.3721387340472294) + | > loader_time: 0.0072 (0.005977337176983172) + + + --> TIME: 2025-04-19 03:48:13 -- STEP: 129/142 -- GLOBAL_STEP: 420875 + | > loss_disc: 2.440934419631958 (2.4301197325536443) + | > loss_disc_real_0: 0.10812163352966309 (0.12472734909302505) + | > loss_disc_real_1: 0.2130696177482605 (0.2038616330124611) + | > loss_disc_real_2: 0.23861654102802277 (0.2230803919161937) + | > loss_disc_real_3: 0.23091505467891693 (0.22557199867658836) + | > loss_disc_real_4: 0.23314088582992554 (0.2281832475532857) + | > loss_disc_real_5: 0.1893446296453476 (0.22792743054009224) + | > loss_0: 2.440934419631958 (2.4301197325536443) + | > grad_norm_0: tensor(4.8992, device='cuda:0') (tensor(10.9387, device='cuda:0')) + | > loss_gen: 2.3954052925109863 (2.3804476630780114) + | > loss_kl: 1.9763332605361938 (1.8433381577794865) + | > loss_feat: 7.792686462402344 (7.174898203029189) + | > loss_mel: 16.719783782958984 (17.688252279000682) + | > loss_duration: 1.4899574518203735 (1.4664537213569464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.37416648864746 (30.553390103717184) + | > grad_norm_1: tensor(44.2949, device='cuda:0') (tensor(233.5715, device='cuda:0')) + | > current_lr_0: 0.0001928311788459725 + | > current_lr_1: 0.0001928311788459725 + | > step_time: 0.4448 (0.3836136289345201) + | > loader_time: 0.0077 (0.006240142408267472) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4833521842956543 (2.4833521842956543) + | > loss_disc_real_0: 0.12114842236042023 (0.12114842236042023) + | > loss_disc_real_1: 0.17993129789829254 (0.17993129789829254) + | > loss_disc_real_2: 0.2282465249300003 (0.2282465249300003) + | > loss_disc_real_3: 0.26820236444473267 (0.26820236444473267) + | > loss_disc_real_4: 0.2160952240228653 (0.2160952240228653) + | > loss_disc_real_5: 0.25078245997428894 (0.25078245997428894) + | > loss_0: 2.4833521842956543 (2.4833521842956543) + | > loss_gen: 2.2147483825683594 (2.2147483825683594) + | > loss_kl: 2.3991551399230957 (2.3991551399230957) + | > loss_feat: 7.353464126586914 (7.353464126586914) + | > loss_mel: 19.290355682373047 (19.290355682373047) + | > loss_duration: 1.7047369480133057 (1.7047369480133057) + | > loss_1: 32.962459564208984 (32.962459564208984) + + --> STEP: 1 + | > loss_disc: 2.470947027206421 (2.470947027206421) + | > loss_disc_real_0: 0.1248815655708313 (0.1248815655708313) + | > loss_disc_real_1: 0.16362306475639343 (0.16362306475639343) + | > loss_disc_real_2: 0.2001899778842926 (0.2001899778842926) + | > loss_disc_real_3: 0.2066582590341568 (0.2066582590341568) + | > loss_disc_real_4: 0.23080921173095703 (0.23080921173095703) + | > loss_disc_real_5: 0.2387019246816635 (0.2387019246816635) + | > loss_0: 2.470947027206421 (2.470947027206421) + | > loss_gen: 2.164187431335449 (2.164187431335449) + | > loss_kl: 2.6797547340393066 (2.6797547340393066) + | > loss_feat: 7.460311412811279 (7.460311412811279) + | > loss_mel: 19.347537994384766 (19.347537994384766) + | > loss_duration: 1.7741496562957764 (1.7741496562957764) + | > loss_1: 33.425941467285156 (33.425941467285156) + + --> STEP: 2 + | > loss_disc: 2.5700347423553467 (2.520490884780884) + | > loss_disc_real_0: 0.15351217985153198 (0.13919687271118164) + | > loss_disc_real_1: 0.1818690299987793 (0.17274604737758636) + | > loss_disc_real_2: 0.21541936695575714 (0.20780467242002487) + | > loss_disc_real_3: 0.25051572918891907 (0.22858699411153793) + | > loss_disc_real_4: 0.23726233839988708 (0.23403577506542206) + | > loss_disc_real_5: 0.23628847301006317 (0.23749519884586334) + | > loss_0: 2.5700347423553467 (2.520490884780884) + | > loss_gen: 2.0808143615722656 (2.1225008964538574) + | > loss_kl: 2.2186546325683594 (2.449204683303833) + | > loss_feat: 6.880950927734375 (7.170631170272827) + | > loss_mel: 16.246896743774414 (17.79721736907959) + | > loss_duration: 1.6925668716430664 (1.7333582639694214) + | > loss_1: 29.119884490966797 (31.272912979125977) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003453969955444336 (-7.534027099609375e-05) + | > avg_loss_disc: 2.520490884780884 (-0.09402060508728027) + | > avg_loss_disc_real_0: 0.13919687271118164 (-0.05053962022066116) + | > avg_loss_disc_real_1: 0.17274604737758636 (-0.1305384337902069) + | > avg_loss_disc_real_2: 0.20780467242002487 (-0.050224095582962036) + | > avg_loss_disc_real_3: 0.22858699411153793 (-0.0320456400513649) + | > avg_loss_disc_real_4: 0.23403577506542206 (-0.03268280625343323) + | > avg_loss_disc_real_5: 0.23749519884586334 (-0.01886039972305298) + | > avg_loss_0: 2.520490884780884 (-0.09402060508728027) + | > avg_loss_gen: 2.1225008964538574 (-0.3049525022506714) + | > avg_loss_kl: 2.449204683303833 (-0.02651214599609375) + | > avg_loss_feat: 7.170631170272827 (+0.7741327285766602) + | > avg_loss_mel: 17.79721736907959 (+1.3029279708862305) + | > avg_loss_duration: 1.7333582639694214 (+0.007857561111450195) + | > avg_loss_1: 31.272912979125977 (+1.753453254699707) + + + > EPOCH: 293/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:48:20)  + + --> TIME: 2025-04-19 03:48:25 -- STEP: 12/142 -- GLOBAL_STEP: 420900 + | > loss_disc: 2.42034649848938 (2.3571070233980813) + | > loss_disc_real_0: 0.19736304879188538 (0.12233669062455495) + | > loss_disc_real_1: 0.1684468388557434 (0.20294898748397827) + | > loss_disc_real_2: 0.2169000506401062 (0.22524154682954153) + | > loss_disc_real_3: 0.22058261930942535 (0.22047113378842673) + | > loss_disc_real_4: 0.22247347235679626 (0.22178771098454794) + | > loss_disc_real_5: 0.19128254055976868 (0.21206211174527803) + | > loss_0: 2.42034649848938 (2.3571070233980813) + | > grad_norm_0: tensor(9.0338, device='cuda:0') (tensor(6.4692, device='cuda:0')) + | > loss_gen: 2.324469804763794 (2.476534148057302) + | > loss_kl: 1.884799838066101 (1.7861144542694092) + | > loss_feat: 6.801380157470703 (7.7309720913569135) + | > loss_mel: 17.20602798461914 (18.440255641937256) + | > loss_duration: 1.4219608306884766 (1.4142695566018422) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.638639450073242 (31.848145961761475) + | > grad_norm_1: tensor(181.1678, device='cuda:0') (tensor(120.8330, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.3281 (0.32549355427424115) + | > loader_time: 0.005 (0.004816393057505289) + + + --> TIME: 2025-04-19 03:48:34 -- STEP: 37/142 -- GLOBAL_STEP: 420925 + | > loss_disc: 2.503288984298706 (2.4047874308921195) + | > loss_disc_real_0: 0.11477752029895782 (0.11864089704043157) + | > loss_disc_real_1: 0.15359941124916077 (0.2057321051488052) + | > loss_disc_real_2: 0.22691993415355682 (0.22296452482004422) + | > loss_disc_real_3: 0.2477082759141922 (0.2252631654610505) + | > loss_disc_real_4: 0.23042698204517365 (0.22249913296184023) + | > loss_disc_real_5: 0.26801928877830505 (0.2208598405122757) + | > loss_0: 2.503288984298706 (2.4047874308921195) + | > grad_norm_0: tensor(5.7319, device='cuda:0') (tensor(8.5874, device='cuda:0')) + | > loss_gen: 2.2074453830718994 (2.4036506188882365) + | > loss_kl: 1.8477609157562256 (1.8310401343010567) + | > loss_feat: 6.321635723114014 (7.404976161750588) + | > loss_mel: 16.266027450561523 (17.980306676916175) + | > loss_duration: 1.4689819812774658 (1.4296959703033034) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.111852645874023 (31.04966942039696) + | > grad_norm_1: tensor(238.2692, device='cuda:0') (tensor(193.1893, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.3621 (0.34347815126986114) + | > loader_time: 0.0057 (0.005275036837603596) + + + --> TIME: 2025-04-19 03:48:44 -- STEP: 62/142 -- GLOBAL_STEP: 420950 + | > loss_disc: 2.568582773208618 (2.4214900885858834) + | > loss_disc_real_0: 0.10243623703718185 (0.11948161764490989) + | > loss_disc_real_1: 0.2207724004983902 (0.2050066342757594) + | > loss_disc_real_2: 0.2504119575023651 (0.22402925356741873) + | > loss_disc_real_3: 0.24474342167377472 (0.2271491137723769) + | > loss_disc_real_4: 0.2188899666070938 (0.224852948900192) + | > loss_disc_real_5: 0.22808508574962616 (0.22553390961500905) + | > loss_0: 2.568582773208618 (2.4214900885858834) + | > grad_norm_0: tensor(8.4737, device='cuda:0') (tensor(9.0773, device='cuda:0')) + | > loss_gen: 2.3210272789001465 (2.394293123675931) + | > loss_kl: 1.8507272005081177 (1.850664654085713) + | > loss_feat: 7.238729953765869 (7.372504588096373) + | > loss_mel: 17.782766342163086 (17.891322628144295) + | > loss_duration: 1.480149745941162 (1.4416520018731394) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.67340087890625 (30.95043683821155) + | > grad_norm_1: tensor(116.9045, device='cuda:0') (tensor(213.8282, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.3834 (0.3603783461355394) + | > loader_time: 0.006 (0.005634377079625284) + + + --> TIME: 2025-04-19 03:48:54 -- STEP: 87/142 -- GLOBAL_STEP: 420975 + | > loss_disc: 2.4309916496276855 (2.4522908994521204) + | > loss_disc_real_0: 0.12333831936120987 (0.12431909829035573) + | > loss_disc_real_1: 0.21068640053272247 (0.20822446921776072) + | > loss_disc_real_2: 0.22875669598579407 (0.22669096239681902) + | > loss_disc_real_3: 0.21122147142887115 (0.22900134993010554) + | > loss_disc_real_4: 0.22378399968147278 (0.2291014016359702) + | > loss_disc_real_5: 0.24531136453151703 (0.227962467690994) + | > loss_0: 2.4309916496276855 (2.4522908994521204) + | > grad_norm_0: tensor(29.1392, device='cuda:0') (tensor(10.6407, device='cuda:0')) + | > loss_gen: 2.3628313541412354 (2.3778720532340576) + | > loss_kl: 1.7946345806121826 (1.8623103615881382) + | > loss_feat: 6.8984150886535645 (7.240198299802583) + | > loss_mel: 18.086795806884766 (17.812556749102715) + | > loss_duration: 1.4919155836105347 (1.4534964958826697) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.634592056274414 (30.74643369652759) + | > grad_norm_1: tensor(599.2194, device='cuda:0') (tensor(238.8003, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.4006 (0.36973771281626033) + | > loader_time: 0.006 (0.005851460599351203) + + + --> TIME: 2025-04-19 03:49:04 -- STEP: 112/142 -- GLOBAL_STEP: 421000 + | > loss_disc: 2.430710792541504 (2.4493331973041803) + | > loss_disc_real_0: 0.15454022586345673 (0.12452827201091819) + | > loss_disc_real_1: 0.22433507442474365 (0.20766126218118833) + | > loss_disc_real_2: 0.24960565567016602 (0.22595863935670682) + | > loss_disc_real_3: 0.22041770815849304 (0.22845171059348754) + | > loss_disc_real_4: 0.2419041246175766 (0.22928730611290252) + | > loss_disc_real_5: 0.24395082890987396 (0.22736962166215693) + | > loss_0: 2.430710792541504 (2.4493331973041803) + | > grad_norm_0: tensor(15.0920, device='cuda:0') (tensor(11.3681, device='cuda:0')) + | > loss_gen: 2.3497848510742188 (2.3816552332469407) + | > loss_kl: 1.932809591293335 (1.869110073362078) + | > loss_feat: 7.232691287994385 (7.210313094513757) + | > loss_mel: 17.855382919311523 (17.765097660677775) + | > loss_duration: 1.4580752849578857 (1.461115981851305) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.82874298095703 (30.687291877610342) + | > grad_norm_1: tensor(458.2033, device='cuda:0') (tensor(261.6691, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.4256 (0.37893072835036684) + | > loader_time: 0.0073 (0.0060869114739554265) + + + --> TIME: 2025-04-19 03:49:15 -- STEP: 137/142 -- GLOBAL_STEP: 421025 + | > loss_disc: 2.4522929191589355 (2.4596124339277723) + | > loss_disc_real_0: 0.06350165605545044 (0.1252639289863788) + | > loss_disc_real_1: 0.2155005782842636 (0.20860419395196172) + | > loss_disc_real_2: 0.21685011684894562 (0.22707144883427302) + | > loss_disc_real_3: 0.20042966306209564 (0.2289903615295452) + | > loss_disc_real_4: 0.2103961706161499 (0.22902177926832742) + | > loss_disc_real_5: 0.19056513905525208 (0.22755229516621053) + | > loss_0: 2.4522929191589355 (2.4596124339277723) + | > grad_norm_0: tensor(23.2498, device='cuda:0') (tensor(12.7538, device='cuda:0')) + | > loss_gen: 2.40814208984375 (2.3788912905393733) + | > loss_kl: 1.8773534297943115 (1.8787351289804834) + | > loss_feat: 8.361165046691895 (7.181392488688448) + | > loss_mel: 19.2684326171875 (17.74123246130282) + | > loss_duration: 1.4938099384307861 (1.468052749216121) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.40890121459961 (30.64830397167345) + | > grad_norm_1: tensor(517.7021, device='cuda:0') (tensor(280.7247, device='cuda:0')) + | > current_lr_0: 0.00019280707494861676 + | > current_lr_1: 0.00019280707494861676 + | > step_time: 0.4529 (0.3901799369032365) + | > loader_time: 0.0076 (0.006330119432324041) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.234558582305908 (2.234558582305908) + | > loss_disc_real_0: 0.05731621012091637 (0.05731621012091637) + | > loss_disc_real_1: 0.19141414761543274 (0.19141414761543274) + | > loss_disc_real_2: 0.22397540509700775 (0.22397540509700775) + | > loss_disc_real_3: 0.2359185516834259 (0.2359185516834259) + | > loss_disc_real_4: 0.22434741258621216 (0.22434741258621216) + | > loss_disc_real_5: 0.22603899240493774 (0.22603899240493774) + | > loss_0: 2.234558582305908 (2.234558582305908) + | > loss_gen: 2.4621615409851074 (2.4621615409851074) + | > loss_kl: 2.6107099056243896 (2.6107099056243896) + | > loss_feat: 7.69149112701416 (7.69149112701416) + | > loss_mel: 19.35086441040039 (19.35086441040039) + | > loss_duration: 1.7139699459075928 (1.7139699459075928) + | > loss_1: 33.82919692993164 (33.82919692993164) + + --> STEP: 1 + | > loss_disc: 2.4210119247436523 (2.4210119247436523) + | > loss_disc_real_0: 0.10163708031177521 (0.10163708031177521) + | > loss_disc_real_1: 0.24268226325511932 (0.24268226325511932) + | > loss_disc_real_2: 0.2513187825679779 (0.2513187825679779) + | > loss_disc_real_3: 0.19494619965553284 (0.19494619965553284) + | > loss_disc_real_4: 0.23024043440818787 (0.23024043440818787) + | > loss_disc_real_5: 0.18331480026245117 (0.18331480026245117) + | > loss_0: 2.4210119247436523 (2.4210119247436523) + | > loss_gen: 2.3572490215301514 (2.3572490215301514) + | > loss_kl: 2.663511037826538 (2.663511037826538) + | > loss_feat: 7.367222785949707 (7.367222785949707) + | > loss_mel: 18.2703800201416 (18.2703800201416) + | > loss_duration: 1.7551687955856323 (1.7551687955856323) + | > loss_1: 32.41352844238281 (32.41352844238281) + + --> STEP: 2 + | > loss_disc: 2.421786308288574 (2.4213991165161133) + | > loss_disc_real_0: 0.1061793714761734 (0.1039082258939743) + | > loss_disc_real_1: 0.2513481378555298 (0.24701520055532455) + | > loss_disc_real_2: 0.21790291368961334 (0.23461084812879562) + | > loss_disc_real_3: 0.22003807127475739 (0.2074921354651451) + | > loss_disc_real_4: 0.2325403243303299 (0.23139037936925888) + | > loss_disc_real_5: 0.20401905477046967 (0.19366692751646042) + | > loss_0: 2.421786308288574 (2.4213991165161133) + | > loss_gen: 2.1800765991210938 (2.2686628103256226) + | > loss_kl: 2.319171667098999 (2.4913413524627686) + | > loss_feat: 5.935610771179199 (6.651416778564453) + | > loss_mel: 15.359105110168457 (16.81474256515503) + | > loss_duration: 1.6791152954101562 (1.7171420454978943) + | > loss_1: 27.473079681396484 (29.94330406188965) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037442445755004883 (+0.00029027462005615234) + | > avg_loss_disc: 2.4213991165161133 (-0.09909176826477051) + | > avg_loss_disc_real_0: 0.1039082258939743 (-0.035288646817207336) + | > avg_loss_disc_real_1: 0.24701520055532455 (+0.07426915317773819) + | > avg_loss_disc_real_2: 0.23461084812879562 (+0.026806175708770752) + | > avg_loss_disc_real_3: 0.2074921354651451 (-0.021094858646392822) + | > avg_loss_disc_real_4: 0.23139037936925888 (-0.0026453956961631775) + | > avg_loss_disc_real_5: 0.19366692751646042 (-0.043828271329402924) + | > avg_loss_0: 2.4213991165161133 (-0.09909176826477051) + | > avg_loss_gen: 2.2686628103256226 (+0.14616191387176514) + | > avg_loss_kl: 2.4913413524627686 (+0.04213666915893555) + | > avg_loss_feat: 6.651416778564453 (-0.519214391708374) + | > avg_loss_mel: 16.81474256515503 (-0.9824748039245605) + | > avg_loss_duration: 1.7171420454978943 (-0.0162162184715271) + | > avg_loss_1: 29.94330406188965 (-1.3296089172363281) + + + > EPOCH: 294/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:49:19)  + + --> TIME: 2025-04-19 03:49:27 -- STEP: 20/142 -- GLOBAL_STEP: 421050 + | > loss_disc: 2.609729051589966 (2.4508127212524418) + | > loss_disc_real_0: 0.09979970008134842 (0.11943769976496696) + | > loss_disc_real_1: 0.21847951412200928 (0.20615777522325515) + | > loss_disc_real_2: 0.2383205145597458 (0.22701339349150657) + | > loss_disc_real_3: 0.23283050954341888 (0.22986807227134703) + | > loss_disc_real_4: 0.2477700114250183 (0.23107304871082307) + | > loss_disc_real_5: 0.24631541967391968 (0.23327908143401146) + | > loss_0: 2.609729051589966 (2.4508127212524418) + | > grad_norm_0: tensor(11.1093, device='cuda:0') (tensor(10.7550, device='cuda:0')) + | > loss_gen: 2.1030261516571045 (2.376445949077606) + | > loss_kl: 1.8728479146957397 (1.8284967422485352) + | > loss_feat: 5.6614670753479 (7.196522188186646) + | > loss_mel: 16.40330696105957 (17.596657848358156) + | > loss_duration: 1.4733749628067017 (1.4228108286857606) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.514022827148438 (30.420933628082274) + | > grad_norm_1: tensor(124.7319, device='cuda:0') (tensor(277.7522, device='cuda:0')) + | > current_lr_0: 0.00019278297406424816 + | > current_lr_1: 0.00019278297406424816 + | > step_time: 0.3415 (0.33387595415115356) + | > loader_time: 0.0055 (0.0048914670944213865) + + + --> TIME: 2025-04-19 03:49:36 -- STEP: 45/142 -- GLOBAL_STEP: 421075 + | > loss_disc: 2.369722843170166 (2.4308525085449224) + | > loss_disc_real_0: 0.1421162188053131 (0.11912396550178528) + | > loss_disc_real_1: 0.1923833042383194 (0.2036941260099411) + | > loss_disc_real_2: 0.22303257882595062 (0.22247288392649756) + | > loss_disc_real_3: 0.20306530594825745 (0.22814530399110583) + | > loss_disc_real_4: 0.2130081057548523 (0.22889616092046103) + | > loss_disc_real_5: 0.20933973789215088 (0.2305733283360799) + | > loss_0: 2.369722843170166 (2.4308525085449224) + | > grad_norm_0: tensor(5.4008, device='cuda:0') (tensor(8.6936, device='cuda:0')) + | > loss_gen: 2.3384714126586914 (2.365786621305677) + | > loss_kl: 1.8345614671707153 (1.8358407868279352) + | > loss_feat: 6.897658348083496 (7.133731704288059) + | > loss_mel: 18.431095123291016 (17.539460033840605) + | > loss_duration: 1.471227765083313 (1.4322468201319376) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.973012924194336 (30.307066048516166) + | > grad_norm_1: tensor(110.8900, device='cuda:0') (tensor(197.2509, device='cuda:0')) + | > current_lr_0: 0.00019278297406424816 + | > current_lr_1: 0.00019278297406424816 + | > step_time: 0.3659 (0.34520898395114474) + | > loader_time: 0.0061 (0.005278200573391385) + + + --> TIME: 2025-04-19 03:49:45 -- STEP: 70/142 -- GLOBAL_STEP: 421100 + | > loss_disc: 2.3252220153808594 (2.427397758620126) + | > loss_disc_real_0: 0.14962856471538544 (0.11979407870343753) + | > loss_disc_real_1: 0.23222903907299042 (0.20394123643636702) + | > loss_disc_real_2: 0.18271015584468842 (0.22335444007601057) + | > loss_disc_real_3: 0.21293090283870697 (0.22764821201562882) + | > loss_disc_real_4: 0.20367531478405 (0.22778567820787426) + | > loss_disc_real_5: 0.24659030139446259 (0.22943219138043267) + | > loss_0: 2.3252220153808594 (2.427397758620126) + | > grad_norm_0: tensor(6.8010, device='cuda:0') (tensor(8.2177, device='cuda:0')) + | > loss_gen: 2.543768882751465 (2.371082823617117) + | > loss_kl: 1.8460184335708618 (1.8354230472019741) + | > loss_feat: 7.776477813720703 (7.1259659494672505) + | > loss_mel: 18.38981056213379 (17.561670889173243) + | > loss_duration: 1.4780240058898926 (1.44348212821143) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.03409957885742 (30.337624876839772) + | > grad_norm_1: tensor(46.0023, device='cuda:0') (tensor(166.9679, device='cuda:0')) + | > current_lr_0: 0.00019278297406424816 + | > current_lr_1: 0.00019278297406424816 + | > step_time: 0.3872 (0.35647847311837333) + | > loader_time: 0.0064 (0.0055867127009800515) + + + --> TIME: 2025-04-19 03:49:56 -- STEP: 95/142 -- GLOBAL_STEP: 421125 + | > loss_disc: 2.6473278999328613 (2.4437997943476613) + | > loss_disc_real_0: 0.18719467520713806 (0.12377130804877533) + | > loss_disc_real_1: 0.22695112228393555 (0.2065217038518504) + | > loss_disc_real_2: 0.21283265948295593 (0.22476749341738853) + | > loss_disc_real_3: 0.2189873307943344 (0.2282435434429269) + | > loss_disc_real_4: 0.21480977535247803 (0.22775306435007797) + | > loss_disc_real_5: 0.22793957591056824 (0.2289902268271697) + | > loss_0: 2.6473278999328613 (2.4437997943476613) + | > grad_norm_0: tensor(7.1996, device='cuda:0') (tensor(8.4434, device='cuda:0')) + | > loss_gen: 2.04341983795166 (2.3583557379873175) + | > loss_kl: 1.9408562183380127 (1.8495726008164255) + | > loss_feat: 5.923394680023193 (7.058021525332802) + | > loss_mel: 16.507457733154297 (17.571255523280097) + | > loss_duration: 1.4951140880584717 (1.454916302781356) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.910240173339844 (30.292121666356138) + | > grad_norm_1: tensor(231.9159, device='cuda:0') (tensor(162.7393, device='cuda:0')) + | > current_lr_0: 0.00019278297406424816 + | > current_lr_1: 0.00019278297406424816 + | > step_time: 0.4092 (0.36839933395385727) + | > loader_time: 0.0075 (0.005895346089413294) + + + --> TIME: 2025-04-19 03:50:06 -- STEP: 120/142 -- GLOBAL_STEP: 421150 + | > loss_disc: 2.476984977722168 (2.446828357378641) + | > loss_disc_real_0: 0.09898117184638977 (0.1252286737163862) + | > loss_disc_real_1: 0.18655243515968323 (0.20547552468876043) + | > loss_disc_real_2: 0.24887968599796295 (0.22547350749373435) + | > loss_disc_real_3: 0.2530665397644043 (0.22841060894231002) + | > loss_disc_real_4: 0.21958763897418976 (0.22882876619696618) + | > loss_disc_real_5: 0.2350628674030304 (0.2280759363124768) + | > loss_0: 2.476984977722168 (2.446828357378641) + | > grad_norm_0: tensor(9.9895, device='cuda:0') (tensor(8.3057, device='cuda:0')) + | > loss_gen: 2.5306811332702637 (2.3530462721983585) + | > loss_kl: 1.937088131904602 (1.8578867793083191) + | > loss_feat: 7.437853813171387 (7.068296778202057) + | > loss_mel: 18.504106521606445 (17.616364169120796) + | > loss_duration: 1.498594045639038 (1.4626573582490285) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.908323287963867 (30.358251412709553) + | > grad_norm_1: tensor(227.9766, device='cuda:0') (tensor(163.5836, device='cuda:0')) + | > current_lr_0: 0.00019278297406424816 + | > current_lr_1: 0.00019278297406424816 + | > step_time: 0.4351 (0.37912256518999726) + | > loader_time: 0.0074 (0.006142733494440717) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4425742626190186 (2.4425742626190186) + | > loss_disc_real_0: 0.2083500474691391 (0.2083500474691391) + | > loss_disc_real_1: 0.22188685834407806 (0.22188685834407806) + | > loss_disc_real_2: 0.2642219066619873 (0.2642219066619873) + | > loss_disc_real_3: 0.23775985836982727 (0.23775985836982727) + | > loss_disc_real_4: 0.2672453820705414 (0.2672453820705414) + | > loss_disc_real_5: 0.2328750342130661 (0.2328750342130661) + | > loss_0: 2.4425742626190186 (2.4425742626190186) + | > loss_gen: 2.5187182426452637 (2.5187182426452637) + | > loss_kl: 2.3883767127990723 (2.3883767127990723) + | > loss_feat: 6.898030757904053 (6.898030757904053) + | > loss_mel: 18.387310028076172 (18.387310028076172) + | > loss_duration: 1.7124407291412354 (1.7124407291412354) + | > loss_1: 31.904876708984375 (31.904876708984375) + + --> STEP: 1 + | > loss_disc: 2.407012939453125 (2.407012939453125) + | > loss_disc_real_0: 0.17768196761608124 (0.17768196761608124) + | > loss_disc_real_1: 0.19624368846416473 (0.19624368846416473) + | > loss_disc_real_2: 0.22737139463424683 (0.22737139463424683) + | > loss_disc_real_3: 0.21316318213939667 (0.21316318213939667) + | > loss_disc_real_4: 0.2156188040971756 (0.2156188040971756) + | > loss_disc_real_5: 0.1955217719078064 (0.1955217719078064) + | > loss_0: 2.407012939453125 (2.407012939453125) + | > loss_gen: 2.3044798374176025 (2.3044798374176025) + | > loss_kl: 2.8606412410736084 (2.8606412410736084) + | > loss_feat: 7.009214401245117 (7.009214401245117) + | > loss_mel: 18.86040687561035 (18.86040687561035) + | > loss_duration: 1.7281320095062256 (1.7281320095062256) + | > loss_1: 32.762874603271484 (32.762874603271484) + + --> STEP: 2 + | > loss_disc: 2.434985876083374 (2.4209994077682495) + | > loss_disc_real_0: 0.2079765647649765 (0.19282926619052887) + | > loss_disc_real_1: 0.23463048040866852 (0.21543708443641663) + | > loss_disc_real_2: 0.2129988968372345 (0.22018514573574066) + | > loss_disc_real_3: 0.19310979545116425 (0.20313648879528046) + | > loss_disc_real_4: 0.17586691677570343 (0.19574286043643951) + | > loss_disc_real_5: 0.23423191905021667 (0.21487684547901154) + | > loss_0: 2.434985876083374 (2.4209994077682495) + | > loss_gen: 2.3804452419281006 (2.3424625396728516) + | > loss_kl: 2.17496657371521 (2.517803907394409) + | > loss_feat: 7.036463737487793 (7.022839069366455) + | > loss_mel: 19.924306869506836 (19.392356872558594) + | > loss_duration: 1.68630051612854 (1.7072162628173828) + | > loss_1: 33.202484130859375 (32.98267936706543) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032552480697631836 (-0.0004889965057373047) + | > avg_loss_disc: 2.4209994077682495 (-0.00039970874786376953) + | > avg_loss_disc_real_0: 0.19282926619052887 (+0.08892104029655457) + | > avg_loss_disc_real_1: 0.21543708443641663 (-0.03157811611890793) + | > avg_loss_disc_real_2: 0.22018514573574066 (-0.014425702393054962) + | > avg_loss_disc_real_3: 0.20313648879528046 (-0.0043556466698646545) + | > avg_loss_disc_real_4: 0.19574286043643951 (-0.035647518932819366) + | > avg_loss_disc_real_5: 0.21487684547901154 (+0.021209917962551117) + | > avg_loss_0: 2.4209994077682495 (-0.00039970874786376953) + | > avg_loss_gen: 2.3424625396728516 (+0.073799729347229) + | > avg_loss_kl: 2.517803907394409 (+0.026462554931640625) + | > avg_loss_feat: 7.022839069366455 (+0.37142229080200195) + | > avg_loss_mel: 19.392356872558594 (+2.5776143074035645) + | > avg_loss_duration: 1.7072162628173828 (-0.009925782680511475) + | > avg_loss_1: 32.98267936706543 (+3.0393753051757812) + + + > EPOCH: 295/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:50:18)  + + --> TIME: 2025-04-19 03:50:20 -- STEP: 3/142 -- GLOBAL_STEP: 421175 + | > loss_disc: 2.4399056434631348 (2.440380652745565) + | > loss_disc_real_0: 0.08005132526159286 (0.10996591796477635) + | > loss_disc_real_1: 0.2238239198923111 (0.20817133784294128) + | > loss_disc_real_2: 0.24103787541389465 (0.2421194463968277) + | > loss_disc_real_3: 0.2408236414194107 (0.23780861496925354) + | > loss_disc_real_4: 0.29401373863220215 (0.25542496641476947) + | > loss_disc_real_5: 0.25528210401535034 (0.25574814279874164) + | > loss_0: 2.4399056434631348 (2.440380652745565) + | > grad_norm_0: tensor(5.8756, device='cuda:0') (tensor(11.6494, device='cuda:0')) + | > loss_gen: 2.4551095962524414 (2.4095542430877686) + | > loss_kl: 1.769867181777954 (1.7652819951375325) + | > loss_feat: 6.829142093658447 (7.474502722422282) + | > loss_mel: 18.16730499267578 (18.416117350260418) + | > loss_duration: 1.4396610260009766 (1.422789494196574) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.66108512878418 (31.488245010375977) + | > grad_norm_1: tensor(86.6925, device='cuda:0') (tensor(247.1660, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.3236 (0.324582576751709) + | > loader_time: 0.0045 (0.004873196283976237) + + + --> TIME: 2025-04-19 03:50:28 -- STEP: 28/142 -- GLOBAL_STEP: 421200 + | > loss_disc: 2.5321877002716064 (2.424894528729575) + | > loss_disc_real_0: 0.08791565150022507 (0.11843632068485022) + | > loss_disc_real_1: 0.24085494875907898 (0.20457214702452933) + | > loss_disc_real_2: 0.23068849742412567 (0.22398149594664574) + | > loss_disc_real_3: 0.23069992661476135 (0.2258963074002947) + | > loss_disc_real_4: 0.22277021408081055 (0.2276871321456773) + | > loss_disc_real_5: 0.23156651854515076 (0.22821258061698504) + | > loss_0: 2.5321877002716064 (2.424894528729575) + | > grad_norm_0: tensor(6.7436, device='cuda:0') (tensor(7.2193, device='cuda:0')) + | > loss_gen: 2.3121259212493896 (2.404298765318734) + | > loss_kl: 1.6939232349395752 (1.799651631287166) + | > loss_feat: 6.901782035827637 (7.31822977747236) + | > loss_mel: 16.883258819580078 (17.592657293592186) + | > loss_duration: 1.4780457019805908 (1.4296332001686096) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.269136428833008 (30.544470514569962) + | > grad_norm_1: tensor(70.4445, device='cuda:0') (tensor(141.1152, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.346 (0.33624763999666485) + | > loader_time: 0.0058 (0.005229013306753977) + + + --> TIME: 2025-04-19 03:50:37 -- STEP: 53/142 -- GLOBAL_STEP: 421225 + | > loss_disc: 2.345463275909424 (2.4362942992516285) + | > loss_disc_real_0: 0.16614335775375366 (0.12327958214676606) + | > loss_disc_real_1: 0.21325820684432983 (0.20281506203255564) + | > loss_disc_real_2: 0.2644566297531128 (0.2253977985314603) + | > loss_disc_real_3: 0.2055913656949997 (0.22798145323429467) + | > loss_disc_real_4: 0.23825666308403015 (0.22793667524490716) + | > loss_disc_real_5: 0.19053104519844055 (0.22952501503926404) + | > loss_0: 2.345463275909424 (2.4362942992516285) + | > grad_norm_0: tensor(5.6515, device='cuda:0') (tensor(7.0287, device='cuda:0')) + | > loss_gen: 2.617615222930908 (2.39017043923432) + | > loss_kl: 1.8298721313476562 (1.8275594104011104) + | > loss_feat: 8.538613319396973 (7.290840643756795) + | > loss_mel: 18.20038604736328 (17.674655014613883) + | > loss_duration: 1.4191224575042725 (1.4340958302875735) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.60560607910156 (30.617321050392007) + | > grad_norm_1: tensor(326.6151, device='cuda:0') (tensor(155.4293, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.3672 (0.3476545990638013) + | > loader_time: 0.0062 (0.005590830209120267) + + + --> TIME: 2025-04-19 03:50:47 -- STEP: 78/142 -- GLOBAL_STEP: 421250 + | > loss_disc: 2.4865658283233643 (2.447763485786242) + | > loss_disc_real_0: 0.09338108450174332 (0.12258077322099453) + | > loss_disc_real_1: 0.19212545454502106 (0.20432294771457332) + | > loss_disc_real_2: 0.20478855073451996 (0.22643131170517358) + | > loss_disc_real_3: 0.24505402147769928 (0.22867310410126662) + | > loss_disc_real_4: 0.2207447737455368 (0.22912478275023973) + | > loss_disc_real_5: 0.2070973664522171 (0.232963611300175) + | > loss_0: 2.4865658283233643 (2.447763485786242) + | > grad_norm_0: tensor(14.3980, device='cuda:0') (tensor(8.1517, device='cuda:0')) + | > loss_gen: 2.291144609451294 (2.388138908606309) + | > loss_kl: 1.9313222169876099 (1.8498274317154517) + | > loss_feat: 6.918338775634766 (7.261609285305708) + | > loss_mel: 17.881357192993164 (17.732768450027855) + | > loss_duration: 1.492859959602356 (1.4463577637305627) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.515024185180664 (30.678701694195087) + | > grad_norm_1: tensor(231.1685, device='cuda:0') (tensor(190.0320, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.3927 (0.3595944337355786) + | > loader_time: 0.0077 (0.005860243088159807) + + + --> TIME: 2025-04-19 03:50:58 -- STEP: 103/142 -- GLOBAL_STEP: 421275 + | > loss_disc: 2.4669389724731445 (2.456261310762572) + | > loss_disc_real_0: 0.141758993268013 (0.12382237161102805) + | > loss_disc_real_1: 0.23364591598510742 (0.205852764902763) + | > loss_disc_real_2: 0.23767773807048798 (0.22717294993909817) + | > loss_disc_real_3: 0.22825780510902405 (0.22921997178526757) + | > loss_disc_real_4: 0.25428444147109985 (0.22993060599253015) + | > loss_disc_real_5: 0.21947333216667175 (0.23319113543890055) + | > loss_0: 2.4669389724731445 (2.456261310762572) + | > grad_norm_0: tensor(20.6943, device='cuda:0') (tensor(10.5107, device='cuda:0')) + | > loss_gen: 2.2022626399993896 (2.383949214972339) + | > loss_kl: 1.8028258085250854 (1.8548297835785208) + | > loss_feat: 6.953727722167969 (7.191209654206211) + | > loss_mel: 17.911672592163086 (17.774474505081912) + | > loss_duration: 1.5327117443084717 (1.4576057218810887) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.403200149536133 (30.662068765140273) + | > grad_norm_1: tensor(299.3871, device='cuda:0') (tensor(227.4937, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.4124 (0.37019686560028964) + | > loader_time: 0.0066 (0.006074407725658232) + + + --> TIME: 2025-04-19 03:51:09 -- STEP: 128/142 -- GLOBAL_STEP: 421300 + | > loss_disc: 2.53328275680542 (2.460376091301441) + | > loss_disc_real_0: 0.11784272640943527 (0.12585105589823797) + | > loss_disc_real_1: 0.22603045403957367 (0.2061460284749046) + | > loss_disc_real_2: 0.20647136867046356 (0.22718443349003792) + | > loss_disc_real_3: 0.21923808753490448 (0.2288488803897053) + | > loss_disc_real_4: 0.21356245875358582 (0.2297013874631375) + | > loss_disc_real_5: 0.24111239612102509 (0.2324242489412427) + | > loss_0: 2.53328275680542 (2.460376091301441) + | > grad_norm_0: tensor(6.5883, device='cuda:0') (tensor(11.5107, device='cuda:0')) + | > loss_gen: 2.0995166301727295 (2.379544137045741) + | > loss_kl: 1.8988746404647827 (1.861788997426629) + | > loss_feat: 6.382396221160889 (7.143538992851973) + | > loss_mel: 17.246414184570312 (17.737894609570496) + | > loss_duration: 1.4853969812393188 (1.46515725646168) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.112598419189453 (30.587923854589462) + | > grad_norm_1: tensor(82.0087, device='cuda:0') (tensor(243.5262, device='cuda:0')) + | > current_lr_0: 0.00019275887619249013 + | > current_lr_1: 0.00019275887619249013 + | > step_time: 0.4501 (0.3819130267947912) + | > loader_time: 0.0074 (0.006304208189249039) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.398297071456909 (2.398297071456909) + | > loss_disc_real_0: 0.1964813470840454 (0.1964813470840454) + | > loss_disc_real_1: 0.23278337717056274 (0.23278337717056274) + | > loss_disc_real_2: 0.24563419818878174 (0.24563419818878174) + | > loss_disc_real_3: 0.27467307448387146 (0.27467307448387146) + | > loss_disc_real_4: 0.2521915137767792 (0.2521915137767792) + | > loss_disc_real_5: 0.23102818429470062 (0.23102818429470062) + | > loss_0: 2.398297071456909 (2.398297071456909) + | > loss_gen: 2.7216854095458984 (2.7216854095458984) + | > loss_kl: 2.6607303619384766 (2.6607303619384766) + | > loss_feat: 7.497537612915039 (7.497537612915039) + | > loss_mel: 18.830543518066406 (18.830543518066406) + | > loss_duration: 1.7264817953109741 (1.7264817953109741) + | > loss_1: 33.43697738647461 (33.43697738647461) + + --> STEP: 1 + | > loss_disc: 2.6137354373931885 (2.6137354373931885) + | > loss_disc_real_0: 0.15177904069423676 (0.15177904069423676) + | > loss_disc_real_1: 0.21767064929008484 (0.21767064929008484) + | > loss_disc_real_2: 0.25099465250968933 (0.25099465250968933) + | > loss_disc_real_3: 0.29655736684799194 (0.29655736684799194) + | > loss_disc_real_4: 0.27571091055870056 (0.27571091055870056) + | > loss_disc_real_5: 0.2539094090461731 (0.2539094090461731) + | > loss_0: 2.6137354373931885 (2.6137354373931885) + | > loss_gen: 2.3774118423461914 (2.3774118423461914) + | > loss_kl: 2.74410343170166 (2.74410343170166) + | > loss_feat: 6.668824672698975 (6.668824672698975) + | > loss_mel: 18.2268123626709 (18.2268123626709) + | > loss_duration: 1.7413887977600098 (1.7413887977600098) + | > loss_1: 31.7585391998291 (31.7585391998291) + + --> STEP: 2 + | > loss_disc: 2.5106985569000244 (2.5622169971466064) + | > loss_disc_real_0: 0.13985195755958557 (0.14581549912691116) + | > loss_disc_real_1: 0.17654448747634888 (0.19710756838321686) + | > loss_disc_real_2: 0.2523876428604126 (0.25169114768505096) + | > loss_disc_real_3: 0.28006765246391296 (0.28831250965595245) + | > loss_disc_real_4: 0.24294017255306244 (0.2593255415558815) + | > loss_disc_real_5: 0.2604367136955261 (0.2571730613708496) + | > loss_0: 2.5106985569000244 (2.5622169971466064) + | > loss_gen: 2.439230442047119 (2.4083211421966553) + | > loss_kl: 2.357642650604248 (2.550873041152954) + | > loss_feat: 7.513856410980225 (7.0913405418396) + | > loss_mel: 18.526199340820312 (18.376505851745605) + | > loss_duration: 1.6898303031921387 (1.7156095504760742) + | > loss_1: 32.526756286621094 (32.1426477432251) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003972291946411133 (+0.0007170438766479492) + | > avg_loss_disc: 2.5622169971466064 (+0.14121758937835693) + | > avg_loss_disc_real_0: 0.14581549912691116 (-0.047013767063617706) + | > avg_loss_disc_real_1: 0.19710756838321686 (-0.018329516053199768) + | > avg_loss_disc_real_2: 0.25169114768505096 (+0.0315060019493103) + | > avg_loss_disc_real_3: 0.28831250965595245 (+0.085176020860672) + | > avg_loss_disc_real_4: 0.2593255415558815 (+0.06358268111944199) + | > avg_loss_disc_real_5: 0.2571730613708496 (+0.042296215891838074) + | > avg_loss_0: 2.5622169971466064 (+0.14121758937835693) + | > avg_loss_gen: 2.4083211421966553 (+0.06585860252380371) + | > avg_loss_kl: 2.550873041152954 (+0.03306913375854492) + | > avg_loss_feat: 7.0913405418396 (+0.06850147247314453) + | > avg_loss_mel: 18.376505851745605 (-1.0158510208129883) + | > avg_loss_duration: 1.7156095504760742 (+0.008393287658691406) + | > avg_loss_1: 32.1426477432251 (-0.840031623840332) + + + > EPOCH: 296/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:51:16)  + + --> TIME: 2025-04-19 03:51:21 -- STEP: 11/142 -- GLOBAL_STEP: 421325 + | > loss_disc: 2.4650683403015137 (2.410269087011164) + | > loss_disc_real_0: 0.14789681136608124 (0.1100554032759233) + | > loss_disc_real_1: 0.19306252896785736 (0.21076616644859314) + | > loss_disc_real_2: 0.25008273124694824 (0.22962819852612235) + | > loss_disc_real_3: 0.24613025784492493 (0.22615168582309375) + | > loss_disc_real_4: 0.2364586442708969 (0.22458092461932788) + | > loss_disc_real_5: 0.2503281235694885 (0.233318183909763) + | > loss_0: 2.4650683403015137 (2.410269087011164) + | > grad_norm_0: tensor(6.7960, device='cuda:0') (tensor(9.2053, device='cuda:0')) + | > loss_gen: 2.4399032592773438 (2.4242444688623603) + | > loss_kl: 1.823906421661377 (1.822043852372603) + | > loss_feat: 7.36145544052124 (7.6086789044466885) + | > loss_mel: 18.511320114135742 (18.224502216685902) + | > loss_duration: 1.3983209133148193 (1.4140012914484197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.5349063873291 (31.49347062544389) + | > grad_norm_1: tensor(53.3433, device='cuda:0') (tensor(261.9522, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.3319 (0.3273063356226141) + | > loader_time: 0.005 (0.004718086936257102) + + + --> TIME: 2025-04-19 03:51:29 -- STEP: 36/142 -- GLOBAL_STEP: 421350 + | > loss_disc: 2.373250722885132 (2.4263133472866483) + | > loss_disc_real_0: 0.12631027400493622 (0.11583835548824734) + | > loss_disc_real_1: 0.2364923655986786 (0.20690850458211368) + | > loss_disc_real_2: 0.23517277836799622 (0.224144260916445) + | > loss_disc_real_3: 0.2056877166032791 (0.22540362510416243) + | > loss_disc_real_4: 0.22515419125556946 (0.22730953867236772) + | > loss_disc_real_5: 0.22439368069171906 (0.2317270433737172) + | > loss_0: 2.373250722885132 (2.4263133472866483) + | > grad_norm_0: tensor(11.7355, device='cuda:0') (tensor(8.4752, device='cuda:0')) + | > loss_gen: 2.4243357181549072 (2.3810626996888047) + | > loss_kl: 1.8605597019195557 (1.8186798493067424) + | > loss_feat: 7.784680366516113 (7.330966631571452) + | > loss_mel: 17.846860885620117 (17.903931617736816) + | > loss_duration: 1.4432921409606934 (1.4268304109573364) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.35972785949707 (30.861470964219833) + | > grad_norm_1: tensor(139.2439, device='cuda:0') (tensor(212.1469, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.3535 (0.338872492313385) + | > loader_time: 0.0055 (0.005209710862901475) + + + --> TIME: 2025-04-19 03:51:39 -- STEP: 61/142 -- GLOBAL_STEP: 421375 + | > loss_disc: 2.4729437828063965 (2.4462724044674737) + | > loss_disc_real_0: 0.13742679357528687 (0.11870818902723125) + | > loss_disc_real_1: 0.22218196094036102 (0.20899458449394975) + | > loss_disc_real_2: 0.2277410328388214 (0.2253315886024569) + | > loss_disc_real_3: 0.23272912204265594 (0.22741212537054156) + | > loss_disc_real_4: 0.2556097209453583 (0.2287089685436155) + | > loss_disc_real_5: 0.23579764366149902 (0.23140002542831858) + | > loss_0: 2.4729437828063965 (2.4462724044674737) + | > grad_norm_0: tensor(11.9979, device='cuda:0') (tensor(8.2979, device='cuda:0')) + | > loss_gen: 2.324730396270752 (2.3582780595685597) + | > loss_kl: 1.7733596563339233 (1.837556915205033) + | > loss_feat: 7.115087509155273 (7.196680967925025) + | > loss_mel: 17.615346908569336 (17.82621330511374) + | > loss_duration: 1.4654641151428223 (1.44219997867209) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.293987274169922 (30.6609290701444) + | > grad_norm_1: tensor(194.5658, device='cuda:0') (tensor(178.3972, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.3821 (0.3503557502246294) + | > loader_time: 0.006 (0.005551861934974546) + + + --> TIME: 2025-04-19 03:51:49 -- STEP: 86/142 -- GLOBAL_STEP: 421400 + | > loss_disc: 2.380453109741211 (2.4572204628656076) + | > loss_disc_real_0: 0.15884095430374146 (0.12199935299712558) + | > loss_disc_real_1: 0.18508870899677277 (0.20860316535068113) + | > loss_disc_real_2: 0.22654399275779724 (0.22679489001978276) + | > loss_disc_real_3: 0.2618090808391571 (0.22819420019554537) + | > loss_disc_real_4: 0.22072529792785645 (0.23003156864365867) + | > loss_disc_real_5: 0.20751801133155823 (0.23070607490317765) + | > loss_0: 2.380453109741211 (2.4572204628656076) + | > grad_norm_0: tensor(8.5854, device='cuda:0') (tensor(7.7140, device='cuda:0')) + | > loss_gen: 2.426807165145874 (2.3432919396910545) + | > loss_kl: 1.901808261871338 (1.854006010432576) + | > loss_feat: 7.079054355621338 (7.114253886910372) + | > loss_mel: 18.584062576293945 (17.782460423403005) + | > loss_duration: 1.5030651092529297 (1.4544988662697549) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.49479866027832 (30.548511017200557) + | > grad_norm_1: tensor(196.2673, device='cuda:0') (tensor(156.8599, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.3969 (0.36213812717171606) + | > loader_time: 0.0068 (0.005827155224112577) + + + --> TIME: 2025-04-19 03:52:00 -- STEP: 111/142 -- GLOBAL_STEP: 421425 + | > loss_disc: 2.4405086040496826 (2.460896700352162) + | > loss_disc_real_0: 0.11093121021986008 (0.12474794907344354) + | > loss_disc_real_1: 0.16964933276176453 (0.20731006079428904) + | > loss_disc_real_2: 0.22257038950920105 (0.22688001797005936) + | > loss_disc_real_3: 0.21454784274101257 (0.2275165671163851) + | > loss_disc_real_4: 0.2287365347146988 (0.22911396783751412) + | > loss_disc_real_5: 0.21695978939533234 (0.2312111163193041) + | > loss_0: 2.4405086040496826 (2.460896700352162) + | > grad_norm_0: tensor(6.3182, device='cuda:0') (tensor(7.7015, device='cuda:0')) + | > loss_gen: 2.368317127227783 (2.3360175979030013) + | > loss_kl: 1.889840841293335 (1.8744040641698752) + | > loss_feat: 7.231592178344727 (7.083281151883237) + | > loss_mel: 19.16158676147461 (17.757647376876687) + | > loss_duration: 1.5039963722229004 (1.4619516606803413) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.15533447265625 (30.513301729081988) + | > grad_norm_1: tensor(209.9557, device='cuda:0') (tensor(155.1208, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.4332 (0.37464574650601223) + | > loader_time: 0.008 (0.006057859541059614) + + + --> TIME: 2025-04-19 03:52:11 -- STEP: 136/142 -- GLOBAL_STEP: 421450 + | > loss_disc: 2.442216157913208 (2.466222934863147) + | > loss_disc_real_0: 0.1097998172044754 (0.12661735757308848) + | > loss_disc_real_1: 0.21831831336021423 (0.2085854780367192) + | > loss_disc_real_2: 0.1971355676651001 (0.22731008347781265) + | > loss_disc_real_3: 0.22510099411010742 (0.22830615521353834) + | > loss_disc_real_4: 0.24596981704235077 (0.22904597902122667) + | > loss_disc_real_5: 0.24169392883777618 (0.2322650392923285) + | > loss_0: 2.442216157913208 (2.466222934863147) + | > grad_norm_0: tensor(7.4232, device='cuda:0') (tensor(7.9189, device='cuda:0')) + | > loss_gen: 2.4240713119506836 (2.3381524243775527) + | > loss_kl: 1.9267258644104004 (1.878983215374105) + | > loss_feat: 7.1153178215026855 (7.089666569934172) + | > loss_mel: 17.740293502807617 (17.72835644553689) + | > loss_duration: 1.506052851676941 (1.4678505431203281) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.712461471557617 (30.503009108936084) + | > grad_norm_1: tensor(245.2705, device='cuda:0') (tensor(165.2103, device='cuda:0')) + | > current_lr_0: 0.00019273478133296606 + | > current_lr_1: 0.00019273478133296606 + | > step_time: 0.4491 (0.38725052686298594) + | > loader_time: 0.0078 (0.0063275764970218435) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5200812816619873 (2.5200812816619873) + | > loss_disc_real_0: 0.22541101276874542 (0.22541101276874542) + | > loss_disc_real_1: 0.20970264077186584 (0.20970264077186584) + | > loss_disc_real_2: 0.24551300704479218 (0.24551300704479218) + | > loss_disc_real_3: 0.21315278112888336 (0.21315278112888336) + | > loss_disc_real_4: 0.23044966161251068 (0.23044966161251068) + | > loss_disc_real_5: 0.25300270318984985 (0.25300270318984985) + | > loss_0: 2.5200812816619873 (2.5200812816619873) + | > loss_gen: 2.54396915435791 (2.54396915435791) + | > loss_kl: 2.403663158416748 (2.403663158416748) + | > loss_feat: 7.426779270172119 (7.426779270172119) + | > loss_mel: 19.25701332092285 (19.25701332092285) + | > loss_duration: 1.7138655185699463 (1.7138655185699463) + | > loss_1: 33.34529113769531 (33.34529113769531) + + --> STEP: 1 + | > loss_disc: 2.5843052864074707 (2.5843052864074707) + | > loss_disc_real_0: 0.18773138523101807 (0.18773138523101807) + | > loss_disc_real_1: 0.19726115465164185 (0.19726115465164185) + | > loss_disc_real_2: 0.28282761573791504 (0.28282761573791504) + | > loss_disc_real_3: 0.2574799358844757 (0.2574799358844757) + | > loss_disc_real_4: 0.24490445852279663 (0.24490445852279663) + | > loss_disc_real_5: 0.2593061327934265 (0.2593061327934265) + | > loss_0: 2.5843052864074707 (2.5843052864074707) + | > loss_gen: 2.472797393798828 (2.472797393798828) + | > loss_kl: 2.711043119430542 (2.711043119430542) + | > loss_feat: 7.535825252532959 (7.535825252532959) + | > loss_mel: 18.628084182739258 (18.628084182739258) + | > loss_duration: 1.7537438869476318 (1.7537438869476318) + | > loss_1: 33.10149383544922 (33.10149383544922) + + --> STEP: 2 + | > loss_disc: 2.4601564407348633 (2.522230863571167) + | > loss_disc_real_0: 0.15799248218536377 (0.17286193370819092) + | > loss_disc_real_1: 0.2509942650794983 (0.22412770986557007) + | > loss_disc_real_2: 0.25766250491142273 (0.2702450603246689) + | > loss_disc_real_3: 0.24640122056007385 (0.2519405782222748) + | > loss_disc_real_4: 0.20326468348503113 (0.22408457100391388) + | > loss_disc_real_5: 0.23270030319690704 (0.24600321799516678) + | > loss_0: 2.4601564407348633 (2.522230863571167) + | > loss_gen: 2.5687625408172607 (2.5207799673080444) + | > loss_kl: 2.2432432174682617 (2.477143168449402) + | > loss_feat: 7.594827651977539 (7.565326452255249) + | > loss_mel: 18.77848243713379 (18.703283309936523) + | > loss_duration: 1.6915879249572754 (1.7226659059524536) + | > loss_1: 32.87690353393555 (32.98919868469238) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003441333770751953 (-0.0005309581756591797) + | > avg_loss_disc: 2.522230863571167 (-0.03998613357543945) + | > avg_loss_disc_real_0: 0.17286193370819092 (+0.027046434581279755) + | > avg_loss_disc_real_1: 0.22412770986557007 (+0.02702014148235321) + | > avg_loss_disc_real_2: 0.2702450603246689 (+0.01855391263961792) + | > avg_loss_disc_real_3: 0.2519405782222748 (-0.03637193143367767) + | > avg_loss_disc_real_4: 0.22408457100391388 (-0.03524097055196762) + | > avg_loss_disc_real_5: 0.24600321799516678 (-0.01116984337568283) + | > avg_loss_0: 2.522230863571167 (-0.03998613357543945) + | > avg_loss_gen: 2.5207799673080444 (+0.11245882511138916) + | > avg_loss_kl: 2.477143168449402 (-0.07372987270355225) + | > avg_loss_feat: 7.565326452255249 (+0.4739859104156494) + | > avg_loss_mel: 18.703283309936523 (+0.32677745819091797) + | > avg_loss_duration: 1.7226659059524536 (+0.0070563554763793945) + | > avg_loss_1: 32.98919868469238 (+0.8465509414672852) + + + > EPOCH: 297/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:52:15)  + + --> TIME: 2025-04-19 03:52:22 -- STEP: 19/142 -- GLOBAL_STEP: 421475 + | > loss_disc: 2.5446417331695557 (2.419319667314228) + | > loss_disc_real_0: 0.12659852206707 (0.11294197957766683) + | > loss_disc_real_1: 0.17151452600955963 (0.20216092624162374) + | > loss_disc_real_2: 0.22075924277305603 (0.22975697564451317) + | > loss_disc_real_3: 0.21792279183864594 (0.22275538664115102) + | > loss_disc_real_4: 0.21664902567863464 (0.21872358259401822) + | > loss_disc_real_5: 0.23246335983276367 (0.22863848115268506) + | > loss_0: 2.5446417331695557 (2.419319667314228) + | > grad_norm_0: tensor(16.5262, device='cuda:0') (tensor(8.8542, device='cuda:0')) + | > loss_gen: 2.487781286239624 (2.3926647336859452) + | > loss_kl: 1.7773211002349854 (1.841401965994584) + | > loss_feat: 7.665246963500977 (7.301565220481471) + | > loss_mel: 17.457128524780273 (18.037846414666426) + | > loss_duration: 1.432807445526123 (1.414428899162694) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.82028579711914 (30.987906807347347) + | > grad_norm_1: tensor(360.0392, device='cuda:0') (tensor(194.8048, device='cuda:0')) + | > current_lr_0: 0.00019271068948529944 + | > current_lr_1: 0.00019271068948529944 + | > step_time: 0.3384 (0.33157329810293096) + | > loader_time: 0.0052 (0.005054674650493421) + + + --> TIME: 2025-04-19 03:52:31 -- STEP: 44/142 -- GLOBAL_STEP: 421500 + | > loss_disc: 2.4037234783172607 (2.4256977384740654) + | > loss_disc_real_0: 0.10738294571638107 (0.11553104577416723) + | > loss_disc_real_1: 0.18672233819961548 (0.20526694438674234) + | > loss_disc_real_2: 0.21570318937301636 (0.22518676282329994) + | > loss_disc_real_3: 0.2205936163663864 (0.22510963238098405) + | > loss_disc_real_4: 0.23486079275608063 (0.22520513290708716) + | > loss_disc_real_5: 0.20580480992794037 (0.22992827404629101) + | > loss_0: 2.4037234783172607 (2.4256977384740654) + | > grad_norm_0: tensor(9.1328, device='cuda:0') (tensor(8.8581, device='cuda:0')) + | > loss_gen: 2.399777889251709 (2.3952699737115344) + | > loss_kl: 1.8603545427322388 (1.8465987660668113) + | > loss_feat: 7.464132785797119 (7.325960744510997) + | > loss_mel: 18.068937301635742 (17.935645016756926) + | > loss_duration: 1.4759032726287842 (1.4323181211948395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.269105911254883 (30.935792229392312) + | > grad_norm_1: tensor(418.9481, device='cuda:0') (tensor(217.6164, device='cuda:0')) + | > current_lr_0: 0.00019271068948529944 + | > current_lr_1: 0.00019271068948529944 + | > step_time: 0.3597 (0.34341480515219946) + | > loader_time: 0.0057 (0.005398782816800205) + + + --> TIME: 2025-04-19 03:52:41 -- STEP: 69/142 -- GLOBAL_STEP: 421525 + | > loss_disc: 2.519618034362793 (2.4277624289194737) + | > loss_disc_real_0: 0.12335480749607086 (0.11758741174919018) + | > loss_disc_real_1: 0.26868346333503723 (0.2048784310834995) + | > loss_disc_real_2: 0.2557503581047058 (0.22540036837259927) + | > loss_disc_real_3: 0.24294956028461456 (0.22545546919539355) + | > loss_disc_real_4: 0.24291130900382996 (0.22611524596594382) + | > loss_disc_real_5: 0.25178220868110657 (0.23038876704547717) + | > loss_0: 2.519618034362793 (2.4277624289194737) + | > grad_norm_0: tensor(12.9755, device='cuda:0') (tensor(9.3440, device='cuda:0')) + | > loss_gen: 2.3119051456451416 (2.3869603544041724) + | > loss_kl: 1.8715436458587646 (1.8708244717639426) + | > loss_feat: 6.183160781860352 (7.2078460264897) + | > loss_mel: 16.69650650024414 (17.70453368753627) + | > loss_duration: 1.5107735395431519 (1.4422068699546482) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.573888778686523 (30.612371195917543) + | > grad_norm_1: tensor(289.7108, device='cuda:0') (tensor(239.9465, device='cuda:0')) + | > current_lr_0: 0.00019271068948529944 + | > current_lr_1: 0.00019271068948529944 + | > step_time: 0.3954 (0.35534353878187097) + | > loader_time: 0.0062 (0.005640672600787619) + + + --> TIME: 2025-04-19 03:52:51 -- STEP: 94/142 -- GLOBAL_STEP: 421550 + | > loss_disc: 2.598736047744751 (2.4359444455897554) + | > loss_disc_real_0: 0.1486363708972931 (0.11808701660087768) + | > loss_disc_real_1: 0.18820346891880035 (0.20514303826271219) + | > loss_disc_real_2: 0.23794123530387878 (0.22469717406846107) + | > loss_disc_real_3: 0.24863992631435394 (0.22719069522746066) + | > loss_disc_real_4: 0.23326270282268524 (0.22712495590144016) + | > loss_disc_real_5: 0.2147979438304901 (0.23068838328757185) + | > loss_0: 2.598736047744751 (2.4359444455897554) + | > grad_norm_0: tensor(8.7080, device='cuda:0') (tensor(9.9649, device='cuda:0')) + | > loss_gen: 2.136136293411255 (2.380682846333118) + | > loss_kl: 1.7551227807998657 (1.876489485831971) + | > loss_feat: 6.345332622528076 (7.167701629882163) + | > loss_mel: 16.446252822875977 (17.722753281288966) + | > loss_duration: 1.4985077381134033 (1.4536720273342538) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.181352615356445 (30.601299042397358) + | > grad_norm_1: tensor(64.4909, device='cuda:0') (tensor(267.2157, device='cuda:0')) + | > current_lr_0: 0.00019271068948529944 + | > current_lr_1: 0.00019271068948529944 + | > step_time: 0.3967 (0.3663045827378618) + | > loader_time: 0.0067 (0.00589038463349038) + + + --> TIME: 2025-04-19 03:53:02 -- STEP: 119/142 -- GLOBAL_STEP: 421575 + | > loss_disc: 2.5284640789031982 (2.4427980294748513) + | > loss_disc_real_0: 0.06313977390527725 (0.1223534491507947) + | > loss_disc_real_1: 0.18883952498435974 (0.20509932697320185) + | > loss_disc_real_2: 0.21843287348747253 (0.2250354093413393) + | > loss_disc_real_3: 0.23856163024902344 (0.2269166204608789) + | > loss_disc_real_4: 0.24160324037075043 (0.22771066176791152) + | > loss_disc_real_5: 0.25056782364845276 (0.2306698698456548) + | > loss_0: 2.5284640789031982 (2.4427980294748513) + | > grad_norm_0: tensor(11.7138, device='cuda:0') (tensor(11.8856, device='cuda:0')) + | > loss_gen: 2.1634862422943115 (2.384465744515427) + | > loss_kl: 1.8645904064178467 (1.8818800058685432) + | > loss_feat: 5.530070781707764 (7.146578636489997) + | > loss_mel: 15.849663734436035 (17.70481680220917) + | > loss_duration: 1.5371170043945312 (1.4608380043206095) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.944927215576172 (30.578578965002748) + | > grad_norm_1: tensor(185.4928, device='cuda:0') (tensor(278.3161, device='cuda:0')) + | > current_lr_0: 0.00019271068948529944 + | > current_lr_1: 0.00019271068948529944 + | > step_time: 0.4249 (0.3775453327082786) + | > loader_time: 0.0082 (0.00614125989064449) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3284482955932617 (2.3284482955932617) + | > loss_disc_real_0: 0.1633700430393219 (0.1633700430393219) + | > loss_disc_real_1: 0.21839158236980438 (0.21839158236980438) + | > loss_disc_real_2: 0.23247532546520233 (0.23247532546520233) + | > loss_disc_real_3: 0.2884821593761444 (0.2884821593761444) + | > loss_disc_real_4: 0.21558183431625366 (0.21558183431625366) + | > loss_disc_real_5: 0.2418597936630249 (0.2418597936630249) + | > loss_0: 2.3284482955932617 (2.3284482955932617) + | > loss_gen: 2.604316473007202 (2.604316473007202) + | > loss_kl: 2.4048240184783936 (2.4048240184783936) + | > loss_feat: 6.764279842376709 (6.764279842376709) + | > loss_mel: 19.78986167907715 (19.78986167907715) + | > loss_duration: 1.7226617336273193 (1.7226617336273193) + | > loss_1: 33.28594207763672 (33.28594207763672) + + --> STEP: 1 + | > loss_disc: 2.6264595985412598 (2.6264595985412598) + | > loss_disc_real_0: 0.17582300305366516 (0.17582300305366516) + | > loss_disc_real_1: 0.2231854796409607 (0.2231854796409607) + | > loss_disc_real_2: 0.30393943190574646 (0.30393943190574646) + | > loss_disc_real_3: 0.29122650623321533 (0.29122650623321533) + | > loss_disc_real_4: 0.2774146497249603 (0.2774146497249603) + | > loss_disc_real_5: 0.28676870465278625 (0.28676870465278625) + | > loss_0: 2.6264595985412598 (2.6264595985412598) + | > loss_gen: 2.4324405193328857 (2.4324405193328857) + | > loss_kl: 2.7657761573791504 (2.7657761573791504) + | > loss_feat: 5.892244338989258 (5.892244338989258) + | > loss_mel: 17.359970092773438 (17.359970092773438) + | > loss_duration: 1.774204134941101 (1.774204134941101) + | > loss_1: 30.224634170532227 (30.224634170532227) + + --> STEP: 2 + | > loss_disc: 2.50895357131958 (2.56770658493042) + | > loss_disc_real_0: 0.1931406855583191 (0.18448184430599213) + | > loss_disc_real_1: 0.21455822885036469 (0.2188718542456627) + | > loss_disc_real_2: 0.2433270812034607 (0.2736332565546036) + | > loss_disc_real_3: 0.27593448758125305 (0.2835804969072342) + | > loss_disc_real_4: 0.23287604749202728 (0.2551453486084938) + | > loss_disc_real_5: 0.2817874550819397 (0.284278079867363) + | > loss_0: 2.50895357131958 (2.56770658493042) + | > loss_gen: 2.462406873703003 (2.4474236965179443) + | > loss_kl: 2.398939371109009 (2.5823577642440796) + | > loss_feat: 7.293078899383545 (6.592661619186401) + | > loss_mel: 18.790672302246094 (18.075321197509766) + | > loss_duration: 1.6787867546081543 (1.7264954447746277) + | > loss_1: 32.62388229370117 (31.4242582321167) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032633543014526367 (-0.0001779794692993164) + | > avg_loss_disc: 2.56770658493042 (+0.04547572135925293) + | > avg_loss_disc_real_0: 0.18448184430599213 (+0.011619910597801208) + | > avg_loss_disc_real_1: 0.2188718542456627 (-0.005255855619907379) + | > avg_loss_disc_real_2: 0.2736332565546036 (+0.0033881962299346924) + | > avg_loss_disc_real_3: 0.2835804969072342 (+0.03163991868495941) + | > avg_loss_disc_real_4: 0.2551453486084938 (+0.031060777604579926) + | > avg_loss_disc_real_5: 0.284278079867363 (+0.0382748618721962) + | > avg_loss_0: 2.56770658493042 (+0.04547572135925293) + | > avg_loss_gen: 2.4474236965179443 (-0.0733562707901001) + | > avg_loss_kl: 2.5823577642440796 (+0.10521459579467773) + | > avg_loss_feat: 6.592661619186401 (-0.9726648330688477) + | > avg_loss_mel: 18.075321197509766 (-0.6279621124267578) + | > avg_loss_duration: 1.7264954447746277 (+0.0038295388221740723) + | > avg_loss_1: 31.4242582321167 (-1.5649404525756836) + + + > EPOCH: 298/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:53:14)  + + --> TIME: 2025-04-19 03:53:15 -- STEP: 2/142 -- GLOBAL_STEP: 421600 + | > loss_disc: 2.4134364128112793 (2.4055174589157104) + | > loss_disc_real_0: 0.07478921860456467 (0.09892601147294044) + | > loss_disc_real_1: 0.20616133511066437 (0.2062946856021881) + | > loss_disc_real_2: 0.2599085569381714 (0.2505929172039032) + | > loss_disc_real_3: 0.26151448488235474 (0.26282429695129395) + | > loss_disc_real_4: 0.2851051390171051 (0.2689029276371002) + | > loss_disc_real_5: 0.3098660409450531 (0.2870846539735794) + | > loss_0: 2.4134364128112793 (2.4055174589157104) + | > grad_norm_0: tensor(20.4235, device='cuda:0') (tensor(24.6611, device='cuda:0')) + | > loss_gen: 2.480604887008667 (2.51309072971344) + | > loss_kl: 1.7454556226730347 (1.723943293094635) + | > loss_feat: 7.6949052810668945 (7.9417266845703125) + | > loss_mel: 18.050701141357422 (18.105388641357422) + | > loss_duration: 1.4427752494812012 (1.4173372983932495) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.41444206237793 (31.701486587524414) + | > grad_norm_1: tensor(514.8799, device='cuda:0') (tensor(600.4299, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.3353 (0.3332231044769287) + | > loader_time: 0.004 (0.004153609275817871) + + + --> TIME: 2025-04-19 03:53:24 -- STEP: 27/142 -- GLOBAL_STEP: 421625 + | > loss_disc: 2.3727927207946777 (2.391007997371532) + | > loss_disc_real_0: 0.10454393923282623 (0.11587434869121623) + | > loss_disc_real_1: 0.16828398406505585 (0.1983769624321549) + | > loss_disc_real_2: 0.1901368647813797 (0.219611840667548) + | > loss_disc_real_3: 0.1952337622642517 (0.22484495297626214) + | > loss_disc_real_4: 0.17660890519618988 (0.22704862665247033) + | > loss_disc_real_5: 0.17148840427398682 (0.23120144175158608) + | > loss_0: 2.3727927207946777 (2.391007997371532) + | > grad_norm_0: tensor(15.0742, device='cuda:0') (tensor(12.4874, device='cuda:0')) + | > loss_gen: 2.307382345199585 (2.3929641335098832) + | > loss_kl: 1.6971715688705444 (1.7836390336354573) + | > loss_feat: 8.069367408752441 (7.435090948034216) + | > loss_mel: 18.224817276000977 (17.880129990754305) + | > loss_duration: 1.4430732727050781 (1.4266907903883193) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.741811752319336 (30.918514675564236) + | > grad_norm_1: tensor(424.8185, device='cuda:0') (tensor(290.3125, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.3492 (0.33748043908013237) + | > loader_time: 0.0055 (0.005036407046847873) + + + --> TIME: 2025-04-19 03:53:33 -- STEP: 52/142 -- GLOBAL_STEP: 421650 + | > loss_disc: 2.4260828495025635 (2.402924280900221) + | > loss_disc_real_0: 0.14091269671916962 (0.11479729757859157) + | > loss_disc_real_1: 0.18943572044372559 (0.20177065265866426) + | > loss_disc_real_2: 0.23887422680854797 (0.22306151544818512) + | > loss_disc_real_3: 0.2278224676847458 (0.22803456307603762) + | > loss_disc_real_4: 0.24644510447978973 (0.22847538727980393) + | > loss_disc_real_5: 0.22835102677345276 (0.23030669500048345) + | > loss_0: 2.4260828495025635 (2.402924280900221) + | > grad_norm_0: tensor(11.0659, device='cuda:0') (tensor(10.1480, device='cuda:0')) + | > loss_gen: 2.328646421432495 (2.3947152632933393) + | > loss_kl: 1.8808889389038086 (1.8211814761161804) + | > loss_feat: 6.290735721588135 (7.320228998477642) + | > loss_mel: 17.691062927246094 (17.800165781607987) + | > loss_duration: 1.4952099323272705 (1.4360061838076668) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.68654441833496 (30.772297602433426) + | > grad_norm_1: tensor(202.1693, device='cuda:0') (tensor(246.6532, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.3675 (0.3480369127713717) + | > loader_time: 0.0056 (0.005412766566643348) + + + --> TIME: 2025-04-19 03:53:43 -- STEP: 77/142 -- GLOBAL_STEP: 421675 + | > loss_disc: 2.4745519161224365 (2.416710999104883) + | > loss_disc_real_0: 0.13635902106761932 (0.11680133689146538) + | > loss_disc_real_1: 0.21443477272987366 (0.20302950445707743) + | > loss_disc_real_2: 0.25131499767303467 (0.22260338880799033) + | > loss_disc_real_3: 0.21843136847019196 (0.2263019553252629) + | > loss_disc_real_4: 0.23199902474880219 (0.2282658540583276) + | > loss_disc_real_5: 0.22211477160453796 (0.23201367007447526) + | > loss_0: 2.4745519161224365 (2.416710999104883) + | > grad_norm_0: tensor(9.3455, device='cuda:0') (tensor(9.1681, device='cuda:0')) + | > loss_gen: 2.3568525314331055 (2.3792387757982536) + | > loss_kl: 1.8229615688323975 (1.833047298641948) + | > loss_feat: 6.516124248504639 (7.2834872580193855) + | > loss_mel: 15.311992645263672 (17.744945798601417) + | > loss_duration: 1.4741113185882568 (1.4476682783721333) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.48204231262207 (30.688387486841773) + | > grad_norm_1: tensor(49.9167, device='cuda:0') (tensor(213.3995, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.3897 (0.3597448429503998) + | > loader_time: 0.0061 (0.005701024810989182) + + + --> TIME: 2025-04-19 03:53:53 -- STEP: 102/142 -- GLOBAL_STEP: 421700 + | > loss_disc: 2.4826533794403076 (2.4403984383040793) + | > loss_disc_real_0: 0.19804660975933075 (0.12200523145935115) + | > loss_disc_real_1: 0.20615163445472717 (0.20504411660573063) + | > loss_disc_real_2: 0.21718132495880127 (0.22425025891439587) + | > loss_disc_real_3: 0.23534193634986877 (0.2280492992962108) + | > loss_disc_real_4: 0.2418992817401886 (0.2307709937586504) + | > loss_disc_real_5: 0.21569675207138062 (0.2330610649842842) + | > loss_0: 2.4826533794403076 (2.4403984383040793) + | > grad_norm_0: tensor(10.2057, device='cuda:0') (tensor(8.8489, device='cuda:0')) + | > loss_gen: 2.3185653686523438 (2.3672845901227473) + | > loss_kl: 1.9338680505752563 (1.8487042621070264) + | > loss_feat: 6.994527816772461 (7.19251236261106) + | > loss_mel: 17.679264068603516 (17.722631108527082) + | > loss_duration: 1.4835145473480225 (1.459750887225656) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.409740447998047 (30.590883292403877) + | > grad_norm_1: tensor(79.8939, device='cuda:0') (tensor(195.8629, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.4211 (0.3701467373791863) + | > loader_time: 0.0067 (0.005954473626379873) + + + --> TIME: 2025-04-19 03:54:04 -- STEP: 127/142 -- GLOBAL_STEP: 421725 + | > loss_disc: 2.570237636566162 (2.4535467042697707) + | > loss_disc_real_0: 0.12103860825300217 (0.12554981888044534) + | > loss_disc_real_1: 0.1881149560213089 (0.20663284422375086) + | > loss_disc_real_2: 0.22999726235866547 (0.22563994165480605) + | > loss_disc_real_3: 0.25557905435562134 (0.2283224313277898) + | > loss_disc_real_4: 0.264859676361084 (0.2298793135665533) + | > loss_disc_real_5: 0.2451808899641037 (0.2328796328052761) + | > loss_0: 2.570237636566162 (2.4535467042697707) + | > grad_norm_0: tensor(11.4679, device='cuda:0') (tensor(8.8064, device='cuda:0')) + | > loss_gen: 2.200906276702881 (2.3506959085389396) + | > loss_kl: 1.8523329496383667 (1.8561315339381301) + | > loss_feat: 6.911920070648193 (7.127332154221422) + | > loss_mel: 17.63140106201172 (17.726354486360318) + | > loss_duration: 1.4986047744750977 (1.467013705433823) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.095165252685547 (30.527527861707792) + | > grad_norm_1: tensor(269.0643, device='cuda:0') (tensor(183.8458, device='cuda:0')) + | > current_lr_0: 0.00019268660064911377 + | > current_lr_1: 0.00019268660064911377 + | > step_time: 0.4449 (0.381689174907414) + | > loader_time: 0.0075 (0.006220549110352524) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.395479917526245 (2.395479917526245) + | > loss_disc_real_0: 0.1401328295469284 (0.1401328295469284) + | > loss_disc_real_1: 0.18488527834415436 (0.18488527834415436) + | > loss_disc_real_2: 0.2081957757472992 (0.2081957757472992) + | > loss_disc_real_3: 0.24355731904506683 (0.24355731904506683) + | > loss_disc_real_4: 0.2448342740535736 (0.2448342740535736) + | > loss_disc_real_5: 0.2666703164577484 (0.2666703164577484) + | > loss_0: 2.395479917526245 (2.395479917526245) + | > loss_gen: 2.420780658721924 (2.420780658721924) + | > loss_kl: 2.5694618225097656 (2.5694618225097656) + | > loss_feat: 7.200882434844971 (7.200882434844971) + | > loss_mel: 18.592668533325195 (18.592668533325195) + | > loss_duration: 1.7091233730316162 (1.7091233730316162) + | > loss_1: 32.492916107177734 (32.492916107177734) + + --> STEP: 1 + | > loss_disc: 2.44791579246521 (2.44791579246521) + | > loss_disc_real_0: 0.1572924554347992 (0.1572924554347992) + | > loss_disc_real_1: 0.19426041841506958 (0.19426041841506958) + | > loss_disc_real_2: 0.23587024211883545 (0.23587024211883545) + | > loss_disc_real_3: 0.23101311922073364 (0.23101311922073364) + | > loss_disc_real_4: 0.23208096623420715 (0.23208096623420715) + | > loss_disc_real_5: 0.25956353545188904 (0.25956353545188904) + | > loss_0: 2.44791579246521 (2.44791579246521) + | > loss_gen: 2.3561599254608154 (2.3561599254608154) + | > loss_kl: 2.7284095287323 (2.7284095287323) + | > loss_feat: 6.784336566925049 (6.784336566925049) + | > loss_mel: 18.37581443786621 (18.37581443786621) + | > loss_duration: 1.750646710395813 (1.750646710395813) + | > loss_1: 31.9953670501709 (31.9953670501709) + + --> STEP: 2 + | > loss_disc: 2.5016276836395264 (2.474771738052368) + | > loss_disc_real_0: 0.24479025602340698 (0.2010413557291031) + | > loss_disc_real_1: 0.23045125603675842 (0.212355837225914) + | > loss_disc_real_2: 0.22333867847919464 (0.22960446029901505) + | > loss_disc_real_3: 0.20405423641204834 (0.217533677816391) + | > loss_disc_real_4: 0.22036424279212952 (0.22622260451316833) + | > loss_disc_real_5: 0.266279399394989 (0.262921467423439) + | > loss_0: 2.5016276836395264 (2.474771738052368) + | > loss_gen: 2.3245115280151367 (2.340335726737976) + | > loss_kl: 2.1876308917999268 (2.4580202102661133) + | > loss_feat: 5.689670562744141 (6.237003564834595) + | > loss_mel: 16.286548614501953 (17.331181526184082) + | > loss_duration: 1.688368797302246 (1.7195077538490295) + | > loss_1: 28.17673110961914 (30.08604907989502) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003657698631286621 (+0.0003943443298339844) + | > avg_loss_disc: 2.474771738052368 (-0.09293484687805176) + | > avg_loss_disc_real_0: 0.2010413557291031 (+0.016559511423110962) + | > avg_loss_disc_real_1: 0.212355837225914 (-0.006516017019748688) + | > avg_loss_disc_real_2: 0.22960446029901505 (-0.04402879625558853) + | > avg_loss_disc_real_3: 0.217533677816391 (-0.0660468190908432) + | > avg_loss_disc_real_4: 0.22622260451316833 (-0.02892274409532547) + | > avg_loss_disc_real_5: 0.262921467423439 (-0.02135661244392395) + | > avg_loss_0: 2.474771738052368 (-0.09293484687805176) + | > avg_loss_gen: 2.340335726737976 (-0.10708796977996826) + | > avg_loss_kl: 2.4580202102661133 (-0.12433755397796631) + | > avg_loss_feat: 6.237003564834595 (-0.35565805435180664) + | > avg_loss_mel: 17.331181526184082 (-0.7441396713256836) + | > avg_loss_duration: 1.7195077538490295 (-0.0069876909255981445) + | > avg_loss_1: 30.08604907989502 (-1.3382091522216797) + + + > EPOCH: 299/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:54:12)  + + --> TIME: 2025-04-19 03:54:16 -- STEP: 10/142 -- GLOBAL_STEP: 421750 + | > loss_disc: 2.4079201221466064 (2.43601131439209) + | > loss_disc_real_0: 0.10614677518606186 (0.11442670375108718) + | > loss_disc_real_1: 0.23237772285938263 (0.20721286982297898) + | > loss_disc_real_2: 0.2099640667438507 (0.2213507190346718) + | > loss_disc_real_3: 0.19155380129814148 (0.2202604442834854) + | > loss_disc_real_4: 0.23003996908664703 (0.22506160140037537) + | > loss_disc_real_5: 0.22337649762630463 (0.23678374737501146) + | > loss_0: 2.4079201221466064 (2.43601131439209) + | > grad_norm_0: tensor(6.2314, device='cuda:0') (tensor(11.2727, device='cuda:0')) + | > loss_gen: 2.3770267963409424 (2.392720341682434) + | > loss_kl: 1.7519681453704834 (1.8216275215148925) + | > loss_feat: 7.75788688659668 (7.3993470668792725) + | > loss_mel: 19.087841033935547 (18.053970909118654) + | > loss_duration: 1.401816487312317 (1.419109332561493) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.37654113769531 (31.08677520751953) + | > grad_norm_1: tensor(69.3077, device='cuda:0') (tensor(272.8620, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.3372 (0.3262906312942505) + | > loader_time: 0.0055 (0.004609537124633789) + + + --> TIME: 2025-04-19 03:54:25 -- STEP: 35/142 -- GLOBAL_STEP: 421775 + | > loss_disc: 2.2846128940582275 (2.4338910307203023) + | > loss_disc_real_0: 0.0635313019156456 (0.11382307239941188) + | > loss_disc_real_1: 0.15905307233333588 (0.2072965213230678) + | > loss_disc_real_2: 0.1988011747598648 (0.22166997364589147) + | > loss_disc_real_3: 0.1706957072019577 (0.2238128947360175) + | > loss_disc_real_4: 0.2008436918258667 (0.2280555043901716) + | > loss_disc_real_5: 0.2144593894481659 (0.2320267609187535) + | > loss_0: 2.2846128940582275 (2.4338910307203023) + | > grad_norm_0: tensor(15.4803, device='cuda:0') (tensor(11.2516, device='cuda:0')) + | > loss_gen: 2.5574073791503906 (2.38232513155256) + | > loss_kl: 1.852316975593567 (1.8330058676855905) + | > loss_feat: 8.671339988708496 (7.3471645082746235) + | > loss_mel: 19.979877471923828 (17.836178643362864) + | > loss_duration: 1.451804518699646 (1.4317313705171857) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 34.51274871826172 (30.830405589512416) + | > grad_norm_1: tensor(560.3611, device='cuda:0') (tensor(306.4810, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.359 (0.3393570763724191) + | > loader_time: 0.0058 (0.005244112014770508) + + + --> TIME: 2025-04-19 03:54:35 -- STEP: 60/142 -- GLOBAL_STEP: 421800 + | > loss_disc: 2.4350805282592773 (2.4435498237609874) + | > loss_disc_real_0: 0.14354288578033447 (0.11720036516586939) + | > loss_disc_real_1: 0.1757889837026596 (0.2077146162589391) + | > loss_disc_real_2: 0.19660243391990662 (0.225186138600111) + | > loss_disc_real_3: 0.21545033156871796 (0.22603733018040656) + | > loss_disc_real_4: 0.21253177523612976 (0.2300709200402101) + | > loss_disc_real_5: 0.2577322721481323 (0.2324955316881339) + | > loss_0: 2.4350805282592773 (2.4435498237609874) + | > grad_norm_0: tensor(9.1458, device='cuda:0') (tensor(11.2546, device='cuda:0')) + | > loss_gen: 2.2895328998565674 (2.3799689690272015) + | > loss_kl: 1.9298521280288696 (1.8502711951732635) + | > loss_feat: 6.769745349884033 (7.281155673662822) + | > loss_mel: 17.0134220123291 (17.80972849527995) + | > loss_duration: 1.4748318195343018 (1.4409391482671101) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.477386474609375 (30.762063535054526) + | > grad_norm_1: tensor(198.7878, device='cuda:0') (tensor(297.8017, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.3748 (0.3504777749379476) + | > loader_time: 0.0065 (0.005538864930470786) + + + --> TIME: 2025-04-19 03:54:45 -- STEP: 85/142 -- GLOBAL_STEP: 421825 + | > loss_disc: 2.58333420753479 (2.4522393759559185) + | > loss_disc_real_0: 0.18369875848293304 (0.12105105668306351) + | > loss_disc_real_1: 0.24456797540187836 (0.20904491385992832) + | > loss_disc_real_2: 0.2285492718219757 (0.2255253974129172) + | > loss_disc_real_3: 0.24773164093494415 (0.2273061438518412) + | > loss_disc_real_4: 0.2211647778749466 (0.22876291608109195) + | > loss_disc_real_5: 0.23265643417835236 (0.23121061693219577) + | > loss_0: 2.58333420753479 (2.4522393759559185) + | > grad_norm_0: tensor(17.8278, device='cuda:0') (tensor(10.6563, device='cuda:0')) + | > loss_gen: 2.2637271881103516 (2.3713902024661797) + | > loss_kl: 1.9003214836120605 (1.8675847852931302) + | > loss_feat: 6.927952289581299 (7.208060595568488) + | > loss_mel: 18.5841121673584 (17.777212883444378) + | > loss_duration: 1.4776990413665771 (1.4521145960863902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.153812408447266 (30.676363193287568) + | > grad_norm_1: tensor(465.7012, device='cuda:0') (tensor(272.6861, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.4006 (0.36300089219037235) + | > loader_time: 0.0068 (0.005844940858728745) + + + --> TIME: 2025-04-19 03:54:55 -- STEP: 110/142 -- GLOBAL_STEP: 421850 + | > loss_disc: 2.5358223915100098 (2.455617345463146) + | > loss_disc_real_0: 0.1633024662733078 (0.12419678866863251) + | > loss_disc_real_1: 0.19320297241210938 (0.2089938620274717) + | > loss_disc_real_2: 0.24109125137329102 (0.22523840327154507) + | > loss_disc_real_3: 0.21871957182884216 (0.22738533439961348) + | > loss_disc_real_4: 0.2412162572145462 (0.2290141379291361) + | > loss_disc_real_5: 0.23367692530155182 (0.22946325391530992) + | > loss_0: 2.5358223915100098 (2.455617345463146) + | > grad_norm_0: tensor(45.8091, device='cuda:0') (tensor(12.4181, device='cuda:0')) + | > loss_gen: 2.2212207317352295 (2.3738819989291104) + | > loss_kl: 1.90635085105896 (1.8788665348833258) + | > loss_feat: 6.139582633972168 (7.14474136612632) + | > loss_mel: 15.243402481079102 (17.706772752241644) + | > loss_duration: 1.473314642906189 (1.460333108901978) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.983871459960938 (30.564595898714934) + | > grad_norm_1: tensor(311.3178, device='cuda:0') (tensor(286.4994, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.4305 (0.37456787282770343) + | > loader_time: 0.0086 (0.006128547408364036) + + + --> TIME: 2025-04-19 03:55:07 -- STEP: 135/142 -- GLOBAL_STEP: 421875 + | > loss_disc: 2.435807228088379 (2.4609196821848554) + | > loss_disc_real_0: 0.13853192329406738 (0.12590198423023577) + | > loss_disc_real_1: 0.20676958560943604 (0.20879285357616562) + | > loss_disc_real_2: 0.24377909302711487 (0.22523867068467318) + | > loss_disc_real_3: 0.2327038198709488 (0.22772642076015473) + | > loss_disc_real_4: 0.21851463615894318 (0.2292953316812162) + | > loss_disc_real_5: 0.22534844279289246 (0.23009315265549554) + | > loss_0: 2.435807228088379 (2.4609196821848554) + | > grad_norm_0: tensor(21.3862, device='cuda:0') (tensor(13.5805, device='cuda:0')) + | > loss_gen: 2.2894411087036133 (2.3723987491042524) + | > loss_kl: 1.8463038206100464 (1.8807804708127622) + | > loss_feat: 7.369086265563965 (7.111447641584609) + | > loss_mel: 18.71335792541504 (17.666656903867363) + | > loss_duration: 1.4661579132080078 (1.466241347348249) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.68434715270996 (30.49752522221318) + | > grad_norm_1: tensor(307.4731, device='cuda:0') (tensor(295.0148, device='cuda:0')) + | > current_lr_0: 0.00019266251482403263 + | > current_lr_1: 0.00019266251482403263 + | > step_time: 0.4557 (0.3875435652556244) + | > loader_time: 0.0083 (0.0063895702362060545) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5420918464660645 (2.5420918464660645) + | > loss_disc_real_0: 0.057184506207704544 (0.057184506207704544) + | > loss_disc_real_1: 0.27026131749153137 (0.27026131749153137) + | > loss_disc_real_2: 0.32672494649887085 (0.32672494649887085) + | > loss_disc_real_3: 0.2999458909034729 (0.2999458909034729) + | > loss_disc_real_4: 0.2735585868358612 (0.2735585868358612) + | > loss_disc_real_5: 0.2513769268989563 (0.2513769268989563) + | > loss_0: 2.5420918464660645 (2.5420918464660645) + | > loss_gen: 2.370258331298828 (2.370258331298828) + | > loss_kl: 2.495220899581909 (2.495220899581909) + | > loss_feat: 6.769275665283203 (6.769275665283203) + | > loss_mel: 19.2713680267334 (19.2713680267334) + | > loss_duration: 1.7281919717788696 (1.7281919717788696) + | > loss_1: 32.634315490722656 (32.634315490722656) + + --> STEP: 1 + | > loss_disc: 2.5352413654327393 (2.5352413654327393) + | > loss_disc_real_0: 0.07650282233953476 (0.07650282233953476) + | > loss_disc_real_1: 0.278870552778244 (0.278870552778244) + | > loss_disc_real_2: 0.31255650520324707 (0.31255650520324707) + | > loss_disc_real_3: 0.2750435471534729 (0.2750435471534729) + | > loss_disc_real_4: 0.26811546087265015 (0.26811546087265015) + | > loss_disc_real_5: 0.22244793176651 (0.22244793176651) + | > loss_0: 2.5352413654327393 (2.5352413654327393) + | > loss_gen: 2.4385218620300293 (2.4385218620300293) + | > loss_kl: 2.844320058822632 (2.844320058822632) + | > loss_feat: 7.537047863006592 (7.537047863006592) + | > loss_mel: 18.682714462280273 (18.682714462280273) + | > loss_duration: 1.7626149654388428 (1.7626149654388428) + | > loss_1: 33.265220642089844 (33.265220642089844) + + --> STEP: 2 + | > loss_disc: 2.510838508605957 (2.523039937019348) + | > loss_disc_real_0: 0.07691967487335205 (0.0767112486064434) + | > loss_disc_real_1: 0.272207111120224 (0.275538831949234) + | > loss_disc_real_2: 0.30704784393310547 (0.30980217456817627) + | > loss_disc_real_3: 0.27280393242836 (0.27392373979091644) + | > loss_disc_real_4: 0.2787516117095947 (0.27343353629112244) + | > loss_disc_real_5: 0.24274565279483795 (0.23259679228067398) + | > loss_0: 2.510838508605957 (2.523039937019348) + | > loss_gen: 2.4053473472595215 (2.4219346046447754) + | > loss_kl: 2.331989288330078 (2.588154673576355) + | > loss_feat: 7.977451324462891 (7.757249593734741) + | > loss_mel: 18.536474227905273 (18.609594345092773) + | > loss_duration: 1.6912285089492798 (1.7269217371940613) + | > loss_1: 32.94248962402344 (33.10385513305664) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033527612686157227 (-0.00030493736267089844) + | > avg_loss_disc: 2.523039937019348 (+0.04826819896697998) + | > avg_loss_disc_real_0: 0.0767112486064434 (-0.12433010712265968) + | > avg_loss_disc_real_1: 0.275538831949234 (+0.06318299472332001) + | > avg_loss_disc_real_2: 0.30980217456817627 (+0.08019771426916122) + | > avg_loss_disc_real_3: 0.27392373979091644 (+0.05639006197452545) + | > avg_loss_disc_real_4: 0.27343353629112244 (+0.0472109317779541) + | > avg_loss_disc_real_5: 0.23259679228067398 (-0.030324675142765045) + | > avg_loss_0: 2.523039937019348 (+0.04826819896697998) + | > avg_loss_gen: 2.4219346046447754 (+0.08159887790679932) + | > avg_loss_kl: 2.588154673576355 (+0.1301344633102417) + | > avg_loss_feat: 7.757249593734741 (+1.5202460289001465) + | > avg_loss_mel: 18.609594345092773 (+1.2784128189086914) + | > avg_loss_duration: 1.7269217371940613 (+0.007413983345031738) + | > avg_loss_1: 33.10385513305664 (+3.017806053161621) + + + > EPOCH: 300/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:55:11)  + + --> TIME: 2025-04-19 03:55:18 -- STEP: 18/142 -- GLOBAL_STEP: 421900 + | > loss_disc: 2.521601676940918 (2.4575288693110147) + | > loss_disc_real_0: 0.12224207073450089 (0.11824547913339403) + | > loss_disc_real_1: 0.23129352927207947 (0.2097429426179992) + | > loss_disc_real_2: 0.25463780760765076 (0.23112201193968454) + | > loss_disc_real_3: 0.2314104586839676 (0.22541073544157875) + | > loss_disc_real_4: 0.20517314970493317 (0.2281663277082973) + | > loss_disc_real_5: 0.23220817744731903 (0.23203875952296787) + | > loss_0: 2.521601676940918 (2.4575288693110147) + | > grad_norm_0: tensor(6.2273, device='cuda:0') (tensor(8.4959, device='cuda:0')) + | > loss_gen: 2.3109827041625977 (2.3660549057854547) + | > loss_kl: 1.6375244855880737 (1.823311123583052) + | > loss_feat: 6.831884860992432 (7.02003092235989) + | > loss_mel: 17.02554702758789 (17.462916639116074) + | > loss_duration: 1.4161036014556885 (1.4194285935825772) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.222042083740234 (30.091742091708714) + | > grad_norm_1: tensor(135.4869, device='cuda:0') (tensor(190.3147, device='cuda:0')) + | > current_lr_0: 0.00019263843200967962 + | > current_lr_1: 0.00019263843200967962 + | > step_time: 0.3413 (0.3299710088306003) + | > loader_time: 0.0053 (0.004821499188741048) + + + --> TIME: 2025-04-19 03:55:27 -- STEP: 43/142 -- GLOBAL_STEP: 421925 + | > loss_disc: 2.5495898723602295 (2.4440167116564377) + | > loss_disc_real_0: 0.14971856772899628 (0.11936153124931247) + | > loss_disc_real_1: 0.20547646284103394 (0.2071349783692249) + | > loss_disc_real_2: 0.24462930858135223 (0.22474147795244706) + | > loss_disc_real_3: 0.2122829556465149 (0.2254428964021594) + | > loss_disc_real_4: 0.23906740546226501 (0.2282372238331063) + | > loss_disc_real_5: 0.22273018956184387 (0.2316618648379348) + | > loss_0: 2.5495898723602295 (2.4440167116564377) + | > grad_norm_0: tensor(4.3303, device='cuda:0') (tensor(7.5385, device='cuda:0')) + | > loss_gen: 2.2941179275512695 (2.3599784596021793) + | > loss_kl: 1.8873255252838135 (1.845086047815722) + | > loss_feat: 5.943573474884033 (7.098286362581475) + | > loss_mel: 16.346769332885742 (17.532584988793662) + | > loss_duration: 1.450493574142456 (1.4327440345010092) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 27.922279357910156 (30.26867972972781) + | > grad_norm_1: tensor(63.2521, device='cuda:0') (tensor(143.5410, device='cuda:0')) + | > current_lr_0: 0.00019263843200967962 + | > current_lr_1: 0.00019263843200967962 + | > step_time: 0.3622 (0.34284026123756584) + | > loader_time: 0.0056 (0.005304153575453648) + + + --> TIME: 2025-04-19 03:55:36 -- STEP: 68/142 -- GLOBAL_STEP: 421950 + | > loss_disc: 2.5717766284942627 (2.449324372936698) + | > loss_disc_real_0: 0.15771424770355225 (0.12247048022554201) + | > loss_disc_real_1: 0.21138839423656464 (0.20860380241099527) + | > loss_disc_real_2: 0.23984095454216003 (0.22689216627794154) + | > loss_disc_real_3: 0.24269813299179077 (0.22741271160981236) + | > loss_disc_real_4: 0.23546727001667023 (0.2258847101646311) + | > loss_disc_real_5: 0.2269008457660675 (0.2307405024766922) + | > loss_0: 2.5717766284942627 (2.449324372936698) + | > grad_norm_0: tensor(11.5910, device='cuda:0') (tensor(6.8683, device='cuda:0')) + | > loss_gen: 2.4243485927581787 (2.3673339626368364) + | > loss_kl: 1.9857267141342163 (1.8595267341417425) + | > loss_feat: 7.125714302062988 (7.12211098390467) + | > loss_mel: 16.63783836364746 (17.58842883390539) + | > loss_duration: 1.4600939750671387 (1.4439748402904062) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.63372230529785 (30.38137534085442) + | > grad_norm_1: tensor(47.1410, device='cuda:0') (tensor(122.0282, device='cuda:0')) + | > current_lr_0: 0.00019263843200967962 + | > current_lr_1: 0.00019263843200967962 + | > step_time: 0.3874 (0.3546244607252233) + | > loader_time: 0.0062 (0.005565082325654872) + + + --> TIME: 2025-04-19 03:55:47 -- STEP: 93/142 -- GLOBAL_STEP: 421975 + | > loss_disc: 2.494398832321167 (2.457181917723789) + | > loss_disc_real_0: 0.09665793180465698 (0.12299512527001802) + | > loss_disc_real_1: 0.2576935589313507 (0.20830408219368227) + | > loss_disc_real_2: 0.22705237567424774 (0.22761551410921158) + | > loss_disc_real_3: 0.2190084308385849 (0.22754095478724407) + | > loss_disc_real_4: 0.2494475096464157 (0.22842439240024937) + | > loss_disc_real_5: 0.2205614447593689 (0.23246531508943086) + | > loss_0: 2.494398832321167 (2.457181917723789) + | > grad_norm_0: tensor(7.0014, device='cuda:0') (tensor(6.8061, device='cuda:0')) + | > loss_gen: 2.2833776473999023 (2.3644383671463185) + | > loss_kl: 1.9773147106170654 (1.8787177762677592) + | > loss_feat: 6.877971649169922 (7.1099636836718485) + | > loss_mel: 18.732189178466797 (17.65508495864048) + | > loss_duration: 1.4584753513336182 (1.4556018216635598) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.329328536987305 (30.4638066035445) + | > grad_norm_1: tensor(159.2994, device='cuda:0') (tensor(122.5232, device='cuda:0')) + | > current_lr_0: 0.00019263843200967962 + | > current_lr_1: 0.00019263843200967962 + | > step_time: 0.3988 (0.3659676915855818) + | > loader_time: 0.0077 (0.005854414355370304) + + + --> TIME: 2025-04-19 03:55:57 -- STEP: 118/142 -- GLOBAL_STEP: 422000 + | > loss_disc: 2.5944502353668213 (2.4625474840907726) + | > loss_disc_real_0: 0.18435774743556976 (0.12556975530618328) + | > loss_disc_real_1: 0.20038926601409912 (0.20797138345443597) + | > loss_disc_real_2: 0.26885882019996643 (0.22664529653423923) + | > loss_disc_real_3: 0.24506981670856476 (0.2281972539879508) + | > loss_disc_real_4: 0.3079191744327545 (0.2286214231181953) + | > loss_disc_real_5: 0.2727755606174469 (0.23190233768042873) + | > loss_0: 2.5944502353668213 (2.4625474840907726) + | > grad_norm_0: tensor(10.5387, device='cuda:0') (tensor(6.7787, device='cuda:0')) + | > loss_gen: 2.39471697807312 (2.3577288409410913) + | > loss_kl: 1.9312993288040161 (1.8855696310431271) + | > loss_feat: 7.241219520568848 (7.070224139649989) + | > loss_mel: 18.50714111328125 (17.638744378494003) + | > loss_duration: 1.4584360122680664 (1.4624943773625265) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.532814025878906 (30.414761413962154) + | > grad_norm_1: tensor(256.1452, device='cuda:0') (tensor(123.6874, device='cuda:0')) + | > current_lr_0: 0.00019263843200967962 + | > current_lr_1: 0.00019263843200967962 + | > step_time: 0.4421 (0.37684470920239466) + | > loader_time: 0.0074 (0.006126246209871969) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6345174312591553 (2.6345174312591553) + | > loss_disc_real_0: 0.2426597625017166 (0.2426597625017166) + | > loss_disc_real_1: 0.27590039372444153 (0.27590039372444153) + | > loss_disc_real_2: 0.1919725239276886 (0.1919725239276886) + | > loss_disc_real_3: 0.22381915152072906 (0.22381915152072906) + | > loss_disc_real_4: 0.2557336688041687 (0.2557336688041687) + | > loss_disc_real_5: 0.2591075003147125 (0.2591075003147125) + | > loss_0: 2.6345174312591553 (2.6345174312591553) + | > loss_gen: 2.3083114624023438 (2.3083114624023438) + | > loss_kl: 2.4552152156829834 (2.4552152156829834) + | > loss_feat: 5.435308456420898 (5.435308456420898) + | > loss_mel: 15.29522705078125 (15.29522705078125) + | > loss_duration: 1.7165125608444214 (1.7165125608444214) + | > loss_1: 27.210575103759766 (27.210575103759766) + + --> STEP: 1 + | > loss_disc: 2.6474552154541016 (2.6474552154541016) + | > loss_disc_real_0: 0.28796306252479553 (0.28796306252479553) + | > loss_disc_real_1: 0.3171093165874481 (0.3171093165874481) + | > loss_disc_real_2: 0.20349733531475067 (0.20349733531475067) + | > loss_disc_real_3: 0.23546656966209412 (0.23546656966209412) + | > loss_disc_real_4: 0.2800580561161041 (0.2800580561161041) + | > loss_disc_real_5: 0.23790010809898376 (0.23790010809898376) + | > loss_0: 2.6474552154541016 (2.6474552154541016) + | > loss_gen: 2.6016275882720947 (2.6016275882720947) + | > loss_kl: 2.7824501991271973 (2.7824501991271973) + | > loss_feat: 6.757927894592285 (6.757927894592285) + | > loss_mel: 17.368934631347656 (17.368934631347656) + | > loss_duration: 1.7793238162994385 (1.7793238162994385) + | > loss_1: 31.290264129638672 (31.290264129638672) + + --> STEP: 2 + | > loss_disc: 2.623258113861084 (2.6353566646575928) + | > loss_disc_real_0: 0.18064969778060913 (0.23430638015270233) + | > loss_disc_real_1: 0.2279023975133896 (0.27250585705041885) + | > loss_disc_real_2: 0.20942369103431702 (0.20646051317453384) + | > loss_disc_real_3: 0.2290181964635849 (0.2322423830628395) + | > loss_disc_real_4: 0.2505652904510498 (0.26531167328357697) + | > loss_disc_real_5: 0.26432275772094727 (0.2511114329099655) + | > loss_0: 2.623258113861084 (2.6353566646575928) + | > loss_gen: 2.3763415813446045 (2.4889845848083496) + | > loss_kl: 2.405474901199341 (2.593962550163269) + | > loss_feat: 7.269284248352051 (7.013606071472168) + | > loss_mel: 16.136207580566406 (16.75257110595703) + | > loss_duration: 1.7011957168579102 (1.7402597665786743) + | > loss_1: 29.888504028320312 (30.589384078979492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0030852556228637695 (-0.0002675056457519531) + | > avg_loss_disc: 2.6353566646575928 (+0.11231672763824463) + | > avg_loss_disc_real_0: 0.23430638015270233 (+0.15759513154625893) + | > avg_loss_disc_real_1: 0.27250585705041885 (-0.003032974898815155) + | > avg_loss_disc_real_2: 0.20646051317453384 (-0.10334166139364243) + | > avg_loss_disc_real_3: 0.2322423830628395 (-0.041681356728076935) + | > avg_loss_disc_real_4: 0.26531167328357697 (-0.008121863007545471) + | > avg_loss_disc_real_5: 0.2511114329099655 (+0.018514640629291534) + | > avg_loss_0: 2.6353566646575928 (+0.11231672763824463) + | > avg_loss_gen: 2.4889845848083496 (+0.06704998016357422) + | > avg_loss_kl: 2.593962550163269 (+0.0058078765869140625) + | > avg_loss_feat: 7.013606071472168 (-0.7436435222625732) + | > avg_loss_mel: 16.75257110595703 (-1.8570232391357422) + | > avg_loss_duration: 1.7402597665786743 (+0.013338029384613037) + | > avg_loss_1: 30.589384078979492 (-2.5144710540771484) + + + > EPOCH: 301/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:56:10)  + + --> TIME: 2025-04-19 03:56:11 -- STEP: 1/142 -- GLOBAL_STEP: 422025 + | > loss_disc: 2.413569927215576 (2.413569927215576) + | > loss_disc_real_0: 0.1493539810180664 (0.1493539810180664) + | > loss_disc_real_1: 0.23847171664237976 (0.23847171664237976) + | > loss_disc_real_2: 0.22633329033851624 (0.22633329033851624) + | > loss_disc_real_3: 0.2456498146057129 (0.2456498146057129) + | > loss_disc_real_4: 0.2572651505470276 (0.2572651505470276) + | > loss_disc_real_5: 0.2549997866153717 (0.2549997866153717) + | > loss_0: 2.413569927215576 (2.413569927215576) + | > grad_norm_0: tensor(5.0633, device='cuda:0') (tensor(5.0633, device='cuda:0')) + | > loss_gen: 2.4337430000305176 (2.4337430000305176) + | > loss_kl: 1.7813968658447266 (1.7813968658447266) + | > loss_feat: 7.438906669616699 (7.438906669616699) + | > loss_mel: 18.176292419433594 (18.176292419433594) + | > loss_duration: 1.3975415229797363 (1.3975415229797363) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.22787857055664 (31.22787857055664) + | > grad_norm_1: tensor(432.4247, device='cuda:0') (tensor(432.4247, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.3357 (0.3356907367706299) + | > loader_time: 0.004 (0.003975629806518555) + + + --> TIME: 2025-04-19 03:56:20 -- STEP: 26/142 -- GLOBAL_STEP: 422050 + | > loss_disc: 2.422029495239258 (2.464675417313209) + | > loss_disc_real_0: 0.1444372534751892 (0.11631781235337257) + | > loss_disc_real_1: 0.24611875414848328 (0.20523135593304268) + | > loss_disc_real_2: 0.19812551140785217 (0.22819950030400202) + | > loss_disc_real_3: 0.21035601198673248 (0.22864376982817283) + | > loss_disc_real_4: 0.19828347861766815 (0.23885115465292564) + | > loss_disc_real_5: 0.21817715466022491 (0.23394771321461752) + | > loss_0: 2.422029495239258 (2.464675417313209) + | > grad_norm_0: tensor(11.1219, device='cuda:0') (tensor(12.1416, device='cuda:0')) + | > loss_gen: 2.3619532585144043 (2.3765043799693766) + | > loss_kl: 1.7832293510437012 (1.7999601134887109) + | > loss_feat: 7.451643466949463 (7.262143887006319) + | > loss_mel: 17.358840942382812 (17.8228471829341) + | > loss_duration: 1.442681074142456 (1.4203351140022278) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.398347854614258 (30.68179079202505) + | > grad_norm_1: tensor(308.8136, device='cuda:0') (tensor(242.7255, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.3527 (0.3417984889103816) + | > loader_time: 0.0051 (0.0051681720293485205) + + + --> TIME: 2025-04-19 03:56:29 -- STEP: 51/142 -- GLOBAL_STEP: 422075 + | > loss_disc: 2.5074970722198486 (2.4372859749139515) + | > loss_disc_real_0: 0.08974170684814453 (0.11644013927263372) + | > loss_disc_real_1: 0.18664757907390594 (0.20293253543330173) + | > loss_disc_real_2: 0.23397350311279297 (0.2249797004110673) + | > loss_disc_real_3: 0.22749261558055878 (0.22766529637224533) + | > loss_disc_real_4: 0.22499904036521912 (0.23092305221978357) + | > loss_disc_real_5: 0.2349523901939392 (0.2336576724169301) + | > loss_0: 2.5074970722198486 (2.4372859749139515) + | > grad_norm_0: tensor(8.5194, device='cuda:0') (tensor(10.0665, device='cuda:0')) + | > loss_gen: 2.266000747680664 (2.3815955297619684) + | > loss_kl: 1.8584784269332886 (1.820317969602697) + | > loss_feat: 7.892850399017334 (7.221521845050886) + | > loss_mel: 17.148813247680664 (17.661153288448553) + | > loss_duration: 1.4392187595367432 (1.4304061613830865) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.605361938476562 (30.514994826971314) + | > grad_norm_1: tensor(293.4279, device='cuda:0') (tensor(240.8146, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.3699 (0.35170066590402643) + | > loader_time: 0.0062 (0.005460229574465284) + + + --> TIME: 2025-04-19 03:56:39 -- STEP: 76/142 -- GLOBAL_STEP: 422100 + | > loss_disc: 2.5068867206573486 (2.4390802195197656) + | > loss_disc_real_0: 0.14810927212238312 (0.11972245338716005) + | > loss_disc_real_1: 0.2145867496728897 (0.20541906141136823) + | > loss_disc_real_2: 0.2762259244918823 (0.22611506557778308) + | > loss_disc_real_3: 0.2137058824300766 (0.22671053872296684) + | > loss_disc_real_4: 0.21108393371105194 (0.22946769842191747) + | > loss_disc_real_5: 0.22157815098762512 (0.2305388083975566) + | > loss_0: 2.5068867206573486 (2.4390802195197656) + | > grad_norm_0: tensor(4.1646, device='cuda:0') (tensor(9.9538, device='cuda:0')) + | > loss_gen: 2.285404920578003 (2.387706563660973) + | > loss_kl: 1.9150207042694092 (1.838668898532265) + | > loss_feat: 7.039364814758301 (7.258552043061507) + | > loss_mel: 17.759675979614258 (17.778625312604394) + | > loss_duration: 1.4906672239303589 (1.4448972830646916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.49013328552246 (30.70845011660927) + | > grad_norm_1: tensor(69.2232, device='cuda:0') (tensor(242.5925, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.4107 (0.3620547934582359) + | > loader_time: 0.0067 (0.005735359693828382) + + + --> TIME: 2025-04-19 03:56:49 -- STEP: 101/142 -- GLOBAL_STEP: 422125 + | > loss_disc: 2.460510015487671 (2.4488390884777105) + | > loss_disc_real_0: 0.12114071100950241 (0.12151557369397419) + | > loss_disc_real_1: 0.1959751844406128 (0.20543557021877554) + | > loss_disc_real_2: 0.2141897827386856 (0.22687137628545856) + | > loss_disc_real_3: 0.22409667074680328 (0.22882295244991188) + | > loss_disc_real_4: 0.22739745676517487 (0.22973894895893512) + | > loss_disc_real_5: 0.2137451320886612 (0.23135730742227914) + | > loss_0: 2.460510015487671 (2.4488390884777105) + | > grad_norm_0: tensor(8.1828, device='cuda:0') (tensor(10.0897, device='cuda:0')) + | > loss_gen: 2.240049123764038 (2.3731137419691177) + | > loss_kl: 2.003314256668091 (1.8558093722503963) + | > loss_feat: 6.122514724731445 (7.17303470101687) + | > loss_mel: 16.564931869506836 (17.739970386618424) + | > loss_duration: 1.5134109258651733 (1.455874722782928) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.44422149658203 (30.597802983652247) + | > grad_norm_1: tensor(59.7444, device='cuda:0') (tensor(235.9435, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.4143 (0.37222861299420357) + | > loader_time: 0.0074 (0.0060065548018653795) + + + --> TIME: 2025-04-19 03:57:00 -- STEP: 126/142 -- GLOBAL_STEP: 422150 + | > loss_disc: 2.457484722137451 (2.4508022599750094) + | > loss_disc_real_0: 0.11897414922714233 (0.12261267266576252) + | > loss_disc_real_1: 0.19083279371261597 (0.2054748269064086) + | > loss_disc_real_2: 0.2155761569738388 (0.22650545705405492) + | > loss_disc_real_3: 0.21678902208805084 (0.22804806665295646) + | > loss_disc_real_4: 0.19828732311725616 (0.23005013014115985) + | > loss_disc_real_5: 0.22019830346107483 (0.23068440338921925) + | > loss_0: 2.457484722137451 (2.4508022599750094) + | > grad_norm_0: tensor(10.4302, device='cuda:0') (tensor(9.7454, device='cuda:0')) + | > loss_gen: 2.303377866744995 (2.362363978983864) + | > loss_kl: 1.9614907503128052 (1.8697390092743769) + | > loss_feat: 6.799330234527588 (7.12034963804578) + | > loss_mel: 16.737579345703125 (17.6888521436661) + | > loss_duration: 1.4970388412475586 (1.4636967929582747) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.298816680908203 (30.505001613071986) + | > grad_norm_1: tensor(281.6885, device='cuda:0') (tensor(227.9065, device='cuda:0')) + | > current_lr_0: 0.0001926143522056784 + | > current_lr_1: 0.0001926143522056784 + | > step_time: 0.4382 (0.3833336678762285) + | > loader_time: 0.0072 (0.006288223796420627) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4389429092407227 (2.4389429092407227) + | > loss_disc_real_0: 0.1499791443347931 (0.1499791443347931) + | > loss_disc_real_1: 0.2656535506248474 (0.2656535506248474) + | > loss_disc_real_2: 0.26571276783943176 (0.26571276783943176) + | > loss_disc_real_3: 0.2214908003807068 (0.2214908003807068) + | > loss_disc_real_4: 0.22657306492328644 (0.22657306492328644) + | > loss_disc_real_5: 0.22010214626789093 (0.22010214626789093) + | > loss_0: 2.4389429092407227 (2.4389429092407227) + | > loss_gen: 2.4015605449676514 (2.4015605449676514) + | > loss_kl: 2.5223240852355957 (2.5223240852355957) + | > loss_feat: 5.918946266174316 (5.918946266174316) + | > loss_mel: 14.765473365783691 (14.765473365783691) + | > loss_duration: 1.7145307064056396 (1.7145307064056396) + | > loss_1: 27.32283592224121 (27.32283592224121) + + --> STEP: 1 + | > loss_disc: 2.4884724617004395 (2.4884724617004395) + | > loss_disc_real_0: 0.1593487709760666 (0.1593487709760666) + | > loss_disc_real_1: 0.25098904967308044 (0.25098904967308044) + | > loss_disc_real_2: 0.2586784362792969 (0.2586784362792969) + | > loss_disc_real_3: 0.23948746919631958 (0.23948746919631958) + | > loss_disc_real_4: 0.23936890065670013 (0.23936890065670013) + | > loss_disc_real_5: 0.2011115550994873 (0.2011115550994873) + | > loss_0: 2.4884724617004395 (2.4884724617004395) + | > loss_gen: 2.492316722869873 (2.492316722869873) + | > loss_kl: 2.6523594856262207 (2.6523594856262207) + | > loss_feat: 7.474883556365967 (7.474883556365967) + | > loss_mel: 18.526641845703125 (18.526641845703125) + | > loss_duration: 1.7551343441009521 (1.7551343441009521) + | > loss_1: 32.901336669921875 (32.901336669921875) + + --> STEP: 2 + | > loss_disc: 2.2748847007751465 (2.381678581237793) + | > loss_disc_real_0: 0.1144377738237381 (0.13689327239990234) + | > loss_disc_real_1: 0.2442101538181305 (0.24759960174560547) + | > loss_disc_real_2: 0.2270386964082718 (0.24285856634378433) + | > loss_disc_real_3: 0.21729972958564758 (0.22839359939098358) + | > loss_disc_real_4: 0.24179036915302277 (0.24057963490486145) + | > loss_disc_real_5: 0.1988692432641983 (0.1999903991818428) + | > loss_0: 2.2748847007751465 (2.381678581237793) + | > loss_gen: 2.550027370452881 (2.521172046661377) + | > loss_kl: 2.298780918121338 (2.4755702018737793) + | > loss_feat: 7.181892395019531 (7.328387975692749) + | > loss_mel: 19.422426223754883 (18.974534034729004) + | > loss_duration: 1.7213057279586792 (1.7382200360298157) + | > loss_1: 33.17443084716797 (33.03788375854492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035979747772216797 (+0.0005127191543579102) + | > avg_loss_disc: 2.381678581237793 (-0.2536780834197998) + | > avg_loss_disc_real_0: 0.13689327239990234 (-0.09741310775279999) + | > avg_loss_disc_real_1: 0.24759960174560547 (-0.024906255304813385) + | > avg_loss_disc_real_2: 0.24285856634378433 (+0.03639805316925049) + | > avg_loss_disc_real_3: 0.22839359939098358 (-0.0038487836718559265) + | > avg_loss_disc_real_4: 0.24057963490486145 (-0.024732038378715515) + | > avg_loss_disc_real_5: 0.1999903991818428 (-0.05112103372812271) + | > avg_loss_0: 2.381678581237793 (-0.2536780834197998) + | > avg_loss_gen: 2.521172046661377 (+0.032187461853027344) + | > avg_loss_kl: 2.4755702018737793 (-0.11839234828948975) + | > avg_loss_feat: 7.328387975692749 (+0.31478190422058105) + | > avg_loss_mel: 18.974534034729004 (+2.2219629287719727) + | > avg_loss_duration: 1.7382200360298157 (-0.0020397305488586426) + | > avg_loss_1: 33.03788375854492 (+2.4484996795654297) + + + > EPOCH: 302/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:57:09)  + + --> TIME: 2025-04-19 03:57:12 -- STEP: 9/142 -- GLOBAL_STEP: 422175 + | > loss_disc: 2.5312929153442383 (2.419445594151815) + | > loss_disc_real_0: 0.10324490070343018 (0.11280474273694886) + | > loss_disc_real_1: 0.1506582349538803 (0.19763846033149296) + | > loss_disc_real_2: 0.23132550716400146 (0.22475857039292654) + | > loss_disc_real_3: 0.25880515575408936 (0.22907475464873844) + | > loss_disc_real_4: 0.2528042495250702 (0.22695583601792654) + | > loss_disc_real_5: 0.25257742404937744 (0.22584194938341776) + | > loss_0: 2.5312929153442383 (2.419445594151815) + | > grad_norm_0: tensor(12.6938, device='cuda:0') (tensor(8.5362, device='cuda:0')) + | > loss_gen: 2.3969411849975586 (2.4159237808651395) + | > loss_kl: 2.0295135974884033 (1.8887097835540771) + | > loss_feat: 7.3104119300842285 (7.411375098758274) + | > loss_mel: 17.97304916381836 (17.760867860582138) + | > loss_duration: 1.3987970352172852 (1.416773213280572) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.10871124267578 (30.893649419148762) + | > grad_norm_1: tensor(215.4638, device='cuda:0') (tensor(207.4008, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.3258 (0.3267093234592014) + | > loader_time: 0.0046 (0.004347483317057292) + + + --> TIME: 2025-04-19 03:57:21 -- STEP: 34/142 -- GLOBAL_STEP: 422200 + | > loss_disc: 2.623546600341797 (2.452899862738217) + | > loss_disc_real_0: 0.19082188606262207 (0.12096245093818973) + | > loss_disc_real_1: 0.19275052845478058 (0.20549941545023637) + | > loss_disc_real_2: 0.26181358098983765 (0.22639932614915512) + | > loss_disc_real_3: 0.24113526940345764 (0.23008086444700465) + | > loss_disc_real_4: 0.2593691647052765 (0.22829456539715037) + | > loss_disc_real_5: 0.2551718056201935 (0.23242733233115254) + | > loss_0: 2.623546600341797 (2.452899862738217) + | > grad_norm_0: tensor(7.9493, device='cuda:0') (tensor(8.6986, device='cuda:0')) + | > loss_gen: 2.1829051971435547 (2.3707782801459816) + | > loss_kl: 1.833665132522583 (1.8706118674839245) + | > loss_feat: 6.095276355743408 (7.256743276820464) + | > loss_mel: 16.322311401367188 (17.658225732691147) + | > loss_duration: 1.4520039558410645 (1.4272844581043018) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.88616180419922 (30.583643744973575) + | > grad_norm_1: tensor(59.4524, device='cuda:0') (tensor(213.5074, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.3562 (0.33817027596866384) + | > loader_time: 0.0059 (0.005173935609705308) + + + --> TIME: 2025-04-19 03:57:30 -- STEP: 59/142 -- GLOBAL_STEP: 422225 + | > loss_disc: 2.4434385299682617 (2.451912176811089) + | > loss_disc_real_0: 0.14477236568927765 (0.11868049406399162) + | > loss_disc_real_1: 0.23843704164028168 (0.20642669322127002) + | > loss_disc_real_2: 0.23148506879806519 (0.2262360360157692) + | > loss_disc_real_3: 0.2480122298002243 (0.22823986284813638) + | > loss_disc_real_4: 0.24139463901519775 (0.22885229223865575) + | > loss_disc_real_5: 0.2580163776874542 (0.234417801943876) + | > loss_0: 2.4434385299682617 (2.451912176811089) + | > grad_norm_0: tensor(21.2510, device='cuda:0') (tensor(10.0955, device='cuda:0')) + | > loss_gen: 2.563727378845215 (2.374515832480737) + | > loss_kl: 1.8243173360824585 (1.8650142681800712) + | > loss_feat: 7.276362895965576 (7.1788533905805165) + | > loss_mel: 16.83683204650879 (17.67595777673236) + | > loss_duration: 1.4587066173553467 (1.4387333999245853) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.959945678710938 (30.533074750738628) + | > grad_norm_1: tensor(461.2472, device='cuda:0') (tensor(241.3458, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.3738 (0.3496269411959891) + | > loader_time: 0.0065 (0.005496809038065247) + + + --> TIME: 2025-04-19 03:57:40 -- STEP: 84/142 -- GLOBAL_STEP: 422250 + | > loss_disc: 2.5608339309692383 (2.4545767108599343) + | > loss_disc_real_0: 0.1353958249092102 (0.12042626330540293) + | > loss_disc_real_1: 0.1978968381881714 (0.20525538655264036) + | > loss_disc_real_2: 0.2506808340549469 (0.22587710441578002) + | > loss_disc_real_3: 0.26735082268714905 (0.22892420490582785) + | > loss_disc_real_4: 0.238957017660141 (0.22812864414992787) + | > loss_disc_real_5: 0.24848322570323944 (0.23499037928524472) + | > loss_0: 2.5608339309692383 (2.4545767108599343) + | > grad_norm_0: tensor(5.7041, device='cuda:0') (tensor(10.5228, device='cuda:0')) + | > loss_gen: 2.2281241416931152 (2.3671341822260894) + | > loss_kl: 1.799616813659668 (1.8817082359677268) + | > loss_feat: 6.406439781188965 (7.158061629249936) + | > loss_mel: 16.256757736206055 (17.66293313389733) + | > loss_duration: 1.505462408065796 (1.4504092761448448) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.196401596069336 (30.520246528443835) + | > grad_norm_1: tensor(118.0828, device='cuda:0') (tensor(252.5502, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.3938 (0.3616696851594107) + | > loader_time: 0.0064 (0.005806079932621546) + + + --> TIME: 2025-04-19 03:57:51 -- STEP: 109/142 -- GLOBAL_STEP: 422275 + | > loss_disc: 2.515123128890991 (2.4545223362948905) + | > loss_disc_real_0: 0.11501288414001465 (0.12054206123319242) + | > loss_disc_real_1: 0.18109510838985443 (0.20492825472573623) + | > loss_disc_real_2: 0.23798193037509918 (0.22618010448753287) + | > loss_disc_real_3: 0.2138044387102127 (0.22824187986894484) + | > loss_disc_real_4: 0.23406203091144562 (0.2279798357038323) + | > loss_disc_real_5: 0.23499900102615356 (0.23420005954733683) + | > loss_0: 2.515123128890991 (2.4545223362948905) + | > grad_norm_0: tensor(9.3736, device='cuda:0') (tensor(10.6064, device='cuda:0')) + | > loss_gen: 2.241487741470337 (2.3600880771601966) + | > loss_kl: 1.9091547727584839 (1.8883540630340576) + | > loss_feat: 6.4253034591674805 (7.105181260940132) + | > loss_mel: 17.022171020507812 (17.626448832520644) + | > loss_duration: 1.4894416332244873 (1.459269769694827) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.087556838989258 (30.439342043815405) + | > grad_norm_1: tensor(168.5827, device='cuda:0') (tensor(261.2809, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.4125 (0.3721103383860456) + | > loader_time: 0.0073 (0.006090595087873825) + + + --> TIME: 2025-04-19 03:58:02 -- STEP: 134/142 -- GLOBAL_STEP: 422300 + | > loss_disc: 2.4951558113098145 (2.4567908706949706) + | > loss_disc_real_0: 0.14528046548366547 (0.12261659546352144) + | > loss_disc_real_1: 0.2178817242383957 (0.2049543780844603) + | > loss_disc_real_2: 0.24522702395915985 (0.22581358832209858) + | > loss_disc_real_3: 0.2597329616546631 (0.2285382683152583) + | > loss_disc_real_4: 0.24194224178791046 (0.22798134931432668) + | > loss_disc_real_5: 0.22724846005439758 (0.2349132906589935) + | > loss_0: 2.4951558113098145 (2.4567908706949706) + | > grad_norm_0: tensor(6.2194, device='cuda:0') (tensor(10.2122, device='cuda:0')) + | > loss_gen: 2.3533122539520264 (2.361163032588674) + | > loss_kl: 2.0129170417785645 (1.8908138453070797) + | > loss_feat: 7.75701904296875 (7.103530282404885) + | > loss_mel: 19.218643188476562 (17.68621068925999) + | > loss_duration: 1.5101828575134277 (1.4658723989529396) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.852073669433594 (30.507590279650334) + | > grad_norm_1: tensor(155.8500, device='cuda:0') (tensor(243.1663, device='cuda:0')) + | > current_lr_0: 0.00019259027541165268 + | > current_lr_1: 0.00019259027541165268 + | > step_time: 0.4398 (0.3839825089298077) + | > loader_time: 0.0074 (0.006360714115313628) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.478675127029419 (2.478675127029419) + | > loss_disc_real_0: 0.12389226257801056 (0.12389226257801056) + | > loss_disc_real_1: 0.24444036185741425 (0.24444036185741425) + | > loss_disc_real_2: 0.24059811234474182 (0.24059811234474182) + | > loss_disc_real_3: 0.2535821199417114 (0.2535821199417114) + | > loss_disc_real_4: 0.22483183443546295 (0.22483183443546295) + | > loss_disc_real_5: 0.2248658984899521 (0.2248658984899521) + | > loss_0: 2.478675127029419 (2.478675127029419) + | > loss_gen: 2.2119905948638916 (2.2119905948638916) + | > loss_kl: 2.4093992710113525 (2.4093992710113525) + | > loss_feat: 7.08432674407959 (7.08432674407959) + | > loss_mel: 17.647193908691406 (17.647193908691406) + | > loss_duration: 1.717010259628296 (1.717010259628296) + | > loss_1: 31.06991958618164 (31.06991958618164) + + --> STEP: 1 + | > loss_disc: 2.452343702316284 (2.452343702316284) + | > loss_disc_real_0: 0.1473415344953537 (0.1473415344953537) + | > loss_disc_real_1: 0.22582006454467773 (0.22582006454467773) + | > loss_disc_real_2: 0.2488843947649002 (0.2488843947649002) + | > loss_disc_real_3: 0.2599077522754669 (0.2599077522754669) + | > loss_disc_real_4: 0.2618938982486725 (0.2618938982486725) + | > loss_disc_real_5: 0.22747540473937988 (0.22747540473937988) + | > loss_0: 2.452343702316284 (2.452343702316284) + | > loss_gen: 2.3481364250183105 (2.3481364250183105) + | > loss_kl: 2.7330963611602783 (2.7330963611602783) + | > loss_feat: 6.9299492835998535 (6.9299492835998535) + | > loss_mel: 18.87657356262207 (18.87657356262207) + | > loss_duration: 1.7680339813232422 (1.7680339813232422) + | > loss_1: 32.655792236328125 (32.655792236328125) + + --> STEP: 2 + | > loss_disc: 2.6173439025878906 (2.5348438024520874) + | > loss_disc_real_0: 0.0869893878698349 (0.1171654611825943) + | > loss_disc_real_1: 0.21408246457576752 (0.21995126456022263) + | > loss_disc_real_2: 0.23326775431632996 (0.24107607454061508) + | > loss_disc_real_3: 0.2476561814546585 (0.2537819668650627) + | > loss_disc_real_4: 0.2613794207572937 (0.2616366595029831) + | > loss_disc_real_5: 0.2145433872938156 (0.22100939601659775) + | > loss_0: 2.6173439025878906 (2.5348438024520874) + | > loss_gen: 1.9665387868881226 (2.1573376059532166) + | > loss_kl: 2.3679821491241455 (2.550539255142212) + | > loss_feat: 8.404406547546387 (7.66717791557312) + | > loss_mel: 18.54788589477539 (18.71222972869873) + | > loss_duration: 1.708135724067688 (1.738084852695465) + | > loss_1: 32.99494934082031 (32.82537078857422) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035494565963745117 (-4.851818084716797e-05) + | > avg_loss_disc: 2.5348438024520874 (+0.15316522121429443) + | > avg_loss_disc_real_0: 0.1171654611825943 (-0.019727811217308044) + | > avg_loss_disc_real_1: 0.21995126456022263 (-0.027648337185382843) + | > avg_loss_disc_real_2: 0.24107607454061508 (-0.0017824918031692505) + | > avg_loss_disc_real_3: 0.2537819668650627 (+0.025388367474079132) + | > avg_loss_disc_real_4: 0.2616366595029831 (+0.021057024598121643) + | > avg_loss_disc_real_5: 0.22100939601659775 (+0.021018996834754944) + | > avg_loss_0: 2.5348438024520874 (+0.15316522121429443) + | > avg_loss_gen: 2.1573376059532166 (-0.3638344407081604) + | > avg_loss_kl: 2.550539255142212 (+0.07496905326843262) + | > avg_loss_feat: 7.66717791557312 (+0.3387899398803711) + | > avg_loss_mel: 18.71222972869873 (-0.26230430603027344) + | > avg_loss_duration: 1.738084852695465 (-0.00013518333435058594) + | > avg_loss_1: 32.82537078857422 (-0.21251296997070312) + + + > EPOCH: 303/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:58:07)  + + --> TIME: 2025-04-19 03:58:14 -- STEP: 17/142 -- GLOBAL_STEP: 422325 + | > loss_disc: 2.3714919090270996 (2.392539276796229) + | > loss_disc_real_0: 0.11942765861749649 (0.11577387811506495) + | > loss_disc_real_1: 0.17920327186584473 (0.20660730495172389) + | > loss_disc_real_2: 0.213863804936409 (0.21952469646930695) + | > loss_disc_real_3: 0.19024379551410675 (0.22745400754844441) + | > loss_disc_real_4: 0.15761050581932068 (0.21802939474582672) + | > loss_disc_real_5: 0.2039477527141571 (0.2286302973242367) + | > loss_0: 2.3714919090270996 (2.392539276796229) + | > grad_norm_0: tensor(10.2352, device='cuda:0') (tensor(10.6310, device='cuda:0')) + | > loss_gen: 2.406289577484131 (2.419536688748528) + | > loss_kl: 1.7761120796203613 (1.799774723894456) + | > loss_feat: 6.974095821380615 (7.389130844789393) + | > loss_mel: 17.00191307067871 (17.91082213906681) + | > loss_duration: 1.4261865615844727 (1.4224781569312601) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.584598541259766 (30.94174306532916) + | > grad_norm_1: tensor(67.6564, device='cuda:0') (tensor(217.0312, device='cuda:0')) + | > current_lr_0: 0.00019256620162722623 + | > current_lr_1: 0.00019256620162722623 + | > step_time: 0.3361 (0.32966194433324475) + | > loader_time: 0.0051 (0.004922375959508559) + + + --> TIME: 2025-04-19 03:58:23 -- STEP: 42/142 -- GLOBAL_STEP: 422350 + | > loss_disc: 2.4690661430358887 (2.4301572186606273) + | > loss_disc_real_0: 0.0892430767416954 (0.11704361651624952) + | > loss_disc_real_1: 0.21094858646392822 (0.20678016614346278) + | > loss_disc_real_2: 0.20645086467266083 (0.22462430418956847) + | > loss_disc_real_3: 0.2350824922323227 (0.2289256173230353) + | > loss_disc_real_4: 0.20402081310749054 (0.22780422334160125) + | > loss_disc_real_5: 0.20244775712490082 (0.2330901590841157) + | > loss_0: 2.4690661430358887 (2.4301572186606273) + | > grad_norm_0: tensor(7.7476, device='cuda:0') (tensor(9.8926, device='cuda:0')) + | > loss_gen: 2.2888476848602295 (2.4010455154237293) + | > loss_kl: 1.9410622119903564 (1.8381752712385995) + | > loss_feat: 7.20993185043335 (7.32629302569798) + | > loss_mel: 18.92265510559082 (17.889395395914715) + | > loss_duration: 1.465987205505371 (1.432206148192996) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.82848358154297 (30.887115750994003) + | > grad_norm_1: tensor(73.9717, device='cuda:0') (tensor(231.5853, device='cuda:0')) + | > current_lr_0: 0.00019256620162722623 + | > current_lr_1: 0.00019256620162722623 + | > step_time: 0.3609 (0.34741688909984764) + | > loader_time: 0.0054 (0.00546578566233317) + + + --> TIME: 2025-04-19 03:58:32 -- STEP: 67/142 -- GLOBAL_STEP: 422375 + | > loss_disc: 2.558631420135498 (2.435058981625003) + | > loss_disc_real_0: 0.09193750470876694 (0.11934704644911324) + | > loss_disc_real_1: 0.22307702898979187 (0.2068565817021612) + | > loss_disc_real_2: 0.20853957533836365 (0.224896354016973) + | > loss_disc_real_3: 0.21653637290000916 (0.22591399256862812) + | > loss_disc_real_4: 0.2208922952413559 (0.22498033473740764) + | > loss_disc_real_5: 0.2273104339838028 (0.2306898609915776) + | > loss_0: 2.558631420135498 (2.435058981625003) + | > grad_norm_0: tensor(9.3962, device='cuda:0') (tensor(9.9998, device='cuda:0')) + | > loss_gen: 2.3193233013153076 (2.3846594860304657) + | > loss_kl: 1.8743897676467896 (1.8520687106829972) + | > loss_feat: 6.943922996520996 (7.287000407033892) + | > loss_mel: 17.077096939086914 (17.905873455218416) + | > loss_duration: 1.4033563137054443 (1.4429309368133545) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.618087768554688 (30.8725331434563) + | > grad_norm_1: tensor(182.0468, device='cuda:0') (tensor(217.8923, device='cuda:0')) + | > current_lr_0: 0.00019256620162722623 + | > current_lr_1: 0.00019256620162722623 + | > step_time: 0.3826 (0.35815280231077273) + | > loader_time: 0.0069 (0.005802054903400478) + + + --> TIME: 2025-04-19 03:58:42 -- STEP: 92/142 -- GLOBAL_STEP: 422400 + | > loss_disc: 2.564260721206665 (2.4469287499137558) + | > loss_disc_real_0: 0.12638358771800995 (0.12165146134793758) + | > loss_disc_real_1: 0.255904883146286 (0.20795783870246098) + | > loss_disc_real_2: 0.23671557009220123 (0.22527730432541473) + | > loss_disc_real_3: 0.23856088519096375 (0.22721634772808655) + | > loss_disc_real_4: 0.2206527143716812 (0.22669921378078667) + | > loss_disc_real_5: 0.20008890330791473 (0.23271213925403098) + | > loss_0: 2.564260721206665 (2.4469287499137558) + | > grad_norm_0: tensor(13.1503, device='cuda:0') (tensor(10.2629, device='cuda:0')) + | > loss_gen: 2.1897993087768555 (2.3795823973158137) + | > loss_kl: 1.914140224456787 (1.868415222219799) + | > loss_feat: 6.864749908447266 (7.248691408530526) + | > loss_mel: 17.481595993041992 (17.90253728369009) + | > loss_duration: 1.4373167753219604 (1.4546542154706044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.887603759765625 (30.85388073713883) + | > grad_norm_1: tensor(178.2378, device='cuda:0') (tensor(225.2142, device='cuda:0')) + | > current_lr_0: 0.00019256620162722623 + | > current_lr_1: 0.00019256620162722623 + | > step_time: 0.4108 (0.3682820589646049) + | > loader_time: 0.0069 (0.006041301333385964) + + + --> TIME: 2025-04-19 03:58:53 -- STEP: 117/142 -- GLOBAL_STEP: 422425 + | > loss_disc: 2.5719990730285645 (2.4507737669170413) + | > loss_disc_real_0: 0.08281438797712326 (0.12356889680919485) + | > loss_disc_real_1: 0.2189885526895523 (0.20727486373522344) + | > loss_disc_real_2: 0.2863839566707611 (0.22638966092187116) + | > loss_disc_real_3: 0.2582457363605499 (0.22716514091206413) + | > loss_disc_real_4: 0.2667006552219391 (0.22756749799108913) + | > loss_disc_real_5: 0.22525806725025177 (0.2311705596681334) + | > loss_0: 2.5719990730285645 (2.4507737669170413) + | > grad_norm_0: tensor(14.7007, device='cuda:0') (tensor(10.8123, device='cuda:0')) + | > loss_gen: 2.249852180480957 (2.37611589472518) + | > loss_kl: 1.8419517278671265 (1.8784882665699363) + | > loss_feat: 6.9878387451171875 (7.205306350675404) + | > loss_mel: 18.010927200317383 (17.881100361163806) + | > loss_duration: 1.4886643886566162 (1.4617133802837796) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.579233169555664 (30.802724512214336) + | > grad_norm_1: tensor(369.2040, device='cuda:0') (tensor(242.5705, device='cuda:0')) + | > current_lr_0: 0.00019256620162722623 + | > current_lr_1: 0.00019256620162722623 + | > step_time: 0.4356 (0.37867307662963867) + | > loader_time: 0.0074 (0.0062533545697856146) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5634522438049316 (2.5634522438049316) + | > loss_disc_real_0: 0.26513856649398804 (0.26513856649398804) + | > loss_disc_real_1: 0.2482675313949585 (0.2482675313949585) + | > loss_disc_real_2: 0.2984071671962738 (0.2984071671962738) + | > loss_disc_real_3: 0.24183325469493866 (0.24183325469493866) + | > loss_disc_real_4: 0.1630856990814209 (0.1630856990814209) + | > loss_disc_real_5: 0.23114986717700958 (0.23114986717700958) + | > loss_0: 2.5634522438049316 (2.5634522438049316) + | > loss_gen: 2.5220723152160645 (2.5220723152160645) + | > loss_kl: 2.5315778255462646 (2.5315778255462646) + | > loss_feat: 6.978639125823975 (6.978639125823975) + | > loss_mel: 17.521305084228516 (17.521305084228516) + | > loss_duration: 1.7343196868896484 (1.7343196868896484) + | > loss_1: 31.287912368774414 (31.287912368774414) + + --> STEP: 1 + | > loss_disc: 2.610041618347168 (2.610041618347168) + | > loss_disc_real_0: 0.22105439007282257 (0.22105439007282257) + | > loss_disc_real_1: 0.2281680852174759 (0.2281680852174759) + | > loss_disc_real_2: 0.2712915241718292 (0.2712915241718292) + | > loss_disc_real_3: 0.2436981052160263 (0.2436981052160263) + | > loss_disc_real_4: 0.17816904187202454 (0.17816904187202454) + | > loss_disc_real_5: 0.24587874114513397 (0.24587874114513397) + | > loss_0: 2.610041618347168 (2.610041618347168) + | > loss_gen: 2.2485039234161377 (2.2485039234161377) + | > loss_kl: 2.6299660205841064 (2.6299660205841064) + | > loss_feat: 6.0495805740356445 (6.0495805740356445) + | > loss_mel: 16.051464080810547 (16.051464080810547) + | > loss_duration: 1.773884892463684 (1.773884892463684) + | > loss_1: 28.753400802612305 (28.753400802612305) + + --> STEP: 2 + | > loss_disc: 2.5504956245422363 (2.580268621444702) + | > loss_disc_real_0: 0.25287747383117676 (0.23696593195199966) + | > loss_disc_real_1: 0.26342135667800903 (0.24579472094774246) + | > loss_disc_real_2: 0.2621803879737854 (0.2667359560728073) + | > loss_disc_real_3: 0.19321390986442566 (0.21845600754022598) + | > loss_disc_real_4: 0.18419884145259857 (0.18118394166231155) + | > loss_disc_real_5: 0.2592388093471527 (0.25255877524614334) + | > loss_0: 2.5504956245422363 (2.580268621444702) + | > loss_gen: 2.6126511096954346 (2.430577516555786) + | > loss_kl: 2.508533000946045 (2.5692495107650757) + | > loss_feat: 8.119193077087402 (7.084386825561523) + | > loss_mel: 18.503379821777344 (17.277421951293945) + | > loss_duration: 1.719919204711914 (1.746902048587799) + | > loss_1: 33.46367645263672 (31.10853862762451) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033758878707885742 (-0.0001735687255859375) + | > avg_loss_disc: 2.580268621444702 (+0.045424818992614746) + | > avg_loss_disc_real_0: 0.23696593195199966 (+0.11980047076940536) + | > avg_loss_disc_real_1: 0.24579472094774246 (+0.025843456387519836) + | > avg_loss_disc_real_2: 0.2667359560728073 (+0.02565988153219223) + | > avg_loss_disc_real_3: 0.21845600754022598 (-0.03532595932483673) + | > avg_loss_disc_real_4: 0.18118394166231155 (-0.08045271784067154) + | > avg_loss_disc_real_5: 0.25255877524614334 (+0.03154937922954559) + | > avg_loss_0: 2.580268621444702 (+0.045424818992614746) + | > avg_loss_gen: 2.430577516555786 (+0.2732399106025696) + | > avg_loss_kl: 2.5692495107650757 (+0.01871025562286377) + | > avg_loss_feat: 7.084386825561523 (-0.5827910900115967) + | > avg_loss_mel: 17.277421951293945 (-1.4348077774047852) + | > avg_loss_duration: 1.746902048587799 (+0.008817195892333984) + | > avg_loss_1: 31.10853862762451 (-1.716832160949707) + + + > EPOCH: 304/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 03:59:06)  + + --> TIME: 2025-04-19 03:59:07 -- STEP: 0/142 -- GLOBAL_STEP: 422450 + | > loss_disc: 2.4498419761657715 (2.4498419761657715) + | > loss_disc_real_0: 0.17975682020187378 (0.17975682020187378) + | > loss_disc_real_1: 0.23489363491535187 (0.23489363491535187) + | > loss_disc_real_2: 0.2865215539932251 (0.2865215539932251) + | > loss_disc_real_3: 0.25648927688598633 (0.25648927688598633) + | > loss_disc_real_4: 0.18417300283908844 (0.18417300283908844) + | > loss_disc_real_5: 0.2263813018798828 (0.2263813018798828) + | > loss_0: 2.4498419761657715 (2.4498419761657715) + | > grad_norm_0: tensor(16.4429, device='cuda:0') (tensor(16.4429, device='cuda:0')) + | > loss_gen: 2.7154605388641357 (2.7154605388641357) + | > loss_kl: 1.7944836616516113 (1.7944836616516113) + | > loss_feat: 6.434349536895752 (6.434349536895752) + | > loss_mel: 16.701374053955078 (16.701374053955078) + | > loss_duration: 1.438685417175293 (1.438685417175293) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.0843505859375 (29.0843505859375) + | > grad_norm_1: tensor(86.5233, device='cuda:0') (tensor(86.5233, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.3536 (0.35355567932128906) + | > loader_time: 0.314 (0.314037561416626) + + + --> TIME: 2025-04-19 03:59:15 -- STEP: 25/142 -- GLOBAL_STEP: 422475 + | > loss_disc: 2.4383902549743652 (2.437951545715332) + | > loss_disc_real_0: 0.0982167199254036 (0.1259663496911526) + | > loss_disc_real_1: 0.19383981823921204 (0.20347735583782195) + | > loss_disc_real_2: 0.21572314202785492 (0.23013293802738188) + | > loss_disc_real_3: 0.24510927498340607 (0.23128853142261505) + | > loss_disc_real_4: 0.24139845371246338 (0.226122687458992) + | > loss_disc_real_5: 0.179780051112175 (0.2267391002178192) + | > loss_0: 2.4383902549743652 (2.437951545715332) + | > grad_norm_0: tensor(14.4391, device='cuda:0') (tensor(19.1847, device='cuda:0')) + | > loss_gen: 2.307539701461792 (2.417853889465332) + | > loss_kl: 1.844415307044983 (1.8142076873779296) + | > loss_feat: 7.810742378234863 (7.505776195526123) + | > loss_mel: 17.894956588745117 (18.103687400817872) + | > loss_duration: 1.4685876369476318 (1.4241984176635742) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.32624053955078 (31.265723419189452) + | > grad_norm_1: tensor(398.0419, device='cuda:0') (tensor(345.5861, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.3436 (0.33368225097656257) + | > loader_time: 0.0054 (0.005003690719604492) + + + --> TIME: 2025-04-19 03:59:24 -- STEP: 50/142 -- GLOBAL_STEP: 422500 + | > loss_disc: 2.423621654510498 (2.425518956184387) + | > loss_disc_real_0: 0.14811395108699799 (0.1208617454022169) + | > loss_disc_real_1: 0.18703635036945343 (0.20113350450992584) + | > loss_disc_real_2: 0.22229643166065216 (0.22523146718740464) + | > loss_disc_real_3: 0.23407100141048431 (0.22962697088718415) + | > loss_disc_real_4: 0.24086478352546692 (0.2274041873216629) + | > loss_disc_real_5: 0.24882937967777252 (0.23074566155672074) + | > loss_0: 2.423621654510498 (2.425518956184387) + | > grad_norm_0: tensor(12.9598, device='cuda:0') (tensor(13.7032, device='cuda:0')) + | > loss_gen: 2.4081826210021973 (2.3952091312408434) + | > loss_kl: 1.970429539680481 (1.8112560749053954) + | > loss_feat: 7.05645227432251 (7.3127936840057375) + | > loss_mel: 18.139373779296875 (17.646397933959964) + | > loss_duration: 1.472830057144165 (1.4325195884704596) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.04726791381836 (30.598176460266114) + | > grad_norm_1: tensor(303.9687, device='cuda:0') (tensor(260.2280, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.3702 (0.34570881843566886) + | > loader_time: 0.0081 (0.005453042984008789) + + + --> TIME: 2025-04-19 03:59:34 -- STEP: 75/142 -- GLOBAL_STEP: 422525 + | > loss_disc: 2.4580042362213135 (2.431299161911011) + | > loss_disc_real_0: 0.1288629025220871 (0.1219720562795798) + | > loss_disc_real_1: 0.25113406777381897 (0.20321975529193878) + | > loss_disc_real_2: 0.2277543842792511 (0.225469822883606) + | > loss_disc_real_3: 0.20983478426933289 (0.2272867351770401) + | > loss_disc_real_4: 0.20009388029575348 (0.2255917453765869) + | > loss_disc_real_5: 0.22869010269641876 (0.231078822016716) + | > loss_0: 2.4580042362213135 (2.431299161911011) + | > grad_norm_0: tensor(7.2497, device='cuda:0') (tensor(11.7269, device='cuda:0')) + | > loss_gen: 2.4505913257598877 (2.379228350321451) + | > loss_kl: 1.9413944482803345 (1.8393786017100016) + | > loss_feat: 7.908038139343262 (7.2267690340677895) + | > loss_mel: 18.47276496887207 (17.574043986002604) + | > loss_duration: 1.4315204620361328 (1.4441244188944504) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.204307556152344 (30.46354446411133) + | > grad_norm_1: tensor(180.4633, device='cuda:0') (tensor(227.9924, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.4001 (0.35795464197794585) + | > loader_time: 0.008 (0.00581143061319987) + + + --> TIME: 2025-04-19 03:59:44 -- STEP: 100/142 -- GLOBAL_STEP: 422550 + | > loss_disc: 2.24699330329895 (2.4383827853202815) + | > loss_disc_real_0: 0.10403299331665039 (0.12288512129336596) + | > loss_disc_real_1: 0.18053770065307617 (0.20409027472138405) + | > loss_disc_real_2: 0.19735996425151825 (0.22563446015119556) + | > loss_disc_real_3: 0.20075947046279907 (0.2269604106247425) + | > loss_disc_real_4: 0.19804516434669495 (0.2256608274579048) + | > loss_disc_real_5: 0.19659993052482605 (0.23258092269301414) + | > loss_0: 2.24699330329895 (2.4383827853202815) + | > grad_norm_0: tensor(9.9267, device='cuda:0') (tensor(10.7275, device='cuda:0')) + | > loss_gen: 2.450209140777588 (2.374364013671874) + | > loss_kl: 1.800701379776001 (1.8516701197624206) + | > loss_feat: 7.439200401306152 (7.214929919242859) + | > loss_mel: 17.948259353637695 (17.65613876342774) + | > loss_duration: 1.4759249687194824 (1.455438657999039) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.114294052124023 (30.552541580200195) + | > grad_norm_1: tensor(219.4310, device='cuda:0') (tensor(219.0850, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.416 (0.3692873835563659) + | > loader_time: 0.0074 (0.006093907356262206) + + + --> TIME: 2025-04-19 03:59:56 -- STEP: 125/142 -- GLOBAL_STEP: 422575 + | > loss_disc: 2.6107215881347656 (2.443452777862548) + | > loss_disc_real_0: 0.11553259193897247 (0.12429764679074287) + | > loss_disc_real_1: 0.22794851660728455 (0.20446231079101562) + | > loss_disc_real_2: 0.24507494270801544 (0.2262382963895798) + | > loss_disc_real_3: 0.25858786702156067 (0.22649417781829834) + | > loss_disc_real_4: 0.27168846130371094 (0.22758950543403625) + | > loss_disc_real_5: 0.3005557656288147 (0.23158319854736328) + | > loss_0: 2.6107215881347656 (2.443452777862548) + | > grad_norm_0: tensor(15.9710, device='cuda:0') (tensor(10.5754, device='cuda:0')) + | > loss_gen: 2.3137524127960205 (2.366166017532348) + | > loss_kl: 1.749833106994629 (1.86025216960907) + | > loss_feat: 6.954983711242676 (7.136650421142578) + | > loss_mel: 18.149782180786133 (17.572530677795424) + | > loss_duration: 1.532017469406128 (1.4628686952590946) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.700368881225586 (30.39846809387207) + | > grad_norm_1: tensor(415.6069, device='cuda:0') (tensor(225.5143, device='cuda:0')) + | > current_lr_0: 0.00019254213085202282 + | > current_lr_1: 0.00019254213085202282 + | > step_time: 0.438 (0.38302242088317867) + | > loader_time: 0.0076 (0.006343589782714843) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6417455673217773 (2.6417455673217773) + | > loss_disc_real_0: 0.1479230374097824 (0.1479230374097824) + | > loss_disc_real_1: 0.27100807428359985 (0.27100807428359985) + | > loss_disc_real_2: 0.2642741799354553 (0.2642741799354553) + | > loss_disc_real_3: 0.25932633876800537 (0.25932633876800537) + | > loss_disc_real_4: 0.22815759479999542 (0.22815759479999542) + | > loss_disc_real_5: 0.24005204439163208 (0.24005204439163208) + | > loss_0: 2.6417455673217773 (2.6417455673217773) + | > loss_gen: 2.153233289718628 (2.153233289718628) + | > loss_kl: 2.502565383911133 (2.502565383911133) + | > loss_feat: 5.733573913574219 (5.733573913574219) + | > loss_mel: 18.884191513061523 (18.884191513061523) + | > loss_duration: 1.7246605157852173 (1.7246605157852173) + | > loss_1: 30.99822425842285 (30.99822425842285) + + --> STEP: 1 + | > loss_disc: 2.462002754211426 (2.462002754211426) + | > loss_disc_real_0: 0.09035711735486984 (0.09035711735486984) + | > loss_disc_real_1: 0.2754846215248108 (0.2754846215248108) + | > loss_disc_real_2: 0.23941254615783691 (0.23941254615783691) + | > loss_disc_real_3: 0.26952266693115234 (0.26952266693115234) + | > loss_disc_real_4: 0.2256251722574234 (0.2256251722574234) + | > loss_disc_real_5: 0.18956954777240753 (0.18956954777240753) + | > loss_0: 2.462002754211426 (2.462002754211426) + | > loss_gen: 2.187554359436035 (2.187554359436035) + | > loss_kl: 2.8032546043395996 (2.8032546043395996) + | > loss_feat: 6.895532608032227 (6.895532608032227) + | > loss_mel: 17.084918975830078 (17.084918975830078) + | > loss_duration: 1.7411391735076904 (1.7411391735076904) + | > loss_1: 30.712398529052734 (30.712398529052734) + + --> STEP: 2 + | > loss_disc: 2.352769374847412 (2.407386064529419) + | > loss_disc_real_0: 0.08706575632095337 (0.0887114368379116) + | > loss_disc_real_1: 0.24013811349868774 (0.25781136751174927) + | > loss_disc_real_2: 0.22704140841960907 (0.233226977288723) + | > loss_disc_real_3: 0.2354336380958557 (0.25247815251350403) + | > loss_disc_real_4: 0.23569144308567047 (0.23065830767154694) + | > loss_disc_real_5: 0.21051208674907684 (0.2000408172607422) + | > loss_0: 2.352769374847412 (2.407386064529419) + | > loss_gen: 2.3515970706939697 (2.2695757150650024) + | > loss_kl: 2.1603872776031494 (2.4818209409713745) + | > loss_feat: 7.8475518226623535 (7.37154221534729) + | > loss_mel: 19.459829330444336 (18.272374153137207) + | > loss_duration: 1.6957974433898926 (1.7184683084487915) + | > loss_1: 33.51516342163086 (32.1137809753418) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004041552543640137 (+0.0006656646728515625) + | > avg_loss_disc: 2.407386064529419 (-0.1728825569152832) + | > avg_loss_disc_real_0: 0.0887114368379116 (-0.14825449511408806) + | > avg_loss_disc_real_1: 0.25781136751174927 (+0.012016646564006805) + | > avg_loss_disc_real_2: 0.233226977288723 (-0.03350897878408432) + | > avg_loss_disc_real_3: 0.25247815251350403 (+0.034022144973278046) + | > avg_loss_disc_real_4: 0.23065830767154694 (+0.04947436600923538) + | > avg_loss_disc_real_5: 0.2000408172607422 (-0.052517957985401154) + | > avg_loss_0: 2.407386064529419 (-0.1728825569152832) + | > avg_loss_gen: 2.2695757150650024 (-0.1610018014907837) + | > avg_loss_kl: 2.4818209409713745 (-0.08742856979370117) + | > avg_loss_feat: 7.37154221534729 (+0.2871553897857666) + | > avg_loss_mel: 18.272374153137207 (+0.9949522018432617) + | > avg_loss_duration: 1.7184683084487915 (-0.02843374013900757) + | > avg_loss_1: 32.1137809753418 (+1.0052423477172852) + + + > EPOCH: 305/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:00:05)  + + --> TIME: 2025-04-19 04:00:08 -- STEP: 8/142 -- GLOBAL_STEP: 422600 + | > loss_disc: 2.5583999156951904 (2.472428798675537) + | > loss_disc_real_0: 0.11763067543506622 (0.1279700454324484) + | > loss_disc_real_1: 0.16344113647937775 (0.215035792440176) + | > loss_disc_real_2: 0.24060335755348206 (0.22824284248054028) + | > loss_disc_real_3: 0.2513468265533447 (0.22750725410878658) + | > loss_disc_real_4: 0.23741954565048218 (0.22750933840870857) + | > loss_disc_real_5: 0.21225889027118683 (0.22285140864551067) + | > loss_0: 2.5583999156951904 (2.472428798675537) + | > grad_norm_0: tensor(11.0800, device='cuda:0') (tensor(8.8180, device='cuda:0')) + | > loss_gen: 2.2044620513916016 (2.370213121175766) + | > loss_kl: 1.857413411140442 (1.8026946634054184) + | > loss_feat: 6.909114837646484 (7.183789670467377) + | > loss_mel: 17.069734573364258 (17.97366237640381) + | > loss_duration: 1.4436631202697754 (1.4186926484107971) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.484386444091797 (30.74905228614807) + | > grad_norm_1: tensor(354.2592, device='cuda:0') (tensor(191.8109, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.3314 (0.32608580589294434) + | > loader_time: 0.0048 (0.004597872495651245) + + + --> TIME: 2025-04-19 04:00:17 -- STEP: 33/142 -- GLOBAL_STEP: 422625 + | > loss_disc: 2.5186946392059326 (2.4509083068732056) + | > loss_disc_real_0: 0.1784469336271286 (0.12347986097588684) + | > loss_disc_real_1: 0.2108679562807083 (0.20756205600319486) + | > loss_disc_real_2: 0.28067344427108765 (0.2255500347325296) + | > loss_disc_real_3: 0.27764901518821716 (0.22731141023563617) + | > loss_disc_real_4: 0.2531328499317169 (0.22581999906987854) + | > loss_disc_real_5: 0.2953411340713501 (0.23460687109918305) + | > loss_0: 2.5186946392059326 (2.4509083068732056) + | > grad_norm_0: tensor(5.6331, device='cuda:0') (tensor(9.5417, device='cuda:0')) + | > loss_gen: 2.524674892425537 (2.3874815594066274) + | > loss_kl: 1.8949788808822632 (1.79510645794146) + | > loss_feat: 6.471224784851074 (7.310243534319328) + | > loss_mel: 16.90273666381836 (18.092769160415187) + | > loss_duration: 1.4311635494232178 (1.4254246227668994) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.22477912902832 (31.011025168678977) + | > grad_norm_1: tensor(83.8072, device='cuda:0') (tensor(245.5507, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.3557 (0.3389263153076172) + | > loader_time: 0.0058 (0.005271008520415336) + + + --> TIME: 2025-04-19 04:00:26 -- STEP: 58/142 -- GLOBAL_STEP: 422650 + | > loss_disc: 2.458954334259033 (2.455103775550579) + | > loss_disc_real_0: 0.14651256799697876 (0.12468480241709737) + | > loss_disc_real_1: 0.2014240175485611 (0.208307059931344) + | > loss_disc_real_2: 0.20213498175144196 (0.225349480478928) + | > loss_disc_real_3: 0.23586133122444153 (0.22484991648073854) + | > loss_disc_real_4: 0.19310420751571655 (0.2275836280193822) + | > loss_disc_real_5: 0.25628674030303955 (0.23404825453100533) + | > loss_0: 2.458954334259033 (2.455103775550579) + | > grad_norm_0: tensor(6.0028, device='cuda:0') (tensor(9.0427, device='cuda:0')) + | > loss_gen: 2.389035224914551 (2.37756070186352) + | > loss_kl: 1.851927638053894 (1.8238633727205211) + | > loss_feat: 7.265571594238281 (7.278835132204253) + | > loss_mel: 18.350996017456055 (18.04531460794909) + | > loss_duration: 1.4681007862091064 (1.4366785657816923) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.32563018798828 (30.96225218937315) + | > grad_norm_1: tensor(49.3161, device='cuda:0') (tensor(230.2463, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.3827 (0.35137664038559485) + | > loader_time: 0.0063 (0.005571838082938358) + + + --> TIME: 2025-04-19 04:00:36 -- STEP: 83/142 -- GLOBAL_STEP: 422675 + | > loss_disc: 2.485426425933838 (2.464062535619161) + | > loss_disc_real_0: 0.13241180777549744 (0.12633846098759083) + | > loss_disc_real_1: 0.22576701641082764 (0.20780036804905852) + | > loss_disc_real_2: 0.23668649792671204 (0.2280679562364716) + | > loss_disc_real_3: 0.2349146604537964 (0.22668892683753047) + | > loss_disc_real_4: 0.2360658496618271 (0.2293984760721046) + | > loss_disc_real_5: 0.24197307229042053 (0.23289167432181807) + | > loss_0: 2.485426425933838 (2.464062535619161) + | > grad_norm_0: tensor(4.4663, device='cuda:0') (tensor(8.4053, device='cuda:0')) + | > loss_gen: 2.4315218925476074 (2.365014814468753) + | > loss_kl: 1.8741734027862549 (1.8462672018143067) + | > loss_feat: 6.349329948425293 (7.161680399653423) + | > loss_mel: 17.399188995361328 (17.91332854994809) + | > loss_duration: 1.5317131280899048 (1.449081080505647) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.585927963256836 (30.735371991812464) + | > grad_norm_1: tensor(107.3445, device='cuda:0') (tensor(201.9830, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.3984 (0.3639568047351149) + | > loader_time: 0.0067 (0.005860587200486516) + + + --> TIME: 2025-04-19 04:00:47 -- STEP: 108/142 -- GLOBAL_STEP: 422700 + | > loss_disc: 2.508395195007324 (2.4660279044398554) + | > loss_disc_real_0: 0.17856375873088837 (0.12798220767743057) + | > loss_disc_real_1: 0.20134182274341583 (0.207957595448803) + | > loss_disc_real_2: 0.24621352553367615 (0.2270042617563848) + | > loss_disc_real_3: 0.21533247828483582 (0.22719395850543622) + | > loss_disc_real_4: 0.19585567712783813 (0.22866862722569042) + | > loss_disc_real_5: 0.2620159685611725 (0.232067480545353) + | > loss_0: 2.508395195007324 (2.4660279044398554) + | > grad_norm_0: tensor(15.8643, device='cuda:0') (tensor(8.8759, device='cuda:0')) + | > loss_gen: 2.23793888092041 (2.3574708457346323) + | > loss_kl: 1.8018943071365356 (1.8581853851124093) + | > loss_feat: 6.664254188537598 (7.12696146523511) + | > loss_mel: 18.212871551513672 (17.8511484640616) + | > loss_duration: 1.494386076927185 (1.4587908298880967) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.411344528198242 (30.652556913870352) + | > grad_norm_1: tensor(386.1912, device='cuda:0') (tensor(211.2865, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.42 (0.3739056101551763) + | > loader_time: 0.0072 (0.0061144475583676935) + + + --> TIME: 2025-04-19 04:00:58 -- STEP: 133/142 -- GLOBAL_STEP: 422725 + | > loss_disc: 2.4304070472717285 (2.470840276632094) + | > loss_disc_real_0: 0.10990168899297714 (0.12816361993327177) + | > loss_disc_real_1: 0.20138461887836456 (0.2079134034483056) + | > loss_disc_real_2: 0.21645021438598633 (0.22744442130389966) + | > loss_disc_real_3: 0.23294827342033386 (0.22774535608022733) + | > loss_disc_real_4: 0.20346957445144653 (0.22965820794715022) + | > loss_disc_real_5: 0.24069510400295258 (0.23317504500536093) + | > loss_0: 2.4304070472717285 (2.470840276632094) + | > grad_norm_0: tensor(4.8018, device='cuda:0') (tensor(9.5203, device='cuda:0')) + | > loss_gen: 2.2740633487701416 (2.350493796785972) + | > loss_kl: 1.9845194816589355 (1.8695748587300007) + | > loss_feat: 6.655925273895264 (7.074408072278015) + | > loss_mel: 17.812002182006836 (17.82751270523645) + | > loss_duration: 1.5186147689819336 (1.464615015158976) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.24512481689453 (30.586604397996027) + | > grad_norm_1: tensor(69.6031, device='cuda:0') (tensor(219.2483, device='cuda:0')) + | > current_lr_0: 0.0001925180630856663 + | > current_lr_1: 0.0001925180630856663 + | > step_time: 0.438 (0.3854200822070129) + | > loader_time: 0.0075 (0.006369018913211679) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.538926362991333 (2.538926362991333) + | > loss_disc_real_0: 0.15565061569213867 (0.15565061569213867) + | > loss_disc_real_1: 0.19069930911064148 (0.19069930911064148) + | > loss_disc_real_2: 0.2170742154121399 (0.2170742154121399) + | > loss_disc_real_3: 0.2163708209991455 (0.2163708209991455) + | > loss_disc_real_4: 0.19566728174686432 (0.19566728174686432) + | > loss_disc_real_5: 0.2340339869260788 (0.2340339869260788) + | > loss_0: 2.538926362991333 (2.538926362991333) + | > loss_gen: 2.221829891204834 (2.221829891204834) + | > loss_kl: 2.3956127166748047 (2.3956127166748047) + | > loss_feat: 6.641806125640869 (6.641806125640869) + | > loss_mel: 17.380319595336914 (17.380319595336914) + | > loss_duration: 1.7091668844223022 (1.7091668844223022) + | > loss_1: 30.348735809326172 (30.348735809326172) + + --> STEP: 1 + | > loss_disc: 2.4824304580688477 (2.4824304580688477) + | > loss_disc_real_0: 0.2299242466688156 (0.2299242466688156) + | > loss_disc_real_1: 0.23520667850971222 (0.23520667850971222) + | > loss_disc_real_2: 0.2046252191066742 (0.2046252191066742) + | > loss_disc_real_3: 0.21689912676811218 (0.21689912676811218) + | > loss_disc_real_4: 0.2063884288072586 (0.2063884288072586) + | > loss_disc_real_5: 0.26297879219055176 (0.26297879219055176) + | > loss_0: 2.4824304580688477 (2.4824304580688477) + | > loss_gen: 2.540591239929199 (2.540591239929199) + | > loss_kl: 2.7402541637420654 (2.7402541637420654) + | > loss_feat: 7.108093738555908 (7.108093738555908) + | > loss_mel: 18.67673110961914 (18.67673110961914) + | > loss_duration: 1.750454068183899 (1.750454068183899) + | > loss_1: 32.816123962402344 (32.816123962402344) + + --> STEP: 2 + | > loss_disc: 2.540843963623047 (2.5116372108459473) + | > loss_disc_real_0: 0.2576814293861389 (0.24380283802747726) + | > loss_disc_real_1: 0.20136235654354095 (0.2182845175266266) + | > loss_disc_real_2: 0.2197432518005371 (0.21218423545360565) + | > loss_disc_real_3: 0.2337246537208557 (0.22531189024448395) + | > loss_disc_real_4: 0.2230835258960724 (0.2147359773516655) + | > loss_disc_real_5: 0.2388058453798294 (0.2508923187851906) + | > loss_0: 2.540843963623047 (2.5116372108459473) + | > loss_gen: 2.588766098022461 (2.56467866897583) + | > loss_kl: 2.3383562564849854 (2.5393052101135254) + | > loss_feat: 7.488487243652344 (7.298290491104126) + | > loss_mel: 18.101757049560547 (18.389244079589844) + | > loss_duration: 1.7001124620437622 (1.7252832651138306) + | > loss_1: 32.21747970581055 (32.516801834106445) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003680109977722168 (-0.00036144256591796875) + | > avg_loss_disc: 2.5116372108459473 (+0.10425114631652832) + | > avg_loss_disc_real_0: 0.24380283802747726 (+0.15509140118956566) + | > avg_loss_disc_real_1: 0.2182845175266266 (-0.03952684998512268) + | > avg_loss_disc_real_2: 0.21218423545360565 (-0.02104274183511734) + | > avg_loss_disc_real_3: 0.22531189024448395 (-0.02716626226902008) + | > avg_loss_disc_real_4: 0.2147359773516655 (-0.01592233031988144) + | > avg_loss_disc_real_5: 0.2508923187851906 (+0.050851501524448395) + | > avg_loss_0: 2.5116372108459473 (+0.10425114631652832) + | > avg_loss_gen: 2.56467866897583 (+0.29510295391082764) + | > avg_loss_kl: 2.5393052101135254 (+0.05748426914215088) + | > avg_loss_feat: 7.298290491104126 (-0.07325172424316406) + | > avg_loss_mel: 18.389244079589844 (+0.11686992645263672) + | > avg_loss_duration: 1.7252832651138306 (+0.0068149566650390625) + | > avg_loss_1: 32.516801834106445 (+0.40302085876464844) + + + > EPOCH: 306/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:01:03)  + + --> TIME: 2025-04-19 04:01:09 -- STEP: 16/142 -- GLOBAL_STEP: 422750 + | > loss_disc: 2.4431796073913574 (2.4192195385694504) + | > loss_disc_real_0: 0.12213199585676193 (0.11132101900875568) + | > loss_disc_real_1: 0.2223048210144043 (0.20474018529057503) + | > loss_disc_real_2: 0.2478952258825302 (0.22813139017671347) + | > loss_disc_real_3: 0.258565217256546 (0.2278648531064391) + | > loss_disc_real_4: 0.2613277733325958 (0.23210252914577723) + | > loss_disc_real_5: 0.19518765807151794 (0.2263768371194601) + | > loss_0: 2.4431796073913574 (2.4192195385694504) + | > grad_norm_0: tensor(5.4005, device='cuda:0') (tensor(7.5210, device='cuda:0')) + | > loss_gen: 2.5188961029052734 (2.412656709551811) + | > loss_kl: 1.786681890487671 (1.8096547797322273) + | > loss_feat: 7.500908374786377 (7.487378627061844) + | > loss_mel: 18.573013305664062 (18.15974795818329) + | > loss_duration: 1.4142178297042847 (1.415017917752266) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.793716430664062 (31.284456372261047) + | > grad_norm_1: tensor(262.4661, device='cuda:0') (tensor(191.1424, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.3393 (0.32936136424541473) + | > loader_time: 0.0051 (0.004762321710586549) + + + --> TIME: 2025-04-19 04:01:19 -- STEP: 41/142 -- GLOBAL_STEP: 422775 + | > loss_disc: 2.3886923789978027 (2.4243340550399406) + | > loss_disc_real_0: 0.14162912964820862 (0.11772314404569022) + | > loss_disc_real_1: 0.1888102889060974 (0.20721330221106365) + | > loss_disc_real_2: 0.20892870426177979 (0.22220194012653538) + | > loss_disc_real_3: 0.23487359285354614 (0.22567468481819805) + | > loss_disc_real_4: 0.22003239393234253 (0.22527532606590084) + | > loss_disc_real_5: 0.20740903913974762 (0.2280278445743933) + | > loss_0: 2.3886923789978027 (2.4243340550399406) + | > grad_norm_0: tensor(7.6610, device='cuda:0') (tensor(9.0699, device='cuda:0')) + | > loss_gen: 2.445500373840332 (2.3820095934518952) + | > loss_kl: 1.5753318071365356 (1.8063427064476945) + | > loss_feat: 7.448116779327393 (7.231810139446724) + | > loss_mel: 18.119178771972656 (17.8056926727295) + | > loss_duration: 1.409409999847412 (1.4300479947066889) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.99753761291504 (30.655903234714415) + | > grad_norm_1: tensor(141.7700, device='cuda:0') (tensor(201.9323, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.3702 (0.3468730333374768) + | > loader_time: 0.006 (0.005291595691587867) + + + --> TIME: 2025-04-19 04:01:28 -- STEP: 66/142 -- GLOBAL_STEP: 422800 + | > loss_disc: 2.4563541412353516 (2.436085263888041) + | > loss_disc_real_0: 0.10120377689599991 (0.11950857937335968) + | > loss_disc_real_1: 0.2022240310907364 (0.2084083737749042) + | > loss_disc_real_2: 0.21721547842025757 (0.22352581787289996) + | > loss_disc_real_3: 0.25433677434921265 (0.2270709380055919) + | > loss_disc_real_4: 0.24951153993606567 (0.22583412266138828) + | > loss_disc_real_5: 0.2322852909564972 (0.22916303620193945) + | > loss_0: 2.4563541412353516 (2.436085263888041) + | > grad_norm_0: tensor(9.3480, device='cuda:0') (tensor(9.4170, device='cuda:0')) + | > loss_gen: 2.4566750526428223 (2.3813163258812646) + | > loss_kl: 1.8489943742752075 (1.8275018785939072) + | > loss_feat: 7.401699066162109 (7.211237069332238) + | > loss_mel: 18.111576080322266 (17.797907655889343) + | > loss_duration: 1.4375956058502197 (1.4413149934826475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.256540298461914 (30.659278002652254) + | > grad_norm_1: tensor(265.7466, device='cuda:0') (tensor(214.3971, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.3837 (0.36025913917657104) + | > loader_time: 0.006 (0.005657499486749823) + + + --> TIME: 2025-04-19 04:01:38 -- STEP: 91/142 -- GLOBAL_STEP: 422825 + | > loss_disc: 2.40706729888916 (2.4432577484256615) + | > loss_disc_real_0: 0.15494965016841888 (0.12179913437300986) + | > loss_disc_real_1: 0.19882655143737793 (0.20708234378924736) + | > loss_disc_real_2: 0.2279605120420456 (0.224433557672815) + | > loss_disc_real_3: 0.225169375538826 (0.22659025742457464) + | > loss_disc_real_4: 0.21515104174613953 (0.22740817774128128) + | > loss_disc_real_5: 0.2037637084722519 (0.22842153235451207) + | > loss_0: 2.40706729888916 (2.4432577484256615) + | > grad_norm_0: tensor(22.5369, device='cuda:0') (tensor(9.4700, device='cuda:0')) + | > loss_gen: 2.4734578132629395 (2.374932386062958) + | > loss_kl: 1.824165940284729 (1.836991514478411) + | > loss_feat: 7.440422058105469 (7.141551227359981) + | > loss_mel: 17.51447868347168 (17.714325055971255) + | > loss_duration: 1.4823634624481201 (1.4523732282303186) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.734888076782227 (30.52017341865288) + | > grad_norm_1: tensor(505.6622, device='cuda:0') (tensor(220.7382, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.4026 (0.36974890939481964) + | > loader_time: 0.0067 (0.005928581887549099) + + + --> TIME: 2025-04-19 04:01:49 -- STEP: 116/142 -- GLOBAL_STEP: 422850 + | > loss_disc: 2.414301872253418 (2.4506441416411566) + | > loss_disc_real_0: 0.15453971922397614 (0.12333453706754692) + | > loss_disc_real_1: 0.20669326186180115 (0.20833396718933664) + | > loss_disc_real_2: 0.2395075410604477 (0.2249801262956241) + | > loss_disc_real_3: 0.25490978360176086 (0.22772689690363818) + | > loss_disc_real_4: 0.250403493642807 (0.22744400054216385) + | > loss_disc_real_5: 0.24260373413562775 (0.2288124899155107) + | > loss_0: 2.414301872253418 (2.4506441416411566) + | > grad_norm_0: tensor(7.0930, device='cuda:0') (tensor(10.7707, device='cuda:0')) + | > loss_gen: 2.6655094623565674 (2.371978901583574) + | > loss_kl: 1.8094992637634277 (1.8521819813498135) + | > loss_feat: 6.363404273986816 (7.1060331073300596) + | > loss_mel: 16.55625343322754 (17.64121344993854) + | > loss_duration: 1.472001075744629 (1.4590466330791336) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.866668701171875 (30.430454089723785) + | > grad_norm_1: tensor(48.4332, device='cuda:0') (tensor(227.0612, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.4285 (0.3794809250996031) + | > loader_time: 0.0083 (0.0061662998692742724) + + + --> TIME: 2025-04-19 04:02:00 -- STEP: 141/142 -- GLOBAL_STEP: 422875 + | > loss_disc: 2.0966153144836426 (2.4562584559122715) + | > loss_disc_real_0: 0.060512300580739975 (0.1250385727125702) + | > loss_disc_real_1: 0.16519589722156525 (0.20808028709803914) + | > loss_disc_real_2: 0.2287173569202423 (0.22619260741886518) + | > loss_disc_real_3: 0.20732882618904114 (0.22865004161148206) + | > loss_disc_real_4: 0.19576987624168396 (0.22766868051484968) + | > loss_disc_real_5: 0.22882550954818726 (0.22974703561329674) + | > loss_0: 2.0966153144836426 (2.4562584559122715) + | > grad_norm_0: tensor(5.8071, device='cuda:0') (tensor(10.9030, device='cuda:0')) + | > loss_gen: 2.6937315464019775 (2.363892365854683) + | > loss_kl: 1.6747528314590454 (1.857181818772715) + | > loss_feat: 8.200244903564453 (7.089780120984883) + | > loss_mel: 18.450271606445312 (17.6046031113212) + | > loss_duration: 1.3385261297225952 (1.464980783191978) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.35752868652344 (30.38043824324371) + | > grad_norm_1: tensor(337.7916, device='cuda:0') (tensor(234.3431, device='cuda:0')) + | > current_lr_0: 0.0001924939983277806 + | > current_lr_1: 0.0001924939983277806 + | > step_time: 0.2031 (0.3891405281445659) + | > loader_time: 0.0031 (0.0064005462835866524) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.375272274017334 (2.375272274017334) + | > loss_disc_real_0: 0.09517672657966614 (0.09517672657966614) + | > loss_disc_real_1: 0.20663750171661377 (0.20663750171661377) + | > loss_disc_real_2: 0.20347103476524353 (0.20347103476524353) + | > loss_disc_real_3: 0.24114903807640076 (0.24114903807640076) + | > loss_disc_real_4: 0.2422100156545639 (0.2422100156545639) + | > loss_disc_real_5: 0.23710355162620544 (0.23710355162620544) + | > loss_0: 2.375272274017334 (2.375272274017334) + | > loss_gen: 2.316793203353882 (2.316793203353882) + | > loss_kl: 2.321786403656006 (2.321786403656006) + | > loss_feat: 7.168991565704346 (7.168991565704346) + | > loss_mel: 19.177528381347656 (19.177528381347656) + | > loss_duration: 1.7013493776321411 (1.7013493776321411) + | > loss_1: 32.68645095825195 (32.68645095825195) + + --> STEP: 1 + | > loss_disc: 2.5219054222106934 (2.5219054222106934) + | > loss_disc_real_0: 0.10204621404409409 (0.10204621404409409) + | > loss_disc_real_1: 0.2296106368303299 (0.2296106368303299) + | > loss_disc_real_2: 0.2844293415546417 (0.2844293415546417) + | > loss_disc_real_3: 0.29066985845565796 (0.29066985845565796) + | > loss_disc_real_4: 0.2852865159511566 (0.2852865159511566) + | > loss_disc_real_5: 0.2789224684238434 (0.2789224684238434) + | > loss_0: 2.5219054222106934 (2.5219054222106934) + | > loss_gen: 2.427293300628662 (2.427293300628662) + | > loss_kl: 2.760786533355713 (2.760786533355713) + | > loss_feat: 7.2399067878723145 (7.2399067878723145) + | > loss_mel: 18.662616729736328 (18.662616729736328) + | > loss_duration: 1.7517549991607666 (1.7517549991607666) + | > loss_1: 32.84236145019531 (32.84236145019531) + + --> STEP: 2 + | > loss_disc: 2.5141937732696533 (2.5180495977401733) + | > loss_disc_real_0: 0.12421456724405289 (0.11313039064407349) + | > loss_disc_real_1: 0.2451489269733429 (0.2373797819018364) + | > loss_disc_real_2: 0.2762604355812073 (0.2803448885679245) + | > loss_disc_real_3: 0.2462921440601349 (0.2684810012578964) + | > loss_disc_real_4: 0.2544049620628357 (0.26984573900699615) + | > loss_disc_real_5: 0.2217504233121872 (0.2503364458680153) + | > loss_0: 2.5141937732696533 (2.5180495977401733) + | > loss_gen: 2.4194650650024414 (2.4233791828155518) + | > loss_kl: 2.1550872325897217 (2.4579368829727173) + | > loss_feat: 7.915962219238281 (7.577934503555298) + | > loss_mel: 17.596874237060547 (18.129745483398438) + | > loss_duration: 1.6871206760406494 (1.719437837600708) + | > loss_1: 31.774507522583008 (32.30843448638916) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035191774368286133 (-0.0001609325408935547) + | > avg_loss_disc: 2.5180495977401733 (+0.006412386894226074) + | > avg_loss_disc_real_0: 0.11313039064407349 (-0.13067244738340378) + | > avg_loss_disc_real_1: 0.2373797819018364 (+0.01909526437520981) + | > avg_loss_disc_real_2: 0.2803448885679245 (+0.06816065311431885) + | > avg_loss_disc_real_3: 0.2684810012578964 (+0.043169111013412476) + | > avg_loss_disc_real_4: 0.26984573900699615 (+0.05510976165533066) + | > avg_loss_disc_real_5: 0.2503364458680153 (-0.000555872917175293) + | > avg_loss_0: 2.5180495977401733 (+0.006412386894226074) + | > avg_loss_gen: 2.4233791828155518 (-0.14129948616027832) + | > avg_loss_kl: 2.4579368829727173 (-0.0813683271408081) + | > avg_loss_feat: 7.577934503555298 (+0.2796440124511719) + | > avg_loss_mel: 18.129745483398438 (-0.25949859619140625) + | > avg_loss_duration: 1.719437837600708 (-0.005845427513122559) + | > avg_loss_1: 32.30843448638916 (-0.20836734771728516) + + + > EPOCH: 307/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:02:02)  + + --> TIME: 2025-04-19 04:02:11 -- STEP: 24/142 -- GLOBAL_STEP: 422900 + | > loss_disc: 2.4930014610290527 (2.4110458890597024) + | > loss_disc_real_0: 0.10787635296583176 (0.11041325672219197) + | > loss_disc_real_1: 0.16895753145217896 (0.2080999556928873) + | > loss_disc_real_2: 0.2399182766675949 (0.2234905461470286) + | > loss_disc_real_3: 0.18075399100780487 (0.22418346938987574) + | > loss_disc_real_4: 0.23936422169208527 (0.2293715743968884) + | > loss_disc_real_5: 0.2659572958946228 (0.22815460339188576) + | > loss_0: 2.4930014610290527 (2.4110458890597024) + | > grad_norm_0: tensor(5.5990, device='cuda:0') (tensor(7.9841, device='cuda:0')) + | > loss_gen: 2.1430015563964844 (2.386344055334727) + | > loss_kl: 1.8557156324386597 (1.8019668757915497) + | > loss_feat: 6.747246742248535 (7.277212540308635) + | > loss_mel: 18.112077713012695 (17.746999661127724) + | > loss_duration: 1.4569579362869263 (1.4186532845099766) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.315000534057617 (30.631176153818767) + | > grad_norm_1: tensor(33.9869, device='cuda:0') (tensor(173.7983, device='cuda:0')) + | > current_lr_0: 0.0001924699365779896 + | > current_lr_1: 0.0001924699365779896 + | > step_time: 0.347 (0.33851358294487) + | > loader_time: 0.006 (0.005058685938517253) + + + --> TIME: 2025-04-19 04:02:20 -- STEP: 49/142 -- GLOBAL_STEP: 422925 + | > loss_disc: 2.4423773288726807 (2.415378818706591) + | > loss_disc_real_0: 0.09349038451910019 (0.11355434069219901) + | > loss_disc_real_1: 0.19945189356803894 (0.20639882920956126) + | > loss_disc_real_2: 0.22147315740585327 (0.22364373292241777) + | > loss_disc_real_3: 0.23331710696220398 (0.22445820819358436) + | > loss_disc_real_4: 0.2645035982131958 (0.22924271256339793) + | > loss_disc_real_5: 0.2738730311393738 (0.2272279226050085) + | > loss_0: 2.4423773288726807 (2.415378818706591) + | > grad_norm_0: tensor(7.3107, device='cuda:0') (tensor(8.6520, device='cuda:0')) + | > loss_gen: 2.328394889831543 (2.3864017895289824) + | > loss_kl: 2.0088489055633545 (1.8212618730506118) + | > loss_feat: 7.275033473968506 (7.3081667374591435) + | > loss_mel: 18.190290451049805 (17.944793506544464) + | > loss_duration: 1.4636051654815674 (1.430286490187353) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.266172409057617 (30.89091040163624) + | > grad_norm_1: tensor(434.5482, device='cuda:0') (tensor(206.9017, device='cuda:0')) + | > current_lr_0: 0.0001924699365779896 + | > current_lr_1: 0.0001924699365779896 + | > step_time: 0.371 (0.3479736191885812) + | > loader_time: 0.0059 (0.005413629570785834) + + + --> TIME: 2025-04-19 04:02:30 -- STEP: 74/142 -- GLOBAL_STEP: 422950 + | > loss_disc: 2.5364699363708496 (2.426874144657239) + | > loss_disc_real_0: 0.15394394099712372 (0.11793256712120932) + | > loss_disc_real_1: 0.19315186142921448 (0.20591903558453994) + | > loss_disc_real_2: 0.20202314853668213 (0.22373710673403097) + | > loss_disc_real_3: 0.2509976029396057 (0.22577337719298698) + | > loss_disc_real_4: 0.22155442833900452 (0.22782871650682912) + | > loss_disc_real_5: 0.2339773327112198 (0.22755050155762085) + | > loss_0: 2.5364699363708496 (2.426874144657239) + | > grad_norm_0: tensor(8.3461, device='cuda:0') (tensor(9.1157, device='cuda:0')) + | > loss_gen: 2.2727859020233154 (2.3837905026770922) + | > loss_kl: 1.8552377223968506 (1.8432284738566425) + | > loss_feat: 6.8134355545043945 (7.295640526591121) + | > loss_mel: 18.392349243164062 (18.01629739194303) + | > loss_duration: 1.4705586433410645 (1.442909327713219) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.804367065429688 (30.981866192173314) + | > grad_norm_1: tensor(122.1632, device='cuda:0') (tensor(219.6884, device='cuda:0')) + | > current_lr_0: 0.0001924699365779896 + | > current_lr_1: 0.0001924699365779896 + | > step_time: 0.3867 (0.3590128808408169) + | > loader_time: 0.0066 (0.005697752978350665) + + + --> TIME: 2025-04-19 04:02:40 -- STEP: 99/142 -- GLOBAL_STEP: 422975 + | > loss_disc: 2.467559337615967 (2.4351253437273432) + | > loss_disc_real_0: 0.0968889594078064 (0.11888489955001408) + | > loss_disc_real_1: 0.18900102376937866 (0.2070014065865314) + | > loss_disc_real_2: 0.237395241856575 (0.22425244853954124) + | > loss_disc_real_3: 0.2584281265735626 (0.22497513890266418) + | > loss_disc_real_4: 0.2497972995042801 (0.22804601144308995) + | > loss_disc_real_5: 0.25665372610092163 (0.2277379797564613) + | > loss_0: 2.467559337615967 (2.4351253437273432) + | > grad_norm_0: tensor(6.9294, device='cuda:0') (tensor(9.8844, device='cuda:0')) + | > loss_gen: 2.4733731746673584 (2.3736253772119067) + | > loss_kl: 1.925673007965088 (1.8553543464101927) + | > loss_feat: 7.668997764587402 (7.189823747885348) + | > loss_mel: 19.22000503540039 (17.97889119928533) + | > loss_duration: 1.486572265625 (1.4534028226679028) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.774620056152344 (30.85109749225655) + | > grad_norm_1: tensor(47.5051, device='cuda:0') (tensor(233.0220, device='cuda:0')) + | > current_lr_0: 0.0001924699365779896 + | > current_lr_1: 0.0001924699365779896 + | > step_time: 0.4038 (0.36922196667603757) + | > loader_time: 0.0068 (0.005949441832725448) + + + --> TIME: 2025-04-19 04:02:51 -- STEP: 124/142 -- GLOBAL_STEP: 423000 + | > loss_disc: 2.385148286819458 (2.441666697302172) + | > loss_disc_real_0: 0.10769224166870117 (0.12030671831340559) + | > loss_disc_real_1: 0.21937628090381622 (0.2069364568639186) + | > loss_disc_real_2: 0.23218926787376404 (0.22410774098769312) + | > loss_disc_real_3: 0.23192359507083893 (0.22613712388180918) + | > loss_disc_real_4: 0.23394329845905304 (0.22831016142041452) + | > loss_disc_real_5: 0.23103658854961395 (0.2295125722885132) + | > loss_0: 2.385148286819458 (2.441666697302172) + | > grad_norm_0: tensor(12.4588, device='cuda:0') (tensor(9.9052, device='cuda:0')) + | > loss_gen: 2.379371404647827 (2.364997742637511) + | > loss_kl: 2.09747576713562 (1.8646080099767255) + | > loss_feat: 6.755131244659424 (7.1297702481669765) + | > loss_mel: 18.179235458374023 (17.90219932217751) + | > loss_duration: 1.5160298347473145 (1.4607439531433972) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.927242279052734 (30.722319341474964) + | > grad_norm_1: tensor(259.2067, device='cuda:0') (tensor(230.1099, device='cuda:0')) + | > current_lr_0: 0.0001924699365779896 + | > current_lr_1: 0.0001924699365779896 + | > step_time: 0.4424 (0.3810491350389295) + | > loader_time: 0.0067 (0.006228173932721538) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.479052782058716 (2.479052782058716) + | > loss_disc_real_0: 0.12748880684375763 (0.12748880684375763) + | > loss_disc_real_1: 0.2228393852710724 (0.2228393852710724) + | > loss_disc_real_2: 0.2168866991996765 (0.2168866991996765) + | > loss_disc_real_3: 0.28339746594429016 (0.28339746594429016) + | > loss_disc_real_4: 0.296524316072464 (0.296524316072464) + | > loss_disc_real_5: 0.2669718563556671 (0.2669718563556671) + | > loss_0: 2.479052782058716 (2.479052782058716) + | > loss_gen: 2.442110061645508 (2.442110061645508) + | > loss_kl: 2.5646402835845947 (2.5646402835845947) + | > loss_feat: 6.664572238922119 (6.664572238922119) + | > loss_mel: 18.675914764404297 (18.675914764404297) + | > loss_duration: 1.7365633249282837 (1.7365633249282837) + | > loss_1: 32.08380126953125 (32.08380126953125) + + --> STEP: 1 + | > loss_disc: 2.7446165084838867 (2.7446165084838867) + | > loss_disc_real_0: 0.14258158206939697 (0.14258158206939697) + | > loss_disc_real_1: 0.2505159378051758 (0.2505159378051758) + | > loss_disc_real_2: 0.24303613603115082 (0.24303613603115082) + | > loss_disc_real_3: 0.2925252914428711 (0.2925252914428711) + | > loss_disc_real_4: 0.3010028898715973 (0.3010028898715973) + | > loss_disc_real_5: 0.2837178111076355 (0.2837178111076355) + | > loss_0: 2.7446165084838867 (2.7446165084838867) + | > loss_gen: 2.1568784713745117 (2.1568784713745117) + | > loss_kl: 2.7798495292663574 (2.7798495292663574) + | > loss_feat: 6.570532321929932 (6.570532321929932) + | > loss_mel: 18.041959762573242 (18.041959762573242) + | > loss_duration: 1.749987006187439 (1.749987006187439) + | > loss_1: 31.29920768737793 (31.29920768737793) + + --> STEP: 2 + | > loss_disc: 2.3986353874206543 (2.5716259479522705) + | > loss_disc_real_0: 0.17265768349170685 (0.1576196327805519) + | > loss_disc_real_1: 0.22809076309204102 (0.2393033504486084) + | > loss_disc_real_2: 0.24288855493068695 (0.24296234548091888) + | > loss_disc_real_3: 0.26789724826812744 (0.28021126985549927) + | > loss_disc_real_4: 0.2297845482826233 (0.2653937190771103) + | > loss_disc_real_5: 0.2183392196893692 (0.25102851539850235) + | > loss_0: 2.3986353874206543 (2.5716259479522705) + | > loss_gen: 2.4752256870269775 (2.3160520792007446) + | > loss_kl: 2.2772951126098633 (2.5285723209381104) + | > loss_feat: 7.767446517944336 (7.168989419937134) + | > loss_mel: 18.52034568786621 (18.281152725219727) + | > loss_duration: 1.69270920753479 (1.7213481068611145) + | > loss_1: 32.7330207824707 (32.016114234924316) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032542943954467773 (-0.00026488304138183594) + | > avg_loss_disc: 2.5716259479522705 (+0.05357635021209717) + | > avg_loss_disc_real_0: 0.1576196327805519 (+0.044489242136478424) + | > avg_loss_disc_real_1: 0.2393033504486084 (+0.0019235685467720032) + | > avg_loss_disc_real_2: 0.24296234548091888 (-0.037382543087005615) + | > avg_loss_disc_real_3: 0.28021126985549927 (+0.011730268597602844) + | > avg_loss_disc_real_4: 0.2653937190771103 (-0.004452019929885864) + | > avg_loss_disc_real_5: 0.25102851539850235 (+0.0006920695304870605) + | > avg_loss_0: 2.5716259479522705 (+0.05357635021209717) + | > avg_loss_gen: 2.3160520792007446 (-0.10732710361480713) + | > avg_loss_kl: 2.5285723209381104 (+0.07063543796539307) + | > avg_loss_feat: 7.168989419937134 (-0.40894508361816406) + | > avg_loss_mel: 18.281152725219727 (+0.15140724182128906) + | > avg_loss_duration: 1.7213481068611145 (+0.0019102692604064941) + | > avg_loss_1: 32.016114234924316 (-0.29232025146484375) + + + > EPOCH: 308/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:03:01)  + + --> TIME: 2025-04-19 04:03:04 -- STEP: 7/142 -- GLOBAL_STEP: 423025 + | > loss_disc: 2.4944658279418945 (2.444596290588379) + | > loss_disc_real_0: 0.13278689980506897 (0.12441857159137726) + | > loss_disc_real_1: 0.22594696283340454 (0.21726209350994655) + | > loss_disc_real_2: 0.26658734679222107 (0.23142407195908682) + | > loss_disc_real_3: 0.22856099903583527 (0.2357386669942311) + | > loss_disc_real_4: 0.20050908625125885 (0.23463510828358786) + | > loss_disc_real_5: 0.249130979180336 (0.22506327288491385) + | > loss_0: 2.4944658279418945 (2.444596290588379) + | > grad_norm_0: tensor(12.6455, device='cuda:0') (tensor(10.9469, device='cuda:0')) + | > loss_gen: 2.2463879585266113 (2.393697670527867) + | > loss_kl: 1.8877391815185547 (1.8068149260112218) + | > loss_feat: 6.569797992706299 (7.267994403839111) + | > loss_mel: 16.363609313964844 (17.905837195260183) + | > loss_duration: 1.4203672409057617 (1.4199317523411341) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.487899780273438 (30.79427637372698) + | > grad_norm_1: tensor(168.4322, device='cuda:0') (tensor(228.1422, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.323 (0.3254659516470773) + | > loader_time: 0.0041 (0.004649605069841657) + + + --> TIME: 2025-04-19 04:03:13 -- STEP: 32/142 -- GLOBAL_STEP: 423050 + | > loss_disc: 2.55363392829895 (2.4326373487710953) + | > loss_disc_real_0: 0.16667616367340088 (0.12569342949427667) + | > loss_disc_real_1: 0.21041817963123322 (0.2053316249512136) + | > loss_disc_real_2: 0.20748119056224823 (0.22374923713505268) + | > loss_disc_real_3: 0.2215093970298767 (0.2277830932289362) + | > loss_disc_real_4: 0.2281942218542099 (0.22568314522504807) + | > loss_disc_real_5: 0.2169983983039856 (0.22345132613554597) + | > loss_0: 2.55363392829895 (2.4326373487710953) + | > grad_norm_0: tensor(25.9038, device='cuda:0') (tensor(16.8753, device='cuda:0')) + | > loss_gen: 2.568934917449951 (2.4339849278330803) + | > loss_kl: 1.8311140537261963 (1.8220466896891594) + | > loss_feat: 7.308098316192627 (7.362134158611298) + | > loss_mel: 17.38081932067871 (17.75071480870247) + | > loss_duration: 1.4149391651153564 (1.4262845441699028) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.50390625 (30.795165181159973) + | > grad_norm_1: tensor(514.1718, device='cuda:0') (tensor(302.7418, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.3556 (0.3443647399544716) + | > loader_time: 0.0053 (0.005118049681186677) + + + --> TIME: 2025-04-19 04:03:22 -- STEP: 57/142 -- GLOBAL_STEP: 423075 + | > loss_disc: 2.4795873165130615 (2.4298540123722008) + | > loss_disc_real_0: 0.09788630157709122 (0.1243788875116591) + | > loss_disc_real_1: 0.2194516360759735 (0.20414583933980843) + | > loss_disc_real_2: 0.2413029968738556 (0.22490701732928292) + | > loss_disc_real_3: 0.23199136555194855 (0.22790533853204628) + | > loss_disc_real_4: 0.2349691092967987 (0.22641636823353015) + | > loss_disc_real_5: 0.21454821527004242 (0.22797795581190208) + | > loss_0: 2.4795873165130615 (2.4298540123722008) + | > grad_norm_0: tensor(9.6695, device='cuda:0') (tensor(15.8756, device='cuda:0')) + | > loss_gen: 2.292020082473755 (2.426636156282927) + | > loss_kl: 1.897202968597412 (1.8263778080020034) + | > loss_feat: 6.856998920440674 (7.370957023219058) + | > loss_mel: 16.836977005004883 (17.90049183159544) + | > loss_duration: 1.491541862487793 (1.4371608663023563) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.374740600585938 (30.96162377742299) + | > grad_norm_1: tensor(263.0876, device='cuda:0') (tensor(315.2096, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.3751 (0.3533598916572437) + | > loader_time: 0.0063 (0.0054776668548584) + + + --> TIME: 2025-04-19 04:03:32 -- STEP: 82/142 -- GLOBAL_STEP: 423100 + | > loss_disc: 2.44053316116333 (2.4296911024465793) + | > loss_disc_real_0: 0.09899058192968369 (0.12352220282503744) + | > loss_disc_real_1: 0.22547347843647003 (0.20449464877204196) + | > loss_disc_real_2: 0.20982220768928528 (0.22416233325876841) + | > loss_disc_real_3: 0.23851878941059113 (0.22671128064393997) + | > loss_disc_real_4: 0.21572192013263702 (0.2257058057116299) + | > loss_disc_real_5: 0.23665454983711243 (0.2284254353584313) + | > loss_0: 2.44053316116333 (2.4296911024465793) + | > grad_norm_0: tensor(6.6488, device='cuda:0') (tensor(13.1589, device='cuda:0')) + | > loss_gen: 2.333132266998291 (2.4111019809071603) + | > loss_kl: 1.9855568408966064 (1.8424739430590373) + | > loss_feat: 6.689489841461182 (7.3292768990121235) + | > loss_mel: 17.95733642578125 (17.8487058151059) + | > loss_duration: 1.5391887426376343 (1.4474833869352575) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.504703521728516 (30.879042043918517) + | > grad_norm_1: tensor(121.3555, device='cuda:0') (tensor(262.7354, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.398 (0.36437272734758325) + | > loader_time: 0.0065 (0.005777629410348289) + + + --> TIME: 2025-04-19 04:03:42 -- STEP: 107/142 -- GLOBAL_STEP: 423125 + | > loss_disc: 2.594636917114258 (2.444611297589597) + | > loss_disc_real_0: 0.14693515002727509 (0.12535249340896293) + | > loss_disc_real_1: 0.19849872589111328 (0.20471301438095413) + | > loss_disc_real_2: 0.24017560482025146 (0.22472669950155455) + | > loss_disc_real_3: 0.2490931898355484 (0.2287168108692793) + | > loss_disc_real_4: 0.24998822808265686 (0.22774199152661262) + | > loss_disc_real_5: 0.22828024625778198 (0.2302754521369934) + | > loss_0: 2.594636917114258 (2.444611297589597) + | > grad_norm_0: tensor(6.3303, device='cuda:0') (tensor(12.0344, device='cuda:0')) + | > loss_gen: 2.348785877227783 (2.390132993181174) + | > loss_kl: 1.7793251276016235 (1.8547404347179093) + | > loss_feat: 6.88686466217041 (7.233458438766337) + | > loss_mel: 16.107378005981445 (17.76663168791298) + | > loss_duration: 1.4850773811340332 (1.4570968106528308) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.60742950439453 (30.702060378600503) + | > grad_norm_1: tensor(136.9623, device='cuda:0') (tensor(229.2921, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.416 (0.3741055582171288) + | > loader_time: 0.0069 (0.006041468860947084) + + + --> TIME: 2025-04-19 04:03:54 -- STEP: 132/142 -- GLOBAL_STEP: 423150 + | > loss_disc: 2.4388747215270996 (2.4485854921918935) + | > loss_disc_real_0: 0.10177868604660034 (0.12647197787847483) + | > loss_disc_real_1: 0.22246438264846802 (0.20618696612390605) + | > loss_disc_real_2: 0.25807395577430725 (0.2250314848215291) + | > loss_disc_real_3: 0.22214145958423615 (0.2281279114611221) + | > loss_disc_real_4: 0.20605242252349854 (0.22691144500717972) + | > loss_disc_real_5: 0.20031991600990295 (0.22954841229048642) + | > loss_0: 2.4388747215270996 (2.4485854921918935) + | > grad_norm_0: tensor(5.2192, device='cuda:0') (tensor(10.8850, device='cuda:0')) + | > loss_gen: 2.3895480632781982 (2.377328623424876) + | > loss_kl: 1.872617483139038 (1.8614971430012675) + | > loss_feat: 6.047520637512207 (7.167407285083424) + | > loss_mel: 17.045616149902344 (17.743763562404748) + | > loss_duration: 1.474177598953247 (1.463875331661918) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.82948112487793 (30.61387196454135) + | > grad_norm_1: tensor(143.4458, device='cuda:0') (tensor(209.1495, device='cuda:0')) + | > current_lr_0: 0.00019244587783591734 + | > current_lr_1: 0.00019244587783591734 + | > step_time: 0.4465 (0.3856131398316585) + | > loader_time: 0.0079 (0.0063192284468448524) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.476208209991455 (2.476208209991455) + | > loss_disc_real_0: 0.13160093128681183 (0.13160093128681183) + | > loss_disc_real_1: 0.21759586036205292 (0.21759586036205292) + | > loss_disc_real_2: 0.2775275707244873 (0.2775275707244873) + | > loss_disc_real_3: 0.29259082674980164 (0.29259082674980164) + | > loss_disc_real_4: 0.20340555906295776 (0.20340555906295776) + | > loss_disc_real_5: 0.2404956966638565 (0.2404956966638565) + | > loss_0: 2.476208209991455 (2.476208209991455) + | > loss_gen: 2.4325366020202637 (2.4325366020202637) + | > loss_kl: 2.503272294998169 (2.503272294998169) + | > loss_feat: 8.057915687561035 (8.057915687561035) + | > loss_mel: 20.249134063720703 (20.249134063720703) + | > loss_duration: 1.7102129459381104 (1.7102129459381104) + | > loss_1: 34.95307159423828 (34.95307159423828) + + --> STEP: 1 + | > loss_disc: 2.454238176345825 (2.454238176345825) + | > loss_disc_real_0: 0.13584986329078674 (0.13584986329078674) + | > loss_disc_real_1: 0.22564904391765594 (0.22564904391765594) + | > loss_disc_real_2: 0.23485369980335236 (0.23485369980335236) + | > loss_disc_real_3: 0.2636229395866394 (0.2636229395866394) + | > loss_disc_real_4: 0.2045815885066986 (0.2045815885066986) + | > loss_disc_real_5: 0.20694181323051453 (0.20694181323051453) + | > loss_0: 2.454238176345825 (2.454238176345825) + | > loss_gen: 2.185859203338623 (2.185859203338623) + | > loss_kl: 2.882141590118408 (2.882141590118408) + | > loss_feat: 6.71220588684082 (6.71220588684082) + | > loss_mel: 17.870912551879883 (17.870912551879883) + | > loss_duration: 1.7287452220916748 (1.7287452220916748) + | > loss_1: 31.379863739013672 (31.379863739013672) + + --> STEP: 2 + | > loss_disc: 2.5711607933044434 (2.5126994848251343) + | > loss_disc_real_0: 0.20636680722236633 (0.17110833525657654) + | > loss_disc_real_1: 0.22471608221530914 (0.22518256306648254) + | > loss_disc_real_2: 0.2434377521276474 (0.23914572596549988) + | > loss_disc_real_3: 0.22851362824440002 (0.24606828391551971) + | > loss_disc_real_4: 0.23204565048217773 (0.21831361949443817) + | > loss_disc_real_5: 0.2829209268093109 (0.24493137001991272) + | > loss_0: 2.5711607933044434 (2.5126994848251343) + | > loss_gen: 2.3254711627960205 (2.2556651830673218) + | > loss_kl: 2.3197808265686035 (2.600961208343506) + | > loss_feat: 7.143837928771973 (6.9280219078063965) + | > loss_mel: 17.849584579467773 (17.860248565673828) + | > loss_duration: 1.6928404569625854 (1.7107928395271301) + | > loss_1: 31.33151626586914 (31.355690002441406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035129785537719727 (+0.0002586841583251953) + | > avg_loss_disc: 2.5126994848251343 (-0.05892646312713623) + | > avg_loss_disc_real_0: 0.17110833525657654 (+0.013488702476024628) + | > avg_loss_disc_real_1: 0.22518256306648254 (-0.014120787382125854) + | > avg_loss_disc_real_2: 0.23914572596549988 (-0.0038166195154190063) + | > avg_loss_disc_real_3: 0.24606828391551971 (-0.03414298593997955) + | > avg_loss_disc_real_4: 0.21831361949443817 (-0.04708009958267212) + | > avg_loss_disc_real_5: 0.24493137001991272 (-0.00609714537858963) + | > avg_loss_0: 2.5126994848251343 (-0.05892646312713623) + | > avg_loss_gen: 2.2556651830673218 (-0.06038689613342285) + | > avg_loss_kl: 2.600961208343506 (+0.07238888740539551) + | > avg_loss_feat: 6.9280219078063965 (-0.2409675121307373) + | > avg_loss_mel: 17.860248565673828 (-0.42090415954589844) + | > avg_loss_duration: 1.7107928395271301 (-0.010555267333984375) + | > avg_loss_1: 31.355690002441406 (-0.6604242324829102) + + + > EPOCH: 309/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:04:00)  + + --> TIME: 2025-04-19 04:04:05 -- STEP: 15/142 -- GLOBAL_STEP: 423175 + | > loss_disc: 2.48184871673584 (2.448293987909953) + | > loss_disc_real_0: 0.14134684205055237 (0.12159354786078135) + | > loss_disc_real_1: 0.2104034274816513 (0.20120078027248384) + | > loss_disc_real_2: 0.20996083319187164 (0.2284710228443146) + | > loss_disc_real_3: 0.22404494881629944 (0.2381052871545156) + | > loss_disc_real_4: 0.20994509756565094 (0.23789529204368592) + | > loss_disc_real_5: 0.21174734830856323 (0.2313842495282491) + | > loss_0: 2.48184871673584 (2.448293987909953) + | > grad_norm_0: tensor(8.0812, device='cuda:0') (tensor(9.4739, device='cuda:0')) + | > loss_gen: 2.252922534942627 (2.420395755767822) + | > loss_kl: 1.7594224214553833 (1.7931937297185263) + | > loss_feat: 7.3147382736206055 (7.417305628458659) + | > loss_mel: 16.612394332885742 (17.990157826741537) + | > loss_duration: 1.4178788661956787 (1.4154847621917725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.35735511779785 (31.03653767903646) + | > grad_norm_1: tensor(301.5759, device='cuda:0') (tensor(249.4376, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.333 (0.32778682708740237) + | > loader_time: 0.0051 (0.004736884435017904) + + + --> TIME: 2025-04-19 04:04:14 -- STEP: 40/142 -- GLOBAL_STEP: 423200 + | > loss_disc: 2.4039101600646973 (2.4389478802680964) + | > loss_disc_real_0: 0.16409364342689514 (0.11941799893975258) + | > loss_disc_real_1: 0.21594864130020142 (0.20604615211486815) + | > loss_disc_real_2: 0.2301974594593048 (0.2234004981815815) + | > loss_disc_real_3: 0.23171867430210114 (0.23045929335057735) + | > loss_disc_real_4: 0.2445681244134903 (0.22849209159612655) + | > loss_disc_real_5: 0.2298559695482254 (0.231360624358058) + | > loss_0: 2.4039101600646973 (2.4389478802680964) + | > grad_norm_0: tensor(7.6749, device='cuda:0') (tensor(8.7474, device='cuda:0')) + | > loss_gen: 2.3759281635284424 (2.3979022145271296) + | > loss_kl: 2.000854253768921 (1.81308071911335) + | > loss_feat: 6.4822821617126465 (7.313011300563812) + | > loss_mel: 17.596820831298828 (18.029272985458373) + | > loss_duration: 1.3993960618972778 (1.428620654344559) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.855281829833984 (30.98188796043396) + | > grad_norm_1: tensor(116.4864, device='cuda:0') (tensor(213.2611, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.3615 (0.3408265471458435) + | > loader_time: 0.0062 (0.0053287982940673825) + + + --> TIME: 2025-04-19 04:04:24 -- STEP: 65/142 -- GLOBAL_STEP: 423225 + | > loss_disc: 2.6614012718200684 (2.4479132102085983) + | > loss_disc_real_0: 0.1428094208240509 (0.12174626772220318) + | > loss_disc_real_1: 0.22673918306827545 (0.2065726122030845) + | > loss_disc_real_2: 0.24274058640003204 (0.22378748586544625) + | > loss_disc_real_3: 0.24333463609218597 (0.2306244542965522) + | > loss_disc_real_4: 0.23079241812229156 (0.22953983957950885) + | > loss_disc_real_5: 0.24398936331272125 (0.2316427985062966) + | > loss_0: 2.6614012718200684 (2.4479132102085983) + | > grad_norm_0: tensor(11.3081, device='cuda:0') (tensor(8.2566, device='cuda:0')) + | > loss_gen: 2.3057713508605957 (2.3873518320230325) + | > loss_kl: 1.664899468421936 (1.8301599704302274) + | > loss_feat: 6.638454437255859 (7.25332701756404) + | > loss_mel: 17.082843780517578 (18.003509110670826) + | > loss_duration: 1.4674136638641357 (1.441326636534471) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.159381866455078 (30.9156745323768) + | > grad_norm_1: tensor(123.3018, device='cuda:0') (tensor(187.0068, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.3908 (0.353020873436561) + | > loader_time: 0.0066 (0.005593112798837515) + + + --> TIME: 2025-04-19 04:04:34 -- STEP: 90/142 -- GLOBAL_STEP: 423250 + | > loss_disc: 2.4246973991394043 (2.456966553794012) + | > loss_disc_real_0: 0.12796969711780548 (0.12324371387561163) + | > loss_disc_real_1: 0.15621232986450195 (0.20727247777912353) + | > loss_disc_real_2: 0.2515583336353302 (0.22566090077161788) + | > loss_disc_real_3: 0.19778788089752197 (0.2293277242117458) + | > loss_disc_real_4: 0.2445409595966339 (0.22937998788224326) + | > loss_disc_real_5: 0.21771498024463654 (0.23231243871980242) + | > loss_0: 2.4246973991394043 (2.456966553794012) + | > grad_norm_0: tensor(10.5318, device='cuda:0') (tensor(8.4305, device='cuda:0')) + | > loss_gen: 2.325498104095459 (2.3718875620100226) + | > loss_kl: 1.9208937883377075 (1.8475231647491455) + | > loss_feat: 6.883870601654053 (7.160559219784207) + | > loss_mel: 17.584871292114258 (17.94418307410347) + | > loss_duration: 1.4888839721679688 (1.4518805861473083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.20401954650879 (30.776033613416885) + | > grad_norm_1: tensor(357.3428, device='cuda:0') (tensor(191.5869, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.4037 (0.36402423646714965) + | > loader_time: 0.0071 (0.005843583742777506) + + + --> TIME: 2025-04-19 04:04:44 -- STEP: 115/142 -- GLOBAL_STEP: 423275 + | > loss_disc: 2.3688621520996094 (2.4542179149130114) + | > loss_disc_real_0: 0.13156434893608093 (0.12402646191742109) + | > loss_disc_real_1: 0.21646565198898315 (0.2074453153040098) + | > loss_disc_real_2: 0.19739973545074463 (0.2240533603274304) + | > loss_disc_real_3: 0.2249772995710373 (0.22910588215226713) + | > loss_disc_real_4: 0.19718334078788757 (0.22905189667059028) + | > loss_disc_real_5: 0.20776744186878204 (0.23175181629865066) + | > loss_0: 2.3688621520996094 (2.4542179149130114) + | > grad_norm_0: tensor(9.2703, device='cuda:0') (tensor(9.4170, device='cuda:0')) + | > loss_gen: 2.3771469593048096 (2.369811653054279) + | > loss_kl: 1.9207292795181274 (1.858487350007762) + | > loss_feat: 7.119823455810547 (7.163942656309708) + | > loss_mel: 18.0582332611084 (17.93118473550548) + | > loss_duration: 1.4894373416900635 (1.4591935676077137) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.965370178222656 (30.782619924130646) + | > grad_norm_1: tensor(255.8057, device='cuda:0') (tensor(217.2299, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.4221 (0.37460712557253645) + | > loader_time: 0.0079 (0.006063967165739639) + + + --> TIME: 2025-04-19 04:04:56 -- STEP: 140/142 -- GLOBAL_STEP: 423300 + | > loss_disc: 2.5516233444213867 (2.4539573567254203) + | > loss_disc_real_0: 0.07634838670492172 (0.12422830531639713) + | > loss_disc_real_1: 0.20928983390331268 (0.207851842045784) + | > loss_disc_real_2: 0.21772165596485138 (0.2240138420036861) + | > loss_disc_real_3: 0.24533087015151978 (0.228567131289414) + | > loss_disc_real_4: 0.24661312997341156 (0.2291773886552879) + | > loss_disc_real_5: 0.22262096405029297 (0.2311307480292661) + | > loss_0: 2.5516233444213867 (2.4539573567254203) + | > grad_norm_0: tensor(13.1273, device='cuda:0') (tensor(9.6429, device='cuda:0')) + | > loss_gen: 2.2646548748016357 (2.368650690146855) + | > loss_kl: 1.8881739377975464 (1.8653276171003073) + | > loss_feat: 6.35380220413208 (7.1510863099779405) + | > loss_mel: 16.853607177734375 (17.87749308177403) + | > loss_duration: 1.531653642654419 (1.4657024902956826) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.891891479492188 (30.72826009477888) + | > grad_norm_1: tensor(353.5331, device='cuda:0') (tensor(222.9163, device='cuda:0')) + | > current_lr_0: 0.00019242182210118784 + | > current_lr_1: 0.00019242182210118784 + | > step_time: 0.4703 (0.3873482840401787) + | > loader_time: 0.0078 (0.006349200861794608) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5114030838012695 (2.5114030838012695) + | > loss_disc_real_0: 0.20900094509124756 (0.20900094509124756) + | > loss_disc_real_1: 0.21674001216888428 (0.21674001216888428) + | > loss_disc_real_2: 0.2082195281982422 (0.2082195281982422) + | > loss_disc_real_3: 0.2490597665309906 (0.2490597665309906) + | > loss_disc_real_4: 0.24896197021007538 (0.24896197021007538) + | > loss_disc_real_5: 0.23869696259498596 (0.23869696259498596) + | > loss_0: 2.5114030838012695 (2.5114030838012695) + | > loss_gen: 2.464797019958496 (2.464797019958496) + | > loss_kl: 2.446706771850586 (2.446706771850586) + | > loss_feat: 6.9428324699401855 (6.9428324699401855) + | > loss_mel: 20.11979103088379 (20.11979103088379) + | > loss_duration: 1.7422300577163696 (1.7422300577163696) + | > loss_1: 33.71635818481445 (33.71635818481445) + + --> STEP: 1 + | > loss_disc: 2.6037840843200684 (2.6037840843200684) + | > loss_disc_real_0: 0.21214638650417328 (0.21214638650417328) + | > loss_disc_real_1: 0.20962519943714142 (0.20962519943714142) + | > loss_disc_real_2: 0.26769229769706726 (0.26769229769706726) + | > loss_disc_real_3: 0.24805355072021484 (0.24805355072021484) + | > loss_disc_real_4: 0.24077868461608887 (0.24077868461608887) + | > loss_disc_real_5: 0.21068137884140015 (0.21068137884140015) + | > loss_0: 2.6037840843200684 (2.6037840843200684) + | > loss_gen: 2.219017744064331 (2.219017744064331) + | > loss_kl: 2.731292963027954 (2.731292963027954) + | > loss_feat: 5.081083297729492 (5.081083297729492) + | > loss_mel: 15.538107872009277 (15.538107872009277) + | > loss_duration: 1.7579405307769775 (1.7579405307769775) + | > loss_1: 27.32744026184082 (27.32744026184082) + + --> STEP: 2 + | > loss_disc: 2.548816442489624 (2.576300263404846) + | > loss_disc_real_0: 0.2155778408050537 (0.2138621136546135) + | > loss_disc_real_1: 0.22698135673999786 (0.21830327808856964) + | > loss_disc_real_2: 0.21936501562595367 (0.24352865666151047) + | > loss_disc_real_3: 0.21435700356960297 (0.2312052771449089) + | > loss_disc_real_4: 0.22565965354442596 (0.23321916908025742) + | > loss_disc_real_5: 0.2567494213581085 (0.23371540009975433) + | > loss_0: 2.548816442489624 (2.576300263404846) + | > loss_gen: 2.317082166671753 (2.268049955368042) + | > loss_kl: 2.1719858646392822 (2.451639413833618) + | > loss_feat: 7.115297794342041 (6.098190546035767) + | > loss_mel: 16.795455932617188 (16.166781902313232) + | > loss_duration: 1.6782790422439575 (1.7181097865104675) + | > loss_1: 30.078102111816406 (28.702771186828613) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004155635833740234 (+0.0006426572799682617) + | > avg_loss_disc: 2.576300263404846 (+0.06360077857971191) + | > avg_loss_disc_real_0: 0.2138621136546135 (+0.04275377839803696) + | > avg_loss_disc_real_1: 0.21830327808856964 (-0.006879284977912903) + | > avg_loss_disc_real_2: 0.24352865666151047 (+0.00438293069601059) + | > avg_loss_disc_real_3: 0.2312052771449089 (-0.01486300677061081) + | > avg_loss_disc_real_4: 0.23321916908025742 (+0.014905549585819244) + | > avg_loss_disc_real_5: 0.23371540009975433 (-0.011215969920158386) + | > avg_loss_0: 2.576300263404846 (+0.06360077857971191) + | > avg_loss_gen: 2.268049955368042 (+0.012384772300720215) + | > avg_loss_kl: 2.451639413833618 (-0.1493217945098877) + | > avg_loss_feat: 6.098190546035767 (-0.8298313617706299) + | > avg_loss_mel: 16.166781902313232 (-1.6934666633605957) + | > avg_loss_duration: 1.7181097865104675 (+0.007316946983337402) + | > avg_loss_1: 28.702771186828613 (-2.652918815612793) + + + > EPOCH: 310/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:04:58)  + + --> TIME: 2025-04-19 04:05:07 -- STEP: 23/142 -- GLOBAL_STEP: 423325 + | > loss_disc: 2.3039462566375732 (2.449417632559072) + | > loss_disc_real_0: 0.11326779425144196 (0.1194034670682057) + | > loss_disc_real_1: 0.18240928649902344 (0.20470508933067322) + | > loss_disc_real_2: 0.2184734046459198 (0.23097807816837146) + | > loss_disc_real_3: 0.20195908844470978 (0.23473510275716367) + | > loss_disc_real_4: 0.2250697761774063 (0.22595036936842877) + | > loss_disc_real_5: 0.21863581240177155 (0.23567155327485956) + | > loss_0: 2.3039462566375732 (2.449417632559072) + | > grad_norm_0: tensor(26.9090, device='cuda:0') (tensor(14.8970, device='cuda:0')) + | > loss_gen: 2.4538254737854004 (2.441417994706527) + | > loss_kl: 1.8951493501663208 (1.8033013862112295) + | > loss_feat: 8.62077808380127 (7.330172082652217) + | > loss_mel: 19.37013816833496 (17.842303566310715) + | > loss_duration: 1.4565303325653076 (1.418250172034554) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.796424865722656 (30.83544532112453) + | > grad_norm_1: tensor(531.1606, device='cuda:0') (tensor(301.8273, device='cuda:0')) + | > current_lr_0: 0.00019239776937342518 + | > current_lr_1: 0.00019239776937342518 + | > step_time: 0.3472 (0.332848134248153) + | > loader_time: 0.0056 (0.004977848218834919) + + + --> TIME: 2025-04-19 04:05:16 -- STEP: 48/142 -- GLOBAL_STEP: 423350 + | > loss_disc: 2.4605727195739746 (2.440453370412192) + | > loss_disc_real_0: 0.10767114162445068 (0.11770633817650378) + | > loss_disc_real_1: 0.22640125453472137 (0.2056319114441673) + | > loss_disc_real_2: 0.22946776449680328 (0.22762765362858772) + | > loss_disc_real_3: 0.24395687878131866 (0.2297205232704679) + | > loss_disc_real_4: 0.22782756388187408 (0.22900674771517515) + | > loss_disc_real_5: 0.2360384315252304 (0.23046705716600022) + | > loss_0: 2.4605727195739746 (2.440453370412192) + | > grad_norm_0: tensor(3.7340, device='cuda:0') (tensor(13.5272, device='cuda:0')) + | > loss_gen: 2.4368276596069336 (2.4206305692593246) + | > loss_kl: 1.9832664728164673 (1.8206973572572072) + | > loss_feat: 7.470747947692871 (7.282507807016373) + | > loss_mel: 19.336515426635742 (17.84633068243663) + | > loss_duration: 1.431058645248413 (1.4297442783912022) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.658416748046875 (30.79991074403127) + | > grad_norm_1: tensor(150.8903, device='cuda:0') (tensor(303.5378, device='cuda:0')) + | > current_lr_0: 0.00019239776937342518 + | > current_lr_1: 0.00019239776937342518 + | > step_time: 0.3643 (0.34534088770548504) + | > loader_time: 0.0059 (0.005462308724721272) + + + --> TIME: 2025-04-19 04:05:26 -- STEP: 73/142 -- GLOBAL_STEP: 423375 + | > loss_disc: 2.469695568084717 (2.4543515100871063) + | > loss_disc_real_0: 0.136561781167984 (0.1232294262169975) + | > loss_disc_real_1: 0.2499706894159317 (0.2070150373324956) + | > loss_disc_real_2: 0.21052086353302002 (0.22822466349765047) + | > loss_disc_real_3: 0.21794667840003967 (0.22914720030680094) + | > loss_disc_real_4: 0.20672322809696198 (0.22959196812485996) + | > loss_disc_real_5: 0.24184143543243408 (0.2303563319656947) + | > loss_0: 2.469695568084717 (2.4543515100871063) + | > grad_norm_0: tensor(11.6370, device='cuda:0') (tensor(13.5138, device='cuda:0')) + | > loss_gen: 2.3589701652526855 (2.3922211438009175) + | > loss_kl: 1.8843215703964233 (1.8393799883045563) + | > loss_feat: 6.9063191413879395 (7.159606489416671) + | > loss_mel: 18.391422271728516 (17.771617575867534) + | > loss_duration: 1.4822652339935303 (1.44245621112928) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.023300170898438 (30.60528151629722) + | > grad_norm_1: tensor(312.3131, device='cuda:0') (tensor(288.2260, device='cuda:0')) + | > current_lr_0: 0.00019239776937342518 + | > current_lr_1: 0.00019239776937342518 + | > step_time: 0.3951 (0.3573992611610727) + | > loader_time: 0.0065 (0.005791438768987786) + + + --> TIME: 2025-04-19 04:05:36 -- STEP: 98/142 -- GLOBAL_STEP: 423400 + | > loss_disc: 2.4769716262817383 (2.4558632398138243) + | > loss_disc_real_0: 0.1525942087173462 (0.12527553938633326) + | > loss_disc_real_1: 0.23221731185913086 (0.20660359914205512) + | > loss_disc_real_2: 0.23734822869300842 (0.22712876602095) + | > loss_disc_real_3: 0.2536628246307373 (0.22906059604518267) + | > loss_disc_real_4: 0.25346529483795166 (0.22937886036780417) + | > loss_disc_real_5: 0.225876584649086 (0.22938755048172815) + | > loss_0: 2.4769716262817383 (2.4558632398138243) + | > grad_norm_0: tensor(12.4926, device='cuda:0') (tensor(12.7956, device='cuda:0')) + | > loss_gen: 2.389389991760254 (2.381735487860075) + | > loss_kl: 1.9997047185897827 (1.856904114995684) + | > loss_feat: 7.918424606323242 (7.1591077580743905) + | > loss_mel: 17.639060974121094 (17.77683374833087) + | > loss_duration: 1.4639182090759277 (1.4535034688151613) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.410497665405273 (30.628084630382304) + | > grad_norm_1: tensor(289.6938, device='cuda:0') (tensor(285.5680, device='cuda:0')) + | > current_lr_0: 0.00019239776937342518 + | > current_lr_1: 0.00019239776937342518 + | > step_time: 0.4098 (0.3681128000726505) + | > loader_time: 0.0068 (0.005990787428252551) + + + --> TIME: 2025-04-19 04:05:47 -- STEP: 123/142 -- GLOBAL_STEP: 423425 + | > loss_disc: 2.429387092590332 (2.4551204743424093) + | > loss_disc_real_0: 0.0916844978928566 (0.126406197655734) + | > loss_disc_real_1: 0.17023865878582 (0.2068129760947654) + | > loss_disc_real_2: 0.21548041701316833 (0.2259649104946028) + | > loss_disc_real_3: 0.18849791586399078 (0.22792647730528823) + | > loss_disc_real_4: 0.21888495981693268 (0.22759203346279577) + | > loss_disc_real_5: 0.2274119257926941 (0.23011803457407448) + | > loss_0: 2.429387092590332 (2.4551204743424093) + | > grad_norm_0: tensor(9.0174, device='cuda:0') (tensor(13.0016, device='cuda:0')) + | > loss_gen: 2.4750828742980957 (2.378959766248376) + | > loss_kl: 1.9158202409744263 (1.8713461819703017) + | > loss_feat: 7.9702019691467285 (7.141592506470719) + | > loss_mel: 18.882465362548828 (17.74582323989247) + | > loss_duration: 1.4744257926940918 (1.46082391099232) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.717994689941406 (30.598545617204373) + | > grad_norm_1: tensor(329.0259, device='cuda:0') (tensor(288.1792, device='cuda:0')) + | > current_lr_0: 0.00019239776937342518 + | > current_lr_1: 0.00019239776937342518 + | > step_time: 0.4381 (0.379451507475318) + | > loader_time: 0.0069 (0.006242608636375365) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.687866687774658 (2.687866687774658) + | > loss_disc_real_0: 0.17662201821804047 (0.17662201821804047) + | > loss_disc_real_1: 0.23739230632781982 (0.23739230632781982) + | > loss_disc_real_2: 0.25282442569732666 (0.25282442569732666) + | > loss_disc_real_3: 0.26850810647010803 (0.26850810647010803) + | > loss_disc_real_4: 0.23497949540615082 (0.23497949540615082) + | > loss_disc_real_5: 0.25486400723457336 (0.25486400723457336) + | > loss_0: 2.687866687774658 (2.687866687774658) + | > loss_gen: 2.077579975128174 (2.077579975128174) + | > loss_kl: 2.294872760772705 (2.294872760772705) + | > loss_feat: 4.614765167236328 (4.614765167236328) + | > loss_mel: 14.939248085021973 (14.939248085021973) + | > loss_duration: 1.725840449333191 (1.725840449333191) + | > loss_1: 25.652305603027344 (25.652305603027344) + + --> STEP: 1 + | > loss_disc: 2.5371875762939453 (2.5371875762939453) + | > loss_disc_real_0: 0.1512008160352707 (0.1512008160352707) + | > loss_disc_real_1: 0.26989731192588806 (0.26989731192588806) + | > loss_disc_real_2: 0.2519276738166809 (0.2519276738166809) + | > loss_disc_real_3: 0.20425623655319214 (0.20425623655319214) + | > loss_disc_real_4: 0.22654423117637634 (0.22654423117637634) + | > loss_disc_real_5: 0.24347242712974548 (0.24347242712974548) + | > loss_0: 2.5371875762939453 (2.5371875762939453) + | > loss_gen: 2.269500970840454 (2.269500970840454) + | > loss_kl: 2.699561595916748 (2.699561595916748) + | > loss_feat: 7.576027870178223 (7.576027870178223) + | > loss_mel: 18.102697372436523 (18.102697372436523) + | > loss_duration: 1.7739286422729492 (1.7739286422729492) + | > loss_1: 32.421714782714844 (32.421714782714844) + + --> STEP: 2 + | > loss_disc: 2.6975483894348145 (2.61736798286438) + | > loss_disc_real_0: 0.21864154934883118 (0.18492118269205093) + | > loss_disc_real_1: 0.25962740182876587 (0.26476235687732697) + | > loss_disc_real_2: 0.2897418737411499 (0.2708347737789154) + | > loss_disc_real_3: 0.22360531985759735 (0.21393077820539474) + | > loss_disc_real_4: 0.21978731453418732 (0.22316577285528183) + | > loss_disc_real_5: 0.21145126223564148 (0.22746184468269348) + | > loss_0: 2.6975483894348145 (2.61736798286438) + | > loss_gen: 2.3280787467956543 (2.298789858818054) + | > loss_kl: 2.3472046852111816 (2.523383140563965) + | > loss_feat: 7.490444660186768 (7.533236265182495) + | > loss_mel: 18.633777618408203 (18.368237495422363) + | > loss_duration: 1.7018258571624756 (1.7378772497177124) + | > loss_1: 32.5013313293457 (32.46152305603027) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003835916519165039 (-0.0003197193145751953) + | > avg_loss_disc: 2.61736798286438 (+0.04106771945953369) + | > avg_loss_disc_real_0: 0.18492118269205093 (-0.02894093096256256) + | > avg_loss_disc_real_1: 0.26476235687732697 (+0.046459078788757324) + | > avg_loss_disc_real_2: 0.2708347737789154 (+0.027306117117404938) + | > avg_loss_disc_real_3: 0.21393077820539474 (-0.01727449893951416) + | > avg_loss_disc_real_4: 0.22316577285528183 (-0.010053396224975586) + | > avg_loss_disc_real_5: 0.22746184468269348 (-0.006253555417060852) + | > avg_loss_0: 2.61736798286438 (+0.04106771945953369) + | > avg_loss_gen: 2.298789858818054 (+0.030739903450012207) + | > avg_loss_kl: 2.523383140563965 (+0.07174372673034668) + | > avg_loss_feat: 7.533236265182495 (+1.4350457191467285) + | > avg_loss_mel: 18.368237495422363 (+2.201455593109131) + | > avg_loss_duration: 1.7378772497177124 (+0.019767463207244873) + | > avg_loss_1: 32.46152305603027 (+3.75875186920166) + + + > EPOCH: 311/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:05:57)  + + --> TIME: 2025-04-19 04:05:59 -- STEP: 6/142 -- GLOBAL_STEP: 423450 + | > loss_disc: 2.604633092880249 (2.576394955317179) + | > loss_disc_real_0: 0.18288388848304749 (0.1378601205845674) + | > loss_disc_real_1: 0.2003346085548401 (0.2155764972170194) + | > loss_disc_real_2: 0.25959062576293945 (0.24265079696973166) + | > loss_disc_real_3: 0.23610758781433105 (0.2486781825621923) + | > loss_disc_real_4: 0.25054168701171875 (0.26117441554864246) + | > loss_disc_real_5: 0.23190443217754364 (0.23433664441108704) + | > loss_0: 2.604633092880249 (2.576394955317179) + | > grad_norm_0: tensor(14.1887, device='cuda:0') (tensor(14.2744, device='cuda:0')) + | > loss_gen: 2.290520668029785 (2.3692182302474976) + | > loss_kl: 1.7209924459457397 (1.8380163510640461) + | > loss_feat: 7.491158485412598 (7.324361642201741) + | > loss_mel: 17.68512535095215 (17.89831765492757) + | > loss_duration: 1.3953574895858765 (1.4202971657117207) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.583154678344727 (30.850210507710774) + | > grad_norm_1: tensor(513.8184, device='cuda:0') (tensor(348.8942, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.3313 (0.32415024439493817) + | > loader_time: 0.0044 (0.004573504130045573) + + + --> TIME: 2025-04-19 04:06:08 -- STEP: 31/142 -- GLOBAL_STEP: 423475 + | > loss_disc: 2.358367919921875 (2.4598849973370953) + | > loss_disc_real_0: 0.0692070722579956 (0.11814724918334715) + | > loss_disc_real_1: 0.21346847712993622 (0.20921883515773282) + | > loss_disc_real_2: 0.18029426038265228 (0.22492932696496287) + | > loss_disc_real_3: 0.21257030963897705 (0.2289594950214509) + | > loss_disc_real_4: 0.2012433409690857 (0.23221045203747287) + | > loss_disc_real_5: 0.21788713335990906 (0.23291402430303634) + | > loss_0: 2.358367919921875 (2.4598849973370953) + | > grad_norm_0: tensor(8.4253, device='cuda:0') (tensor(11.7265, device='cuda:0')) + | > loss_gen: 2.3431553840637207 (2.3663403603338433) + | > loss_kl: 1.921307921409607 (1.8607844614213513) + | > loss_feat: 7.972024440765381 (7.2424239958486245) + | > loss_mel: 18.359533309936523 (17.749467295985067) + | > loss_duration: 1.414701223373413 (1.4292183153090938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.010719299316406 (30.64823445966167) + | > grad_norm_1: tensor(331.1553, device='cuda:0') (tensor(334.2384, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.3514 (0.3361028702028336) + | > loader_time: 0.0058 (0.005154255897768082) + + + --> TIME: 2025-04-19 04:06:17 -- STEP: 56/142 -- GLOBAL_STEP: 423500 + | > loss_disc: 2.5388970375061035 (2.454762007508959) + | > loss_disc_real_0: 0.13012802600860596 (0.11957781695361648) + | > loss_disc_real_1: 0.22065679728984833 (0.208345037485872) + | > loss_disc_real_2: 0.23174948990345 (0.2255464390452419) + | > loss_disc_real_3: 0.24140967428684235 (0.22970686294138432) + | > loss_disc_real_4: 0.23648950457572937 (0.23255776773606027) + | > loss_disc_real_5: 0.2518875300884247 (0.231429981599961) + | > loss_0: 2.5388970375061035 (2.454762007508959) + | > grad_norm_0: tensor(16.3618, device='cuda:0') (tensor(12.5139, device='cuda:0')) + | > loss_gen: 2.2392466068267822 (2.373568845646723) + | > loss_kl: 1.9428075551986694 (1.8683274877922875) + | > loss_feat: 6.732137203216553 (7.222199763570513) + | > loss_mel: 17.305082321166992 (17.754391738346644) + | > loss_duration: 1.4593912363052368 (1.4397499625171932) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.67866325378418 (30.658237627574376) + | > grad_norm_1: tensor(220.7632, device='cuda:0') (tensor(331.9052, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.373 (0.3481569460460118) + | > loader_time: 0.0063 (0.005542107990809849) + + + --> TIME: 2025-04-19 04:06:27 -- STEP: 81/142 -- GLOBAL_STEP: 423525 + | > loss_disc: 2.5596885681152344 (2.452514598399032) + | > loss_disc_real_0: 0.11883622407913208 (0.11990922956187049) + | > loss_disc_real_1: 0.24254749715328217 (0.20747514602578718) + | > loss_disc_real_2: 0.23925916850566864 (0.22482163339485356) + | > loss_disc_real_3: 0.23754765093326569 (0.22913704610165256) + | > loss_disc_real_4: 0.24343828856945038 (0.23112884292631974) + | > loss_disc_real_5: 0.2882416546344757 (0.2313326472117577) + | > loss_0: 2.5596885681152344 (2.452514598399032) + | > grad_norm_0: tensor(16.8550, device='cuda:0') (tensor(11.7632, device='cuda:0')) + | > loss_gen: 2.3619420528411865 (2.367124218999604) + | > loss_kl: 1.8682490587234497 (1.86320506349022) + | > loss_feat: 6.446217060089111 (7.204940171889317) + | > loss_mel: 17.355257034301758 (17.73895595691823) + | > loss_duration: 1.4598934650421143 (1.4500570238372423) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.491559982299805 (30.624282365963783) + | > grad_norm_1: tensor(334.2458, device='cuda:0') (tensor(305.4587, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.3964 (0.3610067073209786) + | > loader_time: 0.0067 (0.005865868226981456) + + + --> TIME: 2025-04-19 04:06:38 -- STEP: 106/142 -- GLOBAL_STEP: 423550 + | > loss_disc: 2.4434404373168945 (2.4485751412949472) + | > loss_disc_real_0: 0.11825151741504669 (0.12057795249066262) + | > loss_disc_real_1: 0.20368976891040802 (0.2058383492085169) + | > loss_disc_real_2: 0.22322331368923187 (0.2247094340481848) + | > loss_disc_real_3: 0.2431860864162445 (0.2291613444685936) + | > loss_disc_real_4: 0.22929739952087402 (0.22994852881386596) + | > loss_disc_real_5: 0.22362291812896729 (0.23084576652859742) + | > loss_0: 2.4434404373168945 (2.4485751412949472) + | > grad_norm_0: tensor(5.5882, device='cuda:0') (tensor(11.5245, device='cuda:0')) + | > loss_gen: 2.498352527618408 (2.3656499858172433) + | > loss_kl: 1.8959219455718994 (1.8697972308914617) + | > loss_feat: 6.69858455657959 (7.149892136735736) + | > loss_mel: 17.022003173828125 (17.65724452036732) + | > loss_duration: 1.4796477556228638 (1.4590264817453775) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.59450912475586 (30.50161028808018) + | > grad_norm_1: tensor(180.5028, device='cuda:0') (tensor(298.3301, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.4167 (0.3723060095085288) + | > loader_time: 0.0071 (0.006128030003241771) + + + --> TIME: 2025-04-19 04:06:49 -- STEP: 131/142 -- GLOBAL_STEP: 423575 + | > loss_disc: 2.5691661834716797 (2.4600838941472176) + | > loss_disc_real_0: 0.12118718773126602 (0.1222956843162311) + | > loss_disc_real_1: 0.24280834197998047 (0.20774757543592962) + | > loss_disc_real_2: 0.25228506326675415 (0.22604777408010177) + | > loss_disc_real_3: 0.24884939193725586 (0.229429911338646) + | > loss_disc_real_4: 0.28102824091911316 (0.2307468942785991) + | > loss_disc_real_5: 0.24176962673664093 (0.2306855150306498) + | > loss_0: 2.5691661834716797 (2.4600838941472176) + | > grad_norm_0: tensor(11.0715, device='cuda:0') (tensor(10.9807, device='cuda:0')) + | > loss_gen: 2.4438631534576416 (2.3589695591962974) + | > loss_kl: 1.7463139295578003 (1.870211670416912) + | > loss_feat: 7.202999114990234 (7.089162051222707) + | > loss_mel: 18.925189971923828 (17.624905957520465) + | > loss_duration: 1.490250587463379 (1.465198978212953) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.808616638183594 (30.40844812465988) + | > grad_norm_1: tensor(313.3333, device='cuda:0') (tensor(281.0854, device='cuda:0')) + | > current_lr_0: 0.0001923737196522535 + | > current_lr_1: 0.0001923737196522535 + | > step_time: 0.4504 (0.3843312190689203) + | > loader_time: 0.0075 (0.006377010855055946) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4806196689605713 (2.4806196689605713) + | > loss_disc_real_0: 0.20400623977184296 (0.20400623977184296) + | > loss_disc_real_1: 0.20749184489250183 (0.20749184489250183) + | > loss_disc_real_2: 0.23647046089172363 (0.23647046089172363) + | > loss_disc_real_3: 0.26125937700271606 (0.26125937700271606) + | > loss_disc_real_4: 0.22872719168663025 (0.22872719168663025) + | > loss_disc_real_5: 0.245609849691391 (0.245609849691391) + | > loss_0: 2.4806196689605713 (2.4806196689605713) + | > loss_gen: 2.5185630321502686 (2.5185630321502686) + | > loss_kl: 2.4009995460510254 (2.4009995460510254) + | > loss_feat: 7.5000104904174805 (7.5000104904174805) + | > loss_mel: 18.992341995239258 (18.992341995239258) + | > loss_duration: 1.7368803024291992 (1.7368803024291992) + | > loss_1: 33.14879608154297 (33.14879608154297) + + --> STEP: 1 + | > loss_disc: 2.4580068588256836 (2.4580068588256836) + | > loss_disc_real_0: 0.2225896120071411 (0.2225896120071411) + | > loss_disc_real_1: 0.2044408619403839 (0.2044408619403839) + | > loss_disc_real_2: 0.22242608666419983 (0.22242608666419983) + | > loss_disc_real_3: 0.24879547953605652 (0.24879547953605652) + | > loss_disc_real_4: 0.25005418062210083 (0.25005418062210083) + | > loss_disc_real_5: 0.2439243346452713 (0.2439243346452713) + | > loss_0: 2.4580068588256836 (2.4580068588256836) + | > loss_gen: 2.576373815536499 (2.576373815536499) + | > loss_kl: 2.825805902481079 (2.825805902481079) + | > loss_feat: 7.172398567199707 (7.172398567199707) + | > loss_mel: 18.754594802856445 (18.754594802856445) + | > loss_duration: 1.7676329612731934 (1.7676329612731934) + | > loss_1: 33.096805572509766 (33.096805572509766) + + --> STEP: 2 + | > loss_disc: 2.661104679107666 (2.559555768966675) + | > loss_disc_real_0: 0.29184210300445557 (0.25721585750579834) + | > loss_disc_real_1: 0.20842328667640686 (0.20643207430839539) + | > loss_disc_real_2: 0.23114968836307526 (0.22678788751363754) + | > loss_disc_real_3: 0.3102279007434845 (0.2795116901397705) + | > loss_disc_real_4: 0.23502960801124573 (0.24254189431667328) + | > loss_disc_real_5: 0.36313876509666443 (0.30353154987096786) + | > loss_0: 2.661104679107666 (2.559555768966675) + | > loss_gen: 2.724919319152832 (2.6506465673446655) + | > loss_kl: 2.2375411987304688 (2.531673550605774) + | > loss_feat: 7.008124351501465 (7.090261459350586) + | > loss_mel: 18.753265380859375 (18.75393009185791) + | > loss_duration: 1.7117758989334106 (1.739704430103302) + | > loss_1: 32.43562698364258 (32.76621627807617) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035848617553710938 (-0.0002510547637939453) + | > avg_loss_disc: 2.559555768966675 (-0.05781221389770508) + | > avg_loss_disc_real_0: 0.25721585750579834 (+0.0722946748137474) + | > avg_loss_disc_real_1: 0.20643207430839539 (-0.05833028256893158) + | > avg_loss_disc_real_2: 0.22678788751363754 (-0.04404688626527786) + | > avg_loss_disc_real_3: 0.2795116901397705 (+0.06558091193437576) + | > avg_loss_disc_real_4: 0.24254189431667328 (+0.01937612146139145) + | > avg_loss_disc_real_5: 0.30353154987096786 (+0.07606970518827438) + | > avg_loss_0: 2.559555768966675 (-0.05781221389770508) + | > avg_loss_gen: 2.6506465673446655 (+0.35185670852661133) + | > avg_loss_kl: 2.531673550605774 (+0.008290410041809082) + | > avg_loss_feat: 7.090261459350586 (-0.4429748058319092) + | > avg_loss_mel: 18.75393009185791 (+0.3856925964355469) + | > avg_loss_duration: 1.739704430103302 (+0.0018271803855895996) + | > avg_loss_1: 32.76621627807617 (+0.30469322204589844) + + + > EPOCH: 312/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:06:55)  + + --> TIME: 2025-04-19 04:07:01 -- STEP: 14/142 -- GLOBAL_STEP: 423600 + | > loss_disc: 2.49440336227417 (2.428903136934553) + | > loss_disc_real_0: 0.10506708920001984 (0.11665211724383491) + | > loss_disc_real_1: 0.23385131359100342 (0.2009399767432894) + | > loss_disc_real_2: 0.24177731573581696 (0.2282268192086901) + | > loss_disc_real_3: 0.23876313865184784 (0.2319562147770609) + | > loss_disc_real_4: 0.25488805770874023 (0.2321748829313687) + | > loss_disc_real_5: 0.23271983861923218 (0.23097724254642213) + | > loss_0: 2.49440336227417 (2.428903136934553) + | > grad_norm_0: tensor(14.9454, device='cuda:0') (tensor(13.1458, device='cuda:0')) + | > loss_gen: 2.451528549194336 (2.4037847178322926) + | > loss_kl: 1.8749638795852661 (1.8162436144692558) + | > loss_feat: 7.6619343757629395 (7.484093053000314) + | > loss_mel: 17.04045295715332 (17.957953044346404) + | > loss_duration: 1.4077439308166504 (1.4137585844312395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.436622619628906 (31.075832503182546) + | > grad_norm_1: tensor(410.1494, device='cuda:0') (tensor(306.9652, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.3377 (0.333719049181257) + | > loader_time: 0.0055 (0.005124194281441825) + + + --> TIME: 2025-04-19 04:07:10 -- STEP: 39/142 -- GLOBAL_STEP: 423625 + | > loss_disc: 2.459503650665283 (2.4291117924910326) + | > loss_disc_real_0: 0.09775763005018234 (0.11732405042036986) + | > loss_disc_real_1: 0.16419917345046997 (0.20568226124995795) + | > loss_disc_real_2: 0.2431260198354721 (0.22579058813743103) + | > loss_disc_real_3: 0.2531687617301941 (0.22828453206098998) + | > loss_disc_real_4: 0.24648219347000122 (0.22721105890396315) + | > loss_disc_real_5: 0.26336732506752014 (0.23001277484954932) + | > loss_0: 2.459503650665283 (2.4291117924910326) + | > grad_norm_0: tensor(15.2870, device='cuda:0') (tensor(12.6801, device='cuda:0')) + | > loss_gen: 2.249695301055908 (2.3916561970343957) + | > loss_kl: 2.0071470737457275 (1.8473592568666508) + | > loss_feat: 6.68338680267334 (7.23383301955003) + | > loss_mel: 17.806875228881836 (17.627746582031246) + | > loss_duration: 1.4853389263153076 (1.4295778824732854) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.23244285583496 (30.53017298380534) + | > grad_norm_1: tensor(310.5716, device='cuda:0') (tensor(340.3871, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.3576 (0.3444349765777587) + | > loader_time: 0.0058 (0.005485870899298253) + + + --> TIME: 2025-04-19 04:07:19 -- STEP: 64/142 -- GLOBAL_STEP: 423650 + | > loss_disc: 2.5796589851379395 (2.4448776468634605) + | > loss_disc_real_0: 0.23456548154354095 (0.12167018419131637) + | > loss_disc_real_1: 0.17250028252601624 (0.20495838881470263) + | > loss_disc_real_2: 0.22849278151988983 (0.22513653780333698) + | > loss_disc_real_3: 0.22832338511943817 (0.2284684181213379) + | > loss_disc_real_4: 0.2411709576845169 (0.22896906756795943) + | > loss_disc_real_5: 0.2287648469209671 (0.23159719351679087) + | > loss_0: 2.5796589851379395 (2.4448776468634605) + | > grad_norm_0: tensor(38.8730, device='cuda:0') (tensor(14.4715, device='cuda:0')) + | > loss_gen: 2.230624198913574 (2.3972257897257805) + | > loss_kl: 1.8490626811981201 (1.8702835105359554) + | > loss_feat: 6.094953536987305 (7.218560464680195) + | > loss_mel: 18.190732955932617 (17.692611753940575) + | > loss_duration: 1.4858739376068115 (1.441198067739606) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.85124969482422 (30.619879692792892) + | > grad_norm_1: tensor(285.2291, device='cuda:0') (tensor(336.8630, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.4037 (0.3559797704219817) + | > loader_time: 0.0063 (0.005747016519308092) + + + --> TIME: 2025-04-19 04:07:30 -- STEP: 89/142 -- GLOBAL_STEP: 423675 + | > loss_disc: 2.4382259845733643 (2.450188985031642) + | > loss_disc_real_0: 0.11601953953504562 (0.12189573208602626) + | > loss_disc_real_1: 0.18495412170886993 (0.2054933982618739) + | > loss_disc_real_2: 0.2554306089878082 (0.22618820627083938) + | > loss_disc_real_3: 0.21481235325336456 (0.22826956898978587) + | > loss_disc_real_4: 0.23698316514492035 (0.2288361849744668) + | > loss_disc_real_5: 0.24158480763435364 (0.23194687219148272) + | > loss_0: 2.4382259845733643 (2.450188985031642) + | > grad_norm_0: tensor(15.7858, device='cuda:0') (tensor(15.3836, device='cuda:0')) + | > loss_gen: 2.3169379234313965 (2.40082408604997) + | > loss_kl: 1.7891275882720947 (1.881923789388678) + | > loss_feat: 7.181987285614014 (7.201349960284286) + | > loss_mel: 17.811738967895508 (17.75761915056892) + | > loss_duration: 1.4592396020889282 (1.4521923828660772) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.559032440185547 (30.693909462918057) + | > grad_norm_1: tensor(444.5703, device='cuda:0') (tensor(343.6192, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.4031 (0.3670757486579122) + | > loader_time: 0.0066 (0.006016195490119165) + + + --> TIME: 2025-04-19 04:07:40 -- STEP: 114/142 -- GLOBAL_STEP: 423700 + | > loss_disc: 2.5105233192443848 (2.450185470413744) + | > loss_disc_real_0: 0.1319706290960312 (0.12264354680583142) + | > loss_disc_real_1: 0.17905768752098083 (0.20559125668124148) + | > loss_disc_real_2: 0.20400753617286682 (0.22400260690534324) + | > loss_disc_real_3: 0.23137691617012024 (0.2280511734516997) + | > loss_disc_real_4: 0.20560549199581146 (0.22858424510872155) + | > loss_disc_real_5: 0.2256784290075302 (0.23069739054169572) + | > loss_0: 2.5105233192443848 (2.450185470413744) + | > grad_norm_0: tensor(8.3438, device='cuda:0') (tensor(15.4562, device='cuda:0')) + | > loss_gen: 2.2233946323394775 (2.3908155550036514) + | > loss_kl: 1.959275722503662 (1.8924087871584976) + | > loss_feat: 6.235482692718506 (7.168212886442218) + | > loss_mel: 15.69460391998291 (17.727948130222778) + | > loss_duration: 1.4661452770233154 (1.45952469319628) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.578901290893555 (30.638910142998945) + | > grad_norm_1: tensor(319.1063, device='cuda:0') (tensor(343.2089, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.425 (0.37793787111315796) + | > loader_time: 0.0074 (0.0062774503440187705) + + + --> TIME: 2025-04-19 04:07:52 -- STEP: 139/142 -- GLOBAL_STEP: 423725 + | > loss_disc: 2.4004979133605957 (2.4506739026350943) + | > loss_disc_real_0: 0.11043474823236465 (0.1235218171110685) + | > loss_disc_real_1: 0.20415925979614258 (0.20543046574369608) + | > loss_disc_real_2: 0.24134178459644318 (0.2250637050798471) + | > loss_disc_real_3: 0.1975756585597992 (0.2284069340006053) + | > loss_disc_real_4: 0.21521155536174774 (0.22800228149770832) + | > loss_disc_real_5: 0.25438371300697327 (0.23063711961396308) + | > loss_0: 2.4004979133605957 (2.4506739026350943) + | > grad_norm_0: tensor(19.7575, device='cuda:0') (tensor(14.5084, device='cuda:0')) + | > loss_gen: 2.481330394744873 (2.3817061228717833) + | > loss_kl: 1.8055493831634521 (1.8987036591811146) + | > loss_feat: 7.858473777770996 (7.156836763560343) + | > loss_mel: 17.801807403564453 (17.693152166956605) + | > loss_duration: 1.4919716119766235 (1.4660112257484053) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.43913459777832 (30.59641005152421) + | > grad_norm_1: tensor(507.0830, device='cuda:0') (tensor(322.5418, device='cuda:0')) + | > current_lr_0: 0.00019234967293729698 + | > current_lr_1: 0.00019234967293729698 + | > step_time: 0.4527 (0.38981123279324525) + | > loader_time: 0.0076 (0.006527926424424427) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.344048500061035 (2.344048500061035) + | > loss_disc_real_0: 0.1290145069360733 (0.1290145069360733) + | > loss_disc_real_1: 0.22654230892658234 (0.22654230892658234) + | > loss_disc_real_2: 0.2607770562171936 (0.2607770562171936) + | > loss_disc_real_3: 0.27145978808403015 (0.27145978808403015) + | > loss_disc_real_4: 0.27554604411125183 (0.27554604411125183) + | > loss_disc_real_5: 0.3050939440727234 (0.3050939440727234) + | > loss_0: 2.344048500061035 (2.344048500061035) + | > loss_gen: 2.910874366760254 (2.910874366760254) + | > loss_kl: 2.4818592071533203 (2.4818592071533203) + | > loss_feat: 7.236627578735352 (7.236627578735352) + | > loss_mel: 18.949138641357422 (18.949138641357422) + | > loss_duration: 1.7216562032699585 (1.7216562032699585) + | > loss_1: 33.30015563964844 (33.30015563964844) + + --> STEP: 1 + | > loss_disc: 2.621300220489502 (2.621300220489502) + | > loss_disc_real_0: 0.18338261544704437 (0.18338261544704437) + | > loss_disc_real_1: 0.2608735263347626 (0.2608735263347626) + | > loss_disc_real_2: 0.26334595680236816 (0.26334595680236816) + | > loss_disc_real_3: 0.2566066086292267 (0.2566066086292267) + | > loss_disc_real_4: 0.25484704971313477 (0.25484704971313477) + | > loss_disc_real_5: 0.24788546562194824 (0.24788546562194824) + | > loss_0: 2.621300220489502 (2.621300220489502) + | > loss_gen: 2.291630744934082 (2.291630744934082) + | > loss_kl: 2.636849880218506 (2.636849880218506) + | > loss_feat: 5.770971298217773 (5.770971298217773) + | > loss_mel: 16.659746170043945 (16.659746170043945) + | > loss_duration: 1.7805218696594238 (1.7805218696594238) + | > loss_1: 29.139720916748047 (29.139720916748047) + + --> STEP: 2 + | > loss_disc: 2.582913875579834 (2.602107048034668) + | > loss_disc_real_0: 0.17428618669509888 (0.17883440107107162) + | > loss_disc_real_1: 0.20493721961975098 (0.23290537297725677) + | > loss_disc_real_2: 0.24089978635311127 (0.2521228715777397) + | > loss_disc_real_3: 0.2329060286283493 (0.244756318628788) + | > loss_disc_real_4: 0.22367006540298462 (0.2392585575580597) + | > loss_disc_real_5: 0.20993225276470184 (0.22890885919332504) + | > loss_0: 2.582913875579834 (2.602107048034668) + | > loss_gen: 2.1907660961151123 (2.241198420524597) + | > loss_kl: 2.252690553665161 (2.4447702169418335) + | > loss_feat: 6.603536605834961 (6.187253952026367) + | > loss_mel: 16.054536819458008 (16.357141494750977) + | > loss_duration: 1.711442232131958 (1.745982050895691) + | > loss_1: 28.812971115112305 (28.976346015930176) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003388047218322754 (-0.00019681453704833984) + | > avg_loss_disc: 2.602107048034668 (+0.042551279067993164) + | > avg_loss_disc_real_0: 0.17883440107107162 (-0.07838145643472672) + | > avg_loss_disc_real_1: 0.23290537297725677 (+0.02647329866886139) + | > avg_loss_disc_real_2: 0.2521228715777397 (+0.025334984064102173) + | > avg_loss_disc_real_3: 0.244756318628788 (-0.03475537151098251) + | > avg_loss_disc_real_4: 0.2392585575580597 (-0.0032833367586135864) + | > avg_loss_disc_real_5: 0.22890885919332504 (-0.07462269067764282) + | > avg_loss_0: 2.602107048034668 (+0.042551279067993164) + | > avg_loss_gen: 2.241198420524597 (-0.40944814682006836) + | > avg_loss_kl: 2.4447702169418335 (-0.08690333366394043) + | > avg_loss_feat: 6.187253952026367 (-0.9030075073242188) + | > avg_loss_mel: 16.357141494750977 (-2.3967885971069336) + | > avg_loss_duration: 1.745982050895691 (+0.006277620792388916) + | > avg_loss_1: 28.976346015930176 (-3.789870262145996) + + + > EPOCH: 313/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:07:54)  + + --> TIME: 2025-04-19 04:08:02 -- STEP: 22/142 -- GLOBAL_STEP: 423750 + | > loss_disc: 2.500807046890259 (2.3974120616912846) + | > loss_disc_real_0: 0.12464604526758194 (0.10893476449630478) + | > loss_disc_real_1: 0.2389756143093109 (0.20382869040424173) + | > loss_disc_real_2: 0.24772191047668457 (0.22320634871721268) + | > loss_disc_real_3: 0.19808629155158997 (0.22067590057849884) + | > loss_disc_real_4: 0.21820025146007538 (0.2236795019019734) + | > loss_disc_real_5: 0.2497706413269043 (0.22937706993384796) + | > loss_0: 2.500807046890259 (2.3974120616912846) + | > grad_norm_0: tensor(19.1308, device='cuda:0') (tensor(10.5295, device='cuda:0')) + | > loss_gen: 2.4840173721313477 (2.4162485599517822) + | > loss_kl: 1.8642281293869019 (1.8258798230778088) + | > loss_feat: 7.694982528686523 (7.4575159116224805) + | > loss_mel: 18.39208221435547 (17.894984592090953) + | > loss_duration: 1.4077191352844238 (1.4177777225320989) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.843029022216797 (31.012406089089133) + | > grad_norm_1: tensor(409.4518, device='cuda:0') (tensor(220.3077, device='cuda:0')) + | > current_lr_0: 0.0001923256292281798 + | > current_lr_1: 0.0001923256292281798 + | > step_time: 0.352 (0.33331481976942584) + | > loader_time: 0.0052 (0.005032365972345526) + + + --> TIME: 2025-04-19 04:08:12 -- STEP: 47/142 -- GLOBAL_STEP: 423775 + | > loss_disc: 2.4376473426818848 (2.4221362661808095) + | > loss_disc_real_0: 0.11903039366006851 (0.11780222267546553) + | > loss_disc_real_1: 0.179872527718544 (0.20312623964979293) + | > loss_disc_real_2: 0.24699005484580994 (0.22439280469366846) + | > loss_disc_real_3: 0.23087669909000397 (0.22359434372567116) + | > loss_disc_real_4: 0.2201695442199707 (0.22604569792747498) + | > loss_disc_real_5: 0.20636597275733948 (0.2318112625086561) + | > loss_0: 2.4376473426818848 (2.4221362661808095) + | > grad_norm_0: tensor(4.5061, device='cuda:0') (tensor(8.8347, device='cuda:0')) + | > loss_gen: 2.3115575313568115 (2.3865100931614007) + | > loss_kl: 1.681986689567566 (1.8438652271920062) + | > loss_feat: 7.226202964782715 (7.293795311704595) + | > loss_mel: 16.92806053161621 (17.74849132781333) + | > loss_duration: 1.425089955329895 (1.4306584900998058) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.572898864746094 (30.703320036543175) + | > grad_norm_1: tensor(46.9755, device='cuda:0') (tensor(184.7987, device='cuda:0')) + | > current_lr_0: 0.0001923256292281798 + | > current_lr_1: 0.0001923256292281798 + | > step_time: 0.3713 (0.3493915770916229) + | > loader_time: 0.0061 (0.005495127211225794) + + + --> TIME: 2025-04-19 04:08:22 -- STEP: 72/142 -- GLOBAL_STEP: 423800 + | > loss_disc: 2.4018619060516357 (2.4361012147532564) + | > loss_disc_real_0: 0.16734127700328827 (0.12067409584091769) + | > loss_disc_real_1: 0.24346232414245605 (0.2050265566342407) + | > loss_disc_real_2: 0.22408002614974976 (0.22654764540493488) + | > loss_disc_real_3: 0.24808628857135773 (0.22746149409148428) + | > loss_disc_real_4: 0.21893958747386932 (0.22742901783850458) + | > loss_disc_real_5: 0.2403014600276947 (0.2320596884108252) + | > loss_0: 2.4018619060516357 (2.4361012147532564) + | > grad_norm_0: tensor(9.2308, device='cuda:0') (tensor(8.5315, device='cuda:0')) + | > loss_gen: 2.413111448287964 (2.3853850364685063) + | > loss_kl: 1.8464711904525757 (1.8460139599111345) + | > loss_feat: 7.373969554901123 (7.232708374659221) + | > loss_mel: 17.295455932617188 (17.66840471161736) + | > loss_duration: 1.4378547668457031 (1.4435037208928003) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.36686134338379 (30.576015419430203) + | > grad_norm_1: tensor(207.5117, device='cuda:0') (tensor(177.8350, device='cuda:0')) + | > current_lr_0: 0.0001923256292281798 + | > current_lr_1: 0.0001923256292281798 + | > step_time: 0.4043 (0.3635218706395891) + | > loader_time: 0.0065 (0.005805283784866332) + + + --> TIME: 2025-04-19 04:08:32 -- STEP: 97/142 -- GLOBAL_STEP: 423825 + | > loss_disc: 2.4680275917053223 (2.447087737702831) + | > loss_disc_real_0: 0.13867013156414032 (0.1223616782844681) + | > loss_disc_real_1: 0.21696992218494415 (0.20589629921716512) + | > loss_disc_real_2: 0.2254541665315628 (0.2270671190674772) + | > loss_disc_real_3: 0.26899033784866333 (0.2275291307992542) + | > loss_disc_real_4: 0.23219110071659088 (0.2292426830714511) + | > loss_disc_real_5: 0.2264915406703949 (0.2317469395005826) + | > loss_0: 2.4680275917053223 (2.447087737702831) + | > grad_norm_0: tensor(11.9512, device='cuda:0') (tensor(7.9471, device='cuda:0')) + | > loss_gen: 2.291029930114746 (2.3730601723661136) + | > loss_kl: 1.969159722328186 (1.857469944609809) + | > loss_feat: 7.819855690002441 (7.179963043055584) + | > loss_mel: 18.380565643310547 (17.72410377030519) + | > loss_duration: 1.4895364046096802 (1.45489928034163) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.950149536132812 (30.589495983320415) + | > grad_norm_1: tensor(196.1901, device='cuda:0') (tensor(162.9084, device='cuda:0')) + | > current_lr_0: 0.0001923256292281798 + | > current_lr_1: 0.0001923256292281798 + | > step_time: 0.4228 (0.3756015571122317) + | > loader_time: 0.007 (0.006069023584582139) + + + --> TIME: 2025-04-19 04:08:44 -- STEP: 122/142 -- GLOBAL_STEP: 423850 + | > loss_disc: 2.504599094390869 (2.4483927898719653) + | > loss_disc_real_0: 0.10280406475067139 (0.12382480459379368) + | > loss_disc_real_1: 0.21443477272987366 (0.20701472705504934) + | > loss_disc_real_2: 0.2113441824913025 (0.22598416859009227) + | > loss_disc_real_3: 0.2205900400876999 (0.2268535609616608) + | > loss_disc_real_4: 0.23848989605903625 (0.2290929507036678) + | > loss_disc_real_5: 0.2191731482744217 (0.2307712482135804) + | > loss_0: 2.504599094390869 (2.4483927898719653) + | > grad_norm_0: tensor(6.1934, device='cuda:0') (tensor(8.1868, device='cuda:0')) + | > loss_gen: 2.1203525066375732 (2.363540442263495) + | > loss_kl: 1.8433133363723755 (1.8662736621059355) + | > loss_feat: 5.823090553283691 (7.137439860672247) + | > loss_mel: 14.15550708770752 (17.638152442994656) + | > loss_duration: 1.4798189401626587 (1.4627608207405591) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 25.422080993652344 (30.468167054848593) + | > grad_norm_1: tensor(78.6960, device='cuda:0') (tensor(169.0379, device='cuda:0')) + | > current_lr_0: 0.0001923256292281798 + | > current_lr_1: 0.0001923256292281798 + | > step_time: 0.4534 (0.3893776721641666) + | > loader_time: 0.0073 (0.006324109483937746) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.427934408187866 (2.427934408187866) + | > loss_disc_real_0: 0.20318971574306488 (0.20318971574306488) + | > loss_disc_real_1: 0.22174809873104095 (0.22174809873104095) + | > loss_disc_real_2: 0.22876842319965363 (0.22876842319965363) + | > loss_disc_real_3: 0.19864413142204285 (0.19864413142204285) + | > loss_disc_real_4: 0.21482641994953156 (0.21482641994953156) + | > loss_disc_real_5: 0.21858926117420197 (0.21858926117420197) + | > loss_0: 2.427934408187866 (2.427934408187866) + | > loss_gen: 2.4689033031463623 (2.4689033031463623) + | > loss_kl: 2.58081316947937 (2.58081316947937) + | > loss_feat: 6.861657619476318 (6.861657619476318) + | > loss_mel: 19.58234977722168 (19.58234977722168) + | > loss_duration: 1.7378275394439697 (1.7378275394439697) + | > loss_1: 33.23155212402344 (33.23155212402344) + + --> STEP: 1 + | > loss_disc: 2.4777705669403076 (2.4777705669403076) + | > loss_disc_real_0: 0.19939641654491425 (0.19939641654491425) + | > loss_disc_real_1: 0.24445584416389465 (0.24445584416389465) + | > loss_disc_real_2: 0.2321915179491043 (0.2321915179491043) + | > loss_disc_real_3: 0.20750658214092255 (0.20750658214092255) + | > loss_disc_real_4: 0.24203506112098694 (0.24203506112098694) + | > loss_disc_real_5: 0.23397457599639893 (0.23397457599639893) + | > loss_0: 2.4777705669403076 (2.4777705669403076) + | > loss_gen: 2.416003942489624 (2.416003942489624) + | > loss_kl: 2.695291519165039 (2.695291519165039) + | > loss_feat: 6.850409984588623 (6.850409984588623) + | > loss_mel: 19.311758041381836 (19.311758041381836) + | > loss_duration: 1.7825477123260498 (1.7825477123260498) + | > loss_1: 33.05601119995117 (33.05601119995117) + + --> STEP: 2 + | > loss_disc: 2.499284267425537 (2.4885274171829224) + | > loss_disc_real_0: 0.20858877897262573 (0.20399259775877) + | > loss_disc_real_1: 0.18623261153697968 (0.21534422785043716) + | > loss_disc_real_2: 0.2136002629995346 (0.22289589047431946) + | > loss_disc_real_3: 0.24502506852149963 (0.2262658253312111) + | > loss_disc_real_4: 0.2134944051504135 (0.22776473313570023) + | > loss_disc_real_5: 0.2301621437072754 (0.23206835985183716) + | > loss_0: 2.499284267425537 (2.4885274171829224) + | > loss_gen: 2.3796114921569824 (2.3978077173233032) + | > loss_kl: 2.1711792945861816 (2.4332354068756104) + | > loss_feat: 7.54089879989624 (7.195654392242432) + | > loss_mel: 18.544960021972656 (18.928359031677246) + | > loss_duration: 1.715630292892456 (1.749089002609253) + | > loss_1: 32.35227966308594 (32.704145431518555) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003705620765686035 (+0.00031757354736328125) + | > avg_loss_disc: 2.4885274171829224 (-0.1135796308517456) + | > avg_loss_disc_real_0: 0.20399259775877 (+0.025158196687698364) + | > avg_loss_disc_real_1: 0.21534422785043716 (-0.01756114512681961) + | > avg_loss_disc_real_2: 0.22289589047431946 (-0.029226981103420258) + | > avg_loss_disc_real_3: 0.2262658253312111 (-0.018490493297576904) + | > avg_loss_disc_real_4: 0.22776473313570023 (-0.011493824422359467) + | > avg_loss_disc_real_5: 0.23206835985183716 (+0.0031595006585121155) + | > avg_loss_0: 2.4885274171829224 (-0.1135796308517456) + | > avg_loss_gen: 2.3978077173233032 (+0.15660929679870605) + | > avg_loss_kl: 2.4332354068756104 (-0.011534810066223145) + | > avg_loss_feat: 7.195654392242432 (+1.0084004402160645) + | > avg_loss_mel: 18.928359031677246 (+2.5712175369262695) + | > avg_loss_duration: 1.749089002609253 (+0.0031069517135620117) + | > avg_loss_1: 32.704145431518555 (+3.727799415588379) + + + > EPOCH: 314/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:08:54)  + + --> TIME: 2025-04-19 04:08:56 -- STEP: 5/142 -- GLOBAL_STEP: 423875 + | > loss_disc: 2.5497426986694336 (2.423407459259033) + | > loss_disc_real_0: 0.1260652393102646 (0.13293304592370986) + | > loss_disc_real_1: 0.22078993916511536 (0.20555500388145448) + | > loss_disc_real_2: 0.19570717215538025 (0.2272883415222168) + | > loss_disc_real_3: 0.25662466883659363 (0.2403663367033005) + | > loss_disc_real_4: 0.21611496806144714 (0.23116968870162963) + | > loss_disc_real_5: 0.2214328795671463 (0.2120100349187851) + | > loss_0: 2.5497426986694336 (2.423407459259033) + | > grad_norm_0: tensor(13.4247, device='cuda:0') (tensor(15.1978, device='cuda:0')) + | > loss_gen: 2.1850223541259766 (2.382235860824585) + | > loss_kl: 1.7402452230453491 (1.7405515432357788) + | > loss_feat: 6.838177680969238 (7.38412446975708) + | > loss_mel: 16.340181350708008 (17.47640495300293) + | > loss_duration: 1.4255212545394897 (1.4179190635681151) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.52914810180664 (30.40123596191406) + | > grad_norm_1: tensor(188.9167, device='cuda:0') (tensor(264.9477, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.3253 (0.32396669387817384) + | > loader_time: 0.0044 (0.00449223518371582) + + + --> TIME: 2025-04-19 04:09:05 -- STEP: 30/142 -- GLOBAL_STEP: 423900 + | > loss_disc: 2.4962220191955566 (2.453617970148722) + | > loss_disc_real_0: 0.15519669651985168 (0.12414471507072448) + | > loss_disc_real_1: 0.19156567752361298 (0.20895701150099436) + | > loss_disc_real_2: 0.22674378752708435 (0.22796634932359058) + | > loss_disc_real_3: 0.2444160431623459 (0.22991008112827938) + | > loss_disc_real_4: 0.24217933416366577 (0.22910546412070593) + | > loss_disc_real_5: 0.17774976789951324 (0.21953855206569037) + | > loss_0: 2.4962220191955566 (2.453617970148722) + | > grad_norm_0: tensor(10.1085, device='cuda:0') (tensor(11.3150, device='cuda:0')) + | > loss_gen: 2.3494112491607666 (2.384540303548177) + | > loss_kl: 1.8343435525894165 (1.8139952659606933) + | > loss_feat: 7.505687236785889 (7.353967316945394) + | > loss_mel: 18.901592254638672 (17.94165407816569) + | > loss_duration: 1.4560766220092773 (1.4311959584554037) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.04711151123047 (30.925352732340496) + | > grad_norm_1: tensor(288.9424, device='cuda:0') (tensor(288.4568, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.354 (0.3381442626317342) + | > loader_time: 0.0065 (0.005244652430216472) + + + --> TIME: 2025-04-19 04:09:15 -- STEP: 55/142 -- GLOBAL_STEP: 423925 + | > loss_disc: 2.559293508529663 (2.443421580574729) + | > loss_disc_real_0: 0.11064145714044571 (0.12090704278512435) + | > loss_disc_real_1: 0.24316123127937317 (0.20694059106436644) + | > loss_disc_real_2: 0.2680760622024536 (0.2259639620780945) + | > loss_disc_real_3: 0.224473237991333 (0.22838920869610527) + | > loss_disc_real_4: 0.24245332181453705 (0.2293419000777331) + | > loss_disc_real_5: 0.24221163988113403 (0.2246999591588974) + | > loss_0: 2.559293508529663 (2.443421580574729) + | > grad_norm_0: tensor(8.7892, device='cuda:0') (tensor(11.2622, device='cuda:0')) + | > loss_gen: 2.3679275512695312 (2.3952856887470593) + | > loss_kl: 1.921580195426941 (1.829136644710194) + | > loss_feat: 6.903873443603516 (7.328952277790417) + | > loss_mel: 17.40930938720703 (17.845665862343527) + | > loss_duration: 1.4891955852508545 (1.438350918076255) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.091886520385742 (30.8373912117698) + | > grad_norm_1: tensor(284.4383, device='cuda:0') (tensor(305.1194, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.3728 (0.35243048667907717) + | > loader_time: 0.0063 (0.005644291097467595) + + + --> TIME: 2025-04-19 04:09:25 -- STEP: 80/142 -- GLOBAL_STEP: 423950 + | > loss_disc: 2.3762593269348145 (2.4559590429067617) + | > loss_disc_real_0: 0.0927574634552002 (0.12452938919886947) + | > loss_disc_real_1: 0.21887022256851196 (0.20819231513887643) + | > loss_disc_real_2: 0.2188958078622818 (0.22673362866044044) + | > loss_disc_real_3: 0.18792100250720978 (0.2292124515399337) + | > loss_disc_real_4: 0.24034295976161957 (0.2286549039185047) + | > loss_disc_real_5: 0.18880118429660797 (0.22652904856950046) + | > loss_0: 2.3762593269348145 (2.4559590429067617) + | > grad_norm_0: tensor(10.7107, device='cuda:0') (tensor(12.0193, device='cuda:0')) + | > loss_gen: 2.3333852291107178 (2.3759047031402583) + | > loss_kl: 1.9807761907577515 (1.8387985154986382) + | > loss_feat: 7.385984897613525 (7.204125547409058) + | > loss_mel: 18.20992660522461 (17.85742005109787) + | > loss_duration: 1.4683704376220703 (1.4497292175889016) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.378442764282227 (30.72597782611847) + | > grad_norm_1: tensor(316.3373, device='cuda:0') (tensor(305.9543, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.4084 (0.3634967833757401) + | > loader_time: 0.0066 (0.005901384353637695) + + + --> TIME: 2025-04-19 04:09:35 -- STEP: 105/142 -- GLOBAL_STEP: 423975 + | > loss_disc: 2.4588823318481445 (2.4684386843726758) + | > loss_disc_real_0: 0.09179859608411789 (0.12732817722218373) + | > loss_disc_real_1: 0.20652751624584198 (0.20851781765619914) + | > loss_disc_real_2: 0.1948186606168747 (0.22678075305053166) + | > loss_disc_real_3: 0.2116636484861374 (0.22972872526872726) + | > loss_disc_real_4: 0.18991698324680328 (0.22922106101399375) + | > loss_disc_real_5: 0.2583509087562561 (0.22898554730983006) + | > loss_0: 2.4588823318481445 (2.4684386843726758) + | > grad_norm_0: tensor(19.4958, device='cuda:0') (tensor(12.0181, device='cuda:0')) + | > loss_gen: 2.2164690494537354 (2.3576173169272283) + | > loss_kl: 1.8039703369140625 (1.8511968079067411) + | > loss_feat: 6.895554542541504 (7.087428024836949) + | > loss_mel: 18.527084350585938 (17.7717395964123) + | > loss_duration: 1.518392562866211 (1.4592403184799927) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.961469650268555 (30.527221897670202) + | > grad_norm_1: tensor(275.7622, device='cuda:0') (tensor(295.2625, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.4163 (0.37297321274167017) + | > loader_time: 0.0074 (0.006141369683401924) + + + --> TIME: 2025-04-19 04:09:46 -- STEP: 130/142 -- GLOBAL_STEP: 424000 + | > loss_disc: 2.464372158050537 (2.4663713491879986) + | > loss_disc_real_0: 0.16325822472572327 (0.12782882876121082) + | > loss_disc_real_1: 0.1937609612941742 (0.2078503300364201) + | > loss_disc_real_2: 0.1888207644224167 (0.2263525707217363) + | > loss_disc_real_3: 0.22383885085582733 (0.228743839263916) + | > loss_disc_real_4: 0.23360763490200043 (0.2290140756047689) + | > loss_disc_real_5: 0.23387213051319122 (0.22902687593148305) + | > loss_0: 2.464372158050537 (2.4663713491879986) + | > grad_norm_0: tensor(11.0367, device='cuda:0') (tensor(12.0909, device='cuda:0')) + | > loss_gen: 2.3565826416015625 (2.353876231266902) + | > loss_kl: 1.8746364116668701 (1.860626261967879) + | > loss_feat: 6.8353800773620605 (7.073169568868784) + | > loss_mel: 15.85977840423584 (17.695571525280304) + | > loss_duration: 1.476298213005066 (1.4646726745825551) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.402673721313477 (30.44791614825909) + | > grad_norm_1: tensor(287.1750, device='cuda:0') (tensor(293.1377, device='cuda:0')) + | > current_lr_0: 0.0001923015885245263 + | > current_lr_1: 0.0001923015885245263 + | > step_time: 0.447 (0.38426931087787336) + | > loader_time: 0.0077 (0.006379668529217059) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5819711685180664 (2.5819711685180664) + | > loss_disc_real_0: 0.13147756457328796 (0.13147756457328796) + | > loss_disc_real_1: 0.22698397934436798 (0.22698397934436798) + | > loss_disc_real_2: 0.2544482946395874 (0.2544482946395874) + | > loss_disc_real_3: 0.2397114336490631 (0.2397114336490631) + | > loss_disc_real_4: 0.26404237747192383 (0.26404237747192383) + | > loss_disc_real_5: 0.3135649561882019 (0.3135649561882019) + | > loss_0: 2.5819711685180664 (2.5819711685180664) + | > loss_gen: 2.159966468811035 (2.159966468811035) + | > loss_kl: 2.3578343391418457 (2.3578343391418457) + | > loss_feat: 5.202916145324707 (5.202916145324707) + | > loss_mel: 16.565610885620117 (16.565610885620117) + | > loss_duration: 1.7251049280166626 (1.7251049280166626) + | > loss_1: 28.011432647705078 (28.011432647705078) + + --> STEP: 1 + | > loss_disc: 2.5233891010284424 (2.5233891010284424) + | > loss_disc_real_0: 0.13055485486984253 (0.13055485486984253) + | > loss_disc_real_1: 0.22435443103313446 (0.22435443103313446) + | > loss_disc_real_2: 0.2486339956521988 (0.2486339956521988) + | > loss_disc_real_3: 0.20122842490673065 (0.20122842490673065) + | > loss_disc_real_4: 0.19329558312892914 (0.19329558312892914) + | > loss_disc_real_5: 0.22574983537197113 (0.22574983537197113) + | > loss_0: 2.5233891010284424 (2.5233891010284424) + | > loss_gen: 2.1021058559417725 (2.1021058559417725) + | > loss_kl: 2.6269280910491943 (2.6269280910491943) + | > loss_feat: 6.230384349822998 (6.230384349822998) + | > loss_mel: 17.35747528076172 (17.35747528076172) + | > loss_duration: 1.768267273902893 (1.768267273902893) + | > loss_1: 30.085159301757812 (30.085159301757812) + + --> STEP: 2 + | > loss_disc: 2.4706668853759766 (2.4970279932022095) + | > loss_disc_real_0: 0.1760132908821106 (0.15328407287597656) + | > loss_disc_real_1: 0.21369175612926483 (0.21902309358119965) + | > loss_disc_real_2: 0.23341940343379974 (0.24102669954299927) + | > loss_disc_real_3: 0.19632776081562042 (0.19877809286117554) + | > loss_disc_real_4: 0.24649584293365479 (0.21989571303129196) + | > loss_disc_real_5: 0.2157416194677353 (0.2207457274198532) + | > loss_0: 2.4706668853759766 (2.4970279932022095) + | > loss_gen: 2.3291385173797607 (2.2156221866607666) + | > loss_kl: 2.304938554763794 (2.465933322906494) + | > loss_feat: 7.85574197769165 (7.043063163757324) + | > loss_mel: 19.452720642089844 (18.40509796142578) + | > loss_duration: 1.7015318870544434 (1.7348995804786682) + | > loss_1: 33.64406967163086 (31.864614486694336) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003689885139465332 (-1.5735626220703125e-05) + | > avg_loss_disc: 2.4970279932022095 (+0.00850057601928711) + | > avg_loss_disc_real_0: 0.15328407287597656 (-0.050708524882793427) + | > avg_loss_disc_real_1: 0.21902309358119965 (+0.0036788657307624817) + | > avg_loss_disc_real_2: 0.24102669954299927 (+0.01813080906867981) + | > avg_loss_disc_real_3: 0.19877809286117554 (-0.027487732470035553) + | > avg_loss_disc_real_4: 0.21989571303129196 (-0.007869020104408264) + | > avg_loss_disc_real_5: 0.2207457274198532 (-0.011322632431983948) + | > avg_loss_0: 2.4970279932022095 (+0.00850057601928711) + | > avg_loss_gen: 2.2156221866607666 (-0.18218553066253662) + | > avg_loss_kl: 2.465933322906494 (+0.03269791603088379) + | > avg_loss_feat: 7.043063163757324 (-0.15259122848510742) + | > avg_loss_mel: 18.40509796142578 (-0.5232610702514648) + | > avg_loss_duration: 1.7348995804786682 (-0.014189422130584717) + | > avg_loss_1: 31.864614486694336 (-0.8395309448242188) + + + > EPOCH: 315/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:09:53)  + + --> TIME: 2025-04-19 04:09:58 -- STEP: 13/142 -- GLOBAL_STEP: 424025 + | > loss_disc: 2.3903539180755615 (2.412842475450956) + | > loss_disc_real_0: 0.1112944707274437 (0.11458907734889251) + | > loss_disc_real_1: 0.23139812052249908 (0.20949623447198135) + | > loss_disc_real_2: 0.19941143691539764 (0.21586681214662698) + | > loss_disc_real_3: 0.2132657766342163 (0.2265097086246197) + | > loss_disc_real_4: 0.22005772590637207 (0.23267426972205824) + | > loss_disc_real_5: 0.25486278533935547 (0.23287473618984222) + | > loss_0: 2.3903539180755615 (2.412842475450956) + | > grad_norm_0: tensor(10.6836, device='cuda:0') (tensor(10.3077, device='cuda:0')) + | > loss_gen: 2.445786476135254 (2.4145223177396336) + | > loss_kl: 1.6751478910446167 (1.8234772773889394) + | > loss_feat: 7.42603874206543 (7.34786946956928) + | > loss_mel: 17.173046112060547 (17.747767741863548) + | > loss_duration: 1.4315763711929321 (1.4138451906350942) + | > amp_scaler: 512.0 (315.0769230769231) + | > loss_1: 30.151596069335938 (30.747481712928185) + | > grad_norm_1: tensor(308.1740, device='cuda:0') (tensor(267.7610, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.3274 (0.3268301303570087) + | > loader_time: 0.0051 (0.004699615331796499) + + + --> TIME: 2025-04-19 04:10:07 -- STEP: 38/142 -- GLOBAL_STEP: 424050 + | > loss_disc: 2.557445526123047 (2.4302805850380347) + | > loss_disc_real_0: 0.1133846640586853 (0.11422292280353998) + | > loss_disc_real_1: 0.23286594450473785 (0.20970799146514188) + | > loss_disc_real_2: 0.2224234640598297 (0.22551167521037555) + | > loss_disc_real_3: 0.22788478434085846 (0.226822460952558) + | > loss_disc_real_4: 0.19631367921829224 (0.22708458571057571) + | > loss_disc_real_5: 0.24576862156391144 (0.2315925634221027) + | > loss_0: 2.557445526123047 (2.4302805850380347) + | > grad_norm_0: tensor(14.8195, device='cuda:0') (tensor(9.3366, device='cuda:0')) + | > loss_gen: 2.2639272212982178 (2.399084461362738) + | > loss_kl: 1.83559250831604 (1.817463711688393) + | > loss_feat: 6.868380546569824 (7.263551774777864) + | > loss_mel: 17.68286895751953 (17.717363181867103) + | > loss_duration: 1.4328492879867554 (1.4296535159412183) + | > amp_scaler: 256.0 (323.36842105263156) + | > loss_1: 30.0836181640625 (30.627116755435342) + | > grad_norm_1: tensor(175.3764, device='cuda:0') (tensor(231.4615, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.3606 (0.3433579959367451) + | > loader_time: 0.0056 (0.005304280080293354) + + + --> TIME: 2025-04-19 04:10:16 -- STEP: 63/142 -- GLOBAL_STEP: 424075 + | > loss_disc: 2.4306790828704834 (2.448021442171127) + | > loss_disc_real_0: 0.13072440028190613 (0.11997293787343162) + | > loss_disc_real_1: 0.23511910438537598 (0.21096658351875486) + | > loss_disc_real_2: 0.28474247455596924 (0.22628060976664224) + | > loss_disc_real_3: 0.22173446416854858 (0.2275484622943969) + | > loss_disc_real_4: 0.19642853736877441 (0.22730027233797406) + | > loss_disc_real_5: 0.25961175560951233 (0.230389086026994) + | > loss_0: 2.4306790828704834 (2.448021442171127) + | > grad_norm_0: tensor(16.5069, device='cuda:0') (tensor(9.5530, device='cuda:0')) + | > loss_gen: 2.474403142929077 (2.3864616288079157) + | > loss_kl: 1.7476191520690918 (1.824703331977602) + | > loss_feat: 7.267553806304932 (7.232680593218122) + | > loss_mel: 18.91897964477539 (17.824333296881782) + | > loss_duration: 1.4906681776046753 (1.4412359775058807) + | > amp_scaler: 256.0 (296.63492063492066) + | > loss_1: 31.89922332763672 (30.709414921109637) + | > grad_norm_1: tensor(419.8892, device='cuda:0') (tensor(226.2724, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.399 (0.35790324211120605) + | > loader_time: 0.0064 (0.005704626204475525) + + + --> TIME: 2025-04-19 04:10:27 -- STEP: 88/142 -- GLOBAL_STEP: 424100 + | > loss_disc: 2.5327680110931396 (2.455099314451217) + | > loss_disc_real_0: 0.09890247136354446 (0.12150168952278116) + | > loss_disc_real_1: 0.24504832923412323 (0.2101122157817537) + | > loss_disc_real_2: 0.26591822504997253 (0.22672960009764542) + | > loss_disc_real_3: 0.2294149398803711 (0.22726969488642432) + | > loss_disc_real_4: 0.2513331174850464 (0.22807819379324262) + | > loss_disc_real_5: 0.22697626054286957 (0.2300222311168909) + | > loss_0: 2.5327680110931396 (2.455099314451217) + | > grad_norm_0: tensor(13.8374, device='cuda:0') (tensor(10.4632, device='cuda:0')) + | > loss_gen: 2.4243991374969482 (2.37407966635444) + | > loss_kl: 2.0329678058624268 (1.843289552764459) + | > loss_feat: 7.501855850219727 (7.191127051006664) + | > loss_mel: 19.20884895324707 (17.897834387692537) + | > loss_duration: 1.5113134384155273 (1.4504566720940852) + | > amp_scaler: 256.0 (285.09090909090907) + | > loss_1: 32.67938232421875 (30.756787235086616) + | > grad_norm_1: tensor(388.0988, device='cuda:0') (tensor(241.8522, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.4082 (0.3715572221712634) + | > loader_time: 0.0071 (0.006006129763343118) + + + --> TIME: 2025-04-19 04:10:38 -- STEP: 113/142 -- GLOBAL_STEP: 424125 + | > loss_disc: 2.4870331287384033 (2.462848116866255) + | > loss_disc_real_0: 0.09464084357023239 (0.12465674074086468) + | > loss_disc_real_1: 0.22055096924304962 (0.209435248269444) + | > loss_disc_real_2: 0.1998739391565323 (0.22667076748556794) + | > loss_disc_real_3: 0.20201264321804047 (0.22867107905645287) + | > loss_disc_real_4: 0.23681612312793732 (0.2283648227432133) + | > loss_disc_real_5: 0.2073860615491867 (0.23131188934883185) + | > loss_0: 2.4870331287384033 (2.462848116866255) + | > grad_norm_0: tensor(14.1619, device='cuda:0') (tensor(11.4971, device='cuda:0')) + | > loss_gen: 2.165560722351074 (2.363920891179448) + | > loss_kl: 1.8172037601470947 (1.8534526803852183) + | > loss_feat: 6.764086723327637 (7.119181903062668) + | > loss_mel: 16.749099731445312 (17.805931935268173) + | > loss_duration: 1.4914705753326416 (1.4578924495561991) + | > amp_scaler: 256.0 (278.6548672566371) + | > loss_1: 28.987422943115234 (30.60037987633089) + | > grad_norm_1: tensor(298.1812, device='cuda:0') (tensor(257.9657, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.4306 (0.3827455676762405) + | > loader_time: 0.0073 (0.006269775660692063) + + + --> TIME: 2025-04-19 04:10:49 -- STEP: 138/142 -- GLOBAL_STEP: 424150 + | > loss_disc: 2.48630428314209 (2.461864664934683) + | > loss_disc_real_0: 0.15609775483608246 (0.12569916939389877) + | > loss_disc_real_1: 0.2030302882194519 (0.20886859751265982) + | > loss_disc_real_2: 0.2230117917060852 (0.22623648764430612) + | > loss_disc_real_3: 0.2600694000720978 (0.2278016131857167) + | > loss_disc_real_4: 0.2397388517856598 (0.2273930013179779) + | > loss_disc_real_5: 0.23076681792736053 (0.23184308215327884) + | > loss_0: 2.48630428314209 (2.461864664934683) + | > grad_norm_0: tensor(13.7796, device='cuda:0') (tensor(11.2246, device='cuda:0')) + | > loss_gen: 2.341076374053955 (2.3577423389407173) + | > loss_kl: 1.8725066184997559 (1.8627804042636484) + | > loss_feat: 6.931906700134277 (7.076272107552791) + | > loss_mel: 17.915943145751953 (17.742507913838267) + | > loss_duration: 1.542464256286621 (1.4645385984061425) + | > amp_scaler: 256.0 (274.55072463768107) + | > loss_1: 30.603897094726562 (30.503841358682383) + | > grad_norm_1: tensor(179.1775, device='cuda:0') (tensor(250.9697, device='cuda:0')) + | > current_lr_0: 0.00019227755082596072 + | > current_lr_1: 0.00019227755082596072 + | > step_time: 0.4509 (0.39314574262370244) + | > loader_time: 0.0079 (0.006499202355094579) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.395580291748047 (2.395580291748047) + | > loss_disc_real_0: 0.1097557321190834 (0.1097557321190834) + | > loss_disc_real_1: 0.15408843755722046 (0.15408843755722046) + | > loss_disc_real_2: 0.21769104897975922 (0.21769104897975922) + | > loss_disc_real_3: 0.21740533411502838 (0.21740533411502838) + | > loss_disc_real_4: 0.1595580279827118 (0.1595580279827118) + | > loss_disc_real_5: 0.2231229692697525 (0.2231229692697525) + | > loss_0: 2.395580291748047 (2.395580291748047) + | > loss_gen: 2.1197867393493652 (2.1197867393493652) + | > loss_kl: 2.4418582916259766 (2.4418582916259766) + | > loss_feat: 7.519189834594727 (7.519189834594727) + | > loss_mel: 19.138492584228516 (19.138492584228516) + | > loss_duration: 1.7550615072250366 (1.7550615072250366) + | > loss_1: 32.974388122558594 (32.974388122558594) + + --> STEP: 1 + | > loss_disc: 2.500669479370117 (2.500669479370117) + | > loss_disc_real_0: 0.14638932049274445 (0.14638932049274445) + | > loss_disc_real_1: 0.17325976490974426 (0.17325976490974426) + | > loss_disc_real_2: 0.21503813564777374 (0.21503813564777374) + | > loss_disc_real_3: 0.21549667418003082 (0.21549667418003082) + | > loss_disc_real_4: 0.17927463352680206 (0.17927463352680206) + | > loss_disc_real_5: 0.24618497490882874 (0.24618497490882874) + | > loss_0: 2.500669479370117 (2.500669479370117) + | > loss_gen: 2.0499870777130127 (2.0499870777130127) + | > loss_kl: 2.7787203788757324 (2.7787203788757324) + | > loss_feat: 7.264759063720703 (7.264759063720703) + | > loss_mel: 18.53536605834961 (18.53536605834961) + | > loss_duration: 1.774314522743225 (1.774314522743225) + | > loss_1: 32.40314865112305 (32.40314865112305) + + --> STEP: 2 + | > loss_disc: 2.4395675659179688 (2.470118522644043) + | > loss_disc_real_0: 0.1720864176750183 (0.15923786908388138) + | > loss_disc_real_1: 0.14074142277240753 (0.1570005938410759) + | > loss_disc_real_2: 0.24036528170108795 (0.22770170867443085) + | > loss_disc_real_3: 0.17601977288722992 (0.19575822353363037) + | > loss_disc_real_4: 0.18816259503364563 (0.18371861428022385) + | > loss_disc_real_5: 0.1868504136800766 (0.21651769429445267) + | > loss_0: 2.4395675659179688 (2.470118522644043) + | > loss_gen: 2.2308292388916016 (2.140408158302307) + | > loss_kl: 2.2603235244750977 (2.519521951675415) + | > loss_feat: 7.02708101272583 (7.145920038223267) + | > loss_mel: 16.594501495361328 (17.56493377685547) + | > loss_duration: 1.7174237966537476 (1.7458691596984863) + | > loss_1: 29.83016014099121 (31.11665439605713) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003992915153503418 (+0.00030303001403808594) + | > avg_loss_disc: 2.470118522644043 (-0.026909470558166504) + | > avg_loss_disc_real_0: 0.15923786908388138 (+0.005953796207904816) + | > avg_loss_disc_real_1: 0.1570005938410759 (-0.06202249974012375) + | > avg_loss_disc_real_2: 0.22770170867443085 (-0.01332499086856842) + | > avg_loss_disc_real_3: 0.19575822353363037 (-0.003019869327545166) + | > avg_loss_disc_real_4: 0.18371861428022385 (-0.036177098751068115) + | > avg_loss_disc_real_5: 0.21651769429445267 (-0.004228033125400543) + | > avg_loss_0: 2.470118522644043 (-0.026909470558166504) + | > avg_loss_gen: 2.140408158302307 (-0.07521402835845947) + | > avg_loss_kl: 2.519521951675415 (+0.0535886287689209) + | > avg_loss_feat: 7.145920038223267 (+0.10285687446594238) + | > avg_loss_mel: 17.56493377685547 (-0.8401641845703125) + | > avg_loss_duration: 1.7458691596984863 (+0.010969579219818115) + | > avg_loss_1: 31.11665439605713 (-0.747960090637207) + + + > EPOCH: 316/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:10:52)  + + --> TIME: 2025-04-19 04:11:00 -- STEP: 21/142 -- GLOBAL_STEP: 424175 + | > loss_disc: 2.3811259269714355 (2.4215360709599087) + | > loss_disc_real_0: 0.10595406591892242 (0.11544615846304666) + | > loss_disc_real_1: 0.22204262018203735 (0.20289592444896698) + | > loss_disc_real_2: 0.2183375507593155 (0.2218216537010102) + | > loss_disc_real_3: 0.2680261731147766 (0.23160901949519203) + | > loss_disc_real_4: 0.22023729979991913 (0.22586614035424732) + | > loss_disc_real_5: 0.2567834258079529 (0.23192427839551652) + | > loss_0: 2.3811259269714355 (2.4215360709599087) + | > grad_norm_0: tensor(7.8290, device='cuda:0') (tensor(9.1941, device='cuda:0')) + | > loss_gen: 2.675879955291748 (2.42663083757673) + | > loss_kl: 1.764008641242981 (1.7980411393301827) + | > loss_feat: 8.087087631225586 (7.307120890844436) + | > loss_mel: 18.33311653137207 (17.72892693110875) + | > loss_duration: 1.4375364780426025 (1.4206027076357888) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.297630310058594 (30.68132227943057) + | > grad_norm_1: tensor(245.4933, device='cuda:0') (tensor(191.7351, device='cuda:0')) + | > current_lr_0: 0.00019225351613210746 + | > current_lr_1: 0.00019225351613210746 + | > step_time: 0.3445 (0.33050765310014996) + | > loader_time: 0.0054 (0.004855553309122722) + + + --> TIME: 2025-04-19 04:11:09 -- STEP: 46/142 -- GLOBAL_STEP: 424200 + | > loss_disc: 2.3539257049560547 (2.44132443614628) + | > loss_disc_real_0: 0.09022360295057297 (0.11704851654560669) + | > loss_disc_real_1: 0.16212444007396698 (0.20484716056481653) + | > loss_disc_real_2: 0.20507368445396423 (0.2232008641180785) + | > loss_disc_real_3: 0.1956857144832611 (0.2273575893562773) + | > loss_disc_real_4: 0.16572579741477966 (0.22516302634840427) + | > loss_disc_real_5: 0.20974013209342957 (0.2308926627687786) + | > loss_0: 2.3539257049560547 (2.44132443614628) + | > grad_norm_0: tensor(6.0921, device='cuda:0') (tensor(8.9726, device='cuda:0')) + | > loss_gen: 2.6088383197784424 (2.3802725854127296) + | > loss_kl: 1.8235244750976562 (1.824605967687524) + | > loss_feat: 8.385558128356934 (7.186071934907333) + | > loss_mel: 18.35881233215332 (17.66291475296021) + | > loss_duration: 1.4156835079193115 (1.4301228782405024) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.59241485595703 (30.483988015548043) + | > grad_norm_1: tensor(297.3819, device='cuda:0') (tensor(211.6478, device='cuda:0')) + | > current_lr_0: 0.00019225351613210746 + | > current_lr_1: 0.00019225351613210746 + | > step_time: 0.3703 (0.34439122676849354) + | > loader_time: 0.0064 (0.005253480828326682) + + + --> TIME: 2025-04-19 04:11:19 -- STEP: 71/142 -- GLOBAL_STEP: 424225 + | > loss_disc: 2.5271496772766113 (2.4513162760667404) + | > loss_disc_real_0: 0.2005423754453659 (0.12207483883265038) + | > loss_disc_real_1: 0.2207084745168686 (0.2053016184920996) + | > loss_disc_real_2: 0.264705628156662 (0.22408722655873903) + | > loss_disc_real_3: 0.22456219792366028 (0.22850929326574568) + | > loss_disc_real_4: 0.27356207370758057 (0.2282808089340237) + | > loss_disc_real_5: 0.22383342683315277 (0.23439004975305477) + | > loss_0: 2.5271496772766113 (2.4513162760667404) + | > grad_norm_0: tensor(12.7468, device='cuda:0') (tensor(9.5512, device='cuda:0')) + | > loss_gen: 2.38753342628479 (2.3708244276718347) + | > loss_kl: 1.8921358585357666 (1.8402538752891648) + | > loss_feat: 6.761773586273193 (7.104807846982714) + | > loss_mel: 18.095355987548828 (17.660792820890183) + | > loss_duration: 1.4739878177642822 (1.4404209952958873) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.61078643798828 (30.41709999299385) + | > grad_norm_1: tensor(299.4132, device='cuda:0') (tensor(209.4726, device='cuda:0')) + | > current_lr_0: 0.00019225351613210746 + | > current_lr_1: 0.00019225351613210746 + | > step_time: 0.3874 (0.35944978284164203) + | > loader_time: 0.0062 (0.005695913878964706) + + + --> TIME: 2025-04-19 04:11:29 -- STEP: 96/142 -- GLOBAL_STEP: 424250 + | > loss_disc: 2.505420446395874 (2.4509029214580873) + | > loss_disc_real_0: 0.11718627065420151 (0.12205087901869167) + | > loss_disc_real_1: 0.1822323501110077 (0.20450365962460637) + | > loss_disc_real_2: 0.20109164714813232 (0.22375823355590305) + | > loss_disc_real_3: 0.19822020828723907 (0.2281917294797798) + | > loss_disc_real_4: 0.22470644116401672 (0.22837067923198143) + | > loss_disc_real_5: 0.23252800107002258 (0.2339616360453268) + | > loss_0: 2.505420446395874 (2.4509029214580873) + | > grad_norm_0: tensor(3.6248, device='cuda:0') (tensor(8.6390, device='cuda:0')) + | > loss_gen: 2.254422426223755 (2.358552577594915) + | > loss_kl: 1.7595149278640747 (1.8573362876971562) + | > loss_feat: 7.099996089935303 (7.085041339198749) + | > loss_mel: 17.459943771362305 (17.683668792247765) + | > loss_duration: 1.4751858711242676 (1.4527694086233778) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.049062728881836 (30.43736845254898) + | > grad_norm_1: tensor(44.6067, device='cuda:0') (tensor(184.0516, device='cuda:0')) + | > current_lr_0: 0.00019225351613210746 + | > current_lr_1: 0.00019225351613210746 + | > step_time: 0.4048 (0.3693558424711227) + | > loader_time: 0.0073 (0.005957613388697307) + + + --> TIME: 2025-04-19 04:11:40 -- STEP: 121/142 -- GLOBAL_STEP: 424275 + | > loss_disc: 2.3677926063537598 (2.4513933520671762) + | > loss_disc_real_0: 0.14834092557430267 (0.12346245049814547) + | > loss_disc_real_1: 0.204422727227211 (0.20496162685973585) + | > loss_disc_real_2: 0.1911182552576065 (0.22436503671910152) + | > loss_disc_real_3: 0.18989752233028412 (0.22861356171202069) + | > loss_disc_real_4: 0.2107342779636383 (0.22817331995845827) + | > loss_disc_real_5: 0.1574152708053589 (0.23278568303289493) + | > loss_0: 2.3677926063537598 (2.4513933520671762) + | > grad_norm_0: tensor(6.8713, device='cuda:0') (tensor(8.0640, device='cuda:0')) + | > loss_gen: 2.3711154460906982 (2.3528565887577257) + | > loss_kl: 1.8356469869613647 (1.8641748851981044) + | > loss_feat: 7.712204933166504 (7.081468026500103) + | > loss_mel: 19.699247360229492 (17.72598146801151) + | > loss_duration: 1.5208165645599365 (1.460561327697817) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.13903045654297 (30.48504236709973) + | > grad_norm_1: tensor(244.6507, device='cuda:0') (tensor(165.8510, device='cuda:0')) + | > current_lr_0: 0.00019225351613210746 + | > current_lr_1: 0.00019225351613210746 + | > step_time: 0.441 (0.37995289179904396) + | > loader_time: 0.0071 (0.006243006256986256) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.326328754425049 (2.326328754425049) + | > loss_disc_real_0: 0.14902189373970032 (0.14902189373970032) + | > loss_disc_real_1: 0.20823831856250763 (0.20823831856250763) + | > loss_disc_real_2: 0.237984761595726 (0.237984761595726) + | > loss_disc_real_3: 0.22904600203037262 (0.22904600203037262) + | > loss_disc_real_4: 0.22511564195156097 (0.22511564195156097) + | > loss_disc_real_5: 0.21920108795166016 (0.21920108795166016) + | > loss_0: 2.326328754425049 (2.326328754425049) + | > loss_gen: 2.5194461345672607 (2.5194461345672607) + | > loss_kl: 2.2870302200317383 (2.2870302200317383) + | > loss_feat: 7.208867073059082 (7.208867073059082) + | > loss_mel: 18.8868350982666 (18.8868350982666) + | > loss_duration: 1.7509089708328247 (1.7509089708328247) + | > loss_1: 32.6530876159668 (32.6530876159668) + + --> STEP: 1 + | > loss_disc: 2.378465175628662 (2.378465175628662) + | > loss_disc_real_0: 0.14002251625061035 (0.14002251625061035) + | > loss_disc_real_1: 0.24179083108901978 (0.24179083108901978) + | > loss_disc_real_2: 0.2517242431640625 (0.2517242431640625) + | > loss_disc_real_3: 0.2213921844959259 (0.2213921844959259) + | > loss_disc_real_4: 0.2127421796321869 (0.2127421796321869) + | > loss_disc_real_5: 0.18900606036186218 (0.18900606036186218) + | > loss_0: 2.378465175628662 (2.378465175628662) + | > loss_gen: 2.440613269805908 (2.440613269805908) + | > loss_kl: 2.771143913269043 (2.771143913269043) + | > loss_feat: 7.759927749633789 (7.759927749633789) + | > loss_mel: 18.503480911254883 (18.503480911254883) + | > loss_duration: 1.7674977779388428 (1.7674977779388428) + | > loss_1: 33.2426643371582 (33.2426643371582) + + --> STEP: 2 + | > loss_disc: 2.5223746299743652 (2.4504199028015137) + | > loss_disc_real_0: 0.17068973183631897 (0.15535612404346466) + | > loss_disc_real_1: 0.21438734233379364 (0.2280890867114067) + | > loss_disc_real_2: 0.25825726985931396 (0.25499075651168823) + | > loss_disc_real_3: 0.2374681532382965 (0.2294301688671112) + | > loss_disc_real_4: 0.2282869517803192 (0.22051456570625305) + | > loss_disc_real_5: 0.22786374390125275 (0.20843490213155746) + | > loss_0: 2.5223746299743652 (2.4504199028015137) + | > loss_gen: 2.315500259399414 (2.378056764602661) + | > loss_kl: 2.2460575103759766 (2.5086007118225098) + | > loss_feat: 7.559227466583252 (7.6595776081085205) + | > loss_mel: 18.704469680786133 (18.603975296020508) + | > loss_duration: 1.715044617652893 (1.741271197795868) + | > loss_1: 32.54029846191406 (32.89148139953613) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0039021968841552734 (-9.071826934814453e-05) + | > avg_loss_disc: 2.4504199028015137 (-0.019698619842529297) + | > avg_loss_disc_real_0: 0.15535612404346466 (-0.0038817450404167175) + | > avg_loss_disc_real_1: 0.2280890867114067 (+0.07108849287033081) + | > avg_loss_disc_real_2: 0.25499075651168823 (+0.027289047837257385) + | > avg_loss_disc_real_3: 0.2294301688671112 (+0.033671945333480835) + | > avg_loss_disc_real_4: 0.22051456570625305 (+0.036795951426029205) + | > avg_loss_disc_real_5: 0.20843490213155746 (-0.008082792162895203) + | > avg_loss_0: 2.4504199028015137 (-0.019698619842529297) + | > avg_loss_gen: 2.378056764602661 (+0.237648606300354) + | > avg_loss_kl: 2.5086007118225098 (-0.010921239852905273) + | > avg_loss_feat: 7.6595776081085205 (+0.5136575698852539) + | > avg_loss_mel: 18.603975296020508 (+1.039041519165039) + | > avg_loss_duration: 1.741271197795868 (-0.004597961902618408) + | > avg_loss_1: 32.89148139953613 (+1.774827003479004) + + + > EPOCH: 317/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:11:51)  + + --> TIME: 2025-04-19 04:11:53 -- STEP: 4/142 -- GLOBAL_STEP: 424300 + | > loss_disc: 2.3137316703796387 (2.3884876370429993) + | > loss_disc_real_0: 0.0833282470703125 (0.10517097637057304) + | > loss_disc_real_1: 0.21042679250240326 (0.21063141897320747) + | > loss_disc_real_2: 0.21100878715515137 (0.2172820158302784) + | > loss_disc_real_3: 0.21636122465133667 (0.2211720049381256) + | > loss_disc_real_4: 0.21535679697990417 (0.21949653699994087) + | > loss_disc_real_5: 0.17367224395275116 (0.19220150262117386) + | > loss_0: 2.3137316703796387 (2.3884876370429993) + | > grad_norm_0: tensor(5.0177, device='cuda:0') (tensor(6.2570, device='cuda:0')) + | > loss_gen: 2.6914408206939697 (2.4468111395835876) + | > loss_kl: 1.7925504446029663 (1.7594775259494781) + | > loss_feat: 8.59406852722168 (7.489738464355469) + | > loss_mel: 19.488271713256836 (17.818456172943115) + | > loss_duration: 1.404166340827942 (1.417195349931717) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.970497131347656 (30.931679248809814) + | > grad_norm_1: tensor(76.2383, device='cuda:0') (tensor(105.4563, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.3204 (0.32624876499176025) + | > loader_time: 0.0051 (0.004353642463684082) + + + --> TIME: 2025-04-19 04:12:02 -- STEP: 29/142 -- GLOBAL_STEP: 424325 + | > loss_disc: 2.390610694885254 (2.4460668646056076) + | > loss_disc_real_0: 0.12138203531503677 (0.11916595723094611) + | > loss_disc_real_1: 0.18035121262073517 (0.20402069698119984) + | > loss_disc_real_2: 0.24488162994384766 (0.23140923062275195) + | > loss_disc_real_3: 0.25235602259635925 (0.23068622936462535) + | > loss_disc_real_4: 0.25025680661201477 (0.22784646163726674) + | > loss_disc_real_5: 0.2516193687915802 (0.22805949428985858) + | > loss_0: 2.390610694885254 (2.4460668646056076) + | > grad_norm_0: tensor(5.5926, device='cuda:0') (tensor(6.0719, device='cuda:0')) + | > loss_gen: 2.5674633979797363 (2.395391102494865) + | > loss_kl: 1.9085644483566284 (1.791085148679799) + | > loss_feat: 7.556427478790283 (7.317757343423778) + | > loss_mel: 17.64056968688965 (17.767072776268268) + | > loss_duration: 1.4266718626022339 (1.4241959267649158) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.099695205688477 (30.69550244561557) + | > grad_norm_1: tensor(52.4985, device='cuda:0') (tensor(108.0403, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.3461 (0.33640893574418695) + | > loader_time: 0.0056 (0.005080371067441743) + + + --> TIME: 2025-04-19 04:12:11 -- STEP: 54/142 -- GLOBAL_STEP: 424350 + | > loss_disc: 2.5265541076660156 (2.4358293259585344) + | > loss_disc_real_0: 0.09113957732915878 (0.11799707677629259) + | > loss_disc_real_1: 0.2028106302022934 (0.2052574254296444) + | > loss_disc_real_2: 0.23030288517475128 (0.2260492650998963) + | > loss_disc_real_3: 0.17911870777606964 (0.22556095239188936) + | > loss_disc_real_4: 0.20129179954528809 (0.2261227653534324) + | > loss_disc_real_5: 0.20572613179683685 (0.2248265067184413) + | > loss_0: 2.5265541076660156 (2.4358293259585344) + | > grad_norm_0: tensor(9.2714, device='cuda:0') (tensor(7.1280, device='cuda:0')) + | > loss_gen: 2.367751121520996 (2.3832461259983226) + | > loss_kl: 1.8325297832489014 (1.817139451150541) + | > loss_feat: 7.364660263061523 (7.267544605113842) + | > loss_mel: 18.83682632446289 (17.75890094262582) + | > loss_duration: 1.4510021209716797 (1.4342871440781488) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.85276985168457 (30.66111836610017) + | > grad_norm_1: tensor(148.3134, device='cuda:0') (tensor(141.9650, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.3736 (0.3479216231240166) + | > loader_time: 0.0061 (0.005410203227290401) + + + --> TIME: 2025-04-19 04:12:21 -- STEP: 79/142 -- GLOBAL_STEP: 424375 + | > loss_disc: 2.5792646408081055 (2.457571898834615) + | > loss_disc_real_0: 0.15981219708919525 (0.121309856165059) + | > loss_disc_real_1: 0.1943635642528534 (0.20618618251402168) + | > loss_disc_real_2: 0.26013481616973877 (0.2262538680924645) + | > loss_disc_real_3: 0.19289781153202057 (0.2293734593859202) + | > loss_disc_real_4: 0.18935386836528778 (0.23042702259896677) + | > loss_disc_real_5: 0.2665395438671112 (0.23108832368367835) + | > loss_0: 2.5792646408081055 (2.457571898834615) + | > grad_norm_0: tensor(17.2788, device='cuda:0') (tensor(8.5259, device='cuda:0')) + | > loss_gen: 2.1183416843414307 (2.380579676809192) + | > loss_kl: 1.8512918949127197 (1.837698441517504) + | > loss_feat: 6.123920917510986 (7.210049490385417) + | > loss_mel: 16.21529769897461 (17.752909068819843) + | > loss_duration: 1.5099079608917236 (1.4476742095585111) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.81875991821289 (30.628910958012447) + | > grad_norm_1: tensor(355.4132, device='cuda:0') (tensor(177.8472, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.3874 (0.3606498845015899) + | > loader_time: 0.0066 (0.005752159070365038) + + + --> TIME: 2025-04-19 04:12:31 -- STEP: 104/142 -- GLOBAL_STEP: 424400 + | > loss_disc: 2.407787799835205 (2.459375395224644) + | > loss_disc_real_0: 0.10513291507959366 (0.12053017791074055) + | > loss_disc_real_1: 0.17981918156147003 (0.20601478405296803) + | > loss_disc_real_2: 0.2020539939403534 (0.2274956267613631) + | > loss_disc_real_3: 0.21776796877384186 (0.22871352431292719) + | > loss_disc_real_4: 0.23284406960010529 (0.22998950816690922) + | > loss_disc_real_5: 0.17913904786109924 (0.23052100980511078) + | > loss_0: 2.407787799835205 (2.459375395224644) + | > grad_norm_0: tensor(11.3333, device='cuda:0') (tensor(9.2378, device='cuda:0')) + | > loss_gen: 2.484670877456665 (2.374956133273933) + | > loss_kl: 1.9118945598602295 (1.8453083577064366) + | > loss_feat: 7.192924499511719 (7.165447964118077) + | > loss_mel: 17.961366653442383 (17.781983577288113) + | > loss_duration: 1.5028166770935059 (1.458241146344405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.053672790527344 (30.62593726011423) + | > grad_norm_1: tensor(220.5669, device='cuda:0') (tensor(202.7101, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.4094 (0.37096578112015355) + | > loader_time: 0.0069 (0.006022600027231069) + + + --> TIME: 2025-04-19 04:12:42 -- STEP: 129/142 -- GLOBAL_STEP: 424425 + | > loss_disc: 2.531492233276367 (2.4611567914947985) + | > loss_disc_real_0: 0.11607009172439575 (0.12280522454385609) + | > loss_disc_real_1: 0.199407160282135 (0.2057667758113654) + | > loss_disc_real_2: 0.24222061038017273 (0.2273033314435057) + | > loss_disc_real_3: 0.23273614048957825 (0.22926185216552528) + | > loss_disc_real_4: 0.2571007013320923 (0.23016145178513933) + | > loss_disc_real_5: 0.2610585689544678 (0.23113130229388096) + | > loss_0: 2.531492233276367 (2.4611567914947985) + | > grad_norm_0: tensor(5.3864, device='cuda:0') (tensor(9.1134, device='cuda:0')) + | > loss_gen: 2.3137693405151367 (2.3718596669130556) + | > loss_kl: 1.9525973796844482 (1.8529161676880002) + | > loss_feat: 7.082170009613037 (7.148908360059871) + | > loss_mel: 17.607210159301758 (17.798306073329247) + | > loss_duration: 1.4865162372589111 (1.464678544406743) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.442262649536133 (30.636668892793878) + | > grad_norm_1: tensor(192.9092, device='cuda:0') (tensor(203.6508, device='cuda:0')) + | > current_lr_0: 0.00019222948444259095 + | > current_lr_1: 0.00019222948444259095 + | > step_time: 0.4451 (0.38261932920115865) + | > loader_time: 0.0072 (0.006254898485287215) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4828009605407715 (2.4828009605407715) + | > loss_disc_real_0: 0.185614213347435 (0.185614213347435) + | > loss_disc_real_1: 0.2507658898830414 (0.2507658898830414) + | > loss_disc_real_2: 0.29154860973358154 (0.29154860973358154) + | > loss_disc_real_3: 0.2805047929286957 (0.2805047929286957) + | > loss_disc_real_4: 0.24726347625255585 (0.24726347625255585) + | > loss_disc_real_5: 0.27847567200660706 (0.27847567200660706) + | > loss_0: 2.4828009605407715 (2.4828009605407715) + | > loss_gen: 2.6997809410095215 (2.6997809410095215) + | > loss_kl: 2.305619478225708 (2.305619478225708) + | > loss_feat: 6.54620885848999 (6.54620885848999) + | > loss_mel: 17.120502471923828 (17.120502471923828) + | > loss_duration: 1.747820258140564 (1.747820258140564) + | > loss_1: 30.419933319091797 (30.419933319091797) + + --> STEP: 1 + | > loss_disc: 2.3408455848693848 (2.3408455848693848) + | > loss_disc_real_0: 0.20219922065734863 (0.20219922065734863) + | > loss_disc_real_1: 0.24988558888435364 (0.24988558888435364) + | > loss_disc_real_2: 0.2685248553752899 (0.2685248553752899) + | > loss_disc_real_3: 0.2375939041376114 (0.2375939041376114) + | > loss_disc_real_4: 0.20339953899383545 (0.20339953899383545) + | > loss_disc_real_5: 0.2362014502286911 (0.2362014502286911) + | > loss_0: 2.3408455848693848 (2.3408455848693848) + | > loss_gen: 2.8859503269195557 (2.8859503269195557) + | > loss_kl: 2.6155664920806885 (2.6155664920806885) + | > loss_feat: 6.931711196899414 (6.931711196899414) + | > loss_mel: 19.073528289794922 (19.073528289794922) + | > loss_duration: 1.7407641410827637 (1.7407641410827637) + | > loss_1: 33.247520446777344 (33.247520446777344) + + --> STEP: 2 + | > loss_disc: 2.491325855255127 (2.416085720062256) + | > loss_disc_real_0: 0.25882795453071594 (0.2305135875940323) + | > loss_disc_real_1: 0.23964320123195648 (0.24476439505815506) + | > loss_disc_real_2: 0.2783752679824829 (0.2734500616788864) + | > loss_disc_real_3: 0.24382396042346954 (0.24070893228054047) + | > loss_disc_real_4: 0.2710733115673065 (0.23723642528057098) + | > loss_disc_real_5: 0.22052916884422302 (0.22836530953645706) + | > loss_0: 2.491325855255127 (2.416085720062256) + | > loss_gen: 2.640737771987915 (2.7633440494537354) + | > loss_kl: 2.146480083465576 (2.3810232877731323) + | > loss_feat: 6.338160514831543 (6.6349358558654785) + | > loss_mel: 17.227781295776367 (18.150654792785645) + | > loss_duration: 1.7259855270385742 (1.733374834060669) + | > loss_1: 30.079143524169922 (31.663331985473633) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037134885787963867 (-0.00018870830535888672) + | > avg_loss_disc: 2.416085720062256 (-0.03433418273925781) + | > avg_loss_disc_real_0: 0.2305135875940323 (+0.07515746355056763) + | > avg_loss_disc_real_1: 0.24476439505815506 (+0.016675308346748352) + | > avg_loss_disc_real_2: 0.2734500616788864 (+0.01845930516719818) + | > avg_loss_disc_real_3: 0.24070893228054047 (+0.01127876341342926) + | > avg_loss_disc_real_4: 0.23723642528057098 (+0.016721859574317932) + | > avg_loss_disc_real_5: 0.22836530953645706 (+0.019930407404899597) + | > avg_loss_0: 2.416085720062256 (-0.03433418273925781) + | > avg_loss_gen: 2.7633440494537354 (+0.3852872848510742) + | > avg_loss_kl: 2.3810232877731323 (-0.12757742404937744) + | > avg_loss_feat: 6.6349358558654785 (-1.024641752243042) + | > avg_loss_mel: 18.150654792785645 (-0.4533205032348633) + | > avg_loss_duration: 1.733374834060669 (-0.007896363735198975) + | > avg_loss_1: 31.663331985473633 (-1.2281494140625) + + + > EPOCH: 318/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:12:49)  + + --> TIME: 2025-04-19 04:12:54 -- STEP: 12/142 -- GLOBAL_STEP: 424450 + | > loss_disc: 2.3539657592773438 (2.4206882913907366) + | > loss_disc_real_0: 0.11328846216201782 (0.11963564405838649) + | > loss_disc_real_1: 0.2520126402378082 (0.21598434820771217) + | > loss_disc_real_2: 0.22993071377277374 (0.2288472019135952) + | > loss_disc_real_3: 0.21106363832950592 (0.22409075001875559) + | > loss_disc_real_4: 0.2272803634405136 (0.22411802038550377) + | > loss_disc_real_5: 0.20160967111587524 (0.2318028969069322) + | > loss_0: 2.3539657592773438 (2.4206882913907366) + | > grad_norm_0: tensor(6.6998, device='cuda:0') (tensor(7.3017, device='cuda:0')) + | > loss_gen: 2.3084158897399902 (2.4045590559641523) + | > loss_kl: 1.8434700965881348 (1.7827937404314678) + | > loss_feat: 6.553563594818115 (7.289900938669841) + | > loss_mel: 15.803351402282715 (17.577269315719604) + | > loss_duration: 1.415755271911621 (1.416042685508728) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.924556732177734 (30.470565636952717) + | > grad_norm_1: tensor(139.1097, device='cuda:0') (tensor(144.5487, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.3294 (0.3261621395746867) + | > loader_time: 0.0053 (0.004717429478963216) + + + --> TIME: 2025-04-19 04:13:03 -- STEP: 37/142 -- GLOBAL_STEP: 424475 + | > loss_disc: 2.5402488708496094 (2.468955555477658) + | > loss_disc_real_0: 0.10183451324701309 (0.1171449088567012) + | > loss_disc_real_1: 0.23691150546073914 (0.21314253557372737) + | > loss_disc_real_2: 0.22218623757362366 (0.22925702583145452) + | > loss_disc_real_3: 0.2572070062160492 (0.22909004261364807) + | > loss_disc_real_4: 0.25855714082717896 (0.22957061754690633) + | > loss_disc_real_5: 0.26924267411231995 (0.2335552053677069) + | > loss_0: 2.5402488708496094 (2.468955555477658) + | > grad_norm_0: tensor(5.9632, device='cuda:0') (tensor(7.5860, device='cuda:0')) + | > loss_gen: 2.4225196838378906 (2.3707800749185926) + | > loss_kl: 1.8653773069381714 (1.8090292891940556) + | > loss_feat: 6.554689884185791 (7.227079675004289) + | > loss_mel: 17.330129623413086 (17.76719049505285) + | > loss_duration: 1.466308355331421 (1.4294242665574357) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63902473449707 (30.603503871608424) + | > grad_norm_1: tensor(92.1933, device='cuda:0') (tensor(149.9338, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.3526 (0.33873683697468526) + | > loader_time: 0.0059 (0.005218196559596706) + + + --> TIME: 2025-04-19 04:13:12 -- STEP: 62/142 -- GLOBAL_STEP: 424500 + | > loss_disc: 2.4807682037353516 (2.452780488998659) + | > loss_disc_real_0: 0.12350740283727646 (0.1187501358649423) + | > loss_disc_real_1: 0.2006562352180481 (0.20835798113576828) + | > loss_disc_real_2: 0.2558981776237488 (0.22674257260176442) + | > loss_disc_real_3: 0.248456209897995 (0.22768834161181603) + | > loss_disc_real_4: 0.2577573359012604 (0.22886725563195445) + | > loss_disc_real_5: 0.22821496427059174 (0.23127709232991742) + | > loss_0: 2.4807682037353516 (2.452780488998659) + | > grad_norm_0: tensor(5.8615, device='cuda:0') (tensor(7.4746, device='cuda:0')) + | > loss_gen: 2.445788860321045 (2.359546838268156) + | > loss_kl: 1.8810830116271973 (1.8363286352926684) + | > loss_feat: 7.162327289581299 (7.14286192001835) + | > loss_mel: 16.81211280822754 (17.661955741144) + | > loss_duration: 1.5006787776947021 (1.442101751604388) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.801990509033203 (30.4427950151505) + | > grad_norm_1: tensor(218.7363, device='cuda:0') (tensor(166.5205, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.3816 (0.35038022841176675) + | > loader_time: 0.0074 (0.0055596059368502706) + + + --> TIME: 2025-04-19 04:13:23 -- STEP: 87/142 -- GLOBAL_STEP: 424525 + | > loss_disc: 2.5010268688201904 (2.4573699633280435) + | > loss_disc_real_0: 0.16918347775936127 (0.1205906964752866) + | > loss_disc_real_1: 0.2267822027206421 (0.20761385679930106) + | > loss_disc_real_2: 0.21866348385810852 (0.22667258412673555) + | > loss_disc_real_3: 0.24012728035449982 (0.2289074601113111) + | > loss_disc_real_4: 0.2530469298362732 (0.22955181081404633) + | > loss_disc_real_5: 0.26016953587532043 (0.23305993107543593) + | > loss_0: 2.5010268688201904 (2.4573699633280435) + | > grad_norm_0: tensor(11.1111, device='cuda:0') (tensor(8.0413, device='cuda:0')) + | > loss_gen: 2.2708685398101807 (2.352884807805905) + | > loss_kl: 1.7572407722473145 (1.8552321853308842) + | > loss_feat: 6.719202995300293 (7.090550954314484) + | > loss_mel: 18.29192352294922 (17.647628773217914) + | > loss_duration: 1.483834981918335 (1.4524998664855955) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.5230712890625 (30.398796629631658) + | > grad_norm_1: tensor(236.6967, device='cuda:0') (tensor(196.4003, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.4042 (0.3650614228741875) + | > loader_time: 0.0067 (0.005910407537701486) + + + --> TIME: 2025-04-19 04:13:33 -- STEP: 112/142 -- GLOBAL_STEP: 424550 + | > loss_disc: 2.4307587146759033 (2.4619846514293124) + | > loss_disc_real_0: 0.08698639273643494 (0.12242707689957959) + | > loss_disc_real_1: 0.18214455246925354 (0.206976018846035) + | > loss_disc_real_2: 0.2014293372631073 (0.226785539782473) + | > loss_disc_real_3: 0.23476119339466095 (0.22884471235530718) + | > loss_disc_real_4: 0.20899748802185059 (0.2286831076656069) + | > loss_disc_real_5: 0.21245965361595154 (0.23210764516677176) + | > loss_0: 2.4307587146759033 (2.4619846514293124) + | > grad_norm_0: tensor(9.1452, device='cuda:0') (tensor(8.6528, device='cuda:0')) + | > loss_gen: 2.3733017444610596 (2.344509837882858) + | > loss_kl: 1.970937967300415 (1.8722025847860746) + | > loss_feat: 6.430149078369141 (7.025811740330288) + | > loss_mel: 17.520288467407227 (17.63746679680688) + | > loss_duration: 1.4538789987564087 (1.4603570795484953) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.74855613708496 (30.34034812450409) + | > grad_norm_1: tensor(228.8767, device='cuda:0') (tensor(208.5330, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.4318 (0.37610534259251177) + | > loader_time: 0.0074 (0.006169638463429042) + + + --> TIME: 2025-04-19 04:13:45 -- STEP: 137/142 -- GLOBAL_STEP: 424575 + | > loss_disc: 2.5895845890045166 (2.4697063004013398) + | > loss_disc_real_0: 0.13503392040729523 (0.12356775841356193) + | > loss_disc_real_1: 0.2726852297782898 (0.20761999890317012) + | > loss_disc_real_2: 0.28845468163490295 (0.2274858253280612) + | > loss_disc_real_3: 0.2661219835281372 (0.22927739085072146) + | > loss_disc_real_4: 0.24970903992652893 (0.22926831462957564) + | > loss_disc_real_5: 0.23319077491760254 (0.23357826645356894) + | > loss_0: 2.5895845890045166 (2.4697063004013398) + | > grad_norm_0: tensor(7.6535, device='cuda:0') (tensor(9.7510, device='cuda:0')) + | > loss_gen: 2.4594573974609375 (2.3474464225072924) + | > loss_kl: 1.8357006311416626 (1.8825432410205367) + | > loss_feat: 6.423927307128906 (7.022176290080495) + | > loss_mel: 17.75322914123535 (17.610436843259492) + | > loss_duration: 1.4914900064468384 (1.4666065186479667) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.963804244995117 (30.329209404270145) + | > grad_norm_1: tensor(37.5294, device='cuda:0') (tensor(223.6344, device='cuda:0')) + | > current_lr_0: 0.0001922054557570356 + | > current_lr_1: 0.0001922054557570356 + | > step_time: 0.4522 (0.3888898818162235) + | > loader_time: 0.0076 (0.00638921939543564) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6468796730041504 (2.6468796730041504) + | > loss_disc_real_0: 0.1932932585477829 (0.1932932585477829) + | > loss_disc_real_1: 0.24004164338111877 (0.24004164338111877) + | > loss_disc_real_2: 0.25288742780685425 (0.25288742780685425) + | > loss_disc_real_3: 0.2843521535396576 (0.2843521535396576) + | > loss_disc_real_4: 0.2693673372268677 (0.2693673372268677) + | > loss_disc_real_5: 0.26241403818130493 (0.26241403818130493) + | > loss_0: 2.6468796730041504 (2.6468796730041504) + | > loss_gen: 2.2626285552978516 (2.2626285552978516) + | > loss_kl: 2.484053611755371 (2.484053611755371) + | > loss_feat: 7.028682708740234 (7.028682708740234) + | > loss_mel: 16.81361198425293 (16.81361198425293) + | > loss_duration: 1.7160612344741821 (1.7160612344741821) + | > loss_1: 30.305036544799805 (30.305036544799805) + + --> STEP: 1 + | > loss_disc: 2.6050453186035156 (2.6050453186035156) + | > loss_disc_real_0: 0.17199814319610596 (0.17199814319610596) + | > loss_disc_real_1: 0.21830473840236664 (0.21830473840236664) + | > loss_disc_real_2: 0.24321849644184113 (0.24321849644184113) + | > loss_disc_real_3: 0.2753235101699829 (0.2753235101699829) + | > loss_disc_real_4: 0.2801251709461212 (0.2801251709461212) + | > loss_disc_real_5: 0.2591717541217804 (0.2591717541217804) + | > loss_0: 2.6050453186035156 (2.6050453186035156) + | > loss_gen: 2.3648788928985596 (2.3648788928985596) + | > loss_kl: 2.82932448387146 (2.82932448387146) + | > loss_feat: 7.3394365310668945 (7.3394365310668945) + | > loss_mel: 19.103778839111328 (19.103778839111328) + | > loss_duration: 1.7802599668502808 (1.7802599668502808) + | > loss_1: 33.41767883300781 (33.41767883300781) + + --> STEP: 2 + | > loss_disc: 2.5074446201324463 (2.556244969367981) + | > loss_disc_real_0: 0.13930827379226685 (0.1556532084941864) + | > loss_disc_real_1: 0.1877688318490982 (0.20303678512573242) + | > loss_disc_real_2: 0.2043679654598236 (0.22379323095083237) + | > loss_disc_real_3: 0.22128911316394806 (0.24830631166696548) + | > loss_disc_real_4: 0.19837938249111176 (0.23925227671861649) + | > loss_disc_real_5: 0.23991666734218597 (0.24954421073198318) + | > loss_0: 2.5074446201324463 (2.556244969367981) + | > loss_gen: 2.2402477264404297 (2.3025633096694946) + | > loss_kl: 2.6000773906707764 (2.714700937271118) + | > loss_feat: 9.047961235046387 (8.19369888305664) + | > loss_mel: 20.533950805664062 (19.818864822387695) + | > loss_duration: 1.713100790977478 (1.7466803789138794) + | > loss_1: 36.13534164428711 (34.77651023864746) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038061141967773438 (+9.262561798095703e-05) + | > avg_loss_disc: 2.556244969367981 (+0.1401592493057251) + | > avg_loss_disc_real_0: 0.1556532084941864 (-0.07486037909984589) + | > avg_loss_disc_real_1: 0.20303678512573242 (-0.04172760993242264) + | > avg_loss_disc_real_2: 0.22379323095083237 (-0.04965683072805405) + | > avg_loss_disc_real_3: 0.24830631166696548 (+0.007597379386425018) + | > avg_loss_disc_real_4: 0.23925227671861649 (+0.0020158514380455017) + | > avg_loss_disc_real_5: 0.24954421073198318 (+0.021178901195526123) + | > avg_loss_0: 2.556244969367981 (+0.1401592493057251) + | > avg_loss_gen: 2.3025633096694946 (-0.4607807397842407) + | > avg_loss_kl: 2.714700937271118 (+0.33367764949798584) + | > avg_loss_feat: 8.19369888305664 (+1.558763027191162) + | > avg_loss_mel: 19.818864822387695 (+1.6682100296020508) + | > avg_loss_duration: 1.7466803789138794 (+0.01330554485321045) + | > avg_loss_1: 34.77651023864746 (+3.113178253173828) + + + > EPOCH: 319/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:13:48)  + + --> TIME: 2025-04-19 04:13:56 -- STEP: 20/142 -- GLOBAL_STEP: 424600 + | > loss_disc: 2.5217604637145996 (2.481227791309357) + | > loss_disc_real_0: 0.08523489534854889 (0.13324044905602933) + | > loss_disc_real_1: 0.19533908367156982 (0.20583404675126077) + | > loss_disc_real_2: 0.22279499471187592 (0.2315814509987831) + | > loss_disc_real_3: 0.25926926732063293 (0.2351418651640415) + | > loss_disc_real_4: 0.22823169827461243 (0.23582318872213365) + | > loss_disc_real_5: 0.23540538549423218 (0.23522550165653228) + | > loss_0: 2.5217604637145996 (2.481227791309357) + | > grad_norm_0: tensor(13.9862, device='cuda:0') (tensor(12.9140, device='cuda:0')) + | > loss_gen: 2.1264328956604004 (2.338452196121216) + | > loss_kl: 1.9062063694000244 (1.8438904404640197) + | > loss_feat: 5.897283554077148 (6.950380897521972) + | > loss_mel: 15.772013664245605 (17.323187732696532) + | > loss_duration: 1.4747509956359863 (1.4237485468387603) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.176687240600586 (29.879659938812257) + | > grad_norm_1: tensor(195.4713, device='cuda:0') (tensor(300.4439, device='cuda:0')) + | > current_lr_0: 0.00019218143007506598 + | > current_lr_1: 0.00019218143007506598 + | > step_time: 0.3425 (0.3320621013641357) + | > loader_time: 0.0055 (0.0049500703811645504) + + + --> TIME: 2025-04-19 04:14:05 -- STEP: 45/142 -- GLOBAL_STEP: 424625 + | > loss_disc: 2.406359910964966 (2.4490281158023413) + | > loss_disc_real_0: 0.10985348373651505 (0.12585532714923217) + | > loss_disc_real_1: 0.1723988652229309 (0.20541663567225138) + | > loss_disc_real_2: 0.20808827877044678 (0.22752280996905433) + | > loss_disc_real_3: 0.1886286586523056 (0.22680775854322646) + | > loss_disc_real_4: 0.17207954823970795 (0.22565506100654603) + | > loss_disc_real_5: 0.210445374250412 (0.23284066948625776) + | > loss_0: 2.406359910964966 (2.4490281158023413) + | > grad_norm_0: tensor(10.0207, device='cuda:0') (tensor(12.7144, device='cuda:0')) + | > loss_gen: 2.207085609436035 (2.355905559327868) + | > loss_kl: 1.7434649467468262 (1.8599454641342164) + | > loss_feat: 7.23043966293335 (7.1008837699890135) + | > loss_mel: 17.815689086914062 (17.582491344875766) + | > loss_duration: 1.4578219652175903 (1.432986929681566) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.454500198364258 (30.33221329583062) + | > grad_norm_1: tensor(355.1136, device='cuda:0') (tensor(319.8098, device='cuda:0')) + | > current_lr_0: 0.00019218143007506598 + | > current_lr_1: 0.00019218143007506598 + | > step_time: 0.3675 (0.3444690863291423) + | > loader_time: 0.0061 (0.00537469122144911) + + + --> TIME: 2025-04-19 04:14:15 -- STEP: 70/142 -- GLOBAL_STEP: 424650 + | > loss_disc: 2.2198970317840576 (2.4532504081726074) + | > loss_disc_real_0: 0.13056263327598572 (0.12387789913586207) + | > loss_disc_real_1: 0.1673099398612976 (0.20423786895615714) + | > loss_disc_real_2: 0.2358454167842865 (0.2268825929079737) + | > loss_disc_real_3: 0.21222874522209167 (0.22855754494667052) + | > loss_disc_real_4: 0.20900535583496094 (0.22654329942805426) + | > loss_disc_real_5: 0.22148409485816956 (0.23552446237632207) + | > loss_0: 2.2198970317840576 (2.4532504081726074) + | > grad_norm_0: tensor(9.6098, device='cuda:0') (tensor(13.7126, device='cuda:0')) + | > loss_gen: 2.818033218383789 (2.38568685735975) + | > loss_kl: 1.911887288093567 (1.870083338873727) + | > loss_feat: 7.831210136413574 (7.193150506700788) + | > loss_mel: 18.613325119018555 (17.664270264761797) + | > loss_duration: 1.4827208518981934 (1.443680681501116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.65717697143555 (30.556871795654295) + | > grad_norm_1: tensor(342.1739, device='cuda:0') (tensor(317.0900, device='cuda:0')) + | > current_lr_0: 0.00019218143007506598 + | > current_lr_1: 0.00019218143007506598 + | > step_time: 0.3919 (0.35619039535522456) + | > loader_time: 0.0066 (0.005684096472603934) + + + --> TIME: 2025-04-19 04:14:25 -- STEP: 95/142 -- GLOBAL_STEP: 424675 + | > loss_disc: 2.447972059249878 (2.466671035164281) + | > loss_disc_real_0: 0.12305524200201035 (0.12420743531302401) + | > loss_disc_real_1: 0.19056613743305206 (0.2066010495549754) + | > loss_disc_real_2: 0.22767038643360138 (0.22842801056410136) + | > loss_disc_real_3: 0.22010192275047302 (0.22875945034780001) + | > loss_disc_real_4: 0.23554982244968414 (0.22775786158285644) + | > loss_disc_real_5: 0.224216029047966 (0.23578990070443404) + | > loss_0: 2.447972059249878 (2.466671035164281) + | > grad_norm_0: tensor(5.4796, device='cuda:0') (tensor(14.0366, device='cuda:0')) + | > loss_gen: 2.2807130813598633 (2.380025334107248) + | > loss_kl: 1.8787139654159546 (1.8712485426350645) + | > loss_feat: 7.355556488037109 (7.12892101187455) + | > loss_mel: 19.132383346557617 (17.69183181963469) + | > loss_duration: 1.4815784692764282 (1.4535068637446351) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.128944396972656 (30.52553367614746) + | > grad_norm_1: tensor(308.8855, device='cuda:0') (tensor(325.5019, device='cuda:0')) + | > current_lr_0: 0.00019218143007506598 + | > current_lr_1: 0.00019218143007506598 + | > step_time: 0.4033 (0.3666960691150866) + | > loader_time: 0.0071 (0.005936755632099352) + + + --> TIME: 2025-04-19 04:14:35 -- STEP: 120/142 -- GLOBAL_STEP: 424700 + | > loss_disc: 2.517387866973877 (2.4643778165181476) + | > loss_disc_real_0: 0.1092943474650383 (0.12403305986275276) + | > loss_disc_real_1: 0.19846922159194946 (0.2073345709592104) + | > loss_disc_real_2: 0.23555126786231995 (0.22771457334359488) + | > loss_disc_real_3: 0.24964262545108795 (0.22920721831421056) + | > loss_disc_real_4: 0.19091804325580597 (0.22766295485198498) + | > loss_disc_real_5: 0.22027719020843506 (0.23492261233429115) + | > loss_0: 2.517387866973877 (2.4643778165181476) + | > grad_norm_0: tensor(4.6261, device='cuda:0') (tensor(13.2011, device='cuda:0')) + | > loss_gen: 2.077731132507324 (2.3651596426963795) + | > loss_kl: 1.9443776607513428 (1.8723549336194991) + | > loss_feat: 6.901243686676025 (7.067938172817231) + | > loss_mel: 18.581260681152344 (17.603678393363953) + | > loss_duration: 1.4975852966308594 (1.4607127706209817) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.002197265625 (30.36984395980835) + | > grad_norm_1: tensor(102.0282, device='cuda:0') (tensor(308.8288, device='cuda:0')) + | > current_lr_0: 0.00019218143007506598 + | > current_lr_1: 0.00019218143007506598 + | > step_time: 0.432 (0.3775682111581167) + | > loader_time: 0.0071 (0.0061549166838328045) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.601224899291992 (2.601224899291992) + | > loss_disc_real_0: 0.2473454475402832 (0.2473454475402832) + | > loss_disc_real_1: 0.23576517403125763 (0.23576517403125763) + | > loss_disc_real_2: 0.2614492177963257 (0.2614492177963257) + | > loss_disc_real_3: 0.2702491283416748 (0.2702491283416748) + | > loss_disc_real_4: 0.25961294770240784 (0.25961294770240784) + | > loss_disc_real_5: 0.254107266664505 (0.254107266664505) + | > loss_0: 2.601224899291992 (2.601224899291992) + | > loss_gen: 2.5282981395721436 (2.5282981395721436) + | > loss_kl: 2.390789747238159 (2.390789747238159) + | > loss_feat: 6.111304759979248 (6.111304759979248) + | > loss_mel: 16.78791618347168 (16.78791618347168) + | > loss_duration: 1.7376930713653564 (1.7376930713653564) + | > loss_1: 29.556001663208008 (29.556001663208008) + + --> STEP: 1 + | > loss_disc: 2.596158981323242 (2.596158981323242) + | > loss_disc_real_0: 0.23031802475452423 (0.23031802475452423) + | > loss_disc_real_1: 0.2372250109910965 (0.2372250109910965) + | > loss_disc_real_2: 0.23286424577236176 (0.23286424577236176) + | > loss_disc_real_3: 0.2538304030895233 (0.2538304030895233) + | > loss_disc_real_4: 0.26132553815841675 (0.26132553815841675) + | > loss_disc_real_5: 0.22858422994613647 (0.22858422994613647) + | > loss_0: 2.596158981323242 (2.596158981323242) + | > loss_gen: 2.472533702850342 (2.472533702850342) + | > loss_kl: 2.611830472946167 (2.611830472946167) + | > loss_feat: 6.502217769622803 (6.502217769622803) + | > loss_mel: 17.784561157226562 (17.784561157226562) + | > loss_duration: 1.7770733833312988 (1.7770733833312988) + | > loss_1: 31.148216247558594 (31.148216247558594) + + --> STEP: 2 + | > loss_disc: 2.5511562824249268 (2.5736576318740845) + | > loss_disc_real_0: 0.20702509582042694 (0.21867156028747559) + | > loss_disc_real_1: 0.2684943377971649 (0.2528596743941307) + | > loss_disc_real_2: 0.2784426212310791 (0.25565343350172043) + | > loss_disc_real_3: 0.2642505466938019 (0.2590404748916626) + | > loss_disc_real_4: 0.25024110078811646 (0.2557833194732666) + | > loss_disc_real_5: 0.24316534399986267 (0.23587478697299957) + | > loss_0: 2.5511562824249268 (2.5736576318740845) + | > loss_gen: 2.5970873832702637 (2.5348105430603027) + | > loss_kl: 2.2745165824890137 (2.4431735277175903) + | > loss_feat: 7.958287715911865 (7.230252742767334) + | > loss_mel: 18.530452728271484 (18.157506942749023) + | > loss_duration: 1.7194199562072754 (1.748246669769287) + | > loss_1: 33.07976531982422 (32.113990783691406) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003459334373474121 (-0.00034677982330322266) + | > avg_loss_disc: 2.5736576318740845 (+0.017412662506103516) + | > avg_loss_disc_real_0: 0.21867156028747559 (+0.06301835179328918) + | > avg_loss_disc_real_1: 0.2528596743941307 (+0.049822889268398285) + | > avg_loss_disc_real_2: 0.25565343350172043 (+0.03186020255088806) + | > avg_loss_disc_real_3: 0.2590404748916626 (+0.010734163224697113) + | > avg_loss_disc_real_4: 0.2557833194732666 (+0.016531042754650116) + | > avg_loss_disc_real_5: 0.23587478697299957 (-0.013669423758983612) + | > avg_loss_0: 2.5736576318740845 (+0.017412662506103516) + | > avg_loss_gen: 2.5348105430603027 (+0.2322472333908081) + | > avg_loss_kl: 2.4431735277175903 (-0.27152740955352783) + | > avg_loss_feat: 7.230252742767334 (-0.9634461402893066) + | > avg_loss_mel: 18.157506942749023 (-1.6613578796386719) + | > avg_loss_duration: 1.748246669769287 (+0.0015662908554077148) + | > avg_loss_1: 32.113990783691406 (-2.6625194549560547) + + + > EPOCH: 320/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:14:47)  + + --> TIME: 2025-04-19 04:14:48 -- STEP: 3/142 -- GLOBAL_STEP: 424725 + | > loss_disc: 2.409862518310547 (2.4314815203348794) + | > loss_disc_real_0: 0.0831420049071312 (0.11743059506018956) + | > loss_disc_real_1: 0.17781490087509155 (0.1970341702302297) + | > loss_disc_real_2: 0.21990551054477692 (0.2367403507232666) + | > loss_disc_real_3: 0.20766380429267883 (0.24114076296488443) + | > loss_disc_real_4: 0.22436654567718506 (0.2342759519815445) + | > loss_disc_real_5: 0.2207062542438507 (0.24390735725561777) + | > loss_0: 2.409862518310547 (2.4314815203348794) + | > grad_norm_0: tensor(7.2365, device='cuda:0') (tensor(7.4331, device='cuda:0')) + | > loss_gen: 2.3715286254882812 (2.349877039591471) + | > loss_kl: 1.7733325958251953 (1.8192545970280964) + | > loss_feat: 6.724414825439453 (7.348723411560059) + | > loss_mel: 17.584184646606445 (17.94310124715169) + | > loss_duration: 1.411421537399292 (1.4087021350860596) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.86488151550293 (30.869657516479492) + | > grad_norm_1: tensor(175.9693, device='cuda:0') (tensor(167.6105, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.3284 (0.3272240956624349) + | > loader_time: 0.0041 (0.004109382629394531) + + + --> TIME: 2025-04-19 04:14:57 -- STEP: 28/142 -- GLOBAL_STEP: 424750 + | > loss_disc: 2.4345109462738037 (2.4337596467563083) + | > loss_disc_real_0: 0.1136641725897789 (0.1194442981587989) + | > loss_disc_real_1: 0.21147210896015167 (0.20340806086148536) + | > loss_disc_real_2: 0.2703932225704193 (0.22295585381133215) + | > loss_disc_real_3: 0.26684296131134033 (0.23169449929680144) + | > loss_disc_real_4: 0.24377873539924622 (0.2344091912465436) + | > loss_disc_real_5: 0.2614046335220337 (0.23540193853633745) + | > loss_0: 2.4345109462738037 (2.4337596467563083) + | > grad_norm_0: tensor(7.2391, device='cuda:0') (tensor(6.8873, device='cuda:0')) + | > loss_gen: 2.5368080139160156 (2.4219921060970853) + | > loss_kl: 2.0080971717834473 (1.7894491595881326) + | > loss_feat: 7.44281530380249 (7.401416710444859) + | > loss_mel: 18.418655395507812 (17.81077790260315) + | > loss_duration: 1.4543012380599976 (1.423523451600756) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.86067771911621 (30.847159113202775) + | > grad_norm_1: tensor(366.3032, device='cuda:0') (tensor(133.7971, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.3524 (0.3372357657977513) + | > loader_time: 0.0055 (0.005084574222564697) + + + --> TIME: 2025-04-19 04:15:07 -- STEP: 53/142 -- GLOBAL_STEP: 424775 + | > loss_disc: 2.5285685062408447 (2.4354500590630304) + | > loss_disc_real_0: 0.19459982216358185 (0.11815096409815662) + | > loss_disc_real_1: 0.16739660501480103 (0.20452212809391743) + | > loss_disc_real_2: 0.2502853572368622 (0.2252144107841096) + | > loss_disc_real_3: 0.21525053679943085 (0.22723085897148781) + | > loss_disc_real_4: 0.2472279816865921 (0.22976261068065212) + | > loss_disc_real_5: 0.24705444276332855 (0.23229381498300805) + | > loss_0: 2.5285685062408447 (2.4354500590630304) + | > grad_norm_0: tensor(13.3813, device='cuda:0') (tensor(7.9072, device='cuda:0')) + | > loss_gen: 2.3489975929260254 (2.39828078701811) + | > loss_kl: 1.7103091478347778 (1.8101016755373973) + | > loss_feat: 7.274745941162109 (7.297332034920746) + | > loss_mel: 17.539621353149414 (17.776984178795004) + | > loss_duration: 1.4476807117462158 (1.4319435987832412) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.32135581970215 (30.714642290799123) + | > grad_norm_1: tensor(327.7185, device='cuda:0') (tensor(186.7233, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.3775 (0.35242684832159077) + | > loader_time: 0.0065 (0.0055374604351115676) + + + --> TIME: 2025-04-19 04:15:17 -- STEP: 78/142 -- GLOBAL_STEP: 424800 + | > loss_disc: 2.543464183807373 (2.448108453017015) + | > loss_disc_real_0: 0.1174980029463768 (0.12099483284430626) + | > loss_disc_real_1: 0.19840793311595917 (0.2048858916148161) + | > loss_disc_real_2: 0.21384012699127197 (0.22594538713112855) + | > loss_disc_real_3: 0.21796010434627533 (0.22897675881783167) + | > loss_disc_real_4: 0.18899770081043243 (0.2297486519584289) + | > loss_disc_real_5: 0.2067844420671463 (0.23283305496741563) + | > loss_0: 2.543464183807373 (2.448108453017015) + | > grad_norm_0: tensor(17.0532, device='cuda:0') (tensor(8.0957, device='cuda:0')) + | > loss_gen: 2.1666393280029297 (2.37133540556981) + | > loss_kl: 1.6419585943222046 (1.8338975952221797) + | > loss_feat: 6.891005516052246 (7.2180116543403035) + | > loss_mel: 17.010698318481445 (17.70892219054393) + | > loss_duration: 1.4875165224075317 (1.4443377348092887) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.197818756103516 (30.57650446280455) + | > grad_norm_1: tensor(332.3489, device='cuda:0') (tensor(188.6565, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.4077 (0.3663710355758667) + | > loader_time: 0.0067 (0.005888235874665089) + + + --> TIME: 2025-04-19 04:15:27 -- STEP: 103/142 -- GLOBAL_STEP: 424825 + | > loss_disc: 2.434626817703247 (2.453984633232784) + | > loss_disc_real_0: 0.10966463387012482 (0.12265710474796665) + | > loss_disc_real_1: 0.191842183470726 (0.20489763128526003) + | > loss_disc_real_2: 0.21941277384757996 (0.2258160583023886) + | > loss_disc_real_3: 0.2175462245941162 (0.22871622197257663) + | > loss_disc_real_4: 0.2528511881828308 (0.23038446425812917) + | > loss_disc_real_5: 0.24262581765651703 (0.23210473025886758) + | > loss_0: 2.434626817703247 (2.453984633232784) + | > grad_norm_0: tensor(7.6079, device='cuda:0') (tensor(9.3587, device='cuda:0')) + | > loss_gen: 2.3451948165893555 (2.3583212639521633) + | > loss_kl: 1.7009313106536865 (1.8472344643861345) + | > loss_feat: 7.168463230133057 (7.144091620028598) + | > loss_mel: 17.40102195739746 (17.702844295686887) + | > loss_duration: 1.5022103786468506 (1.454899716145784) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.117822647094727 (30.507391327793158) + | > grad_norm_1: tensor(162.3524, device='cuda:0') (tensor(212.4274, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.4153 (0.3794622004610821) + | > loader_time: 0.0065 (0.006142944965547729) + + + --> TIME: 2025-04-19 04:15:38 -- STEP: 128/142 -- GLOBAL_STEP: 424850 + | > loss_disc: 2.4972944259643555 (2.46215083450079) + | > loss_disc_real_0: 0.2204008549451828 (0.1248735988046974) + | > loss_disc_real_1: 0.24387215077877045 (0.20554194622673094) + | > loss_disc_real_2: 0.26449328660964966 (0.2270366520388052) + | > loss_disc_real_3: 0.2564562261104584 (0.23040553857572377) + | > loss_disc_real_4: 0.2697247266769409 (0.23081285180523992) + | > loss_disc_real_5: 0.28177139163017273 (0.23311127722263336) + | > loss_0: 2.4972944259643555 (2.46215083450079) + | > grad_norm_0: tensor(10.6429, device='cuda:0') (tensor(9.7166, device='cuda:0')) + | > loss_gen: 2.4216766357421875 (2.3508549425750975) + | > loss_kl: 1.8502951860427856 (1.8551505105569959) + | > loss_feat: 6.779637813568115 (7.101925011724234) + | > loss_mel: 18.52150535583496 (17.624046243727204) + | > loss_duration: 1.4927423000335693 (1.4610679922625422) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.065858840942383 (30.393044739961624) + | > grad_norm_1: tensor(372.9325, device='cuda:0') (tensor(222.8686, device='cuda:0')) + | > current_lr_0: 0.0001921574073963066 + | > current_lr_1: 0.0001921574073963066 + | > step_time: 0.4384 (0.3900082129985094) + | > loader_time: 0.0076 (0.006376191973686218) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.345486640930176 (2.345486640930176) + | > loss_disc_real_0: 0.09484729915857315 (0.09484729915857315) + | > loss_disc_real_1: 0.2050355076789856 (0.2050355076789856) + | > loss_disc_real_2: 0.2413366734981537 (0.2413366734981537) + | > loss_disc_real_3: 0.2352285534143448 (0.2352285534143448) + | > loss_disc_real_4: 0.1988600492477417 (0.1988600492477417) + | > loss_disc_real_5: 0.26634204387664795 (0.26634204387664795) + | > loss_0: 2.345486640930176 (2.345486640930176) + | > loss_gen: 2.4256293773651123 (2.4256293773651123) + | > loss_kl: 2.4051339626312256 (2.4051339626312256) + | > loss_feat: 7.415622711181641 (7.415622711181641) + | > loss_mel: 19.280826568603516 (19.280826568603516) + | > loss_duration: 1.7236037254333496 (1.7236037254333496) + | > loss_1: 33.250816345214844 (33.250816345214844) + + --> STEP: 1 + | > loss_disc: 2.55108642578125 (2.55108642578125) + | > loss_disc_real_0: 0.15987326204776764 (0.15987326204776764) + | > loss_disc_real_1: 0.24539339542388916 (0.24539339542388916) + | > loss_disc_real_2: 0.25574299693107605 (0.25574299693107605) + | > loss_disc_real_3: 0.2541245222091675 (0.2541245222091675) + | > loss_disc_real_4: 0.20899924635887146 (0.20899924635887146) + | > loss_disc_real_5: 0.1983802169561386 (0.1983802169561386) + | > loss_0: 2.55108642578125 (2.55108642578125) + | > loss_gen: 2.300506591796875 (2.300506591796875) + | > loss_kl: 2.777362823486328 (2.777362823486328) + | > loss_feat: 7.4293670654296875 (7.4293670654296875) + | > loss_mel: 19.250999450683594 (19.250999450683594) + | > loss_duration: 1.7525033950805664 (1.7525033950805664) + | > loss_1: 33.510738372802734 (33.510738372802734) + + --> STEP: 2 + | > loss_disc: 2.327759265899658 (2.439422845840454) + | > loss_disc_real_0: 0.15408721566200256 (0.1569802388548851) + | > loss_disc_real_1: 0.23503364622592926 (0.2402135208249092) + | > loss_disc_real_2: 0.21588587760925293 (0.2358144372701645) + | > loss_disc_real_3: 0.21775877475738525 (0.23594164848327637) + | > loss_disc_real_4: 0.22374363243579865 (0.21637143939733505) + | > loss_disc_real_5: 0.2230725884437561 (0.21072640269994736) + | > loss_0: 2.327759265899658 (2.439422845840454) + | > loss_gen: 2.55560564994812 (2.4280561208724976) + | > loss_kl: 2.4058942794799805 (2.5916285514831543) + | > loss_feat: 8.239298820495605 (7.8343329429626465) + | > loss_mel: 18.743080139160156 (18.997039794921875) + | > loss_duration: 1.7187755107879639 (1.7356394529342651) + | > loss_1: 33.662654876708984 (33.58669662475586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003806591033935547 (+0.0003472566604614258) + | > avg_loss_disc: 2.439422845840454 (-0.13423478603363037) + | > avg_loss_disc_real_0: 0.1569802388548851 (-0.061691321432590485) + | > avg_loss_disc_real_1: 0.2402135208249092 (-0.012646153569221497) + | > avg_loss_disc_real_2: 0.2358144372701645 (-0.01983899623155594) + | > avg_loss_disc_real_3: 0.23594164848327637 (-0.02309882640838623) + | > avg_loss_disc_real_4: 0.21637143939733505 (-0.03941188007593155) + | > avg_loss_disc_real_5: 0.21072640269994736 (-0.025148384273052216) + | > avg_loss_0: 2.439422845840454 (-0.13423478603363037) + | > avg_loss_gen: 2.4280561208724976 (-0.10675442218780518) + | > avg_loss_kl: 2.5916285514831543 (+0.14845502376556396) + | > avg_loss_feat: 7.8343329429626465 (+0.6040802001953125) + | > avg_loss_mel: 18.997039794921875 (+0.8395328521728516) + | > avg_loss_duration: 1.7356394529342651 (-0.012607216835021973) + | > avg_loss_1: 33.58669662475586 (+1.4727058410644531) + + + > EPOCH: 321/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:15:46)  + + --> TIME: 2025-04-19 04:15:51 -- STEP: 11/142 -- GLOBAL_STEP: 424875 + | > loss_disc: 2.499720573425293 (2.4407145326787774) + | > loss_disc_real_0: 0.10492773354053497 (0.12612065537409348) + | > loss_disc_real_1: 0.2003522515296936 (0.21449082277037881) + | > loss_disc_real_2: 0.24319881200790405 (0.22448715973984112) + | > loss_disc_real_3: 0.2720915377140045 (0.23318920487707312) + | > loss_disc_real_4: 0.2849072515964508 (0.23218251222913916) + | > loss_disc_real_5: 0.27131709456443787 (0.22768724100156265) + | > loss_0: 2.499720573425293 (2.4407145326787774) + | > grad_norm_0: tensor(19.8088, device='cuda:0') (tensor(14.4742, device='cuda:0')) + | > loss_gen: 2.4706242084503174 (2.4455818262967197) + | > loss_kl: 1.8037303686141968 (1.8267496065659956) + | > loss_feat: 7.793921947479248 (7.356769388372248) + | > loss_mel: 19.32237434387207 (17.776076923717152) + | > loss_duration: 1.4022703170776367 (1.4089907841248945) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.79292297363281 (30.81416858326305) + | > grad_norm_1: tensor(311.2953, device='cuda:0') (tensor(296.4987, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.3298 (0.32637088949030096) + | > loader_time: 0.0047 (0.004505114121870561) + + + --> TIME: 2025-04-19 04:15:59 -- STEP: 36/142 -- GLOBAL_STEP: 424900 + | > loss_disc: 2.481982707977295 (2.430113368564182) + | > loss_disc_real_0: 0.13613301515579224 (0.12114380237956841) + | > loss_disc_real_1: 0.271668016910553 (0.20506478307975662) + | > loss_disc_real_2: 0.21377816796302795 (0.22279048131571877) + | > loss_disc_real_3: 0.22387799620628357 (0.2245371863245964) + | > loss_disc_real_4: 0.2403714805841446 (0.22507574492030674) + | > loss_disc_real_5: 0.23104684054851532 (0.2291567586362362) + | > loss_0: 2.481982707977295 (2.430113368564182) + | > grad_norm_0: tensor(15.2152, device='cuda:0') (tensor(13.5038, device='cuda:0')) + | > loss_gen: 2.3936145305633545 (2.4119997289445667) + | > loss_kl: 1.9379080533981323 (1.8444038894441392) + | > loss_feat: 6.875638008117676 (7.231499579217699) + | > loss_mel: 17.851863861083984 (17.638302220238582) + | > loss_duration: 1.4320515394210815 (1.4240205155478582) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.491073608398438 (30.550225734710693) + | > grad_norm_1: tensor(377.5903, device='cuda:0') (tensor(276.7882, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.3539 (0.3392458226945665) + | > loader_time: 0.006 (0.005125721295674641) + + + --> TIME: 2025-04-19 04:16:09 -- STEP: 61/142 -- GLOBAL_STEP: 424925 + | > loss_disc: 2.588055372238159 (2.4426185303047054) + | > loss_disc_real_0: 0.13818074762821198 (0.12182588875293732) + | > loss_disc_real_1: 0.2035209983587265 (0.20530185738547904) + | > loss_disc_real_2: 0.2401064932346344 (0.22342026404670026) + | > loss_disc_real_3: 0.24935002624988556 (0.22704896110980238) + | > loss_disc_real_4: 0.21538254618644714 (0.2281954366652692) + | > loss_disc_real_5: 0.20482008159160614 (0.22993098223795655) + | > loss_0: 2.588055372238159 (2.4426185303047054) + | > grad_norm_0: tensor(12.5874, device='cuda:0') (tensor(12.1572, device='cuda:0')) + | > loss_gen: 2.1755712032318115 (2.39004138649487) + | > loss_kl: 1.773982048034668 (1.8521549174043) + | > loss_feat: 6.845557689666748 (7.146604006407691) + | > loss_mel: 16.42445945739746 (17.58917592783444) + | > loss_duration: 1.4604887962341309 (1.4382584427223832) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.6800594329834 (30.41623462614466) + | > grad_norm_1: tensor(292.7458, device='cuda:0') (tensor(250.7462, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.3849 (0.3504177859572114) + | > loader_time: 0.0064 (0.005506245816340212) + + + --> TIME: 2025-04-19 04:16:19 -- STEP: 86/142 -- GLOBAL_STEP: 424950 + | > loss_disc: 2.604992151260376 (2.447254549625309) + | > loss_disc_real_0: 0.1157093271613121 (0.1215674982861031) + | > loss_disc_real_1: 0.23657597601413727 (0.20605385736670606) + | > loss_disc_real_2: 0.23277533054351807 (0.22413193677053894) + | > loss_disc_real_3: 0.22923478484153748 (0.22813783256813538) + | > loss_disc_real_4: 0.22131659090518951 (0.22800645821316298) + | > loss_disc_real_5: 0.2362460345029831 (0.231726820212464) + | > loss_0: 2.604992151260376 (2.447254549625309) + | > grad_norm_0: tensor(10.6326, device='cuda:0') (tensor(11.0161, device='cuda:0')) + | > loss_gen: 2.171462059020996 (2.388810318569804) + | > loss_kl: 2.0220954418182373 (1.861442084922347) + | > loss_feat: 6.510751724243164 (7.159571897151858) + | > loss_mel: 16.373188018798828 (17.684343138406447) + | > loss_duration: 1.5099838972091675 (1.4502072625382008) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.587482452392578 (30.54437462119169) + | > grad_norm_1: tensor(99.7920, device='cuda:0') (tensor(234.9574, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.4018 (0.3626067222550859) + | > loader_time: 0.0078 (0.005818425222884777) + + + --> TIME: 2025-04-19 04:16:29 -- STEP: 111/142 -- GLOBAL_STEP: 424975 + | > loss_disc: 2.479050397872925 (2.452025843096209) + | > loss_disc_real_0: 0.13126182556152344 (0.1236222285676647) + | > loss_disc_real_1: 0.20879776775836945 (0.20626765838614455) + | > loss_disc_real_2: 0.24119918048381805 (0.2241183264298482) + | > loss_disc_real_3: 0.24380652606487274 (0.22829219094804815) + | > loss_disc_real_4: 0.21770356595516205 (0.22806598850198695) + | > loss_disc_real_5: 0.21696129441261292 (0.23084966690690667) + | > loss_0: 2.479050397872925 (2.452025843096209) + | > grad_norm_0: tensor(5.4051, device='cuda:0') (tensor(10.1383, device='cuda:0')) + | > loss_gen: 2.29817533493042 (2.373957670486725) + | > loss_kl: 1.8531992435455322 (1.869728989429302) + | > loss_feat: 6.569906711578369 (7.130044352900875) + | > loss_mel: 17.42103385925293 (17.709542265883453) + | > loss_duration: 1.5128262042999268 (1.4583938895045105) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.655141830444336 (30.541667113433014) + | > grad_norm_1: tensor(85.8993, device='cuda:0') (tensor(211.3122, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.4266 (0.3740504625681285) + | > loader_time: 0.0075 (0.006068755914499094) + + + --> TIME: 2025-04-19 04:16:41 -- STEP: 136/142 -- GLOBAL_STEP: 425000 + | > loss_disc: 2.3916711807250977 (2.4626421490136314) + | > loss_disc_real_0: 0.13211369514465332 (0.12607803617549293) + | > loss_disc_real_1: 0.1854628175497055 (0.2081455978400567) + | > loss_disc_real_2: 0.21278241276741028 (0.22525784022667827) + | > loss_disc_real_3: 0.21634134650230408 (0.22865180134334984) + | > loss_disc_real_4: 0.2230815887451172 (0.22859571972752318) + | > loss_disc_real_5: 0.27988672256469727 (0.23150351895567248) + | > loss_0: 2.3916711807250977 (2.4626421490136314) + | > grad_norm_0: tensor(5.2721, device='cuda:0') (tensor(9.7402, device='cuda:0')) + | > loss_gen: 2.37559175491333 (2.361767060616437) + | > loss_kl: 1.9321494102478027 (1.8757151531822542) + | > loss_feat: 7.48389196395874 (7.054154462674084) + | > loss_mel: 17.943870544433594 (17.627271554049337) + | > loss_duration: 1.5086894035339355 (1.4648821555516303) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.24419403076172 (30.383790338740628) + | > grad_norm_1: tensor(202.9165, device='cuda:0') (tensor(200.8885, device='cuda:0')) + | > current_lr_0: 0.00019213338772038204 + | > current_lr_1: 0.00019213338772038204 + | > step_time: 0.4499 (0.3859588048037362) + | > loader_time: 0.0078 (0.006323369110331815) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4065914154052734 (2.4065914154052734) + | > loss_disc_real_0: 0.11738500744104385 (0.11738500744104385) + | > loss_disc_real_1: 0.20350411534309387 (0.20350411534309387) + | > loss_disc_real_2: 0.2120860517024994 (0.2120860517024994) + | > loss_disc_real_3: 0.2521010935306549 (0.2521010935306549) + | > loss_disc_real_4: 0.24841651320457458 (0.24841651320457458) + | > loss_disc_real_5: 0.2651112377643585 (0.2651112377643585) + | > loss_0: 2.4065914154052734 (2.4065914154052734) + | > loss_gen: 2.3448195457458496 (2.3448195457458496) + | > loss_kl: 2.442329168319702 (2.442329168319702) + | > loss_feat: 6.623009204864502 (6.623009204864502) + | > loss_mel: 17.286705017089844 (17.286705017089844) + | > loss_duration: 1.735849380493164 (1.735849380493164) + | > loss_1: 30.43271255493164 (30.43271255493164) + + --> STEP: 1 + | > loss_disc: 2.4797163009643555 (2.4797163009643555) + | > loss_disc_real_0: 0.16534672677516937 (0.16534672677516937) + | > loss_disc_real_1: 0.22344017028808594 (0.22344017028808594) + | > loss_disc_real_2: 0.23263190686702728 (0.23263190686702728) + | > loss_disc_real_3: 0.22905857861042023 (0.22905857861042023) + | > loss_disc_real_4: 0.2292504906654358 (0.2292504906654358) + | > loss_disc_real_5: 0.20272403955459595 (0.20272403955459595) + | > loss_0: 2.4797163009643555 (2.4797163009643555) + | > loss_gen: 2.2619426250457764 (2.2619426250457764) + | > loss_kl: 2.675297737121582 (2.675297737121582) + | > loss_feat: 6.354025363922119 (6.354025363922119) + | > loss_mel: 16.338146209716797 (16.338146209716797) + | > loss_duration: 1.7536511421203613 (1.7536511421203613) + | > loss_1: 29.3830623626709 (29.3830623626709) + + --> STEP: 2 + | > loss_disc: 2.4136157035827637 (2.4466660022735596) + | > loss_disc_real_0: 0.12885290384292603 (0.1470998153090477) + | > loss_disc_real_1: 0.24033640325069427 (0.2318882867693901) + | > loss_disc_real_2: 0.25197842717170715 (0.24230516701936722) + | > loss_disc_real_3: 0.24965022504329681 (0.23935440182685852) + | > loss_disc_real_4: 0.25394323468208313 (0.24159686267375946) + | > loss_disc_real_5: 0.17804399132728577 (0.19038401544094086) + | > loss_0: 2.4136157035827637 (2.4466660022735596) + | > loss_gen: 2.3371596336364746 (2.2995511293411255) + | > loss_kl: 2.472831964492798 (2.57406485080719) + | > loss_feat: 6.7891740798950195 (6.571599721908569) + | > loss_mel: 16.663118362426758 (16.500632286071777) + | > loss_duration: 1.6965510845184326 (1.725101113319397) + | > loss_1: 29.95883560180664 (29.67094898223877) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034579038619995117 (-0.00034868717193603516) + | > avg_loss_disc: 2.4466660022735596 (+0.007243156433105469) + | > avg_loss_disc_real_0: 0.1470998153090477 (-0.009880423545837402) + | > avg_loss_disc_real_1: 0.2318882867693901 (-0.008325234055519104) + | > avg_loss_disc_real_2: 0.24230516701936722 (+0.006490729749202728) + | > avg_loss_disc_real_3: 0.23935440182685852 (+0.0034127533435821533) + | > avg_loss_disc_real_4: 0.24159686267375946 (+0.025225423276424408) + | > avg_loss_disc_real_5: 0.19038401544094086 (-0.0203423872590065) + | > avg_loss_0: 2.4466660022735596 (+0.007243156433105469) + | > avg_loss_gen: 2.2995511293411255 (-0.12850499153137207) + | > avg_loss_kl: 2.57406485080719 (-0.017563700675964355) + | > avg_loss_feat: 6.571599721908569 (-1.2627332210540771) + | > avg_loss_mel: 16.500632286071777 (-2.4964075088500977) + | > avg_loss_duration: 1.725101113319397 (-0.010538339614868164) + | > avg_loss_1: 29.67094898223877 (-3.91574764251709) + + + > EPOCH: 322/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:16:45)  + + --> TIME: 2025-04-19 04:16:52 -- STEP: 19/142 -- GLOBAL_STEP: 425025 + | > loss_disc: 2.41996169090271 (2.462295494581524) + | > loss_disc_real_0: 0.12280528992414474 (0.12069119591461985) + | > loss_disc_real_1: 0.2198987603187561 (0.21106231212615967) + | > loss_disc_real_2: 0.21196728944778442 (0.23275033580629448) + | > loss_disc_real_3: 0.21035006642341614 (0.23111445966519809) + | > loss_disc_real_4: 0.22078271210193634 (0.23471467745931526) + | > loss_disc_real_5: 0.2061333954334259 (0.2344568709009572) + | > loss_0: 2.41996169090271 (2.462295494581524) + | > grad_norm_0: tensor(4.6745, device='cuda:0') (tensor(8.3216, device='cuda:0')) + | > loss_gen: 2.312361001968384 (2.4005409918333354) + | > loss_kl: 1.939988613128662 (1.7894682194057263) + | > loss_feat: 7.74762487411499 (7.3284791394283895) + | > loss_mel: 18.305578231811523 (17.714810973719548) + | > loss_duration: 1.4273436069488525 (1.4174685164501792) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.732894897460938 (30.650767878482217) + | > grad_norm_1: tensor(76.6727, device='cuda:0') (tensor(156.3077, device='cuda:0')) + | > current_lr_0: 0.000192109371046917 + | > current_lr_1: 0.000192109371046917 + | > step_time: 0.3386 (0.32978384118331106) + | > loader_time: 0.0055 (0.005073597556666324) + + + --> TIME: 2025-04-19 04:17:01 -- STEP: 44/142 -- GLOBAL_STEP: 425050 + | > loss_disc: 2.423402786254883 (2.4553074023940344) + | > loss_disc_real_0: 0.10700374096632004 (0.12331545454534618) + | > loss_disc_real_1: 0.21798063814640045 (0.2106935161758553) + | > loss_disc_real_2: 0.21002799272537231 (0.22826926003802905) + | > loss_disc_real_3: 0.22154361009597778 (0.2287817211313681) + | > loss_disc_real_4: 0.18050697445869446 (0.23126798461784015) + | > loss_disc_real_5: 0.2185056209564209 (0.23127565160393715) + | > loss_0: 2.423402786254883 (2.4553074023940344) + | > grad_norm_0: tensor(11.7762, device='cuda:0') (tensor(7.7588, device='cuda:0')) + | > loss_gen: 2.2580368518829346 (2.3644147461110894) + | > loss_kl: 1.8952908515930176 (1.837083106691187) + | > loss_feat: 7.388904571533203 (7.157541860233653) + | > loss_mel: 18.46590805053711 (17.713314316489484) + | > loss_duration: 1.4741461277008057 (1.4327054294672878) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.48228645324707 (30.5050593289462) + | > grad_norm_1: tensor(116.7675, device='cuda:0') (tensor(134.0934, device='cuda:0')) + | > current_lr_0: 0.000192109371046917 + | > current_lr_1: 0.000192109371046917 + | > step_time: 0.3589 (0.3492459871552207) + | > loader_time: 0.0053 (0.00538457523692738) + + + --> TIME: 2025-04-19 04:17:11 -- STEP: 69/142 -- GLOBAL_STEP: 425075 + | > loss_disc: 2.5269877910614014 (2.456866402556931) + | > loss_disc_real_0: 0.12124209851026535 (0.12627859143675235) + | > loss_disc_real_1: 0.19670116901397705 (0.20729418876378433) + | > loss_disc_real_2: 0.25463199615478516 (0.22745493183965268) + | > loss_disc_real_3: 0.2680555284023285 (0.2299095243215561) + | > loss_disc_real_4: 0.24799400568008423 (0.23003428764101388) + | > loss_disc_real_5: 0.22272242605686188 (0.2312742467375769) + | > loss_0: 2.5269877910614014 (2.456866402556931) + | > grad_norm_0: tensor(5.7220, device='cuda:0') (tensor(7.2294, device='cuda:0')) + | > loss_gen: 2.405186653137207 (2.367070498673812) + | > loss_kl: 1.8695498704910278 (1.8494035903958306) + | > loss_feat: 7.387144565582275 (7.098453266033228) + | > loss_mel: 17.65875816345215 (17.667642897453874) + | > loss_duration: 1.5082099437713623 (1.4432657767033232) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.82884979248047 (30.425835954970207) + | > grad_norm_1: tensor(65.2296, device='cuda:0') (tensor(124.8078, device='cuda:0')) + | > current_lr_0: 0.000192109371046917 + | > current_lr_1: 0.000192109371046917 + | > step_time: 0.3948 (0.3592561742533808) + | > loader_time: 0.0064 (0.005609999532284944) + + + --> TIME: 2025-04-19 04:17:21 -- STEP: 94/142 -- GLOBAL_STEP: 425100 + | > loss_disc: 2.507099151611328 (2.4659430346590407) + | > loss_disc_real_0: 0.12420959770679474 (0.12780398217604516) + | > loss_disc_real_1: 0.19383862614631653 (0.20739071182114013) + | > loss_disc_real_2: 0.2391497641801834 (0.228488619815796) + | > loss_disc_real_3: 0.26625004410743713 (0.22981567236971348) + | > loss_disc_real_4: 0.27003034949302673 (0.2303426574836386) + | > loss_disc_real_5: 0.26841995120048523 (0.2330679286350595) + | > loss_0: 2.507099151611328 (2.4659430346590407) + | > grad_norm_0: tensor(15.4208, device='cuda:0') (tensor(8.0178, device='cuda:0')) + | > loss_gen: 2.5020148754119873 (2.364224324835107) + | > loss_kl: 1.7408361434936523 (1.8601628554628251) + | > loss_feat: 6.999910354614258 (7.083486663534286) + | > loss_mel: 18.916351318359375 (17.73204837961401) + | > loss_duration: 1.4939361810684204 (1.4543001816627827) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.653047561645508 (30.49422241779084) + | > grad_norm_1: tensor(249.6321, device='cuda:0') (tensor(151.2635, device='cuda:0')) + | > current_lr_0: 0.000192109371046917 + | > current_lr_1: 0.000192109371046917 + | > step_time: 0.4013 (0.36927016745222374) + | > loader_time: 0.0069 (0.005866555457419536) + + + --> TIME: 2025-04-19 04:17:32 -- STEP: 119/142 -- GLOBAL_STEP: 425125 + | > loss_disc: 2.6228272914886475 (2.463901611937194) + | > loss_disc_real_0: 0.2722678780555725 (0.12928804410856318) + | > loss_disc_real_1: 0.1868809312582016 (0.20664156610224427) + | > loss_disc_real_2: 0.25524088740348816 (0.22756250265265712) + | > loss_disc_real_3: 0.2807007431983948 (0.22950233143417775) + | > loss_disc_real_4: 0.2589174211025238 (0.22887008430577127) + | > loss_disc_real_5: 0.26814988255500793 (0.23118774843316117) + | > loss_0: 2.6228272914886475 (2.463901611937194) + | > grad_norm_0: tensor(14.5263, device='cuda:0') (tensor(9.8572, device='cuda:0')) + | > loss_gen: 2.3606112003326416 (2.3702180565906157) + | > loss_kl: 1.9937065839767456 (1.8653102752541293) + | > loss_feat: 6.642339706420898 (7.091018356195017) + | > loss_mel: 16.815332412719727 (17.693204142466328) + | > loss_duration: 1.5410988330841064 (1.4612807636501408) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.35308837890625 (30.481031690325057) + | > grad_norm_1: tensor(421.8017, device='cuda:0') (tensor(188.6864, device='cuda:0')) + | > current_lr_0: 0.000192109371046917 + | > current_lr_1: 0.000192109371046917 + | > step_time: 0.4439 (0.3807193812202005) + | > loader_time: 0.0078 (0.006138268639059627) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3062140941619873 (2.3062140941619873) + | > loss_disc_real_0: 0.14086933434009552 (0.14086933434009552) + | > loss_disc_real_1: 0.23702333867549896 (0.23702333867549896) + | > loss_disc_real_2: 0.2336987406015396 (0.2336987406015396) + | > loss_disc_real_3: 0.20488227903842926 (0.20488227903842926) + | > loss_disc_real_4: 0.21139386296272278 (0.21139386296272278) + | > loss_disc_real_5: 0.23437738418579102 (0.23437738418579102) + | > loss_0: 2.3062140941619873 (2.3062140941619873) + | > loss_gen: 2.5218405723571777 (2.5218405723571777) + | > loss_kl: 2.2401819229125977 (2.2401819229125977) + | > loss_feat: 7.708165168762207 (7.708165168762207) + | > loss_mel: 18.341014862060547 (18.341014862060547) + | > loss_duration: 1.7494174242019653 (1.7494174242019653) + | > loss_1: 32.56061935424805 (32.56061935424805) + + --> STEP: 1 + | > loss_disc: 2.3821070194244385 (2.3821070194244385) + | > loss_disc_real_0: 0.10262878239154816 (0.10262878239154816) + | > loss_disc_real_1: 0.23358958959579468 (0.23358958959579468) + | > loss_disc_real_2: 0.2315232902765274 (0.2315232902765274) + | > loss_disc_real_3: 0.2504013180732727 (0.2504013180732727) + | > loss_disc_real_4: 0.24658635258674622 (0.24658635258674622) + | > loss_disc_real_5: 0.23431499302387238 (0.23431499302387238) + | > loss_0: 2.3821070194244385 (2.3821070194244385) + | > loss_gen: 2.3805413246154785 (2.3805413246154785) + | > loss_kl: 2.887920618057251 (2.887920618057251) + | > loss_feat: 7.583551406860352 (7.583551406860352) + | > loss_mel: 18.28114891052246 (18.28114891052246) + | > loss_duration: 1.7537095546722412 (1.7537095546722412) + | > loss_1: 32.886871337890625 (32.886871337890625) + + --> STEP: 2 + | > loss_disc: 2.515221357345581 (2.4486641883850098) + | > loss_disc_real_0: 0.12479687482118607 (0.11371282860636711) + | > loss_disc_real_1: 0.23096440732479095 (0.23227699846029282) + | > loss_disc_real_2: 0.235571026802063 (0.2335471585392952) + | > loss_disc_real_3: 0.227868914604187 (0.23913511633872986) + | > loss_disc_real_4: 0.21944567561149597 (0.2330160140991211) + | > loss_disc_real_5: 0.2447783648967743 (0.23954667896032333) + | > loss_0: 2.515221357345581 (2.4486641883850098) + | > loss_gen: 2.2417478561401367 (2.3111445903778076) + | > loss_kl: 2.534414529800415 (2.711167573928833) + | > loss_feat: 6.40706729888916 (6.995309352874756) + | > loss_mel: 17.333759307861328 (17.807454109191895) + | > loss_duration: 1.7308027744293213 (1.7422561645507812) + | > loss_1: 30.247793197631836 (31.56733226776123) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003537416458129883 (+7.95125961303711e-05) + | > avg_loss_disc: 2.4486641883850098 (+0.0019981861114501953) + | > avg_loss_disc_real_0: 0.11371282860636711 (-0.03338698670268059) + | > avg_loss_disc_real_1: 0.23227699846029282 (+0.00038871169090270996) + | > avg_loss_disc_real_2: 0.2335471585392952 (-0.008758008480072021) + | > avg_loss_disc_real_3: 0.23913511633872986 (-0.0002192854881286621) + | > avg_loss_disc_real_4: 0.2330160140991211 (-0.008580848574638367) + | > avg_loss_disc_real_5: 0.23954667896032333 (+0.04916266351938248) + | > avg_loss_0: 2.4486641883850098 (+0.0019981861114501953) + | > avg_loss_gen: 2.3111445903778076 (+0.011593461036682129) + | > avg_loss_kl: 2.711167573928833 (+0.13710272312164307) + | > avg_loss_feat: 6.995309352874756 (+0.4237096309661865) + | > avg_loss_mel: 17.807454109191895 (+1.3068218231201172) + | > avg_loss_duration: 1.7422561645507812 (+0.017155051231384277) + | > avg_loss_1: 31.56733226776123 (+1.896383285522461) + + + > EPOCH: 323/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:17:44)  + + --> TIME: 2025-04-19 04:17:45 -- STEP: 2/142 -- GLOBAL_STEP: 425150 + | > loss_disc: 2.573188066482544 (2.500943183898926) + | > loss_disc_real_0: 0.26475802063941956 (0.24198780208826065) + | > loss_disc_real_1: 0.2064330279827118 (0.22218119353055954) + | > loss_disc_real_2: 0.3038811683654785 (0.2830405831336975) + | > loss_disc_real_3: 0.27108460664749146 (0.24646393954753876) + | > loss_disc_real_4: 0.2906269133090973 (0.25078266113996506) + | > loss_disc_real_5: 0.2253407984972 (0.23384052515029907) + | > loss_0: 2.573188066482544 (2.500943183898926) + | > grad_norm_0: tensor(32.8811, device='cuda:0') (tensor(30.1974, device='cuda:0')) + | > loss_gen: 2.4935646057128906 (2.5621848106384277) + | > loss_kl: 1.8349937200546265 (1.8153767585754395) + | > loss_feat: 7.0004048347473145 (7.405331611633301) + | > loss_mel: 17.942935943603516 (18.223715782165527) + | > loss_duration: 1.431929588317871 (1.408990204334259) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.70383071899414 (31.415599822998047) + | > grad_norm_1: tensor(337.6703, device='cuda:0') (tensor(276.5413, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.3237 (0.3260101079940796) + | > loader_time: 0.0045 (0.004296422004699707) + + + --> TIME: 2025-04-19 04:17:54 -- STEP: 27/142 -- GLOBAL_STEP: 425175 + | > loss_disc: 2.4345643520355225 (2.454408804575602) + | > loss_disc_real_0: 0.09356695413589478 (0.1251827005158972) + | > loss_disc_real_1: 0.2001265287399292 (0.20467532840039995) + | > loss_disc_real_2: 0.2235395461320877 (0.23386971762886755) + | > loss_disc_real_3: 0.21128368377685547 (0.23041184412108529) + | > loss_disc_real_4: 0.20003372430801392 (0.22981788714726767) + | > loss_disc_real_5: 0.2066117525100708 (0.2281788388888041) + | > loss_0: 2.4345643520355225 (2.454408804575602) + | > grad_norm_0: tensor(10.1762, device='cuda:0') (tensor(14.5268, device='cuda:0')) + | > loss_gen: 2.322226047515869 (2.3989641931321875) + | > loss_kl: 1.8924009799957275 (1.8390884046201352) + | > loss_feat: 6.746375560760498 (7.386394253483525) + | > loss_mel: 16.95277976989746 (17.70602869104456) + | > loss_duration: 1.4275940656661987 (1.4229585462146335) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.34137535095215 (30.753433933964484) + | > grad_norm_1: tensor(159.8718, device='cuda:0') (tensor(304.5748, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.349 (0.3353691012771041) + | > loader_time: 0.0055 (0.0051315007386384184) + + + --> TIME: 2025-04-19 04:18:03 -- STEP: 52/142 -- GLOBAL_STEP: 425200 + | > loss_disc: 2.4980523586273193 (2.4414075108674855) + | > loss_disc_real_0: 0.14267683029174805 (0.12177872134802431) + | > loss_disc_real_1: 0.23007673025131226 (0.20442501025704238) + | > loss_disc_real_2: 0.24100053310394287 (0.22766373134576356) + | > loss_disc_real_3: 0.24573707580566406 (0.22868268449719137) + | > loss_disc_real_4: 0.24782028794288635 (0.2311259671472586) + | > loss_disc_real_5: 0.2604963183403015 (0.2307764349075464) + | > loss_0: 2.4980523586273193 (2.4414075108674855) + | > grad_norm_0: tensor(12.2698, device='cuda:0') (tensor(11.3818, device='cuda:0')) + | > loss_gen: 2.4336273670196533 (2.3864576128812933) + | > loss_kl: 1.9300506114959717 (1.8470021257033715) + | > loss_feat: 6.446848392486572 (7.273276347380418) + | > loss_mel: 17.534765243530273 (17.652631411185638) + | > loss_duration: 1.4874600172042847 (1.4320236559097579) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.83275032043457 (30.591391269977276) + | > grad_norm_1: tensor(171.0668, device='cuda:0') (tensor(252.9377, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.365 (0.3466472488183243) + | > loader_time: 0.006 (0.005494603743919959) + + + --> TIME: 2025-04-19 04:18:13 -- STEP: 77/142 -- GLOBAL_STEP: 425225 + | > loss_disc: 2.564954996109009 (2.4380148850478136) + | > loss_disc_real_0: 0.18482685089111328 (0.12228160801452476) + | > loss_disc_real_1: 0.21482212841510773 (0.20280875517176344) + | > loss_disc_real_2: 0.24310213327407837 (0.22470599161340044) + | > loss_disc_real_3: 0.23485708236694336 (0.2278253532849349) + | > loss_disc_real_4: 0.20886602997779846 (0.22765536316029436) + | > loss_disc_real_5: 0.20811890065670013 (0.23195643123094137) + | > loss_0: 2.564954996109009 (2.4380148850478136) + | > grad_norm_0: tensor(12.4997, device='cuda:0') (tensor(10.9528, device='cuda:0')) + | > loss_gen: 2.3560996055603027 (2.375410693032401) + | > loss_kl: 1.8118163347244263 (1.8562451848736057) + | > loss_feat: 6.970066070556641 (7.209022881148697) + | > loss_mel: 17.220096588134766 (17.59475207638431) + | > loss_duration: 1.4576408863067627 (1.442702270173407) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.81572151184082 (30.478133288296785) + | > grad_norm_1: tensor(96.0666, device='cuda:0') (tensor(227.0262, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.3916 (0.3585974365085751) + | > loader_time: 0.0064 (0.005770924803498503) + + + --> TIME: 2025-04-19 04:18:23 -- STEP: 102/142 -- GLOBAL_STEP: 425250 + | > loss_disc: 2.4492716789245605 (2.4375699618283453) + | > loss_disc_real_0: 0.12039954960346222 (0.122809317656884) + | > loss_disc_real_1: 0.24613463878631592 (0.20339732222697315) + | > loss_disc_real_2: 0.21469531953334808 (0.22454703920612148) + | > loss_disc_real_3: 0.2292022705078125 (0.22838217735874886) + | > loss_disc_real_4: 0.23315194249153137 (0.2282459078466191) + | > loss_disc_real_5: 0.23690345883369446 (0.23101615116876714) + | > loss_0: 2.4492716789245605 (2.4375699618283453) + | > grad_norm_0: tensor(5.2488, device='cuda:0') (tensor(9.9799, device='cuda:0')) + | > loss_gen: 2.368356704711914 (2.3705594960380996) + | > loss_kl: 1.9895718097686768 (1.8643686362341338) + | > loss_feat: 6.889348030090332 (7.1780757670309026) + | > loss_mel: 18.142192840576172 (17.62192926219866) + | > loss_duration: 1.4663658142089844 (1.4537670366904314) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.8558349609375 (30.48870023091634) + | > grad_norm_1: tensor(177.5473, device='cuda:0') (tensor(193.7902, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.418 (0.3692945288676843) + | > loader_time: 0.0072 (0.006057430716121898) + + + --> TIME: 2025-04-19 04:18:34 -- STEP: 127/142 -- GLOBAL_STEP: 425275 + | > loss_disc: 2.529891014099121 (2.4392354938927605) + | > loss_disc_real_0: 0.14628398418426514 (0.12342686836761753) + | > loss_disc_real_1: 0.21674653887748718 (0.20395858527168514) + | > loss_disc_real_2: 0.19986744225025177 (0.22461196733271982) + | > loss_disc_real_3: 0.23489490151405334 (0.22907048968348917) + | > loss_disc_real_4: 0.24587960541248322 (0.22916444385145593) + | > loss_disc_real_5: 0.21634136140346527 (0.23064856066947848) + | > loss_0: 2.529891014099121 (2.4392354938927605) + | > grad_norm_0: tensor(5.1290, device='cuda:0') (tensor(9.3891, device='cuda:0')) + | > loss_gen: 2.144613265991211 (2.3677201515107638) + | > loss_kl: 1.8946161270141602 (1.8775118852224875) + | > loss_feat: 5.810290336608887 (7.135311948971485) + | > loss_mel: 15.552927017211914 (17.57704973596288) + | > loss_duration: 1.4975364208221436 (1.4606722499441913) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.899982452392578 (30.41826601103535) + | > grad_norm_1: tensor(257.8007, device='cuda:0') (tensor(190.7210, device='cuda:0')) + | > current_lr_0: 0.00019208535737553613 + | > current_lr_1: 0.00019208535737553613 + | > step_time: 0.4393 (0.38073172531728683) + | > loader_time: 0.0075 (0.0063198138409712185) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.422590494155884 (2.422590494155884) + | > loss_disc_real_0: 0.14229832589626312 (0.14229832589626312) + | > loss_disc_real_1: 0.20387037098407745 (0.20387037098407745) + | > loss_disc_real_2: 0.21870645880699158 (0.21870645880699158) + | > loss_disc_real_3: 0.26815202832221985 (0.26815202832221985) + | > loss_disc_real_4: 0.25563785433769226 (0.25563785433769226) + | > loss_disc_real_5: 0.22641631960868835 (0.22641631960868835) + | > loss_0: 2.422590494155884 (2.422590494155884) + | > loss_gen: 2.447070837020874 (2.447070837020874) + | > loss_kl: 2.5670056343078613 (2.5670056343078613) + | > loss_feat: 6.888746738433838 (6.888746738433838) + | > loss_mel: 18.150287628173828 (18.150287628173828) + | > loss_duration: 1.725622534751892 (1.725622534751892) + | > loss_1: 31.77873420715332 (31.77873420715332) + + --> STEP: 1 + | > loss_disc: 2.5736165046691895 (2.5736165046691895) + | > loss_disc_real_0: 0.18083743751049042 (0.18083743751049042) + | > loss_disc_real_1: 0.23028230667114258 (0.23028230667114258) + | > loss_disc_real_2: 0.24601104855537415 (0.24601104855537415) + | > loss_disc_real_3: 0.2308240532875061 (0.2308240532875061) + | > loss_disc_real_4: 0.2211943417787552 (0.2211943417787552) + | > loss_disc_real_5: 0.23909366130828857 (0.23909366130828857) + | > loss_0: 2.5736165046691895 (2.5736165046691895) + | > loss_gen: 2.3661246299743652 (2.3661246299743652) + | > loss_kl: 2.762530565261841 (2.762530565261841) + | > loss_feat: 6.702911376953125 (6.702911376953125) + | > loss_mel: 18.08733558654785 (18.08733558654785) + | > loss_duration: 1.7866239547729492 (1.7866239547729492) + | > loss_1: 31.705524444580078 (31.705524444580078) + + --> STEP: 2 + | > loss_disc: 2.711921453475952 (2.642768979072571) + | > loss_disc_real_0: 0.2381664514541626 (0.2095019444823265) + | > loss_disc_real_1: 0.22007782757282257 (0.22518006712198257) + | > loss_disc_real_2: 0.24793516099452972 (0.24697310477495193) + | > loss_disc_real_3: 0.3058308959007263 (0.2683274745941162) + | > loss_disc_real_4: 0.23509334027767181 (0.2281438410282135) + | > loss_disc_real_5: 0.26615822315216064 (0.2526259422302246) + | > loss_0: 2.711921453475952 (2.642768979072571) + | > loss_gen: 2.4161112308502197 (2.3911179304122925) + | > loss_kl: 2.4765453338623047 (2.6195379495620728) + | > loss_feat: 6.674305438995361 (6.688608407974243) + | > loss_mel: 16.997526168823242 (17.542430877685547) + | > loss_duration: 1.6816620826721191 (1.7341430187225342) + | > loss_1: 30.24614906311035 (30.975836753845215) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003487825393676758 (-4.9591064453125e-05) + | > avg_loss_disc: 2.642768979072571 (+0.19410479068756104) + | > avg_loss_disc_real_0: 0.2095019444823265 (+0.0957891158759594) + | > avg_loss_disc_real_1: 0.22518006712198257 (-0.007096931338310242) + | > avg_loss_disc_real_2: 0.24697310477495193 (+0.013425946235656738) + | > avg_loss_disc_real_3: 0.2683274745941162 (+0.029192358255386353) + | > avg_loss_disc_real_4: 0.2281438410282135 (-0.004872173070907593) + | > avg_loss_disc_real_5: 0.2526259422302246 (+0.013079263269901276) + | > avg_loss_0: 2.642768979072571 (+0.19410479068756104) + | > avg_loss_gen: 2.3911179304122925 (+0.07997334003448486) + | > avg_loss_kl: 2.6195379495620728 (-0.09162962436676025) + | > avg_loss_feat: 6.688608407974243 (-0.3067009449005127) + | > avg_loss_mel: 17.542430877685547 (-0.26502323150634766) + | > avg_loss_duration: 1.7341430187225342 (-0.00811314582824707) + | > avg_loss_1: 30.975836753845215 (-0.5914955139160156) + + + > EPOCH: 324/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:18:42)  + + --> TIME: 2025-04-19 04:18:47 -- STEP: 10/142 -- GLOBAL_STEP: 425300 + | > loss_disc: 2.3792054653167725 (2.3944207191467286) + | > loss_disc_real_0: 0.10325000435113907 (0.10684744417667388) + | > loss_disc_real_1: 0.1884113848209381 (0.2022063910961151) + | > loss_disc_real_2: 0.18377146124839783 (0.21883582174777985) + | > loss_disc_real_3: 0.23896177113056183 (0.23444103747606276) + | > loss_disc_real_4: 0.18882110714912415 (0.2236729994416237) + | > loss_disc_real_5: 0.1907571405172348 (0.22993700057268143) + | > loss_0: 2.3792054653167725 (2.3944207191467286) + | > grad_norm_0: tensor(4.5467, device='cuda:0') (tensor(5.9432, device='cuda:0')) + | > loss_gen: 2.4275994300842285 (2.400101828575134) + | > loss_kl: 1.6912250518798828 (1.808858299255371) + | > loss_feat: 7.956353187561035 (7.561915016174316) + | > loss_mel: 17.84372329711914 (17.647275924682617) + | > loss_duration: 1.39298415184021 (1.4082734346389771) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.311885833740234 (30.826424026489256) + | > grad_norm_1: tensor(60.0740, device='cuda:0') (tensor(140.8924, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.3381 (0.34810714721679686) + | > loader_time: 0.0055 (0.004552316665649414) + + + --> TIME: 2025-04-19 04:18:55 -- STEP: 35/142 -- GLOBAL_STEP: 425325 + | > loss_disc: 2.39538836479187 (2.413491937092372) + | > loss_disc_real_0: 0.1307937651872635 (0.11539470297949654) + | > loss_disc_real_1: 0.19277243316173553 (0.20124676610742295) + | > loss_disc_real_2: 0.20734433829784393 (0.22302761333329338) + | > loss_disc_real_3: 0.19813618063926697 (0.22777380645275117) + | > loss_disc_real_4: 0.20421822369098663 (0.22909411575113023) + | > loss_disc_real_5: 0.17342202365398407 (0.2287841818162373) + | > loss_0: 2.39538836479187 (2.413491937092372) + | > grad_norm_0: tensor(6.2853, device='cuda:0') (tensor(6.2665, device='cuda:0')) + | > loss_gen: 2.381615161895752 (2.4022221633366176) + | > loss_kl: 1.836540937423706 (1.8488285779953002) + | > loss_feat: 8.170446395874023 (7.4638778005327495) + | > loss_mel: 19.124027252197266 (17.801386097499304) + | > loss_duration: 1.4623788595199585 (1.4248337779726301) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.97500991821289 (30.94114821297782) + | > grad_norm_1: tensor(89.0419, device='cuda:0') (tensor(121.7790, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.361 (0.34642151423863005) + | > loader_time: 0.0063 (0.005057062421526227) + + + --> TIME: 2025-04-19 04:19:05 -- STEP: 60/142 -- GLOBAL_STEP: 425350 + | > loss_disc: 2.4593329429626465 (2.43610610961914) + | > loss_disc_real_0: 0.1398632973432541 (0.12141729866464933) + | > loss_disc_real_1: 0.2301536202430725 (0.20409924015402794) + | > loss_disc_real_2: 0.20776033401489258 (0.22394095485409102) + | > loss_disc_real_3: 0.21404635906219482 (0.2272260514398416) + | > loss_disc_real_4: 0.22901684045791626 (0.22830710684259733) + | > loss_disc_real_5: 0.2144218385219574 (0.23135470673441888) + | > loss_0: 2.4593329429626465 (2.43610610961914) + | > grad_norm_0: tensor(5.0400, device='cuda:0') (tensor(6.4184, device='cuda:0')) + | > loss_gen: 2.356567144393921 (2.3799805283546447) + | > loss_kl: 1.8771207332611084 (1.8590028782685597) + | > loss_feat: 6.277604579925537 (7.28139526049296) + | > loss_mel: 17.640182495117188 (17.677269633611047) + | > loss_duration: 1.4676599502563477 (1.4369909981886546) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.61913299560547 (30.634639167785643) + | > grad_norm_1: tensor(136.0202, device='cuda:0') (tensor(118.7874, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.3783 (0.3553053021430969) + | > loader_time: 0.0064 (0.005425457159678141) + + + --> TIME: 2025-04-19 04:19:15 -- STEP: 85/142 -- GLOBAL_STEP: 425375 + | > loss_disc: 2.48103928565979 (2.456609852173749) + | > loss_disc_real_0: 0.1306232511997223 (0.1263418304569581) + | > loss_disc_real_1: 0.22251053154468536 (0.20692402860697579) + | > loss_disc_real_2: 0.2366083264350891 (0.2262506572639241) + | > loss_disc_real_3: 0.24336907267570496 (0.22729014123187347) + | > loss_disc_real_4: 0.2837536633014679 (0.22866695365485024) + | > loss_disc_real_5: 0.21637316048145294 (0.23106463253498077) + | > loss_0: 2.48103928565979 (2.456609852173749) + | > grad_norm_0: tensor(8.0100, device='cuda:0') (tensor(6.9579, device='cuda:0')) + | > loss_gen: 2.378480911254883 (2.365404255249922) + | > loss_kl: 1.9361664056777954 (1.8724250092225916) + | > loss_feat: 6.636314392089844 (7.170610568102668) + | > loss_mel: 17.482681274414062 (17.6446414162131) + | > loss_duration: 1.4995253086090088 (1.4484494966619155) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.933168411254883 (30.501530636058135) + | > grad_norm_1: tensor(215.2105, device='cuda:0') (tensor(141.5514, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.3995 (0.36574630176319795) + | > loader_time: 0.0069 (0.005733487185309915) + + + --> TIME: 2025-04-19 04:19:25 -- STEP: 110/142 -- GLOBAL_STEP: 425400 + | > loss_disc: 2.3435020446777344 (2.4597581018101087) + | > loss_disc_real_0: 0.11074014753103256 (0.12714994434605945) + | > loss_disc_real_1: 0.22052718698978424 (0.20745450380173597) + | > loss_disc_real_2: 0.19341886043548584 (0.22492918304421686) + | > loss_disc_real_3: 0.20805586874485016 (0.22699863314628602) + | > loss_disc_real_4: 0.1896863728761673 (0.228947974876924) + | > loss_disc_real_5: 0.2453414648771286 (0.23172549402171916) + | > loss_0: 2.3435020446777344 (2.4597581018101087) + | > grad_norm_0: tensor(7.3560, device='cuda:0') (tensor(7.7680, device='cuda:0')) + | > loss_gen: 2.365577459335327 (2.3560206738385285) + | > loss_kl: 2.0434157848358154 (1.8817195892333984) + | > loss_feat: 7.095309734344482 (7.1399092197418215) + | > loss_mel: 17.543004989624023 (17.678426855260692) + | > loss_duration: 1.4784759283065796 (1.4572110316970133) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.52578353881836 (30.513287301497027) + | > grad_norm_1: tensor(231.9108, device='cuda:0') (tensor(166.7128, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.4354 (0.3755919976667924) + | > loader_time: 0.0072 (0.005968700755726208) + + + --> TIME: 2025-04-19 04:19:36 -- STEP: 135/142 -- GLOBAL_STEP: 425425 + | > loss_disc: 2.525984525680542 (2.464779585379141) + | > loss_disc_real_0: 0.12000919878482819 (0.12761891142085743) + | > loss_disc_real_1: 0.22856757044792175 (0.2081326503444601) + | > loss_disc_real_2: 0.23626063764095306 (0.22587709404804085) + | > loss_disc_real_3: 0.2477474808692932 (0.2276980851535444) + | > loss_disc_real_4: 0.24519935250282288 (0.22888068446406612) + | > loss_disc_real_5: 0.22608573734760284 (0.23198028924288572) + | > loss_0: 2.525984525680542 (2.464779585379141) + | > grad_norm_0: tensor(8.8973, device='cuda:0') (tensor(8.7258, device='cuda:0')) + | > loss_gen: 2.357133626937866 (2.3535884027127865) + | > loss_kl: 1.876761555671692 (1.8884503399884258) + | > loss_feat: 6.584922790527344 (7.107369995117187) + | > loss_mel: 16.35910987854004 (17.665640619066032) + | > loss_duration: 1.4705458879470825 (1.463072450955709) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.648473739624023 (30.47812175044307) + | > grad_norm_1: tensor(271.3004, device='cuda:0') (tensor(187.0257, device='cuda:0')) + | > current_lr_0: 0.00019206134670586418 + | > current_lr_1: 0.00019206134670586418 + | > step_time: 0.4667 (0.38749891916910806) + | > loader_time: 0.0089 (0.0062720051518193) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.643091917037964 (2.643091917037964) + | > loss_disc_real_0: 0.3644835650920868 (0.3644835650920868) + | > loss_disc_real_1: 0.2344203144311905 (0.2344203144311905) + | > loss_disc_real_2: 0.24992579221725464 (0.24992579221725464) + | > loss_disc_real_3: 0.24884623289108276 (0.24884623289108276) + | > loss_disc_real_4: 0.24851323664188385 (0.24851323664188385) + | > loss_disc_real_5: 0.23247453570365906 (0.23247453570365906) + | > loss_0: 2.643091917037964 (2.643091917037964) + | > loss_gen: 2.7340495586395264 (2.7340495586395264) + | > loss_kl: 2.3325188159942627 (2.3325188159942627) + | > loss_feat: 6.1944990158081055 (6.1944990158081055) + | > loss_mel: 15.268270492553711 (15.268270492553711) + | > loss_duration: 1.7054779529571533 (1.7054779529571533) + | > loss_1: 28.23481559753418 (28.23481559753418) + + --> STEP: 1 + | > loss_disc: 2.664592742919922 (2.664592742919922) + | > loss_disc_real_0: 0.34554582834243774 (0.34554582834243774) + | > loss_disc_real_1: 0.2363254427909851 (0.2363254427909851) + | > loss_disc_real_2: 0.2609614431858063 (0.2609614431858063) + | > loss_disc_real_3: 0.22360342741012573 (0.22360342741012573) + | > loss_disc_real_4: 0.22022032737731934 (0.22022032737731934) + | > loss_disc_real_5: 0.21690168976783752 (0.21690168976783752) + | > loss_0: 2.664592742919922 (2.664592742919922) + | > loss_gen: 2.6096301078796387 (2.6096301078796387) + | > loss_kl: 2.8898568153381348 (2.8898568153381348) + | > loss_feat: 7.132126808166504 (7.132126808166504) + | > loss_mel: 17.645326614379883 (17.645326614379883) + | > loss_duration: 1.7768585681915283 (1.7768585681915283) + | > loss_1: 32.05379867553711 (32.05379867553711) + + --> STEP: 2 + | > loss_disc: 2.478107452392578 (2.57135009765625) + | > loss_disc_real_0: 0.29633408784866333 (0.32093995809555054) + | > loss_disc_real_1: 0.2160494327545166 (0.22618743777275085) + | > loss_disc_real_2: 0.25484219193458557 (0.2579018175601959) + | > loss_disc_real_3: 0.22504112124443054 (0.22432227432727814) + | > loss_disc_real_4: 0.2477634996175766 (0.23399191349744797) + | > loss_disc_real_5: 0.21778419613838196 (0.21734294295310974) + | > loss_0: 2.478107452392578 (2.57135009765625) + | > loss_gen: 2.904031991958618 (2.7568310499191284) + | > loss_kl: 2.3340184688568115 (2.611937642097473) + | > loss_feat: 7.279357433319092 (7.205742120742798) + | > loss_mel: 17.615324020385742 (17.630325317382812) + | > loss_duration: 1.700505256652832 (1.7386819124221802) + | > loss_1: 31.833236694335938 (31.943517684936523) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034399032592773438 (-4.792213439941406e-05) + | > avg_loss_disc: 2.57135009765625 (-0.0714188814163208) + | > avg_loss_disc_real_0: 0.32093995809555054 (+0.11143801361322403) + | > avg_loss_disc_real_1: 0.22618743777275085 (+0.00100737065076828) + | > avg_loss_disc_real_2: 0.2579018175601959 (+0.010928712785243988) + | > avg_loss_disc_real_3: 0.22432227432727814 (-0.044005200266838074) + | > avg_loss_disc_real_4: 0.23399191349744797 (+0.0058480724692344666) + | > avg_loss_disc_real_5: 0.21734294295310974 (-0.03528299927711487) + | > avg_loss_0: 2.57135009765625 (-0.0714188814163208) + | > avg_loss_gen: 2.7568310499191284 (+0.36571311950683594) + | > avg_loss_kl: 2.611937642097473 (-0.007600307464599609) + | > avg_loss_feat: 7.205742120742798 (+0.5171337127685547) + | > avg_loss_mel: 17.630325317382812 (+0.08789443969726562) + | > avg_loss_duration: 1.7386819124221802 (+0.004538893699645996) + | > avg_loss_1: 31.943517684936523 (+0.9676809310913086) + + + > EPOCH: 325/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:19:41)  + + --> TIME: 2025-04-19 04:19:48 -- STEP: 18/142 -- GLOBAL_STEP: 425450 + | > loss_disc: 2.4027905464172363 (2.4069096512264676) + | > loss_disc_real_0: 0.09941863268613815 (0.10928120960791905) + | > loss_disc_real_1: 0.21127484738826752 (0.20743931167655522) + | > loss_disc_real_2: 0.20367558300495148 (0.22325585120254093) + | > loss_disc_real_3: 0.24657900631427765 (0.22323364516099295) + | > loss_disc_real_4: 0.2268589586019516 (0.2251039238439666) + | > loss_disc_real_5: 0.21332436800003052 (0.22837847554021412) + | > loss_0: 2.4027905464172363 (2.4069096512264676) + | > grad_norm_0: tensor(9.3057, device='cuda:0') (tensor(8.6252, device='cuda:0')) + | > loss_gen: 2.3815810680389404 (2.397287474738227) + | > loss_kl: 1.7159548997879028 (1.7922851973109775) + | > loss_feat: 7.064065933227539 (7.553083976109822) + | > loss_mel: 16.92702293395996 (18.024294853210446) + | > loss_duration: 1.4081355333328247 (1.4170966015921698) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.49675941467285 (31.184047910902237) + | > grad_norm_1: tensor(231.1146, device='cuda:0') (tensor(249.1268, device='cuda:0')) + | > current_lr_0: 0.00019203733903752594 + | > current_lr_1: 0.00019203733903752594 + | > step_time: 0.338 (0.3293321000205146) + | > loader_time: 0.005 (0.00483042663998074) + + + --> TIME: 2025-04-19 04:19:57 -- STEP: 43/142 -- GLOBAL_STEP: 425475 + | > loss_disc: 2.353377342224121 (2.4231449448785125) + | > loss_disc_real_0: 0.11697312444448471 (0.11607401124959768) + | > loss_disc_real_1: 0.18595772981643677 (0.2037523776985878) + | > loss_disc_real_2: 0.21249249577522278 (0.22171267104703327) + | > loss_disc_real_3: 0.21032746136188507 (0.22371153360189394) + | > loss_disc_real_4: 0.22310395538806915 (0.22988500983216043) + | > loss_disc_real_5: 0.2178495079278946 (0.23206411995166956) + | > loss_0: 2.353377342224121 (2.4231449448785125) + | > grad_norm_0: tensor(13.3667, device='cuda:0') (tensor(10.2435, device='cuda:0')) + | > loss_gen: 2.5917038917541504 (2.3947103688883224) + | > loss_kl: 1.7489182949066162 (1.8337323222049446) + | > loss_feat: 7.534884929656982 (7.368853081104367) + | > loss_mel: 19.294998168945312 (17.69561346187148) + | > loss_duration: 1.45142662525177 (1.428661731786506) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.62192916870117 (30.72157087991404) + | > grad_norm_1: tensor(76.4695, device='cuda:0') (tensor(274.9226, device='cuda:0')) + | > current_lr_0: 0.00019203733903752594 + | > current_lr_1: 0.00019203733903752594 + | > step_time: 0.3621 (0.3422220973081367) + | > loader_time: 0.0059 (0.005246334297712459) + + + --> TIME: 2025-04-19 04:20:06 -- STEP: 68/142 -- GLOBAL_STEP: 425500 + | > loss_disc: 2.2414820194244385 (2.4358268625596) + | > loss_disc_real_0: 0.08665227144956589 (0.12034482530811254) + | > loss_disc_real_1: 0.22075369954109192 (0.2064614598365391) + | > loss_disc_real_2: 0.19382800161838531 (0.22320113550214207) + | > loss_disc_real_3: 0.19644391536712646 (0.22532723975532196) + | > loss_disc_real_4: 0.19185610115528107 (0.22661500358406236) + | > loss_disc_real_5: 0.22112689912319183 (0.23020902210298708) + | > loss_0: 2.2414820194244385 (2.4358268625596) + | > grad_norm_0: tensor(10.8347, device='cuda:0') (tensor(10.5739, device='cuda:0')) + | > loss_gen: 2.499101400375366 (2.379348299082587) + | > loss_kl: 1.8819458484649658 (1.8453013914472916) + | > loss_feat: 8.393121719360352 (7.2602527141571045) + | > loss_mel: 18.872539520263672 (17.64646862534916) + | > loss_duration: 1.443366527557373 (1.4386266031685997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.09007263183594 (30.569997619180118) + | > grad_norm_1: tensor(248.9351, device='cuda:0') (tensor(271.8613, device='cuda:0')) + | > current_lr_0: 0.00019203733903752594 + | > current_lr_1: 0.00019203733903752594 + | > step_time: 0.3905 (0.3540132010684294) + | > loader_time: 0.0066 (0.005607524338890524) + + + --> TIME: 2025-04-19 04:20:17 -- STEP: 93/142 -- GLOBAL_STEP: 425525 + | > loss_disc: 2.4362385272979736 (2.442620213313771) + | > loss_disc_real_0: 0.15413694083690643 (0.12058645198422094) + | > loss_disc_real_1: 0.2289985716342926 (0.20614038439848092) + | > loss_disc_real_2: 0.22509226202964783 (0.22469274042755044) + | > loss_disc_real_3: 0.24911364912986755 (0.2258984188559235) + | > loss_disc_real_4: 0.24069419503211975 (0.22699273241463527) + | > loss_disc_real_5: 0.21225397288799286 (0.23132706681887308) + | > loss_0: 2.4362385272979736 (2.442620213313771) + | > grad_norm_0: tensor(5.2273, device='cuda:0') (tensor(10.7407, device='cuda:0')) + | > loss_gen: 2.3514485359191895 (2.3787738174520494) + | > loss_kl: 1.9573644399642944 (1.8574926955725557) + | > loss_feat: 7.347585678100586 (7.217935075042068) + | > loss_mel: 18.122644424438477 (17.631241449745758) + | > loss_duration: 1.4738924503326416 (1.4496984956085044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.252933502197266 (30.53514156546644) + | > grad_norm_1: tensor(100.2241, device='cuda:0') (tensor(269.0906, device='cuda:0')) + | > current_lr_0: 0.00019203733903752594 + | > current_lr_1: 0.00019203733903752594 + | > step_time: 0.3988 (0.3653503899933189) + | > loader_time: 0.0071 (0.005894781440816898) + + + --> TIME: 2025-04-19 04:20:27 -- STEP: 118/142 -- GLOBAL_STEP: 425550 + | > loss_disc: 2.4014906883239746 (2.4497650619280544) + | > loss_disc_real_0: 0.12870638072490692 (0.12146087464387134) + | > loss_disc_real_1: 0.21462520956993103 (0.20620246067390607) + | > loss_disc_real_2: 0.21599914133548737 (0.22462566083265564) + | > loss_disc_real_3: 0.22733208537101746 (0.2272073591159562) + | > loss_disc_real_4: 0.208375945687294 (0.22792839334677842) + | > loss_disc_real_5: 0.2591463029384613 (0.23185906129873404) + | > loss_0: 2.4014906883239746 (2.4497650619280544) + | > grad_norm_0: tensor(4.1017, device='cuda:0') (tensor(10.3273, device='cuda:0')) + | > loss_gen: 2.4520373344421387 (2.3734514794107198) + | > loss_kl: 1.9604411125183105 (1.8601314809362768) + | > loss_feat: 7.220440864562988 (7.183841701281273) + | > loss_mel: 19.007017135620117 (17.636948957281593) + | > loss_duration: 1.4592795372009277 (1.4572801630375751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.09921646118164 (30.5116538354906) + | > grad_norm_1: tensor(56.1324, device='cuda:0') (tensor(254.2235, device='cuda:0')) + | > current_lr_0: 0.00019203733903752594 + | > current_lr_1: 0.00019203733903752594 + | > step_time: 0.4406 (0.37620801440740037) + | > loader_time: 0.0072 (0.006152011580386403) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4228899478912354 (2.4228899478912354) + | > loss_disc_real_0: 0.17067061364650726 (0.17067061364650726) + | > loss_disc_real_1: 0.18953801691532135 (0.18953801691532135) + | > loss_disc_real_2: 0.2862228751182556 (0.2862228751182556) + | > loss_disc_real_3: 0.233416348695755 (0.233416348695755) + | > loss_disc_real_4: 0.23627758026123047 (0.23627758026123047) + | > loss_disc_real_5: 0.24360160529613495 (0.24360160529613495) + | > loss_0: 2.4228899478912354 (2.4228899478912354) + | > loss_gen: 2.664991617202759 (2.664991617202759) + | > loss_kl: 2.450331687927246 (2.450331687927246) + | > loss_feat: 7.376880168914795 (7.376880168914795) + | > loss_mel: 19.07267189025879 (19.07267189025879) + | > loss_duration: 1.7462093830108643 (1.7462093830108643) + | > loss_1: 33.31108474731445 (33.31108474731445) + + --> STEP: 1 + | > loss_disc: 2.5975639820098877 (2.5975639820098877) + | > loss_disc_real_0: 0.1540386825799942 (0.1540386825799942) + | > loss_disc_real_1: 0.1920158714056015 (0.1920158714056015) + | > loss_disc_real_2: 0.27257871627807617 (0.27257871627807617) + | > loss_disc_real_3: 0.2687569856643677 (0.2687569856643677) + | > loss_disc_real_4: 0.2783111035823822 (0.2783111035823822) + | > loss_disc_real_5: 0.28326866030693054 (0.28326866030693054) + | > loss_0: 2.5975639820098877 (2.5975639820098877) + | > loss_gen: 2.3598549365997314 (2.3598549365997314) + | > loss_kl: 2.5685184001922607 (2.5685184001922607) + | > loss_feat: 6.4026336669921875 (6.4026336669921875) + | > loss_mel: 18.191682815551758 (18.191682815551758) + | > loss_duration: 1.8215296268463135 (1.8215296268463135) + | > loss_1: 31.344219207763672 (31.344219207763672) + + --> STEP: 2 + | > loss_disc: 2.577488422393799 (2.5875262022018433) + | > loss_disc_real_0: 0.20900940895080566 (0.18152404576539993) + | > loss_disc_real_1: 0.22665421664714813 (0.20933504402637482) + | > loss_disc_real_2: 0.24963901937007904 (0.2611088678240776) + | > loss_disc_real_3: 0.25048014521598816 (0.2596185654401779) + | > loss_disc_real_4: 0.28143975138664246 (0.27987542748451233) + | > loss_disc_real_5: 0.21724283695220947 (0.25025574862957) + | > loss_0: 2.577488422393799 (2.5875262022018433) + | > loss_gen: 2.271061420440674 (2.3154581785202026) + | > loss_kl: 2.255378484725952 (2.4119484424591064) + | > loss_feat: 5.290262699127197 (5.846448183059692) + | > loss_mel: 13.75299072265625 (15.972336769104004) + | > loss_duration: 1.7289211750030518 (1.7752254009246826) + | > loss_1: 25.298614501953125 (28.3214168548584) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003492116928100586 (+5.221366882324219e-05) + | > avg_loss_disc: 2.5875262022018433 (+0.01617610454559326) + | > avg_loss_disc_real_0: 0.18152404576539993 (-0.1394159123301506) + | > avg_loss_disc_real_1: 0.20933504402637482 (-0.016852393746376038) + | > avg_loss_disc_real_2: 0.2611088678240776 (+0.0032070502638816833) + | > avg_loss_disc_real_3: 0.2596185654401779 (+0.03529629111289978) + | > avg_loss_disc_real_4: 0.27987542748451233 (+0.04588351398706436) + | > avg_loss_disc_real_5: 0.25025574862957 (+0.032912805676460266) + | > avg_loss_0: 2.5875262022018433 (+0.01617610454559326) + | > avg_loss_gen: 2.3154581785202026 (-0.4413728713989258) + | > avg_loss_kl: 2.4119484424591064 (-0.1999891996383667) + | > avg_loss_feat: 5.846448183059692 (-1.3592939376831055) + | > avg_loss_mel: 15.972336769104004 (-1.6579885482788086) + | > avg_loss_duration: 1.7752254009246826 (+0.03654348850250244) + | > avg_loss_1: 28.3214168548584 (-3.622100830078125) + + + > EPOCH: 326/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:20:40)  + + --> TIME: 2025-04-19 04:20:41 -- STEP: 1/142 -- GLOBAL_STEP: 425575 + | > loss_disc: 2.3849611282348633 (2.3849611282348633) + | > loss_disc_real_0: 0.09504116326570511 (0.09504116326570511) + | > loss_disc_real_1: 0.15250755846500397 (0.15250755846500397) + | > loss_disc_real_2: 0.221246138215065 (0.221246138215065) + | > loss_disc_real_3: 0.23494748771190643 (0.23494748771190643) + | > loss_disc_real_4: 0.2523345649242401 (0.2523345649242401) + | > loss_disc_real_5: 0.2290416657924652 (0.2290416657924652) + | > loss_0: 2.3849611282348633 (2.3849611282348633) + | > grad_norm_0: tensor(5.1125, device='cuda:0') (tensor(5.1125, device='cuda:0')) + | > loss_gen: 2.3892078399658203 (2.3892078399658203) + | > loss_kl: 1.73417067527771 (1.73417067527771) + | > loss_feat: 7.8997392654418945 (7.8997392654418945) + | > loss_mel: 17.66289710998535 (17.66289710998535) + | > loss_duration: 1.4037439823150635 (1.4037439823150635) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.089757919311523 (31.089757919311523) + | > grad_norm_1: tensor(232.9326, device='cuda:0') (tensor(232.9326, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.3289 (0.3288910388946533) + | > loader_time: 0.004 (0.003956317901611328) + + + --> TIME: 2025-04-19 04:20:49 -- STEP: 26/142 -- GLOBAL_STEP: 425600 + | > loss_disc: 2.5435495376586914 (2.415474818303035) + | > loss_disc_real_0: 0.13449984788894653 (0.11146034042422588) + | > loss_disc_real_1: 0.22210071980953217 (0.2037475796846243) + | > loss_disc_real_2: 0.2200523018836975 (0.22522833886054847) + | > loss_disc_real_3: 0.24073505401611328 (0.22987642597693664) + | > loss_disc_real_4: 0.2331906259059906 (0.2246959043236879) + | > loss_disc_real_5: 0.2582367956638336 (0.23117029437652) + | > loss_0: 2.5435495376586914 (2.415474818303035) + | > grad_norm_0: tensor(14.6451, device='cuda:0') (tensor(7.8513, device='cuda:0')) + | > loss_gen: 2.3873023986816406 (2.404608726501465) + | > loss_kl: 1.7701936960220337 (1.832129139166612) + | > loss_feat: 7.205441474914551 (7.423077308214628) + | > loss_mel: 17.17995834350586 (17.842037824484017) + | > loss_duration: 1.4530929327011108 (1.424403740809514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.995988845825195 (30.926256913405197) + | > grad_norm_1: tensor(382.1972, device='cuda:0') (tensor(242.0416, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.3492 (0.33386967732356143) + | > loader_time: 0.0056 (0.0051102821643535905) + + + --> TIME: 2025-04-19 04:20:58 -- STEP: 51/142 -- GLOBAL_STEP: 425625 + | > loss_disc: 2.4428188800811768 (2.4151551676731486) + | > loss_disc_real_0: 0.05812424048781395 (0.11940706725798401) + | > loss_disc_real_1: 0.21385708451271057 (0.20296336914978774) + | > loss_disc_real_2: 0.19988375902175903 (0.22364295785333596) + | > loss_disc_real_3: 0.2251766175031662 (0.2267439327403611) + | > loss_disc_real_4: 0.1909303218126297 (0.22371982330200718) + | > loss_disc_real_5: 0.1950305998325348 (0.22651351140994652) + | > loss_0: 2.4428188800811768 (2.4151551676731486) + | > grad_norm_0: tensor(16.1061, device='cuda:0') (tensor(12.9302, device='cuda:0')) + | > loss_gen: 2.365291118621826 (2.4187506367178524) + | > loss_kl: 1.8961796760559082 (1.8365289440342025) + | > loss_feat: 7.880784511566162 (7.350127912035175) + | > loss_mel: 19.10824203491211 (17.801252851299203) + | > loss_duration: 1.4402365684509277 (1.431572607919281) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.69073486328125 (30.838233237173043) + | > grad_norm_1: tensor(309.2451, device='cuda:0') (tensor(290.7345, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.3682 (0.34588866607815605) + | > loader_time: 0.0061 (0.005437187120026233) + + + --> TIME: 2025-04-19 04:21:08 -- STEP: 76/142 -- GLOBAL_STEP: 425650 + | > loss_disc: 2.5130953788757324 (2.4308704137802124) + | > loss_disc_real_0: 0.05850660800933838 (0.12617333493146457) + | > loss_disc_real_1: 0.20215603709220886 (0.20331392594073944) + | > loss_disc_real_2: 0.236868754029274 (0.22554144027986026) + | > loss_disc_real_3: 0.24500539898872375 (0.22792655817772212) + | > loss_disc_real_4: 0.24109899997711182 (0.22452942674097262) + | > loss_disc_real_5: 0.2935892641544342 (0.22912785842230446) + | > loss_0: 2.5130953788757324 (2.4308704137802124) + | > grad_norm_0: tensor(15.3744, device='cuda:0') (tensor(16.4029, device='cuda:0')) + | > loss_gen: 2.1899704933166504 (2.4409712741249487) + | > loss_kl: 1.870485782623291 (1.8471761518403103) + | > loss_feat: 7.444054126739502 (7.3544370124214575) + | > loss_mel: 18.506227493286133 (17.84458437718843) + | > loss_duration: 1.4834760427474976 (1.443118743206325) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.494213104248047 (30.930287712498714) + | > grad_norm_1: tensor(341.4676, device='cuda:0') (tensor(336.3436, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.393 (0.35814765252565084) + | > loader_time: 0.0059 (0.005746166957052131) + + + --> TIME: 2025-04-19 04:21:19 -- STEP: 101/142 -- GLOBAL_STEP: 425675 + | > loss_disc: 2.4825799465179443 (2.4374116954236937) + | > loss_disc_real_0: 0.1598646491765976 (0.1270583561520175) + | > loss_disc_real_1: 0.25947585701942444 (0.20307037190045457) + | > loss_disc_real_2: 0.27325671911239624 (0.22547834241154172) + | > loss_disc_real_3: 0.2920607924461365 (0.2289812488131004) + | > loss_disc_real_4: 0.2485574185848236 (0.22636999411158043) + | > loss_disc_real_5: 0.23673473298549652 (0.22852784690290395) + | > loss_0: 2.4825799465179443 (2.4374116954236937) + | > grad_norm_0: tensor(9.1152, device='cuda:0') (tensor(15.6829, device='cuda:0')) + | > loss_gen: 2.4373667240142822 (2.4203783615981) + | > loss_kl: 1.9419790506362915 (1.858424439288602) + | > loss_feat: 6.636226654052734 (7.260246583730868) + | > loss_mel: 17.40854263305664 (17.75682765658538) + | > loss_duration: 1.5148067474365234 (1.4541373359094751) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.938920974731445 (30.750014446749546) + | > grad_norm_1: tensor(204.9288, device='cuda:0') (tensor(316.9734, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.4128 (0.3688197159531093) + | > loader_time: 0.0068 (0.00597028921146204) + + + --> TIME: 2025-04-19 04:21:29 -- STEP: 126/142 -- GLOBAL_STEP: 425700 + | > loss_disc: 2.4007089138031006 (2.442376581449358) + | > loss_disc_real_0: 0.11677005141973495 (0.12699845193752227) + | > loss_disc_real_1: 0.17777135968208313 (0.20437065287241857) + | > loss_disc_real_2: 0.2064165472984314 (0.2247319333846607) + | > loss_disc_real_3: 0.2218960076570511 (0.22842081257748226) + | > loss_disc_real_4: 0.24128398299217224 (0.22661690496736103) + | > loss_disc_real_5: 0.19901680946350098 (0.22837224330693956) + | > loss_0: 2.4007089138031006 (2.442376581449358) + | > grad_norm_0: tensor(4.3464, device='cuda:0') (tensor(14.3350, device='cuda:0')) + | > loss_gen: 2.420701503753662 (2.3986922691738797) + | > loss_kl: 1.8681312799453735 (1.8665294921587383) + | > loss_feat: 7.586552619934082 (7.193335355274261) + | > loss_mel: 19.517404556274414 (17.720674560183557) + | > loss_duration: 1.4993335008621216 (1.461579287809039) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.89212417602539 (30.640811011904763) + | > grad_norm_1: tensor(95.5302, device='cuda:0') (tensor(289.6501, device='cuda:0')) + | > current_lr_0: 0.00019201333437014623 + | > current_lr_1: 0.00019201333437014623 + | > step_time: 0.4372 (0.38022994805896093) + | > loader_time: 0.0076 (0.006265551324874636) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6613593101501465 (2.6613593101501465) + | > loss_disc_real_0: 0.19247964024543762 (0.19247964024543762) + | > loss_disc_real_1: 0.17568348348140717 (0.17568348348140717) + | > loss_disc_real_2: 0.24750442802906036 (0.24750442802906036) + | > loss_disc_real_3: 0.21834349632263184 (0.21834349632263184) + | > loss_disc_real_4: 0.2648813724517822 (0.2648813724517822) + | > loss_disc_real_5: 0.298312783241272 (0.298312783241272) + | > loss_0: 2.6613593101501465 (2.6613593101501465) + | > loss_gen: 2.262122392654419 (2.262122392654419) + | > loss_kl: 2.4175922870635986 (2.4175922870635986) + | > loss_feat: 6.582077503204346 (6.582077503204346) + | > loss_mel: 19.243236541748047 (19.243236541748047) + | > loss_duration: 1.7209886312484741 (1.7209886312484741) + | > loss_1: 32.226016998291016 (32.226016998291016) + + --> STEP: 1 + | > loss_disc: 2.4741203784942627 (2.4741203784942627) + | > loss_disc_real_0: 0.19138634204864502 (0.19138634204864502) + | > loss_disc_real_1: 0.16562217473983765 (0.16562217473983765) + | > loss_disc_real_2: 0.2402399182319641 (0.2402399182319641) + | > loss_disc_real_3: 0.2221481204032898 (0.2221481204032898) + | > loss_disc_real_4: 0.26649343967437744 (0.26649343967437744) + | > loss_disc_real_5: 0.250326007604599 (0.250326007604599) + | > loss_0: 2.4741203784942627 (2.4741203784942627) + | > loss_gen: 2.3701789379119873 (2.3701789379119873) + | > loss_kl: 2.820335626602173 (2.820335626602173) + | > loss_feat: 6.831716060638428 (6.831716060638428) + | > loss_mel: 19.640186309814453 (19.640186309814453) + | > loss_duration: 1.8045122623443604 (1.8045122623443604) + | > loss_1: 33.4669303894043 (33.4669303894043) + + --> STEP: 2 + | > loss_disc: 2.5797312259674072 (2.526925802230835) + | > loss_disc_real_0: 0.15794222056865692 (0.17466428130865097) + | > loss_disc_real_1: 0.16972437500953674 (0.1676732748746872) + | > loss_disc_real_2: 0.221480131149292 (0.23086002469062805) + | > loss_disc_real_3: 0.23581507802009583 (0.2289815992116928) + | > loss_disc_real_4: 0.24687160551548004 (0.25668252259492874) + | > loss_disc_real_5: 0.27392303943634033 (0.26212452352046967) + | > loss_0: 2.5797312259674072 (2.526925802230835) + | > loss_gen: 2.271183490753174 (2.3206812143325806) + | > loss_kl: 2.467061996459961 (2.643698811531067) + | > loss_feat: 6.746975898742676 (6.789345979690552) + | > loss_mel: 16.992359161376953 (18.316272735595703) + | > loss_duration: 1.718519687652588 (1.7615159749984741) + | > loss_1: 30.19610023498535 (31.831515312194824) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036545991897583008 (+0.00016248226165771484) + | > avg_loss_disc: 2.526925802230835 (-0.0606003999710083) + | > avg_loss_disc_real_0: 0.17466428130865097 (-0.006859764456748962) + | > avg_loss_disc_real_1: 0.1676732748746872 (-0.04166176915168762) + | > avg_loss_disc_real_2: 0.23086002469062805 (-0.030248843133449554) + | > avg_loss_disc_real_3: 0.2289815992116928 (-0.030636966228485107) + | > avg_loss_disc_real_4: 0.25668252259492874 (-0.023192904889583588) + | > avg_loss_disc_real_5: 0.26212452352046967 (+0.011868774890899658) + | > avg_loss_0: 2.526925802230835 (-0.0606003999710083) + | > avg_loss_gen: 2.3206812143325806 (+0.00522303581237793) + | > avg_loss_kl: 2.643698811531067 (+0.23175036907196045) + | > avg_loss_feat: 6.789345979690552 (+0.9428977966308594) + | > avg_loss_mel: 18.316272735595703 (+2.343935966491699) + | > avg_loss_duration: 1.7615159749984741 (-0.013709425926208496) + | > avg_loss_1: 31.831515312194824 (+3.510098457336426) + + + > EPOCH: 327/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:21:38)  + + --> TIME: 2025-04-19 04:21:42 -- STEP: 9/142 -- GLOBAL_STEP: 425725 + | > loss_disc: 2.3868601322174072 (2.368802123599582) + | > loss_disc_real_0: 0.12227287143468857 (0.11233418269289865) + | > loss_disc_real_1: 0.21635115146636963 (0.19754288593928018) + | > loss_disc_real_2: 0.22459906339645386 (0.22368465860684714) + | > loss_disc_real_3: 0.20543822646141052 (0.2171730382574929) + | > loss_disc_real_4: 0.2417040318250656 (0.21019438405831656) + | > loss_disc_real_5: 0.24648593366146088 (0.22346871925724876) + | > loss_0: 2.3868601322174072 (2.368802123599582) + | > grad_norm_0: tensor(4.3843, device='cuda:0') (tensor(5.0217, device='cuda:0')) + | > loss_gen: 2.4606332778930664 (2.40687534544203) + | > loss_kl: 1.8330937623977661 (1.859540065129598) + | > loss_feat: 7.463054656982422 (7.319558938344319) + | > loss_mel: 18.328054428100586 (17.66572591993544) + | > loss_duration: 1.3842499256134033 (1.4139337142308552) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.469085693359375 (30.665634155273438) + | > grad_norm_1: tensor(95.8217, device='cuda:0') (tensor(98.4156, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.3656 (0.3311486509111192) + | > loader_time: 0.0077 (0.0048330359988742405) + + + --> TIME: 2025-04-19 04:21:51 -- STEP: 34/142 -- GLOBAL_STEP: 425750 + | > loss_disc: 2.377291202545166 (2.4111061516930077) + | > loss_disc_real_0: 0.12056310474872589 (0.11675411594264648) + | > loss_disc_real_1: 0.20632192492485046 (0.20187453969436533) + | > loss_disc_real_2: 0.2266630083322525 (0.22286739112699733) + | > loss_disc_real_3: 0.24654218554496765 (0.2260272222406724) + | > loss_disc_real_4: 0.23288966715335846 (0.22667266822913112) + | > loss_disc_real_5: 0.22579175233840942 (0.22587046509279923) + | > loss_0: 2.377291202545166 (2.4111061516930077) + | > grad_norm_0: tensor(4.8202, device='cuda:0') (tensor(5.4360, device='cuda:0')) + | > loss_gen: 2.4119532108306885 (2.40145797589246) + | > loss_kl: 1.7730566263198853 (1.8500751397188973) + | > loss_feat: 7.444756031036377 (7.348493449828204) + | > loss_mel: 16.879722595214844 (17.892123278449564) + | > loss_duration: 1.4324095249176025 (1.425455507110147) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.941896438598633 (30.91760528788847) + | > grad_norm_1: tensor(40.3863, device='cuda:0') (tensor(97.2982, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.3651 (0.3449576111400829) + | > loader_time: 0.0056 (0.005425249829011805) + + + --> TIME: 2025-04-19 04:22:00 -- STEP: 59/142 -- GLOBAL_STEP: 425775 + | > loss_disc: 2.32106876373291 (2.418671907004664) + | > loss_disc_real_0: 0.08715464919805527 (0.11860945378824816) + | > loss_disc_real_1: 0.1760726124048233 (0.2030795022087582) + | > loss_disc_real_2: 0.21878042817115784 (0.223396752597922) + | > loss_disc_real_3: 0.2169591635465622 (0.226202909471625) + | > loss_disc_real_4: 0.22479569911956787 (0.22507145596762834) + | > loss_disc_real_5: 0.2237033098936081 (0.2283418782181659) + | > loss_0: 2.32106876373291 (2.418671907004664) + | > grad_norm_0: tensor(6.7903, device='cuda:0') (tensor(5.8453, device='cuda:0')) + | > loss_gen: 2.4394471645355225 (2.3891336594597767) + | > loss_kl: 1.8820149898529053 (1.8541864318362737) + | > loss_feat: 7.647159576416016 (7.3018312050124345) + | > loss_mel: 18.08220863342285 (17.81970664202157) + | > loss_duration: 1.4444382190704346 (1.4353337692002117) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.495269775390625 (30.80019155599303) + | > grad_norm_1: tensor(265.1012, device='cuda:0') (tensor(111.4325, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.387 (0.35806937945091116) + | > loader_time: 0.0063 (0.005725205954858812) + + + --> TIME: 2025-04-19 04:22:11 -- STEP: 84/142 -- GLOBAL_STEP: 425800 + | > loss_disc: 2.4766011238098145 (2.4273969644591924) + | > loss_disc_real_0: 0.14159224927425385 (0.11989902971046311) + | > loss_disc_real_1: 0.21209163963794708 (0.20275308209515752) + | > loss_disc_real_2: 0.21258094906806946 (0.22507952561690694) + | > loss_disc_real_3: 0.24434268474578857 (0.22595128292838731) + | > loss_disc_real_4: 0.20774471759796143 (0.22453325082148826) + | > loss_disc_real_5: 0.2344401776790619 (0.22893120259756133) + | > loss_0: 2.4766011238098145 (2.4273969644591924) + | > grad_norm_0: tensor(4.5578, device='cuda:0') (tensor(6.0185, device='cuda:0')) + | > loss_gen: 2.398350238800049 (2.380535968712399) + | > loss_kl: 1.8885778188705444 (1.8609003225962322) + | > loss_feat: 6.809481143951416 (7.236168185869853) + | > loss_mel: 16.160974502563477 (17.736164183843705) + | > loss_duration: 1.520328164100647 (1.4479759903181162) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.777711868286133 (30.661744412921724) + | > grad_norm_1: tensor(76.8671, device='cuda:0') (tensor(108.4951, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.4031 (0.3711380021912711) + | > loader_time: 0.0068 (0.0060427018574305946) + + + --> TIME: 2025-04-19 04:22:21 -- STEP: 109/142 -- GLOBAL_STEP: 425825 + | > loss_disc: 2.4204225540161133 (2.4438079759615277) + | > loss_disc_real_0: 0.12446048110723495 (0.12249954468613371) + | > loss_disc_real_1: 0.2140732705593109 (0.20490523448231024) + | > loss_disc_real_2: 0.23258821666240692 (0.22606896205779609) + | > loss_disc_real_3: 0.2589721083641052 (0.22728611050395792) + | > loss_disc_real_4: 0.28340280055999756 (0.2272439674226516) + | > loss_disc_real_5: 0.24910017848014832 (0.23012841212640114) + | > loss_0: 2.4204225540161133 (2.4438079759615277) + | > grad_norm_0: tensor(4.1747, device='cuda:0') (tensor(6.4691, device='cuda:0')) + | > loss_gen: 2.538285493850708 (2.3720305994016324) + | > loss_kl: 1.815712571144104 (1.8651455093961242) + | > loss_feat: 7.245020866394043 (7.214276930607787) + | > loss_mel: 19.035858154296875 (17.79654719851432) + | > loss_duration: 1.497494101524353 (1.4567316453391257) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.13237380981445 (30.704731652496058) + | > grad_norm_1: tensor(76.1644, device='cuda:0') (tensor(132.4916, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.4251 (0.38295298322625115) + | > loader_time: 0.0065 (0.006252702223051579) + + + --> TIME: 2025-04-19 04:22:33 -- STEP: 134/142 -- GLOBAL_STEP: 425850 + | > loss_disc: 2.489154577255249 (2.445541657618622) + | > loss_disc_real_0: 0.11500281095504761 (0.12289506080212878) + | > loss_disc_real_1: 0.18602102994918823 (0.20496488829602055) + | > loss_disc_real_2: 0.25359541177749634 (0.22555071593665366) + | > loss_disc_real_3: 0.2233550101518631 (0.22663030706679643) + | > loss_disc_real_4: 0.21390865743160248 (0.22694255767473534) + | > loss_disc_real_5: 0.20984424650669098 (0.2300786856395095) + | > loss_0: 2.489154577255249 (2.445541657618622) + | > grad_norm_0: tensor(9.5723, device='cuda:0') (tensor(7.2716, device='cuda:0')) + | > loss_gen: 2.2324295043945312 (2.362660920442041) + | > loss_kl: 1.8631511926651 (1.8636927969420134) + | > loss_feat: 6.595432758331299 (7.190139343489462) + | > loss_mel: 16.765792846679688 (17.80339141390217) + | > loss_duration: 1.5265979766845703 (1.4633354313337978) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.9834041595459 (30.68321965345696) + | > grad_norm_1: tensor(365.1939, device='cuda:0') (tensor(151.7849, device='cuda:0')) + | > current_lr_0: 0.00019198933270334994 + | > current_lr_1: 0.00019198933270334994 + | > step_time: 0.4407 (0.3959924302884002) + | > loader_time: 0.0073 (0.006494595043694795) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.339398145675659 (2.339398145675659) + | > loss_disc_real_0: 0.1384212076663971 (0.1384212076663971) + | > loss_disc_real_1: 0.19409817457199097 (0.19409817457199097) + | > loss_disc_real_2: 0.23677204549312592 (0.23677204549312592) + | > loss_disc_real_3: 0.21625307202339172 (0.21625307202339172) + | > loss_disc_real_4: 0.21493616700172424 (0.21493616700172424) + | > loss_disc_real_5: 0.23979614675045013 (0.23979614675045013) + | > loss_0: 2.339398145675659 (2.339398145675659) + | > loss_gen: 2.46594500541687 (2.46594500541687) + | > loss_kl: 2.2866227626800537 (2.2866227626800537) + | > loss_feat: 7.513571262359619 (7.513571262359619) + | > loss_mel: 18.88709831237793 (18.88709831237793) + | > loss_duration: 1.7058202028274536 (1.7058202028274536) + | > loss_1: 32.85905838012695 (32.85905838012695) + + --> STEP: 1 + | > loss_disc: 2.532491683959961 (2.532491683959961) + | > loss_disc_real_0: 0.18555359542369843 (0.18555359542369843) + | > loss_disc_real_1: 0.2549348771572113 (0.2549348771572113) + | > loss_disc_real_2: 0.2423163801431656 (0.2423163801431656) + | > loss_disc_real_3: 0.2113867998123169 (0.2113867998123169) + | > loss_disc_real_4: 0.2567940354347229 (0.2567940354347229) + | > loss_disc_real_5: 0.2080325037240982 (0.2080325037240982) + | > loss_0: 2.532491683959961 (2.532491683959961) + | > loss_gen: 2.3068387508392334 (2.3068387508392334) + | > loss_kl: 2.7908387184143066 (2.7908387184143066) + | > loss_feat: 5.980792999267578 (5.980792999267578) + | > loss_mel: 17.24585723876953 (17.24585723876953) + | > loss_duration: 1.7612745761871338 (1.7612745761871338) + | > loss_1: 30.085601806640625 (30.085601806640625) + + --> STEP: 2 + | > loss_disc: 2.481215000152588 (2.5068533420562744) + | > loss_disc_real_0: 0.15407484769821167 (0.16981422156095505) + | > loss_disc_real_1: 0.17447753250598907 (0.2147062048316002) + | > loss_disc_real_2: 0.24661940336227417 (0.24446789175271988) + | > loss_disc_real_3: 0.23210614919662476 (0.22174647450447083) + | > loss_disc_real_4: 0.24876198172569275 (0.2527780085802078) + | > loss_disc_real_5: 0.21947813034057617 (0.2137553170323372) + | > loss_0: 2.481215000152588 (2.5068533420562744) + | > loss_gen: 2.308572292327881 (2.307705521583557) + | > loss_kl: 2.346442937850952 (2.5686408281326294) + | > loss_feat: 7.558223247528076 (6.769508123397827) + | > loss_mel: 18.92326545715332 (18.084561347961426) + | > loss_duration: 1.7068147659301758 (1.7340446710586548) + | > loss_1: 32.843318939208984 (31.464460372924805) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034499168395996094 (-0.0002046823501586914) + | > avg_loss_disc: 2.5068533420562744 (-0.020072460174560547) + | > avg_loss_disc_real_0: 0.16981422156095505 (-0.004850059747695923) + | > avg_loss_disc_real_1: 0.2147062048316002 (+0.047032929956912994) + | > avg_loss_disc_real_2: 0.24446789175271988 (+0.013607867062091827) + | > avg_loss_disc_real_3: 0.22174647450447083 (-0.007235124707221985) + | > avg_loss_disc_real_4: 0.2527780085802078 (-0.0039045140147209167) + | > avg_loss_disc_real_5: 0.2137553170323372 (-0.04836920648813248) + | > avg_loss_0: 2.5068533420562744 (-0.020072460174560547) + | > avg_loss_gen: 2.307705521583557 (-0.012975692749023438) + | > avg_loss_kl: 2.5686408281326294 (-0.0750579833984375) + | > avg_loss_feat: 6.769508123397827 (-0.01983785629272461) + | > avg_loss_mel: 18.084561347961426 (-0.23171138763427734) + | > avg_loss_duration: 1.7340446710586548 (-0.027471303939819336) + | > avg_loss_1: 31.464460372924805 (-0.36705493927001953) + + + > EPOCH: 328/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:22:38)  + + --> TIME: 2025-04-19 04:22:45 -- STEP: 17/142 -- GLOBAL_STEP: 425875 + | > loss_disc: 2.413107395172119 (2.4054941990796266) + | > loss_disc_real_0: 0.1218426376581192 (0.11952979599728304) + | > loss_disc_real_1: 0.1758602261543274 (0.20028375615091884) + | > loss_disc_real_2: 0.246432825922966 (0.2216847670428893) + | > loss_disc_real_3: 0.22748661041259766 (0.22334767439786127) + | > loss_disc_real_4: 0.222666934132576 (0.22543133094030268) + | > loss_disc_real_5: 0.21729539334774017 (0.22497365930501154) + | > loss_0: 2.413107395172119 (2.4054941990796266) + | > grad_norm_0: tensor(4.1859, device='cuda:0') (tensor(7.4577, device='cuda:0')) + | > loss_gen: 2.3124101161956787 (2.378821513232063) + | > loss_kl: 1.7836917638778687 (1.814081724952249) + | > loss_feat: 7.558879852294922 (7.417286928962259) + | > loss_mel: 17.454483032226562 (17.874098272884595) + | > loss_duration: 1.386934518814087 (1.4087681349586039) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.49639892578125 (30.89305642071892) + | > grad_norm_1: tensor(84.0260, device='cuda:0') (tensor(137.9647, device='cuda:0')) + | > current_lr_0: 0.000191965334036762 + | > current_lr_1: 0.000191965334036762 + | > step_time: 0.3342 (0.328552105847527) + | > loader_time: 0.005 (0.00481263329001034) + + + --> TIME: 2025-04-19 04:22:54 -- STEP: 42/142 -- GLOBAL_STEP: 425900 + | > loss_disc: 2.382869005203247 (2.413859804471334) + | > loss_disc_real_0: 0.11990632861852646 (0.11616622142138935) + | > loss_disc_real_1: 0.1963452845811844 (0.20317631605125608) + | > loss_disc_real_2: 0.20362472534179688 (0.2231229781394913) + | > loss_disc_real_3: 0.19733726978302002 (0.2258299287585985) + | > loss_disc_real_4: 0.19949151575565338 (0.2264674532981146) + | > loss_disc_real_5: 0.2279515564441681 (0.22921050872121537) + | > loss_0: 2.382869005203247 (2.413859804471334) + | > grad_norm_0: tensor(7.0905, device='cuda:0') (tensor(6.2891, device='cuda:0')) + | > loss_gen: 2.446445941925049 (2.4026287226449874) + | > loss_kl: 1.8911685943603516 (1.827581749075935) + | > loss_feat: 7.516180038452148 (7.438435951868693) + | > loss_mel: 18.401962280273438 (17.966335114978612) + | > loss_duration: 1.4564229249954224 (1.4250493475369044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.71217918395996 (31.060030800955637) + | > grad_norm_1: tensor(194.3423, device='cuda:0') (tensor(128.0778, device='cuda:0')) + | > current_lr_0: 0.000191965334036762 + | > current_lr_1: 0.000191965334036762 + | > step_time: 0.3626 (0.3415505602246239) + | > loader_time: 0.0055 (0.005259224346705845) + + + --> TIME: 2025-04-19 04:23:03 -- STEP: 67/142 -- GLOBAL_STEP: 425925 + | > loss_disc: 2.4171431064605713 (2.4341804660967927) + | > loss_disc_real_0: 0.1457141637802124 (0.11977458433873618) + | > loss_disc_real_1: 0.1965973973274231 (0.20660786197256686) + | > loss_disc_real_2: 0.205962136387825 (0.22484002896209263) + | > loss_disc_real_3: 0.22499632835388184 (0.22666467881914396) + | > loss_disc_real_4: 0.19974613189697266 (0.22672812818591273) + | > loss_disc_real_5: 0.20984101295471191 (0.22990545519252323) + | > loss_0: 2.4171431064605713 (2.4341804660967927) + | > grad_norm_0: tensor(7.4611, device='cuda:0') (tensor(6.5251, device='cuda:0')) + | > loss_gen: 2.318887710571289 (2.387481870935924) + | > loss_kl: 1.9886964559555054 (1.8375850399928306) + | > loss_feat: 7.396813869476318 (7.315150125702815) + | > loss_mel: 17.83806610107422 (17.925604877187244) + | > loss_duration: 1.4171751737594604 (1.4388408411794633) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.959638595581055 (30.904662687387038) + | > grad_norm_1: tensor(74.9909, device='cuda:0') (tensor(127.0006, device='cuda:0')) + | > current_lr_0: 0.000191965334036762 + | > current_lr_1: 0.000191965334036762 + | > step_time: 0.3829 (0.35313354677228787) + | > loader_time: 0.0066 (0.005554704523798245) + + + --> TIME: 2025-04-19 04:23:13 -- STEP: 92/142 -- GLOBAL_STEP: 425950 + | > loss_disc: 2.4319493770599365 (2.4381079025890524) + | > loss_disc_real_0: 0.10042064636945724 (0.12087060883641243) + | > loss_disc_real_1: 0.22803732752799988 (0.20699630078414213) + | > loss_disc_real_2: 0.23799245059490204 (0.22565620408757872) + | > loss_disc_real_3: 0.2337401807308197 (0.2260625951963922) + | > loss_disc_real_4: 0.2338564097881317 (0.22673267093689545) + | > loss_disc_real_5: 0.2201884686946869 (0.23003963186689044) + | > loss_0: 2.4319493770599365 (2.4381079025890524) + | > grad_norm_0: tensor(9.0347, device='cuda:0') (tensor(6.4366, device='cuda:0')) + | > loss_gen: 2.3167290687561035 (2.373299891534059) + | > loss_kl: 1.8518234491348267 (1.8488639722699705) + | > loss_feat: 7.375899791717529 (7.229344435360121) + | > loss_mel: 16.632320404052734 (17.836929808492247) + | > loss_duration: 1.4327187538146973 (1.4509505292643672) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.60948944091797 (30.73938854880955) + | > grad_norm_1: tensor(418.3473, device='cuda:0') (tensor(135.3279, device='cuda:0')) + | > current_lr_0: 0.000191965334036762 + | > current_lr_1: 0.000191965334036762 + | > step_time: 0.4092 (0.36442266339841095) + | > loader_time: 0.0063 (0.005813443142434825) + + + --> TIME: 2025-04-19 04:23:24 -- STEP: 117/142 -- GLOBAL_STEP: 425975 + | > loss_disc: 2.4913716316223145 (2.449510884081197) + | > loss_disc_real_0: 0.11660096794366837 (0.12261088364399396) + | > loss_disc_real_1: 0.20002083480358124 (0.20650236295838642) + | > loss_disc_real_2: 0.254943311214447 (0.2265165248233029) + | > loss_disc_real_3: 0.22201335430145264 (0.22746142375672984) + | > loss_disc_real_4: 0.276365727186203 (0.22783593718822187) + | > loss_disc_real_5: 0.253058522939682 (0.23051194833894062) + | > loss_0: 2.4913716316223145 (2.449510884081197) + | > grad_norm_0: tensor(16.5993, device='cuda:0') (tensor(7.3052, device='cuda:0')) + | > loss_gen: 2.4727532863616943 (2.3594480640867848) + | > loss_kl: 1.878908634185791 (1.8518054984573624) + | > loss_feat: 7.710868835449219 (7.153395359332745) + | > loss_mel: 17.85736846923828 (17.739626322037125) + | > loss_duration: 1.4869732856750488 (1.458636659842271) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.406871795654297 (30.562911824283436) + | > grad_norm_1: tensor(378.7966, device='cuda:0') (tensor(158.9851, device='cuda:0')) + | > current_lr_0: 0.000191965334036762 + | > current_lr_1: 0.000191965334036762 + | > step_time: 0.434 (0.3753270084022457) + | > loader_time: 0.0072 (0.006058493231096839) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6554367542266846 (2.6554367542266846) + | > loss_disc_real_0: 0.25625234842300415 (0.25625234842300415) + | > loss_disc_real_1: 0.2922453284263611 (0.2922453284263611) + | > loss_disc_real_2: 0.29707589745521545 (0.29707589745521545) + | > loss_disc_real_3: 0.2781457304954529 (0.2781457304954529) + | > loss_disc_real_4: 0.2899198830127716 (0.2899198830127716) + | > loss_disc_real_5: 0.28934943675994873 (0.28934943675994873) + | > loss_0: 2.6554367542266846 (2.6554367542266846) + | > loss_gen: 2.7690677642822266 (2.7690677642822266) + | > loss_kl: 2.400081157684326 (2.400081157684326) + | > loss_feat: 7.318331241607666 (7.318331241607666) + | > loss_mel: 19.598413467407227 (19.598413467407227) + | > loss_duration: 1.7343716621398926 (1.7343716621398926) + | > loss_1: 33.82026290893555 (33.82026290893555) + + --> STEP: 1 + | > loss_disc: 2.522519111633301 (2.522519111633301) + | > loss_disc_real_0: 0.2007828801870346 (0.2007828801870346) + | > loss_disc_real_1: 0.29012006521224976 (0.29012006521224976) + | > loss_disc_real_2: 0.2439212054014206 (0.2439212054014206) + | > loss_disc_real_3: 0.27793624997138977 (0.27793624997138977) + | > loss_disc_real_4: 0.25851187109947205 (0.25851187109947205) + | > loss_disc_real_5: 0.2499237358570099 (0.2499237358570099) + | > loss_0: 2.522519111633301 (2.522519111633301) + | > loss_gen: 2.770324230194092 (2.770324230194092) + | > loss_kl: 2.9501705169677734 (2.9501705169677734) + | > loss_feat: 6.691317081451416 (6.691317081451416) + | > loss_mel: 18.285158157348633 (18.285158157348633) + | > loss_duration: 1.772648572921753 (1.772648572921753) + | > loss_1: 32.46961975097656 (32.46961975097656) + + --> STEP: 2 + | > loss_disc: 2.4538393020629883 (2.4881792068481445) + | > loss_disc_real_0: 0.2041541486978531 (0.20246851444244385) + | > loss_disc_real_1: 0.2567141056060791 (0.27341708540916443) + | > loss_disc_real_2: 0.28569701313972473 (0.26480910927057266) + | > loss_disc_real_3: 0.23277318477630615 (0.25535471737384796) + | > loss_disc_real_4: 0.23303712904453278 (0.2457745000720024) + | > loss_disc_real_5: 0.2700760066509247 (0.2599998712539673) + | > loss_0: 2.4538393020629883 (2.4881792068481445) + | > loss_gen: 2.7985594272613525 (2.784441828727722) + | > loss_kl: 2.551739454269409 (2.7509549856185913) + | > loss_feat: 7.918455123901367 (7.304886102676392) + | > loss_mel: 18.638525009155273 (18.461841583251953) + | > loss_duration: 1.6982932090759277 (1.7354708909988403) + | > loss_1: 33.60557174682617 (33.03759574890137) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003470778465270996 (+2.086162567138672e-05) + | > avg_loss_disc: 2.4881792068481445 (-0.018674135208129883) + | > avg_loss_disc_real_0: 0.20246851444244385 (+0.0326542928814888) + | > avg_loss_disc_real_1: 0.27341708540916443 (+0.05871088057756424) + | > avg_loss_disc_real_2: 0.26480910927057266 (+0.020341217517852783) + | > avg_loss_disc_real_3: 0.25535471737384796 (+0.033608242869377136) + | > avg_loss_disc_real_4: 0.2457745000720024 (-0.007003508508205414) + | > avg_loss_disc_real_5: 0.2599998712539673 (+0.046244554221630096) + | > avg_loss_0: 2.4881792068481445 (-0.018674135208129883) + | > avg_loss_gen: 2.784441828727722 (+0.47673630714416504) + | > avg_loss_kl: 2.7509549856185913 (+0.18231415748596191) + | > avg_loss_feat: 7.304886102676392 (+0.5353779792785645) + | > avg_loss_mel: 18.461841583251953 (+0.37728023529052734) + | > avg_loss_duration: 1.7354708909988403 (+0.0014262199401855469) + | > avg_loss_1: 33.03759574890137 (+1.5731353759765625) + + + > EPOCH: 329/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:23:36)  + + --> TIME: 2025-04-19 04:23:37 -- STEP: 0/142 -- GLOBAL_STEP: 426000 + | > loss_disc: 2.503798246383667 (2.503798246383667) + | > loss_disc_real_0: 0.14661918580532074 (0.14661918580532074) + | > loss_disc_real_1: 0.2712577283382416 (0.2712577283382416) + | > loss_disc_real_2: 0.26391199231147766 (0.26391199231147766) + | > loss_disc_real_3: 0.23089012503623962 (0.23089012503623962) + | > loss_disc_real_4: 0.24328377842903137 (0.24328377842903137) + | > loss_disc_real_5: 0.2968348562717438 (0.2968348562717438) + | > loss_0: 2.503798246383667 (2.503798246383667) + | > grad_norm_0: tensor(5.4403, device='cuda:0') (tensor(5.4403, device='cuda:0')) + | > loss_gen: 2.6439123153686523 (2.6439123153686523) + | > loss_kl: 1.7979408502578735 (1.7979408502578735) + | > loss_feat: 7.333440780639648 (7.333440780639648) + | > loss_mel: 19.07349395751953 (19.07349395751953) + | > loss_duration: 1.4608826637268066 (1.4608826637268066) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.30967330932617 (32.30967330932617) + | > grad_norm_1: tensor(176.9130, device='cuda:0') (tensor(176.9130, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.3464 (0.3464016914367676) + | > loader_time: 0.3059 (0.30593371391296387) + + + --> TIME: 2025-04-19 04:23:46 -- STEP: 25/142 -- GLOBAL_STEP: 426025 + | > loss_disc: 2.5315957069396973 (2.4515275382995605) + | > loss_disc_real_0: 0.16237932443618774 (0.12305704146623611) + | > loss_disc_real_1: 0.19675509631633759 (0.20402309954166412) + | > loss_disc_real_2: 0.23124238848686218 (0.22642947256565094) + | > loss_disc_real_3: 0.2327038049697876 (0.22930675745010376) + | > loss_disc_real_4: 0.25139597058296204 (0.22838401198387145) + | > loss_disc_real_5: 0.2556411027908325 (0.23081137478351593) + | > loss_0: 2.5315957069396973 (2.4515275382995605) + | > grad_norm_0: tensor(20.1975, device='cuda:0') (tensor(11.0300, device='cuda:0')) + | > loss_gen: 2.4692554473876953 (2.389908723831177) + | > loss_kl: 1.93279230594635 (1.8044226121902467) + | > loss_feat: 7.489046096801758 (7.315796146392822) + | > loss_mel: 17.51416015625 (17.828961563110354) + | > loss_duration: 1.4669158458709717 (1.4204397916793823) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.872169494628906 (30.759529190063475) + | > grad_norm_1: tensor(375.3119, device='cuda:0') (tensor(249.1602, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.3534 (0.33449263572692867) + | > loader_time: 0.0055 (0.005070867538452148) + + + --> TIME: 2025-04-19 04:23:55 -- STEP: 50/142 -- GLOBAL_STEP: 426050 + | > loss_disc: 2.425766706466675 (2.436248207092285) + | > loss_disc_real_0: 0.11750378459692001 (0.11827667236328125) + | > loss_disc_real_1: 0.21443107724189758 (0.20150409698486327) + | > loss_disc_real_2: 0.25615811347961426 (0.22811141908168792) + | > loss_disc_real_3: 0.2310042828321457 (0.22920042365789414) + | > loss_disc_real_4: 0.2090671956539154 (0.22834717124700546) + | > loss_disc_real_5: 0.20780496299266815 (0.2281525880098343) + | > loss_0: 2.425766706466675 (2.436248207092285) + | > grad_norm_0: tensor(6.9111, device='cuda:0') (tensor(11.1405, device='cuda:0')) + | > loss_gen: 2.3221933841705322 (2.4035181427001957) + | > loss_kl: 1.8554060459136963 (1.8315299463272094) + | > loss_feat: 6.227797985076904 (7.316544532775879) + | > loss_mel: 16.318540573120117 (17.828742141723634) + | > loss_duration: 1.4578237533569336 (1.4303849697113038) + | > amp_scaler: 256.0 (261.12000000000006) + | > loss_1: 28.1817626953125 (30.810720024108885) + | > grad_norm_1: tensor(245.9395, device='cuda:0') (tensor(281.0647, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.3799 (0.3523240852355956) + | > loader_time: 0.0067 (0.005530924797058106) + + + --> TIME: 2025-04-19 04:24:05 -- STEP: 75/142 -- GLOBAL_STEP: 426075 + | > loss_disc: 2.4689853191375732 (2.4496577262878416) + | > loss_disc_real_0: 0.12443049997091293 (0.12127812465031941) + | > loss_disc_real_1: 0.16453370451927185 (0.20301094412803655) + | > loss_disc_real_2: 0.21222133934497833 (0.2274935334920883) + | > loss_disc_real_3: 0.28012794256210327 (0.2307989772160848) + | > loss_disc_real_4: 0.25501108169555664 (0.23105199257532755) + | > loss_disc_real_5: 0.2843007743358612 (0.2308062134186427) + | > loss_0: 2.4689853191375732 (2.4496577262878416) + | > grad_norm_0: tensor(7.0920, device='cuda:0') (tensor(10.2198, device='cuda:0')) + | > loss_gen: 2.4038171768188477 (2.392943604787191) + | > loss_kl: 1.972519040107727 (1.8505809529622397) + | > loss_feat: 7.162705421447754 (7.236550699869792) + | > loss_mel: 17.454816818237305 (17.73302781422933) + | > loss_duration: 1.4405226707458496 (1.4433736991882327) + | > amp_scaler: 256.0 (259.41333333333336) + | > loss_1: 30.43438148498535 (30.656476796468098) + | > grad_norm_1: tensor(211.3197, device='cuda:0') (tensor(245.4171, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.4104 (0.3663010946909587) + | > loader_time: 0.0072 (0.005860061645507812) + + + --> TIME: 2025-04-19 04:24:16 -- STEP: 100/142 -- GLOBAL_STEP: 426100 + | > loss_disc: 2.5212152004241943 (2.4602678823471074) + | > loss_disc_real_0: 0.08968313783407211 (0.12376553036272525) + | > loss_disc_real_1: 0.1984550803899765 (0.20486356139183048) + | > loss_disc_real_2: 0.22226695716381073 (0.22709666833281517) + | > loss_disc_real_3: 0.19430309534072876 (0.22931675001978874) + | > loss_disc_real_4: 0.21486711502075195 (0.23011274129152298) + | > loss_disc_real_5: 0.19074593484401703 (0.23148580461740495) + | > loss_0: 2.5212152004241943 (2.4602678823471074) + | > grad_norm_0: tensor(6.6406, device='cuda:0') (tensor(9.9843, device='cuda:0')) + | > loss_gen: 2.3151917457580566 (2.3739501237869267) + | > loss_kl: 1.9392023086547852 (1.857394700050354) + | > loss_feat: 7.289943695068359 (7.1159615802764895) + | > loss_mel: 18.054506301879883 (17.63683130264282) + | > loss_duration: 1.492794394493103 (1.4551229703426367) + | > amp_scaler: 256.0 (258.55999999999983) + | > loss_1: 31.091638565063477 (30.439260635375977) + | > grad_norm_1: tensor(259.3505, device='cuda:0') (tensor(229.9744, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.4128 (0.37822692394256596) + | > loader_time: 0.0073 (0.006086404323577881) + + + --> TIME: 2025-04-19 04:24:27 -- STEP: 125/142 -- GLOBAL_STEP: 426125 + | > loss_disc: 2.5248911380767822 (2.4626403274536135) + | > loss_disc_real_0: 0.1441064178943634 (0.12658693438768384) + | > loss_disc_real_1: 0.21529966592788696 (0.20561823713779454) + | > loss_disc_real_2: 0.23721687495708466 (0.22663515317440033) + | > loss_disc_real_3: 0.26102304458618164 (0.23000218117237092) + | > loss_disc_real_4: 0.23723141849040985 (0.23011398780345918) + | > loss_disc_real_5: 0.28590553998947144 (0.23145038771629334) + | > loss_0: 2.5248911380767822 (2.4626403274536135) + | > grad_norm_0: tensor(15.0038, device='cuda:0') (tensor(11.0257, device='cuda:0')) + | > loss_gen: 2.363168478012085 (2.370907384872437) + | > loss_kl: 1.8568190336227417 (1.8604083337783814) + | > loss_feat: 5.807887554168701 (7.056787490844727) + | > loss_mel: 15.732706069946289 (17.573828964233403) + | > loss_duration: 1.5141973495483398 (1.4626042537689214) + | > amp_scaler: 256.0 (258.04799999999983) + | > loss_1: 27.2747802734375 (30.324536376953127) + | > grad_norm_1: tensor(209.0091, device='cuda:0') (tensor(246.7834, device='cuda:0')) + | > current_lr_0: 0.0001919413383700074 + | > current_lr_1: 0.0001919413383700074 + | > step_time: 0.4345 (0.3878968753814698) + | > loader_time: 0.0081 (0.006327035903930664) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.543002128601074 (2.543002128601074) + | > loss_disc_real_0: 0.17698757350444794 (0.17698757350444794) + | > loss_disc_real_1: 0.25987547636032104 (0.25987547636032104) + | > loss_disc_real_2: 0.26954683661460876 (0.26954683661460876) + | > loss_disc_real_3: 0.24587386846542358 (0.24587386846542358) + | > loss_disc_real_4: 0.25663191080093384 (0.25663191080093384) + | > loss_disc_real_5: 0.2560286521911621 (0.2560286521911621) + | > loss_0: 2.543002128601074 (2.543002128601074) + | > loss_gen: 2.572736978530884 (2.572736978530884) + | > loss_kl: 2.503726005554199 (2.503726005554199) + | > loss_feat: 7.304919242858887 (7.304919242858887) + | > loss_mel: 19.641265869140625 (19.641265869140625) + | > loss_duration: 1.7005608081817627 (1.7005608081817627) + | > loss_1: 33.723209381103516 (33.723209381103516) + + --> STEP: 1 + | > loss_disc: 2.4508941173553467 (2.4508941173553467) + | > loss_disc_real_0: 0.1724395900964737 (0.1724395900964737) + | > loss_disc_real_1: 0.2291729748249054 (0.2291729748249054) + | > loss_disc_real_2: 0.24943120777606964 (0.24943120777606964) + | > loss_disc_real_3: 0.2298024445772171 (0.2298024445772171) + | > loss_disc_real_4: 0.2876875102519989 (0.2876875102519989) + | > loss_disc_real_5: 0.2535523474216461 (0.2535523474216461) + | > loss_0: 2.4508941173553467 (2.4508941173553467) + | > loss_gen: 2.6316466331481934 (2.6316466331481934) + | > loss_kl: 2.5891075134277344 (2.5891075134277344) + | > loss_feat: 6.944993495941162 (6.944993495941162) + | > loss_mel: 18.132556915283203 (18.132556915283203) + | > loss_duration: 1.786531925201416 (1.786531925201416) + | > loss_1: 32.0848388671875 (32.0848388671875) + + --> STEP: 2 + | > loss_disc: 2.3993215560913086 (2.4251078367233276) + | > loss_disc_real_0: 0.1495370864868164 (0.16098833829164505) + | > loss_disc_real_1: 0.18142206966876984 (0.20529752224683762) + | > loss_disc_real_2: 0.26031792163848877 (0.2548745647072792) + | > loss_disc_real_3: 0.2432401329278946 (0.23652128875255585) + | > loss_disc_real_4: 0.19226104021072388 (0.2399742752313614) + | > loss_disc_real_5: 0.18008001148700714 (0.21681617945432663) + | > loss_0: 2.3993215560913086 (2.4251078367233276) + | > loss_gen: 2.4487464427948 (2.5401965379714966) + | > loss_kl: 2.411595582962036 (2.5003515481948853) + | > loss_feat: 8.384931564331055 (7.664962530136108) + | > loss_mel: 18.22605323791504 (18.17930507659912) + | > loss_duration: 1.7084019184112549 (1.7474669218063354) + | > loss_1: 33.179725646972656 (32.63228225708008) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003757476806640625 (+0.0002866983413696289) + | > avg_loss_disc: 2.4251078367233276 (-0.0630713701248169) + | > avg_loss_disc_real_0: 0.16098833829164505 (-0.0414801761507988) + | > avg_loss_disc_real_1: 0.20529752224683762 (-0.06811956316232681) + | > avg_loss_disc_real_2: 0.2548745647072792 (-0.009934544563293457) + | > avg_loss_disc_real_3: 0.23652128875255585 (-0.018833428621292114) + | > avg_loss_disc_real_4: 0.2399742752313614 (-0.005800224840641022) + | > avg_loss_disc_real_5: 0.21681617945432663 (-0.043183691799640656) + | > avg_loss_0: 2.4251078367233276 (-0.0630713701248169) + | > avg_loss_gen: 2.5401965379714966 (-0.24424529075622559) + | > avg_loss_kl: 2.5003515481948853 (-0.25060343742370605) + | > avg_loss_feat: 7.664962530136108 (+0.3600764274597168) + | > avg_loss_mel: 18.17930507659912 (-0.28253650665283203) + | > avg_loss_duration: 1.7474669218063354 (+0.011996030807495117) + | > avg_loss_1: 32.63228225708008 (-0.40531349182128906) + + + > EPOCH: 330/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:24:36)  + + --> TIME: 2025-04-19 04:24:39 -- STEP: 8/142 -- GLOBAL_STEP: 426150 + | > loss_disc: 2.4152255058288574 (2.387699544429779) + | > loss_disc_real_0: 0.10349936783313751 (0.11028117686510086) + | > loss_disc_real_1: 0.1880037784576416 (0.2094175685197115) + | > loss_disc_real_2: 0.2719872295856476 (0.23218977078795433) + | > loss_disc_real_3: 0.2247382402420044 (0.21916301548480988) + | > loss_disc_real_4: 0.20775817334651947 (0.21371128596365452) + | > loss_disc_real_5: 0.21713688969612122 (0.21856234781444073) + | > loss_0: 2.4152255058288574 (2.387699544429779) + | > grad_norm_0: tensor(13.8154, device='cuda:0') (tensor(11.5539, device='cuda:0')) + | > loss_gen: 2.392152786254883 (2.433209717273712) + | > loss_kl: 1.8261576890945435 (1.8048547357320786) + | > loss_feat: 6.884575843811035 (7.581583499908447) + | > loss_mel: 17.37104606628418 (17.59338128566742) + | > loss_duration: 1.418311595916748 (1.4191633760929108) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.892244338989258 (30.83219265937805) + | > grad_norm_1: tensor(341.4541, device='cuda:0') (tensor(324.6154, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.3243 (0.32420897483825684) + | > loader_time: 0.0051 (0.004681199789047241) + + + --> TIME: 2025-04-19 04:24:48 -- STEP: 33/142 -- GLOBAL_STEP: 426175 + | > loss_disc: 2.2850918769836426 (2.431725935502486) + | > loss_disc_real_0: 0.1152530238032341 (0.11614681057857745) + | > loss_disc_real_1: 0.1685570329427719 (0.20641157121369333) + | > loss_disc_real_2: 0.19528143107891083 (0.22963862346880365) + | > loss_disc_real_3: 0.20537741482257843 (0.22810951055902423) + | > loss_disc_real_4: 0.2186793088912964 (0.22867909209294754) + | > loss_disc_real_5: 0.18671347200870514 (0.22492530625877957) + | > loss_0: 2.2850918769836426 (2.431725935502486) + | > grad_norm_0: tensor(13.9231, device='cuda:0') (tensor(15.4670, device='cuda:0')) + | > loss_gen: 2.2547214031219482 (2.450863383033059) + | > loss_kl: 1.747743844985962 (1.8019286972103696) + | > loss_feat: 7.3088226318359375 (7.615161462263628) + | > loss_mel: 17.914745330810547 (17.984208684979063) + | > loss_duration: 1.418245553970337 (1.4248263727534898) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.644277572631836 (31.276988462968305) + | > grad_norm_1: tensor(292.1843, device='cuda:0') (tensor(367.5380, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.355 (0.3385062651200728) + | > loader_time: 0.0055 (0.0052472822593920155) + + + --> TIME: 2025-04-19 04:24:57 -- STEP: 58/142 -- GLOBAL_STEP: 426200 + | > loss_disc: 2.4238343238830566 (2.419702517575231) + | > loss_disc_real_0: 0.1082482859492302 (0.11717359695968957) + | > loss_disc_real_1: 0.2520504295825958 (0.20670952365316195) + | > loss_disc_real_2: 0.20911180973052979 (0.2259326703075705) + | > loss_disc_real_3: 0.20138084888458252 (0.22491648741837206) + | > loss_disc_real_4: 0.20918764173984528 (0.22502801207632855) + | > loss_disc_real_5: 0.2093435376882553 (0.22307476195795783) + | > loss_0: 2.4238343238830566 (2.419702517575231) + | > grad_norm_0: tensor(6.6986, device='cuda:0') (tensor(14.0736, device='cuda:0')) + | > loss_gen: 2.318495988845825 (2.4169867614219926) + | > loss_kl: 1.8448878526687622 (1.8059936494662845) + | > loss_feat: 6.575716018676758 (7.397399713253153) + | > loss_mel: 17.448945999145508 (17.788115912470317) + | > loss_duration: 1.4707467555999756 (1.4355435145312343) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65879249572754 (30.844039522368334) + | > grad_norm_1: tensor(143.6791, device='cuda:0') (tensor(312.4786, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.3753 (0.3493348976661419) + | > loader_time: 0.0063 (0.005542171412500842) + + + --> TIME: 2025-04-19 04:25:07 -- STEP: 83/142 -- GLOBAL_STEP: 426225 + | > loss_disc: 2.3848607540130615 (2.4227029099521875) + | > loss_disc_real_0: 0.12153496593236923 (0.11855717021298695) + | > loss_disc_real_1: 0.2076857089996338 (0.20571801569088405) + | > loss_disc_real_2: 0.20708727836608887 (0.22591061566967563) + | > loss_disc_real_3: 0.22605563700199127 (0.2250268902764263) + | > loss_disc_real_4: 0.1989540010690689 (0.22696798321712447) + | > loss_disc_real_5: 0.19667266309261322 (0.2255868845316301) + | > loss_0: 2.3848607540130615 (2.4227029099521875) + | > grad_norm_0: tensor(6.1593, device='cuda:0') (tensor(12.1433, device='cuda:0')) + | > loss_gen: 2.3637428283691406 (2.405518640954811) + | > loss_kl: 1.9325159788131714 (1.8238595867731484) + | > loss_feat: 7.599966526031494 (7.370116883013622) + | > loss_mel: 19.205347061157227 (17.81924271870808) + | > loss_duration: 1.5412812232971191 (1.4468157004161055) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.642852783203125 (30.8655535916248) + | > grad_norm_1: tensor(179.1317, device='cuda:0') (tensor(275.2530, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.3967 (0.36119898830551705) + | > loader_time: 0.0067 (0.005828190998858717) + + + --> TIME: 2025-04-19 04:25:18 -- STEP: 108/142 -- GLOBAL_STEP: 426250 + | > loss_disc: 2.3903348445892334 (2.4244501723183527) + | > loss_disc_real_0: 0.0942377820611 (0.1211984079607107) + | > loss_disc_real_1: 0.20000231266021729 (0.20475288577101855) + | > loss_disc_real_2: 0.24067482352256775 (0.2251729078038975) + | > loss_disc_real_3: 0.2383721023797989 (0.22589174975399617) + | > loss_disc_real_4: 0.245815247297287 (0.22729956220697473) + | > loss_disc_real_5: 0.2175842672586441 (0.22542651853075735) + | > loss_0: 2.3903348445892334 (2.4244501723183527) + | > grad_norm_0: tensor(3.7690, device='cuda:0') (tensor(11.1899, device='cuda:0')) + | > loss_gen: 2.5263569355010986 (2.395661104608465) + | > loss_kl: 2.008415699005127 (1.8445765905910068) + | > loss_feat: 7.806400775909424 (7.325358889721058) + | > loss_mel: 19.715709686279297 (17.779441153561624) + | > loss_duration: 1.483414888381958 (1.4557183528387991) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.5402946472168 (30.800756101255065) + | > grad_norm_1: tensor(128.4430, device='cuda:0') (tensor(245.9774, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.4275 (0.3716295030381944) + | > loader_time: 0.0101 (0.0060810888255084005) + + + --> TIME: 2025-04-19 04:25:29 -- STEP: 133/142 -- GLOBAL_STEP: 426275 + | > loss_disc: 2.511864185333252 (2.432701689856393) + | > loss_disc_real_0: 0.1383260041475296 (0.1228564303172262) + | > loss_disc_real_1: 0.19880743324756622 (0.2057992400307405) + | > loss_disc_real_2: 0.23638537526130676 (0.22607642546632237) + | > loss_disc_real_3: 0.2229231745004654 (0.22596328554296852) + | > loss_disc_real_4: 0.21373972296714783 (0.22772524701921562) + | > loss_disc_real_5: 0.22817422449588776 (0.22565298028906486) + | > loss_0: 2.511864185333252 (2.432701689856393) + | > grad_norm_0: tensor(7.4521, device='cuda:0') (tensor(10.3792, device='cuda:0')) + | > loss_gen: 2.162583827972412 (2.3861960988295707) + | > loss_kl: 2.0971906185150146 (1.8559885957187279) + | > loss_feat: 6.495171070098877 (7.282832149276159) + | > loss_mel: 16.71637725830078 (17.76628755268298) + | > loss_duration: 1.5184297561645508 (1.4617705139002408) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.98975372314453 (30.753074932815437) + | > grad_norm_1: tensor(109.7618, device='cuda:0') (tensor(228.9510, device='cuda:0')) + | > current_lr_0: 0.00019191734570271117 + | > current_lr_1: 0.00019191734570271117 + | > step_time: 0.435 (0.38370020228221) + | > loader_time: 0.0076 (0.006352659454919342) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.407252311706543 (2.407252311706543) + | > loss_disc_real_0: 0.1885158121585846 (0.1885158121585846) + | > loss_disc_real_1: 0.2005058079957962 (0.2005058079957962) + | > loss_disc_real_2: 0.283757746219635 (0.283757746219635) + | > loss_disc_real_3: 0.25057491660118103 (0.25057491660118103) + | > loss_disc_real_4: 0.22693690657615662 (0.22693690657615662) + | > loss_disc_real_5: 0.22842314839363098 (0.22842314839363098) + | > loss_0: 2.407252311706543 (2.407252311706543) + | > loss_gen: 2.7732131481170654 (2.7732131481170654) + | > loss_kl: 2.4846982955932617 (2.4846982955932617) + | > loss_feat: 7.97011137008667 (7.97011137008667) + | > loss_mel: 19.251598358154297 (19.251598358154297) + | > loss_duration: 1.7042129039764404 (1.7042129039764404) + | > loss_1: 34.183834075927734 (34.183834075927734) + + --> STEP: 1 + | > loss_disc: 2.3441123962402344 (2.3441123962402344) + | > loss_disc_real_0: 0.20320415496826172 (0.20320415496826172) + | > loss_disc_real_1: 0.21408578753471375 (0.21408578753471375) + | > loss_disc_real_2: 0.2736315131187439 (0.2736315131187439) + | > loss_disc_real_3: 0.23265893757343292 (0.23265893757343292) + | > loss_disc_real_4: 0.2667708396911621 (0.2667708396911621) + | > loss_disc_real_5: 0.21058563888072968 (0.21058563888072968) + | > loss_0: 2.3441123962402344 (2.3441123962402344) + | > loss_gen: 2.721133232116699 (2.721133232116699) + | > loss_kl: 2.7497646808624268 (2.7497646808624268) + | > loss_feat: 7.402285099029541 (7.402285099029541) + | > loss_mel: 19.014373779296875 (19.014373779296875) + | > loss_duration: 1.791129231452942 (1.791129231452942) + | > loss_1: 33.678688049316406 (33.678688049316406) + + --> STEP: 2 + | > loss_disc: 2.533547878265381 (2.4388301372528076) + | > loss_disc_real_0: 0.17171408236026764 (0.18745911866426468) + | > loss_disc_real_1: 0.24308305978775024 (0.228584423661232) + | > loss_disc_real_2: 0.2968525290489197 (0.2852420210838318) + | > loss_disc_real_3: 0.24391241371631622 (0.23828567564487457) + | > loss_disc_real_4: 0.21238356828689575 (0.23957720398902893) + | > loss_disc_real_5: 0.23873844742774963 (0.22466204315423965) + | > loss_0: 2.533547878265381 (2.4388301372528076) + | > loss_gen: 2.541975736618042 (2.6315544843673706) + | > loss_kl: 2.326493740081787 (2.538129210472107) + | > loss_feat: 8.201213836669922 (7.8017494678497314) + | > loss_mel: 19.544950485229492 (19.279662132263184) + | > loss_duration: 1.7135790586471558 (1.7523541450500488) + | > loss_1: 34.32821273803711 (34.00345039367676) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034013986587524414 (-0.0003560781478881836) + | > avg_loss_disc: 2.4388301372528076 (+0.01372230052947998) + | > avg_loss_disc_real_0: 0.18745911866426468 (+0.02647078037261963) + | > avg_loss_disc_real_1: 0.228584423661232 (+0.02328690141439438) + | > avg_loss_disc_real_2: 0.2852420210838318 (+0.030367456376552582) + | > avg_loss_disc_real_3: 0.23828567564487457 (+0.0017643868923187256) + | > avg_loss_disc_real_4: 0.23957720398902893 (-0.0003970712423324585) + | > avg_loss_disc_real_5: 0.22466204315423965 (+0.007845863699913025) + | > avg_loss_0: 2.4388301372528076 (+0.01372230052947998) + | > avg_loss_gen: 2.6315544843673706 (+0.09135794639587402) + | > avg_loss_kl: 2.538129210472107 (+0.03777766227722168) + | > avg_loss_feat: 7.8017494678497314 (+0.13678693771362305) + | > avg_loss_mel: 19.279662132263184 (+1.1003570556640625) + | > avg_loss_duration: 1.7523541450500488 (+0.004887223243713379) + | > avg_loss_1: 34.00345039367676 (+1.3711681365966797) + + + > EPOCH: 331/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:25:34)  + + --> TIME: 2025-04-19 04:25:40 -- STEP: 16/142 -- GLOBAL_STEP: 426300 + | > loss_disc: 2.4828011989593506 (2.401431456208229) + | > loss_disc_real_0: 0.164240762591362 (0.11842007609084249) + | > loss_disc_real_1: 0.23427723348140717 (0.19768067076802254) + | > loss_disc_real_2: 0.24491223692893982 (0.22060204204171896) + | > loss_disc_real_3: 0.22794772684574127 (0.22838603518903255) + | > loss_disc_real_4: 0.21024969220161438 (0.2254415899515152) + | > loss_disc_real_5: 0.21961000561714172 (0.22985977865755558) + | > loss_0: 2.4828011989593506 (2.401431456208229) + | > grad_norm_0: tensor(14.8937, device='cuda:0') (tensor(9.5219, device='cuda:0')) + | > loss_gen: 2.3384318351745605 (2.408591002225876) + | > loss_kl: 1.7178491353988647 (1.7820089533925056) + | > loss_feat: 6.4706501960754395 (7.435404002666473) + | > loss_mel: 15.963518142700195 (17.590768814086914) + | > loss_duration: 1.3920276165008545 (1.4097935035824776) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.882476806640625 (30.62656581401825) + | > grad_norm_1: tensor(93.5803, device='cuda:0') (tensor(143.3290, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.338 (0.32782596349716187) + | > loader_time: 0.0053 (0.004818081855773926) + + + --> TIME: 2025-04-19 04:25:49 -- STEP: 41/142 -- GLOBAL_STEP: 426325 + | > loss_disc: 2.3616154193878174 (2.417802607140889) + | > loss_disc_real_0: 0.09600860625505447 (0.11876285639477939) + | > loss_disc_real_1: 0.20142991840839386 (0.20399039765683616) + | > loss_disc_real_2: 0.21942763030529022 (0.22409722136288154) + | > loss_disc_real_3: 0.21691684424877167 (0.22655001282691956) + | > loss_disc_real_4: 0.22732974588871002 (0.22604639755516517) + | > loss_disc_real_5: 0.25980669260025024 (0.22645824747841534) + | > loss_0: 2.3616154193878174 (2.417802607140889) + | > grad_norm_0: tensor(3.3903, device='cuda:0') (tensor(7.5969, device='cuda:0')) + | > loss_gen: 2.4339792728424072 (2.3941506118309213) + | > loss_kl: 1.8071403503417969 (1.83772540673977) + | > loss_feat: 7.121189117431641 (7.317449627853021) + | > loss_mel: 17.537235260009766 (17.67962274318788) + | > loss_duration: 1.410493016242981 (1.424313865056852) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.31003761291504 (30.65326211510635) + | > grad_norm_1: tensor(110.9475, device='cuda:0') (tensor(145.6382, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.3612 (0.3407820841161216) + | > loader_time: 0.0058 (0.005281302987075433) + + + --> TIME: 2025-04-19 04:25:59 -- STEP: 66/142 -- GLOBAL_STEP: 426350 + | > loss_disc: 2.389101505279541 (2.42480954618165) + | > loss_disc_real_0: 0.08810605853796005 (0.12159828649777354) + | > loss_disc_real_1: 0.19797062873840332 (0.20491595792047906) + | > loss_disc_real_2: 0.21806871891021729 (0.22336333606279257) + | > loss_disc_real_3: 0.20802855491638184 (0.22768186235969717) + | > loss_disc_real_4: 0.23226094245910645 (0.22523771677956436) + | > loss_disc_real_5: 0.2216099202632904 (0.22731209659215176) + | > loss_0: 2.389101505279541 (2.42480954618165) + | > grad_norm_0: tensor(13.3728, device='cuda:0') (tensor(9.0397, device='cuda:0')) + | > loss_gen: 2.5047402381896973 (2.39188746250037) + | > loss_kl: 1.8080097436904907 (1.8532863945672007) + | > loss_feat: 7.632469654083252 (7.316406170527141) + | > loss_mel: 17.964780807495117 (17.722138664939184) + | > loss_duration: 1.4590950012207031 (1.438159662665743) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.369094848632812 (30.721878427447695) + | > grad_norm_1: tensor(382.3460, device='cuda:0') (tensor(187.7419, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.3859 (0.3546572128931682) + | > loader_time: 0.0057 (0.005580707029862838) + + + --> TIME: 2025-04-19 04:26:09 -- STEP: 91/142 -- GLOBAL_STEP: 426375 + | > loss_disc: 2.420397996902466 (2.4366255749712935) + | > loss_disc_real_0: 0.08902689069509506 (0.12274896721918505) + | > loss_disc_real_1: 0.2596735954284668 (0.20558226567048293) + | > loss_disc_real_2: 0.21301895380020142 (0.22398881041086638) + | > loss_disc_real_3: 0.1946556270122528 (0.22854950712932334) + | > loss_disc_real_4: 0.21458905935287476 (0.22727129420081338) + | > loss_disc_real_5: 0.19403745234012604 (0.2285303078837447) + | > loss_0: 2.420397996902466 (2.4366255749712935) + | > grad_norm_0: tensor(4.3152, device='cuda:0') (tensor(8.8136, device='cuda:0')) + | > loss_gen: 2.3865153789520264 (2.3817798855540513) + | > loss_kl: 1.8656704425811768 (1.8594172616581341) + | > loss_feat: 7.772549152374268 (7.261460482419192) + | > loss_mel: 17.953834533691406 (17.739612705104946) + | > loss_duration: 1.4776798486709595 (1.4501940273976588) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.456249237060547 (30.692464472173334) + | > grad_norm_1: tensor(141.8216, device='cuda:0') (tensor(186.1807, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.4012 (0.36570643592666807) + | > loader_time: 0.0071 (0.005848517784705526) + + + --> TIME: 2025-04-19 04:26:20 -- STEP: 116/142 -- GLOBAL_STEP: 426400 + | > loss_disc: 2.553887367248535 (2.4446288408904246) + | > loss_disc_real_0: 0.10556085407733917 (0.12319731301274793) + | > loss_disc_real_1: 0.2159128189086914 (0.20564317176568098) + | > loss_disc_real_2: 0.24343182146549225 (0.22535886037452468) + | > loss_disc_real_3: 0.24218395352363586 (0.22936637404149976) + | > loss_disc_real_4: 0.21202832460403442 (0.22892290957528968) + | > loss_disc_real_5: 0.26071980595588684 (0.22858801728178715) + | > loss_0: 2.553887367248535 (2.4446288408904246) + | > grad_norm_0: tensor(19.7158, device='cuda:0') (tensor(9.1013, device='cuda:0')) + | > loss_gen: 2.3205182552337646 (2.3713836484941937) + | > loss_kl: 1.9456332921981812 (1.8664753550085529) + | > loss_feat: 6.672935962677002 (7.221865177154541) + | > loss_mel: 17.411827087402344 (17.73718426145356) + | > loss_duration: 1.473360538482666 (1.4579403729274356) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.82427406311035 (30.65484887155993) + | > grad_norm_1: tensor(336.6028, device='cuda:0') (tensor(200.0639, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.429 (0.37622335861469147) + | > loader_time: 0.0071 (0.006126451081243053) + + + --> TIME: 2025-04-19 04:26:31 -- STEP: 141/142 -- GLOBAL_STEP: 426425 + | > loss_disc: 2.257477045059204 (2.4476730772789494) + | > loss_disc_real_0: 0.061756573617458344 (0.12469401591001673) + | > loss_disc_real_1: 0.19893787801265717 (0.20588764791370284) + | > loss_disc_real_2: 0.17013615369796753 (0.2255256583927371) + | > loss_disc_real_3: 0.1900038868188858 (0.228947233540792) + | > loss_disc_real_4: 0.1997750997543335 (0.22812650907546916) + | > loss_disc_real_5: 0.19768132269382477 (0.22803520775855857) + | > loss_0: 2.257477045059204 (2.4476730772789494) + | > grad_norm_0: tensor(14.0403, device='cuda:0') (tensor(9.5002, device='cuda:0')) + | > loss_gen: 2.530289649963379 (2.3671358308048114) + | > loss_kl: 1.5734752416610718 (1.8733756795842598) + | > loss_feat: 8.931721687316895 (7.1910306747923505) + | > loss_mel: 18.77422523498535 (17.767671382173575) + | > loss_duration: 1.3324785232543945 (1.4636730691219897) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.142189025878906 (30.662886734549883) + | > grad_norm_1: tensor(309.4374, device='cuda:0') (tensor(203.6471, device='cuda:0')) + | > current_lr_0: 0.00019189335603449832 + | > current_lr_1: 0.00019189335603449832 + | > step_time: 0.2017 (0.38654896384435355) + | > loader_time: 0.0028 (0.006316315197775548) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.387690782546997 (2.387690782546997) + | > loss_disc_real_0: 0.10130085796117783 (0.10130085796117783) + | > loss_disc_real_1: 0.24470070004463196 (0.24470070004463196) + | > loss_disc_real_2: 0.22314032912254333 (0.22314032912254333) + | > loss_disc_real_3: 0.21897639334201813 (0.21897639334201813) + | > loss_disc_real_4: 0.24183958768844604 (0.24183958768844604) + | > loss_disc_real_5: 0.2320321798324585 (0.2320321798324585) + | > loss_0: 2.387690782546997 (2.387690782546997) + | > loss_gen: 2.255824565887451 (2.255824565887451) + | > loss_kl: 2.4162328243255615 (2.4162328243255615) + | > loss_feat: 5.8934006690979 (5.8934006690979) + | > loss_mel: 16.296051025390625 (16.296051025390625) + | > loss_duration: 1.7333705425262451 (1.7333705425262451) + | > loss_1: 28.594879150390625 (28.594879150390625) + + --> STEP: 1 + | > loss_disc: 2.500420570373535 (2.500420570373535) + | > loss_disc_real_0: 0.1565399020910263 (0.1565399020910263) + | > loss_disc_real_1: 0.26952674984931946 (0.26952674984931946) + | > loss_disc_real_2: 0.23210975527763367 (0.23210975527763367) + | > loss_disc_real_3: 0.2613057494163513 (0.2613057494163513) + | > loss_disc_real_4: 0.26203298568725586 (0.26203298568725586) + | > loss_disc_real_5: 0.2675163149833679 (0.2675163149833679) + | > loss_0: 2.500420570373535 (2.500420570373535) + | > loss_gen: 2.5895934104919434 (2.5895934104919434) + | > loss_kl: 2.7540154457092285 (2.7540154457092285) + | > loss_feat: 7.332238674163818 (7.332238674163818) + | > loss_mel: 18.27838134765625 (18.27838134765625) + | > loss_duration: 1.7493983507156372 (1.7493983507156372) + | > loss_1: 32.70362854003906 (32.70362854003906) + + --> STEP: 2 + | > loss_disc: 2.396623373031616 (2.4485219717025757) + | > loss_disc_real_0: 0.10721029341220856 (0.13187509775161743) + | > loss_disc_real_1: 0.2546268701553345 (0.26207681000232697) + | > loss_disc_real_2: 0.2159746140241623 (0.22404218465089798) + | > loss_disc_real_3: 0.22782686352729797 (0.24456630647182465) + | > loss_disc_real_4: 0.21231091022491455 (0.2371719479560852) + | > loss_disc_real_5: 0.1539009064435959 (0.2107086107134819) + | > loss_0: 2.396623373031616 (2.4485219717025757) + | > loss_gen: 2.1900713443756104 (2.389832377433777) + | > loss_kl: 2.196713447570801 (2.4753644466400146) + | > loss_feat: 7.729229927062988 (7.530734300613403) + | > loss_mel: 17.83163070678711 (18.05500602722168) + | > loss_duration: 1.7027316093444824 (1.7260649800300598) + | > loss_1: 31.65037727355957 (32.177002906799316) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034426450729370117 (+4.124641418457031e-05) + | > avg_loss_disc: 2.4485219717025757 (+0.009691834449768066) + | > avg_loss_disc_real_0: 0.13187509775161743 (-0.05558402091264725) + | > avg_loss_disc_real_1: 0.26207681000232697 (+0.03349238634109497) + | > avg_loss_disc_real_2: 0.22404218465089798 (-0.06119983643293381) + | > avg_loss_disc_real_3: 0.24456630647182465 (+0.006280630826950073) + | > avg_loss_disc_real_4: 0.2371719479560852 (-0.0024052560329437256) + | > avg_loss_disc_real_5: 0.2107086107134819 (-0.013953432440757751) + | > avg_loss_0: 2.4485219717025757 (+0.009691834449768066) + | > avg_loss_gen: 2.389832377433777 (-0.24172210693359375) + | > avg_loss_kl: 2.4753644466400146 (-0.06276476383209229) + | > avg_loss_feat: 7.530734300613403 (-0.2710151672363281) + | > avg_loss_mel: 18.05500602722168 (-1.224656105041504) + | > avg_loss_duration: 1.7260649800300598 (-0.026289165019989014) + | > avg_loss_1: 32.177002906799316 (-1.8264474868774414) + + + > EPOCH: 332/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:26:33)  + + --> TIME: 2025-04-19 04:26:42 -- STEP: 24/142 -- GLOBAL_STEP: 426450 + | > loss_disc: 2.4288487434387207 (2.4367575347423553) + | > loss_disc_real_0: 0.11366087198257446 (0.11970928994317849) + | > loss_disc_real_1: 0.220285102725029 (0.20533715623120466) + | > loss_disc_real_2: 0.23371563851833344 (0.22833153357108435) + | > loss_disc_real_3: 0.23352311551570892 (0.2310291063040495) + | > loss_disc_real_4: 0.2472933828830719 (0.23380444198846817) + | > loss_disc_real_5: 0.21245844662189484 (0.22678539529442787) + | > loss_0: 2.4288487434387207 (2.4367575347423553) + | > grad_norm_0: tensor(15.4117, device='cuda:0') (tensor(13.3142, device='cuda:0')) + | > loss_gen: 2.430288791656494 (2.386074155569077) + | > loss_kl: 1.8621678352355957 (1.8408889174461365) + | > loss_feat: 5.988436698913574 (7.227274735768636) + | > loss_mel: 15.960389137268066 (17.35629113515218) + | > loss_duration: 1.4616190195083618 (1.4224733958641689) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.702899932861328 (30.233002026875813) + | > grad_norm_1: tensor(202.8003, device='cuda:0') (tensor(306.7660, device='cuda:0')) + | > current_lr_0: 0.000191869369364994 + | > current_lr_1: 0.000191869369364994 + | > step_time: 0.3505 (0.3337273001670838) + | > loader_time: 0.0054 (0.004972706238428752) + + + --> TIME: 2025-04-19 04:26:51 -- STEP: 49/142 -- GLOBAL_STEP: 426475 + | > loss_disc: 2.463569164276123 (2.419623219237036) + | > loss_disc_real_0: 0.1389918178319931 (0.12025206748928342) + | > loss_disc_real_1: 0.23701129853725433 (0.20579939046684576) + | > loss_disc_real_2: 0.19902336597442627 (0.2223122375352042) + | > loss_disc_real_3: 0.23165908455848694 (0.22649367396928827) + | > loss_disc_real_4: 0.2187519371509552 (0.2294062844344548) + | > loss_disc_real_5: 0.24886299669742584 (0.22582833103987635) + | > loss_0: 2.463569164276123 (2.419623219237036) + | > grad_norm_0: tensor(5.6421, device='cuda:0') (tensor(11.1089, device='cuda:0')) + | > loss_gen: 2.269139528274536 (2.4004956118914547) + | > loss_kl: 1.9224282503128052 (1.858893092797727) + | > loss_feat: 7.6111955642700195 (7.283372071324562) + | > loss_mel: 17.889081954956055 (17.537656219638123) + | > loss_duration: 1.4454171657562256 (1.4315304950791965) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.137264251708984 (30.51194735935756) + | > grad_norm_1: tensor(265.1884, device='cuda:0') (tensor(260.5930, device='cuda:0')) + | > current_lr_0: 0.000191869369364994 + | > current_lr_1: 0.000191869369364994 + | > step_time: 0.3689 (0.352884443438783) + | > loader_time: 0.0062 (0.005379277832654058) + + + --> TIME: 2025-04-19 04:27:01 -- STEP: 74/142 -- GLOBAL_STEP: 426500 + | > loss_disc: 2.4570677280426025 (2.450228549338676) + | > loss_disc_real_0: 0.11961298435926437 (0.12309443054569734) + | > loss_disc_real_1: 0.21303382515907288 (0.20773140482000402) + | > loss_disc_real_2: 0.20753787457942963 (0.22522185400531097) + | > loss_disc_real_3: 0.208816260099411 (0.2282695673607491) + | > loss_disc_real_4: 0.19457222521305084 (0.23054128784585642) + | > loss_disc_real_5: 0.25551989674568176 (0.2310151153722325) + | > loss_0: 2.4570677280426025 (2.450228549338676) + | > grad_norm_0: tensor(9.4756, device='cuda:0') (tensor(10.1156, device='cuda:0')) + | > loss_gen: 2.2955331802368164 (2.3794850336538773) + | > loss_kl: 1.867964267730713 (1.866007448853673) + | > loss_feat: 7.108087539672852 (7.215617727588963) + | > loss_mel: 17.0659122467041 (17.601738504461334) + | > loss_duration: 1.4578677415847778 (1.4430506116635093) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.795366287231445 (30.50589924889642) + | > grad_norm_1: tensor(473.7144, device='cuda:0') (tensor(236.4471, device='cuda:0')) + | > current_lr_0: 0.000191869369364994 + | > current_lr_1: 0.000191869369364994 + | > step_time: 0.3864 (0.3624369292645841) + | > loader_time: 0.0066 (0.005727458644557645) + + + --> TIME: 2025-04-19 04:27:11 -- STEP: 99/142 -- GLOBAL_STEP: 426525 + | > loss_disc: 2.625485420227051 (2.461308563598479) + | > loss_disc_real_0: 0.3009371757507324 (0.1269828322710413) + | > loss_disc_real_1: 0.2529142200946808 (0.20992635295848655) + | > loss_disc_real_2: 0.27580931782722473 (0.22663684384991425) + | > loss_disc_real_3: 0.2447880357503891 (0.22759772852213697) + | > loss_disc_real_4: 0.21653391420841217 (0.22906864035611202) + | > loss_disc_real_5: 0.23158159852027893 (0.22892873124642807) + | > loss_0: 2.625485420227051 (2.461308563598479) + | > grad_norm_0: tensor(34.0221, device='cuda:0') (tensor(11.3519, device='cuda:0')) + | > loss_gen: 2.448190689086914 (2.374063766363896) + | > loss_kl: 1.9683891534805298 (1.8735085220047922) + | > loss_feat: 6.686522483825684 (7.13196933630741) + | > loss_mel: 17.711854934692383 (17.60641862888529) + | > loss_duration: 1.4937316179275513 (1.4533586586364593) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.308691024780273 (30.439318917014383) + | > grad_norm_1: tensor(378.1430, device='cuda:0') (tensor(252.5177, device='cuda:0')) + | > current_lr_0: 0.000191869369364994 + | > current_lr_1: 0.000191869369364994 + | > step_time: 0.4027 (0.3717409528867162) + | > loader_time: 0.0068 (0.005998168328795771) + + + --> TIME: 2025-04-19 04:27:22 -- STEP: 124/142 -- GLOBAL_STEP: 426550 + | > loss_disc: 2.449265480041504 (2.4630078557998902) + | > loss_disc_real_0: 0.12370345741510391 (0.12648363555631323) + | > loss_disc_real_1: 0.22347143292427063 (0.20864319404767406) + | > loss_disc_real_2: 0.2658519744873047 (0.22645925430040206) + | > loss_disc_real_3: 0.22966556251049042 (0.22895047693483292) + | > loss_disc_real_4: 0.2286422848701477 (0.22913937547033833) + | > loss_disc_real_5: 0.25270119309425354 (0.23128873085783375) + | > loss_0: 2.449265480041504 (2.4630078557998902) + | > grad_norm_0: tensor(23.8150, device='cuda:0') (tensor(12.4023, device='cuda:0')) + | > loss_gen: 2.3798892498016357 (2.3818338647965462) + | > loss_kl: 2.0288331508636475 (1.8797136400976489) + | > loss_feat: 6.898741722106934 (7.11850715837171) + | > loss_mel: 18.450157165527344 (17.59334865693123) + | > loss_duration: 1.5149034261703491 (1.4603259592287001) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.272523880004883 (30.43372924866215) + | > grad_norm_1: tensor(337.5462, device='cuda:0') (tensor(268.0308, device='cuda:0')) + | > current_lr_0: 0.000191869369364994 + | > current_lr_1: 0.000191869369364994 + | > step_time: 0.4404 (0.38267323855430846) + | > loader_time: 0.0074 (0.006222667232636483) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.445589542388916 (2.445589542388916) + | > loss_disc_real_0: 0.1615648716688156 (0.1615648716688156) + | > loss_disc_real_1: 0.24036988615989685 (0.24036988615989685) + | > loss_disc_real_2: 0.24163679778575897 (0.24163679778575897) + | > loss_disc_real_3: 0.2150340974330902 (0.2150340974330902) + | > loss_disc_real_4: 0.27948805689811707 (0.27948805689811707) + | > loss_disc_real_5: 0.25048115849494934 (0.25048115849494934) + | > loss_0: 2.445589542388916 (2.445589542388916) + | > loss_gen: 2.640991449356079 (2.640991449356079) + | > loss_kl: 2.256103515625 (2.256103515625) + | > loss_feat: 6.894739627838135 (6.894739627838135) + | > loss_mel: 16.2607421875 (16.2607421875) + | > loss_duration: 1.7421283721923828 (1.7421283721923828) + | > loss_1: 29.79470443725586 (29.79470443725586) + + --> STEP: 1 + | > loss_disc: 2.537356376647949 (2.537356376647949) + | > loss_disc_real_0: 0.24421410262584686 (0.24421410262584686) + | > loss_disc_real_1: 0.24201864004135132 (0.24201864004135132) + | > loss_disc_real_2: 0.26929301023483276 (0.26929301023483276) + | > loss_disc_real_3: 0.23741216957569122 (0.23741216957569122) + | > loss_disc_real_4: 0.2797224223613739 (0.2797224223613739) + | > loss_disc_real_5: 0.21341422200202942 (0.21341422200202942) + | > loss_0: 2.537356376647949 (2.537356376647949) + | > loss_gen: 2.580934524536133 (2.580934524536133) + | > loss_kl: 2.7336227893829346 (2.7336227893829346) + | > loss_feat: 6.323027610778809 (6.323027610778809) + | > loss_mel: 16.632116317749023 (16.632116317749023) + | > loss_duration: 1.7693817615509033 (1.7693817615509033) + | > loss_1: 30.03908348083496 (30.03908348083496) + + --> STEP: 2 + | > loss_disc: 2.525068759918213 (2.531212568283081) + | > loss_disc_real_0: 0.1932162344455719 (0.21871516853570938) + | > loss_disc_real_1: 0.27020931243896484 (0.2561139762401581) + | > loss_disc_real_2: 0.2561836838722229 (0.26273834705352783) + | > loss_disc_real_3: 0.26562196016311646 (0.25151706486940384) + | > loss_disc_real_4: 0.30899903178215027 (0.2943607270717621) + | > loss_disc_real_5: 0.2588607370853424 (0.2361374795436859) + | > loss_0: 2.525068759918213 (2.531212568283081) + | > loss_gen: 2.6779496669769287 (2.6294420957565308) + | > loss_kl: 2.394296407699585 (2.5639595985412598) + | > loss_feat: 7.183905124664307 (6.753466367721558) + | > loss_mel: 18.201126098632812 (17.416621208190918) + | > loss_duration: 1.7153842449188232 (1.7423830032348633) + | > loss_1: 32.17266082763672 (31.10587215423584) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003858804702758789 (+0.00041615962982177734) + | > avg_loss_disc: 2.531212568283081 (+0.08269059658050537) + | > avg_loss_disc_real_0: 0.21871516853570938 (+0.08684007078409195) + | > avg_loss_disc_real_1: 0.2561139762401581 (-0.005962833762168884) + | > avg_loss_disc_real_2: 0.26273834705352783 (+0.03869616240262985) + | > avg_loss_disc_real_3: 0.25151706486940384 (+0.006950758397579193) + | > avg_loss_disc_real_4: 0.2943607270717621 (+0.05718877911567688) + | > avg_loss_disc_real_5: 0.2361374795436859 (+0.02542886883020401) + | > avg_loss_0: 2.531212568283081 (+0.08269059658050537) + | > avg_loss_gen: 2.6294420957565308 (+0.2396097183227539) + | > avg_loss_kl: 2.5639595985412598 (+0.08859515190124512) + | > avg_loss_feat: 6.753466367721558 (-0.7772679328918457) + | > avg_loss_mel: 17.416621208190918 (-0.6383848190307617) + | > avg_loss_duration: 1.7423830032348633 (+0.016318023204803467) + | > avg_loss_1: 31.10587215423584 (-1.0711307525634766) + + + > EPOCH: 333/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:27:32)  + + --> TIME: 2025-04-19 04:27:35 -- STEP: 7/142 -- GLOBAL_STEP: 426575 + | > loss_disc: 2.4372591972351074 (2.4375839233398438) + | > loss_disc_real_0: 0.06763657182455063 (0.1154805485691343) + | > loss_disc_real_1: 0.22920572757720947 (0.21695442497730255) + | > loss_disc_real_2: 0.20712344348430634 (0.22421079022543772) + | > loss_disc_real_3: 0.22117429971694946 (0.22856797916548593) + | > loss_disc_real_4: 0.1962158977985382 (0.24679099023342133) + | > loss_disc_real_5: 0.21217094361782074 (0.23089722437517984) + | > loss_0: 2.4372591972351074 (2.4375839233398438) + | > grad_norm_0: tensor(12.4210, device='cuda:0') (tensor(17.7561, device='cuda:0')) + | > loss_gen: 2.313016891479492 (2.401463304247175) + | > loss_kl: 1.9378678798675537 (1.8387708493641444) + | > loss_feat: 7.218172550201416 (7.098202637263706) + | > loss_mel: 18.05376434326172 (17.63335200718471) + | > loss_duration: 1.4065473079681396 (1.4187823023114885) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.92936897277832 (30.39057104928153) + | > grad_norm_1: tensor(417.3385, device='cuda:0') (tensor(356.1300, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.3282 (0.3257997376578195) + | > loader_time: 0.0041 (0.004334109170096261) + + + --> TIME: 2025-04-19 04:27:44 -- STEP: 32/142 -- GLOBAL_STEP: 426600 + | > loss_disc: 2.5359408855438232 (2.437014728784561) + | > loss_disc_real_0: 0.1225653812289238 (0.11620902211870998) + | > loss_disc_real_1: 0.22287270426750183 (0.20634409692138433) + | > loss_disc_real_2: 0.23020988702774048 (0.22808016696944833) + | > loss_disc_real_3: 0.2505607604980469 (0.22842397447675467) + | > loss_disc_real_4: 0.25459280610084534 (0.23421485908329487) + | > loss_disc_real_5: 0.23114873468875885 (0.22645053220912814) + | > loss_0: 2.5359408855438232 (2.437014728784561) + | > grad_norm_0: tensor(5.0574, device='cuda:0') (tensor(12.6494, device='cuda:0')) + | > loss_gen: 2.329052448272705 (2.3829378709197044) + | > loss_kl: 1.9047174453735352 (1.8396353721618652) + | > loss_feat: 6.686814308166504 (7.288796707987785) + | > loss_mel: 15.860279083251953 (17.68811750411987) + | > loss_duration: 1.4247052669525146 (1.4205640070140362) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.205568313598633 (30.620051562786102) + | > grad_norm_1: tensor(208.2869, device='cuda:0') (tensor(300.7729, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.3556 (0.3391980156302452) + | > loader_time: 0.0057 (0.0051437318325042725) + + + --> TIME: 2025-04-19 04:27:53 -- STEP: 57/142 -- GLOBAL_STEP: 426625 + | > loss_disc: 2.365936756134033 (2.4237419011300063) + | > loss_disc_real_0: 0.12266325205564499 (0.11733095733481541) + | > loss_disc_real_1: 0.1961131989955902 (0.20467347507936912) + | > loss_disc_real_2: 0.224142923951149 (0.22465048285952785) + | > loss_disc_real_3: 0.2116711288690567 (0.22669498151854464) + | > loss_disc_real_4: 0.22741511464118958 (0.23212968440432297) + | > loss_disc_real_5: 0.21637584269046783 (0.2281115909940318) + | > loss_0: 2.365936756134033 (2.4237419011300063) + | > grad_norm_0: tensor(6.6085, device='cuda:0') (tensor(10.7293, device='cuda:0')) + | > loss_gen: 2.397843360900879 (2.377627627891407) + | > loss_kl: 1.9086802005767822 (1.8479461167988025) + | > loss_feat: 7.466060161590576 (7.266646686353181) + | > loss_mel: 17.432613372802734 (17.729110868353583) + | > loss_duration: 1.4891737699508667 (1.4341058751993012) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.69437026977539 (30.655437168322113) + | > grad_norm_1: tensor(115.8114, device='cuda:0') (tensor(254.5852, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.3757 (0.3497080384639273) + | > loader_time: 0.0061 (0.005464716961509303) + + + --> TIME: 2025-04-19 04:28:03 -- STEP: 82/142 -- GLOBAL_STEP: 426650 + | > loss_disc: 2.471479892730713 (2.4399633960026055) + | > loss_disc_real_0: 0.13674303889274597 (0.11919695379712232) + | > loss_disc_real_1: 0.18413329124450684 (0.20542452502541425) + | > loss_disc_real_2: 0.22547949850559235 (0.22582938612961187) + | > loss_disc_real_3: 0.2430453896522522 (0.22719162920626199) + | > loss_disc_real_4: 0.21925169229507446 (0.23208292083042423) + | > loss_disc_real_5: 0.25947490334510803 (0.22960740709450186) + | > loss_0: 2.471479892730713 (2.4399633960026055) + | > grad_norm_0: tensor(7.8062, device='cuda:0') (tensor(9.7215, device='cuda:0')) + | > loss_gen: 2.428407669067383 (2.3664691477287105) + | > loss_kl: 2.02156400680542 (1.8641052609536706) + | > loss_feat: 6.889512062072754 (7.199006196929187) + | > loss_mel: 17.666187286376953 (17.678517364874118) + | > loss_duration: 1.527336835861206 (1.4454486108407743) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.53300666809082 (30.55354655661234) + | > grad_norm_1: tensor(220.9998, device='cuda:0') (tensor(232.6398, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.3999 (0.3628130191709938) + | > loader_time: 0.0073 (0.005732725306255062) + + + --> TIME: 2025-04-19 04:28:13 -- STEP: 107/142 -- GLOBAL_STEP: 426675 + | > loss_disc: 2.388847589492798 (2.443761444537438) + | > loss_disc_real_0: 0.10970323532819748 (0.12251718373638447) + | > loss_disc_real_1: 0.20405313372612 (0.20518461078683906) + | > loss_disc_real_2: 0.20964553952217102 (0.22455398794089523) + | > loss_disc_real_3: 0.18805848062038422 (0.22710390430744562) + | > loss_disc_real_4: 0.1915675848722458 (0.2313507685315943) + | > loss_disc_real_5: 0.22725644707679749 (0.2292826785662464) + | > loss_0: 2.388847589492798 (2.443761444537438) + | > grad_norm_0: tensor(9.7010, device='cuda:0') (tensor(9.6338, device='cuda:0')) + | > loss_gen: 2.2979092597961426 (2.355031040227302) + | > loss_kl: 1.929335355758667 (1.8691557679220894) + | > loss_feat: 7.326057434082031 (7.154176658559068) + | > loss_mel: 16.889219284057617 (17.651793328401084) + | > loss_duration: 1.4771710634231567 (1.4551687875640726) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.91969108581543 (30.48532553699529) + | > grad_norm_1: tensor(343.7079, device='cuda:0') (tensor(228.1698, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.4151 (0.37281947269617954) + | > loader_time: 0.0069 (0.006009699028229045) + + + --> TIME: 2025-04-19 04:28:24 -- STEP: 132/142 -- GLOBAL_STEP: 426700 + | > loss_disc: 2.5037906169891357 (2.448632549155841) + | > loss_disc_real_0: 0.10484059154987335 (0.12326655579222874) + | > loss_disc_real_1: 0.20585615932941437 (0.20574187803449054) + | > loss_disc_real_2: 0.21954619884490967 (0.2257115620781075) + | > loss_disc_real_3: 0.21455609798431396 (0.22720961927464514) + | > loss_disc_real_4: 0.22710029780864716 (0.23063661293549972) + | > loss_disc_real_5: 0.2478533387184143 (0.2296067655763843) + | > loss_0: 2.5037906169891357 (2.448632549155841) + | > grad_norm_0: tensor(8.2617, device='cuda:0') (tensor(9.6440, device='cuda:0')) + | > loss_gen: 2.3568875789642334 (2.3523617585500083) + | > loss_kl: 1.820923089981079 (1.8760592287236995) + | > loss_feat: 6.838374137878418 (7.114542393973379) + | > loss_mel: 17.442293167114258 (17.591047496506665) + | > loss_duration: 1.4643383026123047 (1.4610165818171068) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.922815322875977 (30.395027362939082) + | > grad_norm_1: tensor(290.5031, device='cuda:0') (tensor(232.7347, device='cuda:0')) + | > current_lr_0: 0.00019184538569382337 + | > current_lr_1: 0.00019184538569382337 + | > step_time: 0.4479 (0.3844826943946607) + | > loader_time: 0.0081 (0.006290414116599343) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4603211879730225 (2.4603211879730225) + | > loss_disc_real_0: 0.1632707566022873 (0.1632707566022873) + | > loss_disc_real_1: 0.21066156029701233 (0.21066156029701233) + | > loss_disc_real_2: 0.24108828604221344 (0.24108828604221344) + | > loss_disc_real_3: 0.22949719429016113 (0.22949719429016113) + | > loss_disc_real_4: 0.24188734591007233 (0.24188734591007233) + | > loss_disc_real_5: 0.26030072569847107 (0.26030072569847107) + | > loss_0: 2.4603211879730225 (2.4603211879730225) + | > loss_gen: 2.331920623779297 (2.331920623779297) + | > loss_kl: 2.5110692977905273 (2.5110692977905273) + | > loss_feat: 5.685643196105957 (5.685643196105957) + | > loss_mel: 14.720168113708496 (14.720168113708496) + | > loss_duration: 1.7458739280700684 (1.7458739280700684) + | > loss_1: 26.99467658996582 (26.99467658996582) + + --> STEP: 1 + | > loss_disc: 2.415384292602539 (2.415384292602539) + | > loss_disc_real_0: 0.1361011117696762 (0.1361011117696762) + | > loss_disc_real_1: 0.2031729370355606 (0.2031729370355606) + | > loss_disc_real_2: 0.2670409679412842 (0.2670409679412842) + | > loss_disc_real_3: 0.2312457263469696 (0.2312457263469696) + | > loss_disc_real_4: 0.2936210334300995 (0.2936210334300995) + | > loss_disc_real_5: 0.23476937413215637 (0.23476937413215637) + | > loss_0: 2.415384292602539 (2.415384292602539) + | > loss_gen: 2.533604860305786 (2.533604860305786) + | > loss_kl: 2.829613208770752 (2.829613208770752) + | > loss_feat: 7.184544563293457 (7.184544563293457) + | > loss_mel: 19.00033950805664 (19.00033950805664) + | > loss_duration: 1.775915503501892 (1.775915503501892) + | > loss_1: 33.32402038574219 (33.32402038574219) + + --> STEP: 2 + | > loss_disc: 2.6230063438415527 (2.519195318222046) + | > loss_disc_real_0: 0.13591910898685455 (0.13601011037826538) + | > loss_disc_real_1: 0.22303462028503418 (0.2131037786602974) + | > loss_disc_real_2: 0.25080493092536926 (0.2589229494333267) + | > loss_disc_real_3: 0.21720153093338013 (0.22422362864017487) + | > loss_disc_real_4: 0.22391289472579956 (0.2587669640779495) + | > loss_disc_real_5: 0.23122677206993103 (0.2329980731010437) + | > loss_0: 2.6230063438415527 (2.519195318222046) + | > loss_gen: 2.0141215324401855 (2.273863196372986) + | > loss_kl: 2.2560837268829346 (2.5428484678268433) + | > loss_feat: 6.836756229400635 (7.010650396347046) + | > loss_mel: 18.82647132873535 (18.913405418395996) + | > loss_duration: 1.7048606872558594 (1.7403880953788757) + | > loss_1: 31.638294219970703 (32.481157302856445) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003812074661254883 (-4.673004150390625e-05) + | > avg_loss_disc: 2.519195318222046 (-0.012017250061035156) + | > avg_loss_disc_real_0: 0.13601011037826538 (-0.082705058157444) + | > avg_loss_disc_real_1: 0.2131037786602974 (-0.04301019757986069) + | > avg_loss_disc_real_2: 0.2589229494333267 (-0.003815397620201111) + | > avg_loss_disc_real_3: 0.22422362864017487 (-0.027293436229228973) + | > avg_loss_disc_real_4: 0.2587669640779495 (-0.03559376299381256) + | > avg_loss_disc_real_5: 0.2329980731010437 (-0.003139406442642212) + | > avg_loss_0: 2.519195318222046 (-0.012017250061035156) + | > avg_loss_gen: 2.273863196372986 (-0.3555788993835449) + | > avg_loss_kl: 2.5428484678268433 (-0.021111130714416504) + | > avg_loss_feat: 7.010650396347046 (+0.2571840286254883) + | > avg_loss_mel: 18.913405418395996 (+1.4967842102050781) + | > avg_loss_duration: 1.7403880953788757 (-0.001994907855987549) + | > avg_loss_1: 32.481157302856445 (+1.3752851486206055) + + + > EPOCH: 334/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:28:30)  + + --> TIME: 2025-04-19 04:28:36 -- STEP: 15/142 -- GLOBAL_STEP: 426725 + | > loss_disc: 2.386000633239746 (2.4290462176005048) + | > loss_disc_real_0: 0.07723528146743774 (0.12023089230060577) + | > loss_disc_real_1: 0.17884789407253265 (0.20056053002675375) + | > loss_disc_real_2: 0.25351420044898987 (0.22879898051420847) + | > loss_disc_real_3: 0.21611185371875763 (0.23046831091245015) + | > loss_disc_real_4: 0.19903446733951569 (0.232036758462588) + | > loss_disc_real_5: 0.18178372085094452 (0.23719953795274099) + | > loss_0: 2.386000633239746 (2.4290462176005048) + | > grad_norm_0: tensor(10.1092, device='cuda:0') (tensor(9.2209, device='cuda:0')) + | > loss_gen: 2.366934299468994 (2.3956697305043537) + | > loss_kl: 1.705264925956726 (1.8068482875823975) + | > loss_feat: 6.590446472167969 (7.205506896972656) + | > loss_mel: 15.3404541015625 (17.452856508890786) + | > loss_duration: 1.4095661640167236 (1.4113953431447348) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.41266632080078 (30.272276560465496) + | > grad_norm_1: tensor(160.8558, device='cuda:0') (tensor(257.2018, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.3334 (0.32921490669250486) + | > loader_time: 0.0055 (0.004751332600911458) + + + --> TIME: 2025-04-19 04:28:45 -- STEP: 40/142 -- GLOBAL_STEP: 426750 + | > loss_disc: 2.3609044551849365 (2.405341500043869) + | > loss_disc_real_0: 0.11722099781036377 (0.11354658398777247) + | > loss_disc_real_1: 0.21869738399982452 (0.20257959701120853) + | > loss_disc_real_2: 0.23607894778251648 (0.223202458396554) + | > loss_disc_real_3: 0.20251679420471191 (0.22614045441150665) + | > loss_disc_real_4: 0.25214266777038574 (0.22597610354423522) + | > loss_disc_real_5: 0.21702836453914642 (0.23243423216044903) + | > loss_0: 2.3609044551849365 (2.405341500043869) + | > grad_norm_0: tensor(4.7177, device='cuda:0') (tensor(8.2975, device='cuda:0')) + | > loss_gen: 2.50500226020813 (2.410199707746506) + | > loss_kl: 1.847948670387268 (1.819043630361557) + | > loss_feat: 7.342738151550293 (7.285954737663269) + | > loss_mel: 17.70735740661621 (17.655137228965756) + | > loss_duration: 1.4005786180496216 (1.4259740859270094) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.803627014160156 (30.59630951881409) + | > grad_norm_1: tensor(44.1677, device='cuda:0') (tensor(213.6124, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.3599 (0.34107959866523746) + | > loader_time: 0.006 (0.005253612995147706) + + + --> TIME: 2025-04-19 04:28:55 -- STEP: 65/142 -- GLOBAL_STEP: 426775 + | > loss_disc: 2.4182963371276855 (2.4299578630007233) + | > loss_disc_real_0: 0.09764840453863144 (0.1185153039602133) + | > loss_disc_real_1: 0.16145065426826477 (0.20365915160912734) + | > loss_disc_real_2: 0.20490975677967072 (0.2246184161076179) + | > loss_disc_real_3: 0.22463862597942352 (0.22694385991646693) + | > loss_disc_real_4: 0.22466081380844116 (0.2276750032718365) + | > loss_disc_real_5: 0.25309133529663086 (0.23295068717919865) + | > loss_0: 2.4182963371276855 (2.4299578630007233) + | > grad_norm_0: tensor(5.9104, device='cuda:0') (tensor(8.3063, device='cuda:0')) + | > loss_gen: 2.3949549198150635 (2.381396843836858) + | > loss_kl: 1.9226794242858887 (1.8364657512077918) + | > loss_feat: 7.161339282989502 (7.192918418003963) + | > loss_mel: 17.4359130859375 (17.607379399813137) + | > loss_duration: 1.445108413696289 (1.438230303617624) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.359994888305664 (30.45639082101675) + | > grad_norm_1: tensor(107.5842, device='cuda:0') (tensor(208.0044, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.3895 (0.3528672695159913) + | > loader_time: 0.0061 (0.00560449820298415) + + + --> TIME: 2025-04-19 04:29:05 -- STEP: 90/142 -- GLOBAL_STEP: 426800 + | > loss_disc: 2.5773544311523438 (2.4445961157480864) + | > loss_disc_real_0: 0.22426581382751465 (0.12260499290294118) + | > loss_disc_real_1: 0.26043131947517395 (0.20555605275763406) + | > loss_disc_real_2: 0.243085578083992 (0.22502408772706986) + | > loss_disc_real_3: 0.2330293357372284 (0.22715415938032998) + | > loss_disc_real_4: 0.25911831855773926 (0.22931843333774143) + | > loss_disc_real_5: 0.25876185297966003 (0.23519179423650105) + | > loss_0: 2.5773544311523438 (2.4445961157480864) + | > grad_norm_0: tensor(13.9305, device='cuda:0') (tensor(8.4615, device='cuda:0')) + | > loss_gen: 2.2921667098999023 (2.3733424716525606) + | > loss_kl: 1.8367125988006592 (1.8546106696128846) + | > loss_feat: 5.941084861755371 (7.1549401919047035) + | > loss_mel: 14.591078758239746 (17.67312022315131) + | > loss_duration: 1.4967849254608154 (1.4504449274804858) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 26.15782928466797 (30.506458557976618) + | > grad_norm_1: tensor(103.5865, device='cuda:0') (tensor(199.7802, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.4025 (0.36442046960194924) + | > loader_time: 0.0065 (0.005875929196675619) + + + --> TIME: 2025-04-19 04:29:15 -- STEP: 115/142 -- GLOBAL_STEP: 426825 + | > loss_disc: 2.5201940536499023 (2.4540842159934657) + | > loss_disc_real_0: 0.1549549400806427 (0.12522293568953227) + | > loss_disc_real_1: 0.2441900372505188 (0.20629891481088555) + | > loss_disc_real_2: 0.20560333132743835 (0.22463958030161651) + | > loss_disc_real_3: 0.21317455172538757 (0.22717290779818658) + | > loss_disc_real_4: 0.19138634204864502 (0.22925754267236462) + | > loss_disc_real_5: 0.26618319749832153 (0.23402978492819745) + | > loss_0: 2.5201940536499023 (2.4540842159934657) + | > grad_norm_0: tensor(8.9738, device='cuda:0') (tensor(8.5625, device='cuda:0')) + | > loss_gen: 2.3335886001586914 (2.361974222763725) + | > loss_kl: 1.9757261276245117 (1.8687889804010807) + | > loss_feat: 7.257739067077637 (7.142520154040793) + | > loss_mel: 18.828184127807617 (17.70627678580906) + | > loss_duration: 1.4918177127838135 (1.4580254274865856) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.887054443359375 (30.537585581903873) + | > grad_norm_1: tensor(88.0587, device='cuda:0') (tensor(190.6410, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.4256 (0.3752503415812618) + | > loader_time: 0.0078 (0.00612518891044285) + + + --> TIME: 2025-04-19 04:29:27 -- STEP: 140/142 -- GLOBAL_STEP: 426850 + | > loss_disc: 2.4780168533325195 (2.4555574434144147) + | > loss_disc_real_0: 0.09245027601718903 (0.12621668539941305) + | > loss_disc_real_1: 0.2043764591217041 (0.20643332600593567) + | > loss_disc_real_2: 0.21492250263690948 (0.22406622054321423) + | > loss_disc_real_3: 0.21174722909927368 (0.22726033266101564) + | > loss_disc_real_4: 0.22926245629787445 (0.22892505494611604) + | > loss_disc_real_5: 0.20774225890636444 (0.23299351707100868) + | > loss_0: 2.4780168533325195 (2.4555574434144147) + | > grad_norm_0: tensor(13.4474, device='cuda:0') (tensor(9.2005, device='cuda:0')) + | > loss_gen: 2.353896379470825 (2.356507304736546) + | > loss_kl: 1.9721909761428833 (1.873984719174249) + | > loss_feat: 7.30830192565918 (7.1047955513000485) + | > loss_mel: 17.631513595581055 (17.648700502940585) + | > loss_duration: 1.5142676830291748 (1.4645005651882717) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.780170440673828 (30.448488644191197) + | > grad_norm_1: tensor(328.5060, device='cuda:0') (tensor(203.4167, device='cuda:0')) + | > current_lr_0: 0.00019182140502061165 + | > current_lr_1: 0.00019182140502061165 + | > step_time: 0.4571 (0.38773616552352913) + | > loader_time: 0.0077 (0.006351845605032784) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.589750051498413 (2.589750051498413) + | > loss_disc_real_0: 0.2141932100057602 (0.2141932100057602) + | > loss_disc_real_1: 0.2282414585351944 (0.2282414585351944) + | > loss_disc_real_2: 0.17481009662151337 (0.17481009662151337) + | > loss_disc_real_3: 0.22893425822257996 (0.22893425822257996) + | > loss_disc_real_4: 0.20230643451213837 (0.20230643451213837) + | > loss_disc_real_5: 0.2504200339317322 (0.2504200339317322) + | > loss_0: 2.589750051498413 (2.589750051498413) + | > loss_gen: 2.2129058837890625 (2.2129058837890625) + | > loss_kl: 2.628666639328003 (2.628666639328003) + | > loss_feat: 7.694812774658203 (7.694812774658203) + | > loss_mel: 19.241378784179688 (19.241378784179688) + | > loss_duration: 1.7252724170684814 (1.7252724170684814) + | > loss_1: 33.50303649902344 (33.50303649902344) + + --> STEP: 1 + | > loss_disc: 2.5143871307373047 (2.5143871307373047) + | > loss_disc_real_0: 0.16192181408405304 (0.16192181408405304) + | > loss_disc_real_1: 0.21908698976039886 (0.21908698976039886) + | > loss_disc_real_2: 0.23211006820201874 (0.23211006820201874) + | > loss_disc_real_3: 0.2351069450378418 (0.2351069450378418) + | > loss_disc_real_4: 0.22877086699008942 (0.22877086699008942) + | > loss_disc_real_5: 0.2670959532260895 (0.2670959532260895) + | > loss_0: 2.5143871307373047 (2.5143871307373047) + | > loss_gen: 2.3494009971618652 (2.3494009971618652) + | > loss_kl: 2.797624349594116 (2.797624349594116) + | > loss_feat: 6.755457401275635 (6.755457401275635) + | > loss_mel: 18.397809982299805 (18.397809982299805) + | > loss_duration: 1.7938156127929688 (1.7938156127929688) + | > loss_1: 32.09410858154297 (32.09410858154297) + + --> STEP: 2 + | > loss_disc: 2.61114764213562 (2.5627673864364624) + | > loss_disc_real_0: 0.20931831002235413 (0.18562006205320358) + | > loss_disc_real_1: 0.26214784383773804 (0.24061741679906845) + | > loss_disc_real_2: 0.21530114114284515 (0.22370560467243195) + | > loss_disc_real_3: 0.23154501616954803 (0.23332598060369492) + | > loss_disc_real_4: 0.2657986283302307 (0.24728474766016006) + | > loss_disc_real_5: 0.2162233293056488 (0.24165964126586914) + | > loss_0: 2.61114764213562 (2.5627673864364624) + | > loss_gen: 2.3443689346313477 (2.3468849658966064) + | > loss_kl: 2.406975746154785 (2.6023000478744507) + | > loss_feat: 5.995588302612305 (6.37552285194397) + | > loss_mel: 15.719655990600586 (17.058732986450195) + | > loss_duration: 1.7184195518493652 (1.756117582321167) + | > loss_1: 28.185007095336914 (30.13955783843994) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032154321670532227 (-0.0005966424942016602) + | > avg_loss_disc: 2.5627673864364624 (+0.043572068214416504) + | > avg_loss_disc_real_0: 0.18562006205320358 (+0.0496099516749382) + | > avg_loss_disc_real_1: 0.24061741679906845 (+0.027513638138771057) + | > avg_loss_disc_real_2: 0.22370560467243195 (-0.035217344760894775) + | > avg_loss_disc_real_3: 0.23332598060369492 (+0.00910235196352005) + | > avg_loss_disc_real_4: 0.24728474766016006 (-0.01148221641778946) + | > avg_loss_disc_real_5: 0.24165964126586914 (+0.00866156816482544) + | > avg_loss_0: 2.5627673864364624 (+0.043572068214416504) + | > avg_loss_gen: 2.3468849658966064 (+0.0730217695236206) + | > avg_loss_kl: 2.6023000478744507 (+0.05945158004760742) + | > avg_loss_feat: 6.37552285194397 (-0.6351275444030762) + | > avg_loss_mel: 17.058732986450195 (-1.8546724319458008) + | > avg_loss_duration: 1.756117582321167 (+0.01572948694229126) + | > avg_loss_1: 30.13955783843994 (-2.341599464416504) + + + > EPOCH: 335/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:29:29)  + + --> TIME: 2025-04-19 04:29:37 -- STEP: 23/142 -- GLOBAL_STEP: 426875 + | > loss_disc: 2.367021083831787 (2.447079223135243) + | > loss_disc_real_0: 0.10633444786071777 (0.12185538236213767) + | > loss_disc_real_1: 0.1850699484348297 (0.203567399278931) + | > loss_disc_real_2: 0.2144651710987091 (0.2303588312605153) + | > loss_disc_real_3: 0.21995000541210175 (0.23007452099219614) + | > loss_disc_real_4: 0.2021653950214386 (0.2257206783346508) + | > loss_disc_real_5: 0.2032156139612198 (0.2316723653803701) + | > loss_0: 2.367021083831787 (2.447079223135243) + | > grad_norm_0: tensor(9.0602, device='cuda:0') (tensor(8.5333, device='cuda:0')) + | > loss_gen: 2.300554037094116 (2.3821908287380054) + | > loss_kl: 1.9174138307571411 (1.8296306444251018) + | > loss_feat: 7.353909015655518 (7.2517366616622265) + | > loss_mel: 17.002513885498047 (17.776975507321563) + | > loss_duration: 1.4815144538879395 (1.423118917838387) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.055906295776367 (30.66365275175675) + | > grad_norm_1: tensor(215.1128, device='cuda:0') (tensor(221.1201, device='cuda:0')) + | > current_lr_0: 0.00019179742734498406 + | > current_lr_1: 0.00019179742734498406 + | > step_time: 0.3423 (0.33085767082546075) + | > loader_time: 0.0049 (0.005074687626050866) + + + --> TIME: 2025-04-19 04:29:46 -- STEP: 48/142 -- GLOBAL_STEP: 426900 + | > loss_disc: 2.47855806350708 (2.4467027187347408) + | > loss_disc_real_0: 0.14154882729053497 (0.12411810519794624) + | > loss_disc_real_1: 0.21918097138404846 (0.2044929793725411) + | > loss_disc_real_2: 0.2017677277326584 (0.22644554047534862) + | > loss_disc_real_3: 0.24408702552318573 (0.22815372484425703) + | > loss_disc_real_4: 0.20981866121292114 (0.2269069297860066) + | > loss_disc_real_5: 0.2200973927974701 (0.22974918565402427) + | > loss_0: 2.47855806350708 (2.4467027187347408) + | > grad_norm_0: tensor(5.3103, device='cuda:0') (tensor(7.8877, device='cuda:0')) + | > loss_gen: 2.2009994983673096 (2.3611713300148645) + | > loss_kl: 1.845054030418396 (1.8362891400853794) + | > loss_feat: 6.944482803344727 (7.159899612267812) + | > loss_mel: 17.208364486694336 (17.732926706473034) + | > loss_duration: 1.4269143342971802 (1.4323212827245395) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.625816345214844 (30.522608319918316) + | > grad_norm_1: tensor(43.4795, device='cuda:0') (tensor(172.5233, device='cuda:0')) + | > current_lr_0: 0.00019179742734498406 + | > current_lr_1: 0.00019179742734498406 + | > step_time: 0.3626 (0.3433954070011775) + | > loader_time: 0.006 (0.005358085036277768) + + + --> TIME: 2025-04-19 04:29:56 -- STEP: 73/142 -- GLOBAL_STEP: 426925 + | > loss_disc: 2.5668747425079346 (2.4519623958901176) + | > loss_disc_real_0: 0.14714935421943665 (0.12426989484731465) + | > loss_disc_real_1: 0.20275600254535675 (0.20607628887646817) + | > loss_disc_real_2: 0.22158746421337128 (0.22693576653526254) + | > loss_disc_real_3: 0.24158258736133575 (0.22938089562605504) + | > loss_disc_real_4: 0.27884283661842346 (0.22869494700268522) + | > loss_disc_real_5: 0.24779213964939117 (0.23007949900953736) + | > loss_0: 2.5668747425079346 (2.4519623958901176) + | > grad_norm_0: tensor(12.2693, device='cuda:0') (tensor(8.4674, device='cuda:0')) + | > loss_gen: 2.109471321105957 (2.3640714083632384) + | > loss_kl: 1.9565485715866089 (1.8523197794613773) + | > loss_feat: 7.131411075592041 (7.116319852332547) + | > loss_mel: 16.95930290222168 (17.67741984537203) + | > loss_duration: 1.4966518878936768 (1.4440268947653574) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.65338706970215 (30.454157972989016) + | > grad_norm_1: tensor(209.2045, device='cuda:0') (tensor(179.9696, device='cuda:0')) + | > current_lr_0: 0.00019179742734498406 + | > current_lr_1: 0.00019179742734498406 + | > step_time: 0.3941 (0.3587232583189664) + | > loader_time: 0.0066 (0.005642512073255564) + + + --> TIME: 2025-04-19 04:30:07 -- STEP: 98/142 -- GLOBAL_STEP: 426950 + | > loss_disc: 2.4181301593780518 (2.4605287410775007) + | > loss_disc_real_0: 0.164045512676239 (0.1254318665941151) + | > loss_disc_real_1: 0.1974879950284958 (0.2059708423152262) + | > loss_disc_real_2: 0.2121279239654541 (0.2262986291732107) + | > loss_disc_real_3: 0.2585211396217346 (0.22843244185253064) + | > loss_disc_real_4: 0.20817594230175018 (0.2286046232495989) + | > loss_disc_real_5: 0.21680466830730438 (0.23138633887378537) + | > loss_0: 2.4181301593780518 (2.4605287410775007) + | > grad_norm_0: tensor(25.4179, device='cuda:0') (tensor(9.6082, device='cuda:0')) + | > loss_gen: 2.486187219619751 (2.3567390831149364) + | > loss_kl: 1.823966383934021 (1.8596599406125593) + | > loss_feat: 7.682017803192139 (7.07056205126704) + | > loss_mel: 18.821279525756836 (17.603557820222814) + | > loss_duration: 1.4532907009124756 (1.4533285997351815) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.26673889160156 (30.343847625109614) + | > grad_norm_1: tensor(397.1555, device='cuda:0') (tensor(210.5202, device='cuda:0')) + | > current_lr_0: 0.00019179742734498406 + | > current_lr_1: 0.00019179742734498406 + | > step_time: 0.4254 (0.36962586762953775) + | > loader_time: 0.0078 (0.005926686890271243) + + + --> TIME: 2025-04-19 04:30:18 -- STEP: 123/142 -- GLOBAL_STEP: 426975 + | > loss_disc: 2.4766030311584473 (2.46249170031974) + | > loss_disc_real_0: 0.08568352460861206 (0.12612606821263708) + | > loss_disc_real_1: 0.20903204381465912 (0.20695765583011194) + | > loss_disc_real_2: 0.23531480133533478 (0.22645384642651412) + | > loss_disc_real_3: 0.21321699023246765 (0.22778276905296294) + | > loss_disc_real_4: 0.19450287520885468 (0.22883520438903715) + | > loss_disc_real_5: 0.19060036540031433 (0.23047699930706644) + | > loss_0: 2.4766030311584473 (2.46249170031974) + | > grad_norm_0: tensor(9.4708, device='cuda:0') (tensor(10.8918, device='cuda:0')) + | > loss_gen: 2.281496286392212 (2.3586249332117846) + | > loss_kl: 1.9631224870681763 (1.8719996262372025) + | > loss_feat: 7.346194744110107 (7.03907939864368) + | > loss_mel: 17.01178550720215 (17.534022811951676) + | > loss_duration: 1.4796863794326782 (1.4598534107208248) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.082284927368164 (30.263580291251827) + | > grad_norm_1: tensor(287.6006, device='cuda:0') (tensor(233.1609, device='cuda:0')) + | > current_lr_0: 0.00019179742734498406 + | > current_lr_1: 0.00019179742734498406 + | > step_time: 0.4394 (0.3814126107750869) + | > loader_time: 0.0077 (0.006229832889587897) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4583258628845215 (2.4583258628845215) + | > loss_disc_real_0: 0.1087879166007042 (0.1087879166007042) + | > loss_disc_real_1: 0.17581726610660553 (0.17581726610660553) + | > loss_disc_real_2: 0.22457510232925415 (0.22457510232925415) + | > loss_disc_real_3: 0.2192457765340805 (0.2192457765340805) + | > loss_disc_real_4: 0.1689237803220749 (0.1689237803220749) + | > loss_disc_real_5: 0.2147103250026703 (0.2147103250026703) + | > loss_0: 2.4583258628845215 (2.4583258628845215) + | > loss_gen: 1.9364838600158691 (1.9364838600158691) + | > loss_kl: 2.4412524700164795 (2.4412524700164795) + | > loss_feat: 6.361226558685303 (6.361226558685303) + | > loss_mel: 16.489938735961914 (16.489938735961914) + | > loss_duration: 1.7453289031982422 (1.7453289031982422) + | > loss_1: 28.97422981262207 (28.97422981262207) + + --> STEP: 1 + | > loss_disc: 2.598784923553467 (2.598784923553467) + | > loss_disc_real_0: 0.13255521655082703 (0.13255521655082703) + | > loss_disc_real_1: 0.24289725720882416 (0.24289725720882416) + | > loss_disc_real_2: 0.24379366636276245 (0.24379366636276245) + | > loss_disc_real_3: 0.22717416286468506 (0.22717416286468506) + | > loss_disc_real_4: 0.2120058238506317 (0.2120058238506317) + | > loss_disc_real_5: 0.25081929564476013 (0.25081929564476013) + | > loss_0: 2.598784923553467 (2.598784923553467) + | > loss_gen: 1.9839563369750977 (1.9839563369750977) + | > loss_kl: 2.957303285598755 (2.957303285598755) + | > loss_feat: 5.513743877410889 (5.513743877410889) + | > loss_mel: 15.977783203125 (15.977783203125) + | > loss_duration: 1.7793291807174683 (1.7793291807174683) + | > loss_1: 28.212114334106445 (28.212114334106445) + + --> STEP: 2 + | > loss_disc: 2.5129523277282715 (2.555868625640869) + | > loss_disc_real_0: 0.12056637555360794 (0.12656079605221748) + | > loss_disc_real_1: 0.29127421975135803 (0.2670857384800911) + | > loss_disc_real_2: 0.23535378277301788 (0.23957372456789017) + | > loss_disc_real_3: 0.24536384642124176 (0.2362690046429634) + | > loss_disc_real_4: 0.20553654432296753 (0.20877118408679962) + | > loss_disc_real_5: 0.2168983370065689 (0.23385881632566452) + | > loss_0: 2.5129523277282715 (2.555868625640869) + | > loss_gen: 2.231472969055176 (2.1077146530151367) + | > loss_kl: 2.3457791805267334 (2.651541233062744) + | > loss_feat: 6.908116817474365 (6.210930347442627) + | > loss_mel: 17.89179229736328 (16.93478775024414) + | > loss_duration: 1.6995316743850708 (1.7394304275512695) + | > loss_1: 31.076690673828125 (29.644402503967285) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038334131240844727 (+0.00061798095703125) + | > avg_loss_disc: 2.555868625640869 (-0.006898760795593262) + | > avg_loss_disc_real_0: 0.12656079605221748 (-0.0590592660009861) + | > avg_loss_disc_real_1: 0.2670857384800911 (+0.026468321681022644) + | > avg_loss_disc_real_2: 0.23957372456789017 (+0.01586811989545822) + | > avg_loss_disc_real_3: 0.2362690046429634 (+0.0029430240392684937) + | > avg_loss_disc_real_4: 0.20877118408679962 (-0.03851356357336044) + | > avg_loss_disc_real_5: 0.23385881632566452 (-0.00780082494020462) + | > avg_loss_0: 2.555868625640869 (-0.006898760795593262) + | > avg_loss_gen: 2.1077146530151367 (-0.23917031288146973) + | > avg_loss_kl: 2.651541233062744 (+0.04924118518829346) + | > avg_loss_feat: 6.210930347442627 (-0.16459250450134277) + | > avg_loss_mel: 16.93478775024414 (-0.12394523620605469) + | > avg_loss_duration: 1.7394304275512695 (-0.01668715476989746) + | > avg_loss_1: 29.644402503967285 (-0.49515533447265625) + + + > EPOCH: 336/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:30:28)  + + --> TIME: 2025-04-19 04:30:30 -- STEP: 6/142 -- GLOBAL_STEP: 427000 + | > loss_disc: 2.4450948238372803 (2.4127277533213296) + | > loss_disc_real_0: 0.12674109637737274 (0.11885758116841316) + | > loss_disc_real_1: 0.19134677946567535 (0.20341411232948303) + | > loss_disc_real_2: 0.20483103394508362 (0.2217808539668719) + | > loss_disc_real_3: 0.21772783994674683 (0.22207051267226538) + | > loss_disc_real_4: 0.23622730374336243 (0.2315231884519259) + | > loss_disc_real_5: 0.20460864901542664 (0.21188372125228247) + | > loss_0: 2.4450948238372803 (2.4127277533213296) + | > grad_norm_0: tensor(9.4382, device='cuda:0') (tensor(11.4105, device='cuda:0')) + | > loss_gen: 2.2911062240600586 (2.4127617279688516) + | > loss_kl: 1.8071269989013672 (1.8391362428665161) + | > loss_feat: 6.992090702056885 (7.461377700169881) + | > loss_mel: 16.899629592895508 (17.737274487813313) + | > loss_duration: 1.4248933792114258 (1.4207668900489807) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.414844512939453 (30.87131627400716) + | > grad_norm_1: tensor(198.8119, device='cuda:0') (tensor(292.6017, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.3282 (0.32474811871846515) + | > loader_time: 0.0042 (0.004672686258951823) + + + --> TIME: 2025-04-19 04:30:39 -- STEP: 31/142 -- GLOBAL_STEP: 427025 + | > loss_disc: 2.323452949523926 (2.3862057347451486) + | > loss_disc_real_0: 0.11708708852529526 (0.11634630301306324) + | > loss_disc_real_1: 0.18425098061561584 (0.19777085511915146) + | > loss_disc_real_2: 0.2349710464477539 (0.22381292860354146) + | > loss_disc_real_3: 0.23225563764572144 (0.22477557678376475) + | > loss_disc_real_4: 0.20854251086711884 (0.22617431177246955) + | > loss_disc_real_5: 0.23423922061920166 (0.22139867371128452) + | > loss_0: 2.323452949523926 (2.3862057347451486) + | > grad_norm_0: tensor(14.1521, device='cuda:0') (tensor(8.3317, device='cuda:0')) + | > loss_gen: 2.546478271484375 (2.4361138420720256) + | > loss_kl: 1.7676517963409424 (1.8148103952407837) + | > loss_feat: 7.841624736785889 (7.471400199397918) + | > loss_mel: 18.35483169555664 (17.76366679899154) + | > loss_duration: 1.423354148864746 (1.4259159718790362) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.933940887451172 (30.91190707299017) + | > grad_norm_1: tensor(499.2068, device='cuda:0') (tensor(228.6493, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.3511 (0.33650135224865324) + | > loader_time: 0.0056 (0.005187165352606004) + + + --> TIME: 2025-04-19 04:30:48 -- STEP: 56/142 -- GLOBAL_STEP: 427050 + | > loss_disc: 2.4955387115478516 (2.4099878541060846) + | > loss_disc_real_0: 0.12697923183441162 (0.12013218472046512) + | > loss_disc_real_1: 0.1762208193540573 (0.20074236020445824) + | > loss_disc_real_2: 0.2188301682472229 (0.22196017817727157) + | > loss_disc_real_3: 0.21039895713329315 (0.22493758238852024) + | > loss_disc_real_4: 0.2547350525856018 (0.22724123778087751) + | > loss_disc_real_5: 0.23141250014305115 (0.22426981638584817) + | > loss_0: 2.4955387115478516 (2.4099878541060846) + | > grad_norm_0: tensor(5.0763, device='cuda:0') (tensor(9.5829, device='cuda:0')) + | > loss_gen: 2.320949077606201 (2.4073299637862626) + | > loss_kl: 1.8739712238311768 (1.8350521922111511) + | > loss_feat: 6.977006912231445 (7.366130002907345) + | > loss_mel: 16.820154190063477 (17.780250702585498) + | > loss_duration: 1.457107424736023 (1.4349009288208823) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.449188232421875 (30.82366371154785) + | > grad_norm_1: tensor(214.5710, device='cuda:0') (tensor(249.9451, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.3755 (0.3481172280652181) + | > loader_time: 0.0061 (0.005516248089926583) + + + --> TIME: 2025-04-19 04:30:58 -- STEP: 81/142 -- GLOBAL_STEP: 427075 + | > loss_disc: 2.5104310512542725 (2.4247928754782966) + | > loss_disc_real_0: 0.11276519298553467 (0.12158708568708396) + | > loss_disc_real_1: 0.2249080389738083 (0.20281385786739398) + | > loss_disc_real_2: 0.23769934475421906 (0.22368947738482628) + | > loss_disc_real_3: 0.23695319890975952 (0.22673933005627292) + | > loss_disc_real_4: 0.20556846261024475 (0.22713270709838396) + | > loss_disc_real_5: 0.23890717327594757 (0.22650056727874426) + | > loss_0: 2.5104310512542725 (2.4247928754782966) + | > grad_norm_0: tensor(10.8626, device='cuda:0') (tensor(10.1455, device='cuda:0')) + | > loss_gen: 2.3913395404815674 (2.4035611299820894) + | > loss_kl: 1.6658931970596313 (1.8363375163372653) + | > loss_feat: 6.618096351623535 (7.32238745983736) + | > loss_mel: 17.512632369995117 (17.76367981051221) + | > loss_duration: 1.4544579982757568 (1.44651131718247) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.64242172241211 (30.77247718528465) + | > grad_norm_1: tensor(223.5366, device='cuda:0') (tensor(248.5708, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.3944 (0.3605526877038272) + | > loader_time: 0.0066 (0.005797309640013143) + + + --> TIME: 2025-04-19 04:31:09 -- STEP: 106/142 -- GLOBAL_STEP: 427100 + | > loss_disc: 2.316214084625244 (2.4346619417082587) + | > loss_disc_real_0: 0.1179586872458458 (0.12297762491090118) + | > loss_disc_real_1: 0.16972869634628296 (0.2030038702881561) + | > loss_disc_real_2: 0.22779779136180878 (0.22474871252505285) + | > loss_disc_real_3: 0.20203858613967896 (0.22638264987266288) + | > loss_disc_real_4: 0.1824212372303009 (0.22850863832347798) + | > loss_disc_real_5: 0.19869284331798553 (0.22817949782002647) + | > loss_0: 2.316214084625244 (2.4346619417082587) + | > grad_norm_0: tensor(6.4326, device='cuda:0') (tensor(10.4095, device='cuda:0')) + | > loss_gen: 2.409104108810425 (2.3889798488257066) + | > loss_kl: 1.9072030782699585 (1.847348298666612) + | > loss_feat: 7.83372688293457 (7.2346377237787785) + | > loss_mel: 18.032438278198242 (17.69739432604807) + | > loss_duration: 1.4753416776657104 (1.457374398438435) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.657814025878906 (30.625734509162182) + | > grad_norm_1: tensor(125.0344, device='cuda:0') (tensor(250.1981, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.4082 (0.3709823545419943) + | > loader_time: 0.0073 (0.006059923261966346) + + + --> TIME: 2025-04-19 04:31:20 -- STEP: 131/142 -- GLOBAL_STEP: 427125 + | > loss_disc: 2.3381731510162354 (2.442917183155322) + | > loss_disc_real_0: 0.08687776327133179 (0.124228889051512) + | > loss_disc_real_1: 0.20811307430267334 (0.20422022181157848) + | > loss_disc_real_2: 0.2303732931613922 (0.22535781232455304) + | > loss_disc_real_3: 0.2281327247619629 (0.22761137851776966) + | > loss_disc_real_4: 0.23217740654945374 (0.22874148303770836) + | > loss_disc_real_5: 0.23764964938163757 (0.2289764252100282) + | > loss_0: 2.3381731510162354 (2.442917183155322) + | > grad_norm_0: tensor(12.4953, device='cuda:0') (tensor(10.4489, device='cuda:0')) + | > loss_gen: 2.477935314178467 (2.3762337670071436) + | > loss_kl: 1.97126305103302 (1.8559774406083667) + | > loss_feat: 7.426445484161377 (7.15094067304189) + | > loss_mel: 18.75445556640625 (17.612135421228764) + | > loss_duration: 1.489321231842041 (1.4632341361227832) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.119422912597656 (30.45852135519945) + | > grad_norm_1: tensor(274.5518, device='cuda:0') (tensor(262.9688, device='cuda:0')) + | > current_lr_0: 0.00019177345266656591 + | > current_lr_1: 0.00019177345266656591 + | > step_time: 0.4473 (0.3828330895372928) + | > loader_time: 0.0076 (0.006312373940271276) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.323732852935791 (2.323732852935791) + | > loss_disc_real_0: 0.06939566135406494 (0.06939566135406494) + | > loss_disc_real_1: 0.20123963057994843 (0.20123963057994843) + | > loss_disc_real_2: 0.1863083839416504 (0.1863083839416504) + | > loss_disc_real_3: 0.20549681782722473 (0.20549681782722473) + | > loss_disc_real_4: 0.19305762648582458 (0.19305762648582458) + | > loss_disc_real_5: 0.20767459273338318 (0.20767459273338318) + | > loss_0: 2.323732852935791 (2.323732852935791) + | > loss_gen: 2.1322498321533203 (2.1322498321533203) + | > loss_kl: 2.3883121013641357 (2.3883121013641357) + | > loss_feat: 7.519928932189941 (7.519928932189941) + | > loss_mel: 18.69209098815918 (18.69209098815918) + | > loss_duration: 1.7369096279144287 (1.7369096279144287) + | > loss_1: 32.46949005126953 (32.46949005126953) + + --> STEP: 1 + | > loss_disc: 2.5001578330993652 (2.5001578330993652) + | > loss_disc_real_0: 0.07734555006027222 (0.07734555006027222) + | > loss_disc_real_1: 0.20970188081264496 (0.20970188081264496) + | > loss_disc_real_2: 0.19555293023586273 (0.19555293023586273) + | > loss_disc_real_3: 0.2259901911020279 (0.2259901911020279) + | > loss_disc_real_4: 0.20192885398864746 (0.20192885398864746) + | > loss_disc_real_5: 0.20574761927127838 (0.20574761927127838) + | > loss_0: 2.5001578330993652 (2.5001578330993652) + | > loss_gen: 2.036827325820923 (2.036827325820923) + | > loss_kl: 2.8312792778015137 (2.8312792778015137) + | > loss_feat: 6.554625511169434 (6.554625511169434) + | > loss_mel: 18.68329620361328 (18.68329620361328) + | > loss_duration: 1.7483012676239014 (1.7483012676239014) + | > loss_1: 31.854328155517578 (31.854328155517578) + + --> STEP: 2 + | > loss_disc: 2.415034770965576 (2.4575963020324707) + | > loss_disc_real_0: 0.08892790973186493 (0.08313672989606857) + | > loss_disc_real_1: 0.22296880185604095 (0.21633534133434296) + | > loss_disc_real_2: 0.20982906222343445 (0.2026909962296486) + | > loss_disc_real_3: 0.18261735141277313 (0.2043037712574005) + | > loss_disc_real_4: 0.20636363327503204 (0.20414624363183975) + | > loss_disc_real_5: 0.18366393446922302 (0.1947057768702507) + | > loss_0: 2.415034770965576 (2.4575963020324707) + | > loss_gen: 2.0698044300079346 (2.0533158779144287) + | > loss_kl: 2.155120849609375 (2.4932000637054443) + | > loss_feat: 8.40074634552002 (7.477685928344727) + | > loss_mel: 19.087522506713867 (18.885409355163574) + | > loss_duration: 1.722236156463623 (1.7352687120437622) + | > loss_1: 33.43543243408203 (32.644880294799805) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033162832260131836 (-0.0005171298980712891) + | > avg_loss_disc: 2.4575963020324707 (-0.09827232360839844) + | > avg_loss_disc_real_0: 0.08313672989606857 (-0.04342406615614891) + | > avg_loss_disc_real_1: 0.21633534133434296 (-0.05075039714574814) + | > avg_loss_disc_real_2: 0.2026909962296486 (-0.03688272833824158) + | > avg_loss_disc_real_3: 0.2043037712574005 (-0.0319652333855629) + | > avg_loss_disc_real_4: 0.20414624363183975 (-0.004624940454959869) + | > avg_loss_disc_real_5: 0.1947057768702507 (-0.03915303945541382) + | > avg_loss_0: 2.4575963020324707 (-0.09827232360839844) + | > avg_loss_gen: 2.0533158779144287 (-0.05439877510070801) + | > avg_loss_kl: 2.4932000637054443 (-0.1583411693572998) + | > avg_loss_feat: 7.477685928344727 (+1.2667555809020996) + | > avg_loss_mel: 18.885409355163574 (+1.9506216049194336) + | > avg_loss_duration: 1.7352687120437622 (-0.004161715507507324) + | > avg_loss_1: 32.644880294799805 (+3.0004777908325195) + + + > EPOCH: 337/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:31:26)  + + --> TIME: 2025-04-19 04:31:32 -- STEP: 14/142 -- GLOBAL_STEP: 427150 + | > loss_disc: 2.341052770614624 (2.4180438348225186) + | > loss_disc_real_0: 0.09622093290090561 (0.12090599430458886) + | > loss_disc_real_1: 0.2220078706741333 (0.20997760657753264) + | > loss_disc_real_2: 0.168588325381279 (0.2197128108569554) + | > loss_disc_real_3: 0.19116051495075226 (0.22273752838373184) + | > loss_disc_real_4: 0.20245783030986786 (0.23304364936692373) + | > loss_disc_real_5: 0.19842593371868134 (0.22977655913148606) + | > loss_0: 2.341052770614624 (2.4180438348225186) + | > grad_norm_0: tensor(4.0459, device='cuda:0') (tensor(7.8101, device='cuda:0')) + | > loss_gen: 2.4654910564422607 (2.456824336733137) + | > loss_kl: 1.7359634637832642 (1.7722428015300207) + | > loss_feat: 8.347490310668945 (7.408479043415615) + | > loss_mel: 19.02423095703125 (18.001788003104075) + | > loss_duration: 1.426614761352539 (1.4162866728646415) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.99979019165039 (31.055620874677384) + | > grad_norm_1: tensor(126.3987, device='cuda:0') (tensor(161.3910, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.3316 (0.3281443289348057) + | > loader_time: 0.0055 (0.004783051354544503) + + + --> TIME: 2025-04-19 04:31:41 -- STEP: 39/142 -- GLOBAL_STEP: 427175 + | > loss_disc: 2.3556277751922607 (2.44158697128296) + | > loss_disc_real_0: 0.11500883102416992 (0.12012144063527767) + | > loss_disc_real_1: 0.19640639424324036 (0.20960860298230097) + | > loss_disc_real_2: 0.2179293930530548 (0.22456355201892364) + | > loss_disc_real_3: 0.22151179611682892 (0.22673620551060408) + | > loss_disc_real_4: 0.21389055252075195 (0.2300186317700606) + | > loss_disc_real_5: 0.22142891585826874 (0.23444171517323226) + | > loss_0: 2.3556277751922607 (2.44158697128296) + | > grad_norm_0: tensor(5.8852, device='cuda:0') (tensor(8.5059, device='cuda:0')) + | > loss_gen: 2.312690258026123 (2.402191638946533) + | > loss_kl: 1.8250805139541626 (1.8028817176818848) + | > loss_feat: 7.3726654052734375 (7.353430552360339) + | > loss_mel: 19.048582077026367 (18.0254823244535) + | > loss_duration: 1.4473466873168945 (1.4275213449429243) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.00636672973633 (31.01150762117826) + | > grad_norm_1: tensor(115.9425, device='cuda:0') (tensor(200.6711, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.3562 (0.34064779526148087) + | > loader_time: 0.0062 (0.005277132376646385) + + + --> TIME: 2025-04-19 04:31:50 -- STEP: 64/142 -- GLOBAL_STEP: 427200 + | > loss_disc: 2.4319796562194824 (2.45009795203805) + | > loss_disc_real_0: 0.1813812553882599 (0.12196804489940405) + | > loss_disc_real_1: 0.21520356833934784 (0.2081215267535299) + | > loss_disc_real_2: 0.19823628664016724 (0.22531813010573387) + | > loss_disc_real_3: 0.21908524632453918 (0.22548472718335688) + | > loss_disc_real_4: 0.23984752595424652 (0.23095342447049916) + | > loss_disc_real_5: 0.21784278750419617 (0.2318849670700729) + | > loss_0: 2.4319796562194824 (2.45009795203805) + | > grad_norm_0: tensor(7.9094, device='cuda:0') (tensor(8.5715, device='cuda:0')) + | > loss_gen: 2.4961376190185547 (2.3885981105267997) + | > loss_kl: 1.9590634107589722 (1.832641500979662) + | > loss_feat: 6.985864639282227 (7.230941563844681) + | > loss_mel: 18.565847396850586 (17.816795647144318) + | > loss_duration: 1.483825445175171 (1.4398247189819813) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.490739822387695 (30.708801448345184) + | > grad_norm_1: tensor(132.0267, device='cuda:0') (tensor(199.8503, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.3845 (0.3522266112267971) + | > loader_time: 0.0062 (0.005586940795183182) + + + --> TIME: 2025-04-19 04:32:00 -- STEP: 89/142 -- GLOBAL_STEP: 427225 + | > loss_disc: 2.4829866886138916 (2.4451267371016945) + | > loss_disc_real_0: 0.15044474601745605 (0.12343064678853817) + | > loss_disc_real_1: 0.21605953574180603 (0.20499506475550405) + | > loss_disc_real_2: 0.2280721217393875 (0.22458975485871346) + | > loss_disc_real_3: 0.2333998829126358 (0.22658283606673893) + | > loss_disc_real_4: 0.22629684209823608 (0.228893386011713) + | > loss_disc_real_5: 0.207034170627594 (0.23076376955160935) + | > loss_0: 2.4829866886138916 (2.4451267371016945) + | > grad_norm_0: tensor(15.4634, device='cuda:0') (tensor(9.1539, device='cuda:0')) + | > loss_gen: 2.284557342529297 (2.3877386644984906) + | > loss_kl: 1.977626919746399 (1.8432709393876323) + | > loss_feat: 6.511117458343506 (7.211646519350202) + | > loss_mel: 16.565826416015625 (17.844259390670263) + | > loss_duration: 1.4580223560333252 (1.4508341055237846) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.797151565551758 (30.73774961407265) + | > grad_norm_1: tensor(326.9244, device='cuda:0') (tensor(211.7624, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.4019 (0.36350029238154385) + | > loader_time: 0.0075 (0.0058503338460172164) + + + --> TIME: 2025-04-19 04:32:11 -- STEP: 114/142 -- GLOBAL_STEP: 427250 + | > loss_disc: 2.366588592529297 (2.4537230353606376) + | > loss_disc_real_0: 0.08434773236513138 (0.12401693980944783) + | > loss_disc_real_1: 0.2052372843027115 (0.2058764391562395) + | > loss_disc_real_2: 0.20656397938728333 (0.22615380911973484) + | > loss_disc_real_3: 0.19877538084983826 (0.22756072656627288) + | > loss_disc_real_4: 0.21068723499774933 (0.22971632449250473) + | > loss_disc_real_5: 0.18687084317207336 (0.23146386549138187) + | > loss_0: 2.366588592529297 (2.4537230353606376) + | > grad_norm_0: tensor(8.6580, device='cuda:0') (tensor(10.2183, device='cuda:0')) + | > loss_gen: 2.1899044513702393 (2.3735503167436836) + | > loss_kl: 1.814598560333252 (1.8564337408333493) + | > loss_feat: 7.011148929595947 (7.116161158210353) + | > loss_mel: 17.5063533782959 (17.804244192023024) + | > loss_duration: 1.4938793182373047 (1.4575907619375932) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.015884399414062 (30.60798014255992) + | > grad_norm_1: tensor(168.0173, device='cuda:0') (tensor(237.4690, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.4286 (0.37430499729357275) + | > loader_time: 0.0077 (0.006104697260940284) + + + --> TIME: 2025-04-19 04:32:22 -- STEP: 139/142 -- GLOBAL_STEP: 427275 + | > loss_disc: 2.381251096725464 (2.449431254709367) + | > loss_disc_real_0: 0.12766680121421814 (0.12376480394130131) + | > loss_disc_real_1: 0.1552138775587082 (0.20518228658240475) + | > loss_disc_real_2: 0.19880908727645874 (0.22531158514588856) + | > loss_disc_real_3: 0.19825395941734314 (0.22759993813878338) + | > loss_disc_real_4: 0.20369106531143188 (0.22902776397389474) + | > loss_disc_real_5: 0.2410980761051178 (0.23163163093783015) + | > loss_0: 2.381251096725464 (2.449431254709367) + | > grad_norm_0: tensor(31.0881, device='cuda:0') (tensor(11.1662, device='cuda:0')) + | > loss_gen: 2.463139057159424 (2.376341637947577) + | > loss_kl: 1.9804335832595825 (1.859517747549702) + | > loss_feat: 8.39668083190918 (7.1168302975112585) + | > loss_mel: 18.829322814941406 (17.761561743647075) + | > loss_duration: 1.507819414138794 (1.464307007172125) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.17739486694336 (30.57855844840729) + | > grad_norm_1: tensor(657.6476, device='cuda:0') (tensor(259.3005, device='cuda:0')) + | > current_lr_0: 0.0001917494809849826 + | > current_lr_1: 0.0001917494809849826 + | > step_time: 0.4538 (0.38699334988491146) + | > loader_time: 0.0078 (0.006408164827086085) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4411401748657227 (2.4411401748657227) + | > loss_disc_real_0: 0.09815987199544907 (0.09815987199544907) + | > loss_disc_real_1: 0.2658802270889282 (0.2658802270889282) + | > loss_disc_real_2: 0.28483912348747253 (0.28483912348747253) + | > loss_disc_real_3: 0.22850337624549866 (0.22850337624549866) + | > loss_disc_real_4: 0.2373136430978775 (0.2373136430978775) + | > loss_disc_real_5: 0.23166954517364502 (0.23166954517364502) + | > loss_0: 2.4411401748657227 (2.4411401748657227) + | > loss_gen: 2.489450454711914 (2.489450454711914) + | > loss_kl: 2.520299196243286 (2.520299196243286) + | > loss_feat: 7.0503106117248535 (7.0503106117248535) + | > loss_mel: 18.53656005859375 (18.53656005859375) + | > loss_duration: 1.7124825716018677 (1.7124825716018677) + | > loss_1: 32.309104919433594 (32.309104919433594) + + --> STEP: 1 + | > loss_disc: 2.661654472351074 (2.661654472351074) + | > loss_disc_real_0: 0.12678629159927368 (0.12678629159927368) + | > loss_disc_real_1: 0.2998282313346863 (0.2998282313346863) + | > loss_disc_real_2: 0.30613571405410767 (0.30613571405410767) + | > loss_disc_real_3: 0.20271870493888855 (0.20271870493888855) + | > loss_disc_real_4: 0.22430294752120972 (0.22430294752120972) + | > loss_disc_real_5: 0.2318667322397232 (0.2318667322397232) + | > loss_0: 2.661654472351074 (2.661654472351074) + | > loss_gen: 2.1850945949554443 (2.1850945949554443) + | > loss_kl: 2.747084379196167 (2.747084379196167) + | > loss_feat: 6.4689130783081055 (6.4689130783081055) + | > loss_mel: 17.62335968017578 (17.62335968017578) + | > loss_duration: 1.7763522863388062 (1.7763522863388062) + | > loss_1: 30.800804138183594 (30.800804138183594) + + --> STEP: 2 + | > loss_disc: 2.451202869415283 (2.5564286708831787) + | > loss_disc_real_0: 0.1568606197834015 (0.14182345569133759) + | > loss_disc_real_1: 0.22391578555107117 (0.2618720084428787) + | > loss_disc_real_2: 0.24964696168899536 (0.2778913378715515) + | > loss_disc_real_3: 0.20767459273338318 (0.20519664883613586) + | > loss_disc_real_4: 0.18495073914527893 (0.20462684333324432) + | > loss_disc_real_5: 0.21469524502754211 (0.22328098863363266) + | > loss_0: 2.451202869415283 (2.5564286708831787) + | > loss_gen: 2.2942848205566406 (2.2396897077560425) + | > loss_kl: 2.3094875812530518 (2.5282859802246094) + | > loss_feat: 7.654327869415283 (7.061620473861694) + | > loss_mel: 18.082685470581055 (17.853022575378418) + | > loss_duration: 1.7218213081359863 (1.7490867972373962) + | > loss_1: 32.06260681152344 (31.431705474853516) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003457307815551758 (+0.00014102458953857422) + | > avg_loss_disc: 2.5564286708831787 (+0.09883236885070801) + | > avg_loss_disc_real_0: 0.14182345569133759 (+0.05868672579526901) + | > avg_loss_disc_real_1: 0.2618720084428787 (+0.04553666710853577) + | > avg_loss_disc_real_2: 0.2778913378715515 (+0.07520034164190292) + | > avg_loss_disc_real_3: 0.20519664883613586 (+0.0008928775787353516) + | > avg_loss_disc_real_4: 0.20462684333324432 (+0.00048059970140457153) + | > avg_loss_disc_real_5: 0.22328098863363266 (+0.028575211763381958) + | > avg_loss_0: 2.5564286708831787 (+0.09883236885070801) + | > avg_loss_gen: 2.2396897077560425 (+0.18637382984161377) + | > avg_loss_kl: 2.5282859802246094 (+0.03508591651916504) + | > avg_loss_feat: 7.061620473861694 (-0.4160654544830322) + | > avg_loss_mel: 17.853022575378418 (-1.0323867797851562) + | > avg_loss_duration: 1.7490867972373962 (+0.013818085193634033) + | > avg_loss_1: 31.431705474853516 (-1.213174819946289) + + + > EPOCH: 338/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:32:25)  + + --> TIME: 2025-04-19 04:32:33 -- STEP: 22/142 -- GLOBAL_STEP: 427300 + | > loss_disc: 2.3932178020477295 (2.4598778702995996) + | > loss_disc_real_0: 0.10397928208112717 (0.11906188353896141) + | > loss_disc_real_1: 0.20851723849773407 (0.2077964801679958) + | > loss_disc_real_2: 0.185969740152359 (0.2316704602404074) + | > loss_disc_real_3: 0.24656519293785095 (0.22983366792852228) + | > loss_disc_real_4: 0.2399565577507019 (0.23270973021333868) + | > loss_disc_real_5: 0.25705358386039734 (0.227754031392661) + | > loss_0: 2.3932178020477295 (2.4598778702995996) + | > grad_norm_0: tensor(12.1450, device='cuda:0') (tensor(10.3161, device='cuda:0')) + | > loss_gen: 2.439990997314453 (2.345517461950129) + | > loss_kl: 1.8762147426605225 (1.8148727796294473) + | > loss_feat: 8.113141059875488 (7.230441548607566) + | > loss_mel: 19.081071853637695 (17.58476105603305) + | > loss_duration: 1.3999998569488525 (1.4205577536062763) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.91041946411133 (30.39615076238459) + | > grad_norm_1: tensor(521.1360, device='cuda:0') (tensor(264.9559, device='cuda:0')) + | > current_lr_0: 0.00019172551229985947 + | > current_lr_1: 0.00019172551229985947 + | > step_time: 0.3443 (0.34362553466450085) + | > loader_time: 0.0053 (0.004974690350619229) + + + --> TIME: 2025-04-19 04:32:42 -- STEP: 47/142 -- GLOBAL_STEP: 427325 + | > loss_disc: 2.377426862716675 (2.451781825816378) + | > loss_disc_real_0: 0.12325794994831085 (0.1206568495390263) + | > loss_disc_real_1: 0.26465773582458496 (0.20696454289111685) + | > loss_disc_real_2: 0.25300347805023193 (0.22660178928933244) + | > loss_disc_real_3: 0.18901799619197845 (0.22872421335666737) + | > loss_disc_real_4: 0.22206458449363708 (0.22985312025597754) + | > loss_disc_real_5: 0.19697310030460358 (0.23280287391327797) + | > loss_0: 2.377426862716675 (2.451781825816378) + | > grad_norm_0: tensor(3.2082, device='cuda:0') (tensor(10.5322, device='cuda:0')) + | > loss_gen: 2.509232759475708 (2.3681219638662143) + | > loss_kl: 1.9394123554229736 (1.8349781594377883) + | > loss_feat: 7.930986404418945 (7.241364560228713) + | > loss_mel: 17.491687774658203 (17.561557120465217) + | > loss_duration: 1.4067895412445068 (1.4298923751141166) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.27811050415039 (30.435914465721616) + | > grad_norm_1: tensor(230.0333, device='cuda:0') (tensor(261.1646, device='cuda:0')) + | > current_lr_0: 0.00019172551229985947 + | > current_lr_1: 0.00019172551229985947 + | > step_time: 0.366 (0.3509126216807264) + | > loader_time: 0.0062 (0.005477306690621883) + + + --> TIME: 2025-04-19 04:32:52 -- STEP: 72/142 -- GLOBAL_STEP: 427350 + | > loss_disc: 2.3978097438812256 (2.463211645682653) + | > loss_disc_real_0: 0.11228039115667343 (0.11950717731896374) + | > loss_disc_real_1: 0.1759977489709854 (0.2076251314332088) + | > loss_disc_real_2: 0.23663118481636047 (0.22884849024315676) + | > loss_disc_real_3: 0.22325147688388824 (0.2307969644251797) + | > loss_disc_real_4: 0.2185174524784088 (0.23132344956199327) + | > loss_disc_real_5: 0.24819308519363403 (0.23263214383688238) + | > loss_0: 2.3978097438812256 (2.463211645682653) + | > grad_norm_0: tensor(5.9753, device='cuda:0') (tensor(9.9511, device='cuda:0')) + | > loss_gen: 2.445734977722168 (2.3574586113293963) + | > loss_kl: 1.9767446517944336 (1.8601799342367384) + | > loss_feat: 8.065162658691406 (7.200435241063436) + | > loss_mel: 18.75986671447754 (17.659936401579117) + | > loss_duration: 1.4442262649536133 (1.440993928247028) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.691734313964844 (30.519004265467327) + | > grad_norm_1: tensor(51.8458, device='cuda:0') (tensor(229.7792, device='cuda:0')) + | > current_lr_0: 0.00019172551229985947 + | > current_lr_1: 0.00019172551229985947 + | > step_time: 0.3913 (0.36082481675677835) + | > loader_time: 0.0061 (0.005757368273205227) + + + --> TIME: 2025-04-19 04:33:02 -- STEP: 97/142 -- GLOBAL_STEP: 427375 + | > loss_disc: 2.39491605758667 (2.4566559103346366) + | > loss_disc_real_0: 0.10412327200174332 (0.12109662945737544) + | > loss_disc_real_1: 0.20986026525497437 (0.20728147168134906) + | > loss_disc_real_2: 0.23691394925117493 (0.22752670482876375) + | > loss_disc_real_3: 0.2221028059720993 (0.22871141627277294) + | > loss_disc_real_4: 0.21114781498908997 (0.2295809766066443) + | > loss_disc_real_5: 0.25594696402549744 (0.23126351157414546) + | > loss_0: 2.39491605758667 (2.4566559103346366) + | > grad_norm_0: tensor(9.4646, device='cuda:0') (tensor(9.2969, device='cuda:0')) + | > loss_gen: 2.5270583629608154 (2.3529081614976084) + | > loss_kl: 1.8364895582199097 (1.8685009221440738) + | > loss_feat: 7.130439758300781 (7.13887106020426) + | > loss_mel: 17.235702514648438 (17.671625373289764) + | > loss_duration: 1.4797084331512451 (1.4529110512782621) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.20939826965332 (30.484816797000846) + | > grad_norm_1: tensor(236.5025, device='cuda:0') (tensor(200.0267, device='cuda:0')) + | > current_lr_0: 0.00019172551229985947 + | > current_lr_1: 0.00019172551229985947 + | > step_time: 0.4096 (0.37068501207017407) + | > loader_time: 0.0071 (0.006044990008639306) + + + --> TIME: 2025-04-19 04:33:13 -- STEP: 122/142 -- GLOBAL_STEP: 427400 + | > loss_disc: 2.457859992980957 (2.4605098005201005) + | > loss_disc_real_0: 0.12206216156482697 (0.12372224350444606) + | > loss_disc_real_1: 0.19067877531051636 (0.2074272427402559) + | > loss_disc_real_2: 0.19181406497955322 (0.22738796631332303) + | > loss_disc_real_3: 0.20760998129844666 (0.22948209452824514) + | > loss_disc_real_4: 0.21432748436927795 (0.22979365179284675) + | > loss_disc_real_5: 0.20368771255016327 (0.23141233537529335) + | > loss_0: 2.457859992980957 (2.4605098005201005) + | > grad_norm_0: tensor(13.3602, device='cuda:0') (tensor(9.3952, device='cuda:0')) + | > loss_gen: 2.3467493057250977 (2.3510710372299446) + | > loss_kl: 1.8518714904785156 (1.874046913913039) + | > loss_feat: 7.202619552612305 (7.132936848968756) + | > loss_mel: 18.981395721435547 (17.674457440610798) + | > loss_duration: 1.4768099784851074 (1.4601024594463288) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.859447479248047 (30.49261483989778) + | > grad_norm_1: tensor(250.1630, device='cuda:0') (tensor(207.0753, device='cuda:0')) + | > current_lr_0: 0.00019172551229985947 + | > current_lr_1: 0.00019172551229985947 + | > step_time: 0.4544 (0.38152002897418913) + | > loader_time: 0.0075 (0.006307955648078293) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5352261066436768 (2.5352261066436768) + | > loss_disc_real_0: 0.1521792858839035 (0.1521792858839035) + | > loss_disc_real_1: 0.24617290496826172 (0.24617290496826172) + | > loss_disc_real_2: 0.21727164089679718 (0.21727164089679718) + | > loss_disc_real_3: 0.22031620144844055 (0.22031620144844055) + | > loss_disc_real_4: 0.24076271057128906 (0.24076271057128906) + | > loss_disc_real_5: 0.24068531394004822 (0.24068531394004822) + | > loss_0: 2.5352261066436768 (2.5352261066436768) + | > loss_gen: 2.2399821281433105 (2.2399821281433105) + | > loss_kl: 2.3426709175109863 (2.3426709175109863) + | > loss_feat: 5.623910903930664 (5.623910903930664) + | > loss_mel: 15.196202278137207 (15.196202278137207) + | > loss_duration: 1.742570161819458 (1.742570161819458) + | > loss_1: 27.14533805847168 (27.14533805847168) + + --> STEP: 1 + | > loss_disc: 2.7014873027801514 (2.7014873027801514) + | > loss_disc_real_0: 0.2295427769422531 (0.2295427769422531) + | > loss_disc_real_1: 0.2779298722743988 (0.2779298722743988) + | > loss_disc_real_2: 0.2332533895969391 (0.2332533895969391) + | > loss_disc_real_3: 0.21608039736747742 (0.21608039736747742) + | > loss_disc_real_4: 0.30197080969810486 (0.30197080969810486) + | > loss_disc_real_5: 0.21385903656482697 (0.21385903656482697) + | > loss_0: 2.7014873027801514 (2.7014873027801514) + | > loss_gen: 2.2012884616851807 (2.2012884616851807) + | > loss_kl: 2.7992987632751465 (2.7992987632751465) + | > loss_feat: 5.1982293128967285 (5.1982293128967285) + | > loss_mel: 14.934786796569824 (14.934786796569824) + | > loss_duration: 1.7761744260787964 (1.7761744260787964) + | > loss_1: 26.909778594970703 (26.909778594970703) + + --> STEP: 2 + | > loss_disc: 2.4221739768981934 (2.5618306398391724) + | > loss_disc_real_0: 0.16420798003673553 (0.19687537848949432) + | > loss_disc_real_1: 0.22832465171813965 (0.2531272619962692) + | > loss_disc_real_2: 0.2274118810892105 (0.2303326353430748) + | > loss_disc_real_3: 0.2162211686372757 (0.21615078300237656) + | > loss_disc_real_4: 0.21025341749191284 (0.25611211359500885) + | > loss_disc_real_5: 0.15731142461299896 (0.18558523058891296) + | > loss_0: 2.4221739768981934 (2.5618306398391724) + | > loss_gen: 2.3397910594940186 (2.2705397605895996) + | > loss_kl: 2.2886335849761963 (2.5439661741256714) + | > loss_feat: 7.99550724029541 (6.596868276596069) + | > loss_mel: 18.96291732788086 (16.948852062225342) + | > loss_duration: 1.7240955829620361 (1.7501350045204163) + | > loss_1: 33.310943603515625 (30.110361099243164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003469705581665039 (+1.239776611328125e-05) + | > avg_loss_disc: 2.5618306398391724 (+0.005401968955993652) + | > avg_loss_disc_real_0: 0.19687537848949432 (+0.05505192279815674) + | > avg_loss_disc_real_1: 0.2531272619962692 (-0.008744746446609497) + | > avg_loss_disc_real_2: 0.2303326353430748 (-0.047558702528476715) + | > avg_loss_disc_real_3: 0.21615078300237656 (+0.010954134166240692) + | > avg_loss_disc_real_4: 0.25611211359500885 (+0.051485270261764526) + | > avg_loss_disc_real_5: 0.18558523058891296 (-0.037695758044719696) + | > avg_loss_0: 2.5618306398391724 (+0.005401968955993652) + | > avg_loss_gen: 2.2705397605895996 (+0.03085005283355713) + | > avg_loss_kl: 2.5439661741256714 (+0.01568019390106201) + | > avg_loss_feat: 6.596868276596069 (-0.464752197265625) + | > avg_loss_mel: 16.948852062225342 (-0.9041705131530762) + | > avg_loss_duration: 1.7501350045204163 (+0.0010482072830200195) + | > avg_loss_1: 30.110361099243164 (-1.3213443756103516) + + + > EPOCH: 339/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:33:24)  + + --> TIME: 2025-04-19 04:33:26 -- STEP: 5/142 -- GLOBAL_STEP: 427425 + | > loss_disc: 2.438793659210205 (2.395504331588745) + | > loss_disc_real_0: 0.15656153857707977 (0.11440430730581283) + | > loss_disc_real_1: 0.1895928680896759 (0.19577236473560333) + | > loss_disc_real_2: 0.2097480148077011 (0.22213263511657716) + | > loss_disc_real_3: 0.24436017870903015 (0.24521789252758025) + | > loss_disc_real_4: 0.19489650428295135 (0.2339466392993927) + | > loss_disc_real_5: 0.24762789905071259 (0.23138062953948973) + | > loss_0: 2.438793659210205 (2.395504331588745) + | > grad_norm_0: tensor(5.4989, device='cuda:0') (tensor(8.1810, device='cuda:0')) + | > loss_gen: 2.4833827018737793 (2.416826677322388) + | > loss_kl: 1.8854544162750244 (1.735704231262207) + | > loss_feat: 7.119062423706055 (7.464708995819092) + | > loss_mel: 16.872957229614258 (17.671340560913087) + | > loss_duration: 1.4172923564910889 (1.414862370491028) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.778148651123047 (30.703441619873047) + | > grad_norm_1: tensor(262.7429, device='cuda:0') (tensor(268.8664, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.3266 (0.3328724384307861) + | > loader_time: 0.0047 (0.0048122406005859375) + + + --> TIME: 2025-04-19 04:33:35 -- STEP: 30/142 -- GLOBAL_STEP: 427450 + | > loss_disc: 2.405322790145874 (2.460383741060893) + | > loss_disc_real_0: 0.17130418121814728 (0.11949262196818987) + | > loss_disc_real_1: 0.18557195365428925 (0.21169569343328476) + | > loss_disc_real_2: 0.1995004415512085 (0.2230408211549123) + | > loss_disc_real_3: 0.19740818440914154 (0.2276458129286766) + | > loss_disc_real_4: 0.2081504911184311 (0.23278422206640242) + | > loss_disc_real_5: 0.19648835062980652 (0.23035960694154103) + | > loss_0: 2.405322790145874 (2.460383741060893) + | > grad_norm_0: tensor(13.4877, device='cuda:0') (tensor(10.4295, device='cuda:0')) + | > loss_gen: 2.3466241359710693 (2.3662917216618857) + | > loss_kl: 1.853662371635437 (1.812267009417216) + | > loss_feat: 7.28787899017334 (7.152559312184652) + | > loss_mel: 17.949235916137695 (17.570874341328945) + | > loss_duration: 1.432533621788025 (1.4231141606966655) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.869935989379883 (30.325106302897137) + | > grad_norm_1: tensor(290.4316, device='cuda:0') (tensor(243.2740, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.3482 (0.3373095194498698) + | > loader_time: 0.0055 (0.005153950055440267) + + + --> TIME: 2025-04-19 04:33:44 -- STEP: 55/142 -- GLOBAL_STEP: 427475 + | > loss_disc: 2.350363254547119 (2.4548859552903606) + | > loss_disc_real_0: 0.10162544250488281 (0.12108551941134713) + | > loss_disc_real_1: 0.19013391435146332 (0.20762063671242106) + | > loss_disc_real_2: 0.23333774507045746 (0.22432570132342253) + | > loss_disc_real_3: 0.2205350697040558 (0.22949559011242607) + | > loss_disc_real_4: 0.2007139027118683 (0.22921595763076436) + | > loss_disc_real_5: 0.22781941294670105 (0.23354531716216695) + | > loss_0: 2.350363254547119 (2.4548859552903606) + | > grad_norm_0: tensor(3.5873, device='cuda:0') (tensor(9.6027, device='cuda:0')) + | > loss_gen: 2.418956995010376 (2.358049839193171) + | > loss_kl: 1.8262240886688232 (1.841775462844155) + | > loss_feat: 7.532033920288086 (7.191056286204945) + | > loss_mel: 18.25767707824707 (17.618238136985095) + | > loss_duration: 1.5043647289276123 (1.4325452197681774) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.539257049560547 (30.44166488647461) + | > grad_norm_1: tensor(267.2306, device='cuda:0') (tensor(236.2505, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.3745 (0.3486796899275346) + | > loader_time: 0.0062 (0.00550848353992809) + + + --> TIME: 2025-04-19 04:33:54 -- STEP: 80/142 -- GLOBAL_STEP: 427500 + | > loss_disc: 2.3838376998901367 (2.464220485091209) + | > loss_disc_real_0: 0.1573489010334015 (0.12339320620521903) + | > loss_disc_real_1: 0.210733562707901 (0.20878734178841113) + | > loss_disc_real_2: 0.22162003815174103 (0.22443679682910442) + | > loss_disc_real_3: 0.22649629414081573 (0.22996164597570895) + | > loss_disc_real_4: 0.266722708940506 (0.23017688579857348) + | > loss_disc_real_5: 0.19909606873989105 (0.23414257057011129) + | > loss_0: 2.3838376998901367 (2.464220485091209) + | > grad_norm_0: tensor(14.0578, device='cuda:0') (tensor(9.8819, device='cuda:0')) + | > loss_gen: 2.5409934520721436 (2.3578263849020002) + | > loss_kl: 1.8810594081878662 (1.861374855041504) + | > loss_feat: 7.906752586364746 (7.147008681297303) + | > loss_mel: 19.145910263061523 (17.58340032100678) + | > loss_duration: 1.4652968645095825 (1.4450145855545997) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.94001388549805 (30.39462478160858) + | > grad_norm_1: tensor(544.1646, device='cuda:0') (tensor(261.3926, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.4083 (0.360500866174698) + | > loader_time: 0.0067 (0.005749773979187011) + + + --> TIME: 2025-04-19 04:34:04 -- STEP: 105/142 -- GLOBAL_STEP: 427525 + | > loss_disc: 2.400585651397705 (2.471568591254098) + | > loss_disc_real_0: 0.13360759615898132 (0.12474578519662223) + | > loss_disc_real_1: 0.18354050815105438 (0.20866674851803552) + | > loss_disc_real_2: 0.20484179258346558 (0.22490564797605786) + | > loss_disc_real_3: 0.224206805229187 (0.2293332563979285) + | > loss_disc_real_4: 0.18547062575817108 (0.23112333147298722) + | > loss_disc_real_5: 0.2480146884918213 (0.2338969757159551) + | > loss_0: 2.400585651397705 (2.471568591254098) + | > grad_norm_0: tensor(12.2077, device='cuda:0') (tensor(11.0345, device='cuda:0')) + | > loss_gen: 2.4587061405181885 (2.3562320436750133) + | > loss_kl: 2.012542486190796 (1.8742170277095975) + | > loss_feat: 7.230770587921143 (7.1206840288071405) + | > loss_mel: 18.426855087280273 (17.641059857323064) + | > loss_duration: 1.5044881105422974 (1.4556407769521078) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.63336181640625 (30.4478336697533) + | > grad_norm_1: tensor(121.2062, device='cuda:0') (tensor(272.8765, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.4155 (0.3707438264574325) + | > loader_time: 0.0072 (0.005985700516473679) + + + --> TIME: 2025-04-19 04:34:15 -- STEP: 130/142 -- GLOBAL_STEP: 427550 + | > loss_disc: 2.5562546253204346 (2.477489733695984) + | > loss_disc_real_0: 0.10767047852277756 (0.1258218130813194) + | > loss_disc_real_1: 0.17418935894966125 (0.20871212837787775) + | > loss_disc_real_2: 0.2280956357717514 (0.22537229129901298) + | > loss_disc_real_3: 0.21691472828388214 (0.22906756825171984) + | > loss_disc_real_4: 0.20150145888328552 (0.23144463140230911) + | > loss_disc_real_5: 0.257224977016449 (0.2354089414844146) + | > loss_0: 2.5562546253204346 (2.477489733695984) + | > grad_norm_0: tensor(17.2763, device='cuda:0') (tensor(12.5430, device='cuda:0')) + | > loss_gen: 2.255943536758423 (2.3649320767476) + | > loss_kl: 1.9474188089370728 (1.8822656493920547) + | > loss_feat: 6.2438483238220215 (7.104573121437659) + | > loss_mel: 16.368148803710938 (17.66098723778358) + | > loss_duration: 1.4830048084259033 (1.4615174146798942) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.298364639282227 (30.474275398254395) + | > grad_norm_1: tensor(397.6188, device='cuda:0') (tensor(287.1454, device='cuda:0')) + | > current_lr_0: 0.000191701546610822 + | > current_lr_1: 0.000191701546610822 + | > step_time: 0.4526 (0.3836014069043674) + | > loader_time: 0.0085 (0.006229433646568886) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4865005016326904 (2.4865005016326904) + | > loss_disc_real_0: 0.11346524208784103 (0.11346524208784103) + | > loss_disc_real_1: 0.21738533675670624 (0.21738533675670624) + | > loss_disc_real_2: 0.2372199147939682 (0.2372199147939682) + | > loss_disc_real_3: 0.2402798980474472 (0.2402798980474472) + | > loss_disc_real_4: 0.24664178490638733 (0.24664178490638733) + | > loss_disc_real_5: 0.22237348556518555 (0.22237348556518555) + | > loss_0: 2.4865005016326904 (2.4865005016326904) + | > loss_gen: 2.1944782733917236 (2.1944782733917236) + | > loss_kl: 2.352426052093506 (2.352426052093506) + | > loss_feat: 7.228888511657715 (7.228888511657715) + | > loss_mel: 19.609739303588867 (19.609739303588867) + | > loss_duration: 1.7223784923553467 (1.7223784923553467) + | > loss_1: 33.10791015625 (33.10791015625) + + --> STEP: 1 + | > loss_disc: 2.491042375564575 (2.491042375564575) + | > loss_disc_real_0: 0.17132191359996796 (0.17132191359996796) + | > loss_disc_real_1: 0.19453996419906616 (0.19453996419906616) + | > loss_disc_real_2: 0.23829683661460876 (0.23829683661460876) + | > loss_disc_real_3: 0.23966655135154724 (0.23966655135154724) + | > loss_disc_real_4: 0.24881580471992493 (0.24881580471992493) + | > loss_disc_real_5: 0.21718348562717438 (0.21718348562717438) + | > loss_0: 2.491042375564575 (2.491042375564575) + | > loss_gen: 2.216012477874756 (2.216012477874756) + | > loss_kl: 2.9755070209503174 (2.9755070209503174) + | > loss_feat: 6.791845798492432 (6.791845798492432) + | > loss_mel: 17.998519897460938 (17.998519897460938) + | > loss_duration: 1.7610681056976318 (1.7610681056976318) + | > loss_1: 31.74295425415039 (31.74295425415039) + + --> STEP: 2 + | > loss_disc: 2.3995614051818848 (2.44530189037323) + | > loss_disc_real_0: 0.07259722054004669 (0.12195956707000732) + | > loss_disc_real_1: 0.16571784019470215 (0.18012890219688416) + | > loss_disc_real_2: 0.20529508590698242 (0.2217959612607956) + | > loss_disc_real_3: 0.23356375098228455 (0.2366151511669159) + | > loss_disc_real_4: 0.17772097885608673 (0.21326839178800583) + | > loss_disc_real_5: 0.23018892109394073 (0.22368620336055756) + | > loss_0: 2.3995614051818848 (2.44530189037323) + | > loss_gen: 2.0370030403137207 (2.1265077590942383) + | > loss_kl: 2.442955255508423 (2.70923113822937) + | > loss_feat: 7.395040512084961 (7.093443155288696) + | > loss_mel: 19.10398292541504 (18.55125141143799) + | > loss_duration: 1.7249261140823364 (1.7429971098899841) + | > loss_1: 32.70390701293945 (32.22343063354492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033941268920898438 (-7.557868957519531e-05) + | > avg_loss_disc: 2.44530189037323 (-0.11652874946594238) + | > avg_loss_disc_real_0: 0.12195956707000732 (-0.074915811419487) + | > avg_loss_disc_real_1: 0.18012890219688416 (-0.07299835979938507) + | > avg_loss_disc_real_2: 0.2217959612607956 (-0.008536674082279205) + | > avg_loss_disc_real_3: 0.2366151511669159 (+0.020464368164539337) + | > avg_loss_disc_real_4: 0.21326839178800583 (-0.04284372180700302) + | > avg_loss_disc_real_5: 0.22368620336055756 (+0.03810097277164459) + | > avg_loss_0: 2.44530189037323 (-0.11652874946594238) + | > avg_loss_gen: 2.1265077590942383 (-0.14403200149536133) + | > avg_loss_kl: 2.70923113822937 (+0.16526496410369873) + | > avg_loss_feat: 7.093443155288696 (+0.49657487869262695) + | > avg_loss_mel: 18.55125141143799 (+1.6023993492126465) + | > avg_loss_duration: 1.7429971098899841 (-0.007137894630432129) + | > avg_loss_1: 32.22343063354492 (+2.113069534301758) + + + > EPOCH: 340/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:34:22)  + + --> TIME: 2025-04-19 04:34:27 -- STEP: 13/142 -- GLOBAL_STEP: 427575 + | > loss_disc: 2.299729108810425 (2.3558968030489407) + | > loss_disc_real_0: 0.09485570341348648 (0.10934603558136867) + | > loss_disc_real_1: 0.20051506161689758 (0.20168922039178702) + | > loss_disc_real_2: 0.22890612483024597 (0.22372550918505743) + | > loss_disc_real_3: 0.2001774162054062 (0.2208096281840251) + | > loss_disc_real_4: 0.231659397482872 (0.21874516285382783) + | > loss_disc_real_5: 0.23985013365745544 (0.21938617068987626) + | > loss_0: 2.299729108810425 (2.3558968030489407) + | > grad_norm_0: tensor(10.2871, device='cuda:0') (tensor(8.2487, device='cuda:0')) + | > loss_gen: 2.5031511783599854 (2.436829530275785) + | > loss_kl: 1.811562418937683 (1.8198751211166382) + | > loss_feat: 7.386335372924805 (7.477447436406062) + | > loss_mel: 17.485694885253906 (18.08630605844351) + | > loss_duration: 1.4321060180664062 (1.4171328728015606) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.618850708007812 (31.237590936514046) + | > grad_norm_1: tensor(171.8071, device='cuda:0') (tensor(253.3472, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.3328 (0.32673192024230957) + | > loader_time: 0.0053 (0.0047971835503211385) + + + --> TIME: 2025-04-19 04:34:36 -- STEP: 38/142 -- GLOBAL_STEP: 427600 + | > loss_disc: 2.4244606494903564 (2.4128703757336267) + | > loss_disc_real_0: 0.06536122411489487 (0.11197293255674212) + | > loss_disc_real_1: 0.17613652348518372 (0.20071494540101603) + | > loss_disc_real_2: 0.2376624047756195 (0.22583179293494476) + | > loss_disc_real_3: 0.22308452427387238 (0.22804090027746401) + | > loss_disc_real_4: 0.2524133324623108 (0.22655778025325976) + | > loss_disc_real_5: 0.24065646529197693 (0.23415402597502658) + | > loss_0: 2.4244606494903564 (2.4128703757336267) + | > grad_norm_0: tensor(9.1030, device='cuda:0') (tensor(10.4917, device='cuda:0')) + | > loss_gen: 2.3333981037139893 (2.4138865784594885) + | > loss_kl: 1.7896243333816528 (1.8115598841717369) + | > loss_feat: 7.908788681030273 (7.45634368846291) + | > loss_mel: 18.328857421875 (18.012196239672203) + | > loss_duration: 1.4572157859802246 (1.4314773302329213) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.817886352539062 (31.125463686491315) + | > grad_norm_1: tensor(285.8615, device='cuda:0') (tensor(304.7490, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.3542 (0.3424503238577592) + | > loader_time: 0.0059 (0.005355922799361379) + + + --> TIME: 2025-04-19 04:34:46 -- STEP: 63/142 -- GLOBAL_STEP: 427625 + | > loss_disc: 2.229158878326416 (2.4176829126146107) + | > loss_disc_real_0: 0.12434699386358261 (0.11589096545700044) + | > loss_disc_real_1: 0.17589332163333893 (0.19958674836726414) + | > loss_disc_real_2: 0.22595274448394775 (0.22379801079394326) + | > loss_disc_real_3: 0.19694867730140686 (0.22739648227653805) + | > loss_disc_real_4: 0.19389232993125916 (0.22539315904889787) + | > loss_disc_real_5: 0.18404003977775574 (0.2336233946539107) + | > loss_0: 2.229158878326416 (2.4176829126146107) + | > grad_norm_0: tensor(9.2333, device='cuda:0') (tensor(9.5714, device='cuda:0')) + | > loss_gen: 2.579235553741455 (2.387220991982354) + | > loss_kl: 1.9397077560424805 (1.8237398219487024) + | > loss_feat: 7.875731945037842 (7.309860607934376) + | > loss_mel: 19.217926025390625 (17.81217229933965) + | > loss_duration: 1.4846985340118408 (1.4413293297328647) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.09729766845703 (30.774322963896253) + | > grad_norm_1: tensor(192.0943, device='cuda:0') (tensor(266.8925, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.3859 (0.353585958480835) + | > loader_time: 0.0067 (0.005651492921132888) + + + --> TIME: 2025-04-19 04:34:56 -- STEP: 88/142 -- GLOBAL_STEP: 427650 + | > loss_disc: 2.462618350982666 (2.440035811879418) + | > loss_disc_real_0: 0.09181057661771774 (0.12079927282915874) + | > loss_disc_real_1: 0.2035493552684784 (0.20234342088753535) + | > loss_disc_real_2: 0.22760315239429474 (0.22537009248679335) + | > loss_disc_real_3: 0.2156776636838913 (0.22801058062098242) + | > loss_disc_real_4: 0.22232945263385773 (0.22631229511038822) + | > loss_disc_real_5: 0.2334507554769516 (0.23472842235456814) + | > loss_0: 2.462618350982666 (2.440035811879418) + | > grad_norm_0: tensor(9.4762, device='cuda:0') (tensor(9.7354, device='cuda:0')) + | > loss_gen: 2.2626562118530273 (2.372047191316432) + | > loss_kl: 1.9953358173370361 (1.8416424244642258) + | > loss_feat: 6.446810245513916 (7.19802311333743) + | > loss_mel: 17.59284019470215 (17.69494751366701) + | > loss_duration: 1.5197335481643677 (1.4515437863089824) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.81737518310547 (30.558204000646416) + | > grad_norm_1: tensor(189.8789, device='cuda:0') (tensor(255.2518, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.3978 (0.3645913384177468) + | > loader_time: 0.007 (0.005930586294694379) + + + --> TIME: 2025-04-19 04:35:06 -- STEP: 113/142 -- GLOBAL_STEP: 427675 + | > loss_disc: 2.426913261413574 (2.4422113789921314) + | > loss_disc_real_0: 0.14748819172382355 (0.12285449718479562) + | > loss_disc_real_1: 0.2105124443769455 (0.20318627449790994) + | > loss_disc_real_2: 0.20672790706157684 (0.2244916769255579) + | > loss_disc_real_3: 0.22248022258281708 (0.22798333468690382) + | > loss_disc_real_4: 0.21695350110530853 (0.22625538799087558) + | > loss_disc_real_5: 0.2287340760231018 (0.23418152371866513) + | > loss_0: 2.426913261413574 (2.4422113789921314) + | > grad_norm_0: tensor(16.1958, device='cuda:0') (tensor(9.3967, device='cuda:0')) + | > loss_gen: 2.2595319747924805 (2.363746830847412) + | > loss_kl: 1.796891450881958 (1.8525604843038372) + | > loss_feat: 6.59337854385376 (7.141758593837772) + | > loss_mel: 16.880884170532227 (17.65129676751331) + | > loss_duration: 1.484549641609192 (1.4587225935100456) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.015235900878906 (30.468085246803486) + | > grad_norm_1: tensor(440.0233, device='cuda:0') (tensor(237.7487, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.4279 (0.3749680392510068) + | > loader_time: 0.0072 (0.006181130367042743) + + + --> TIME: 2025-04-19 04:35:18 -- STEP: 138/142 -- GLOBAL_STEP: 427700 + | > loss_disc: 2.486142635345459 (2.4539128956587417) + | > loss_disc_real_0: 0.12491651624441147 (0.1250921589848788) + | > loss_disc_real_1: 0.1918211579322815 (0.20465257493914044) + | > loss_disc_real_2: 0.22405125200748444 (0.22511210400557172) + | > loss_disc_real_3: 0.22112460434436798 (0.22894568622544192) + | > loss_disc_real_4: 0.20947884023189545 (0.22728677493506583) + | > loss_disc_real_5: 0.19246268272399902 (0.2344605320173761) + | > loss_0: 2.486142635345459 (2.4539128956587417) + | > grad_norm_0: tensor(14.4408, device='cuda:0') (tensor(10.1806, device='cuda:0')) + | > loss_gen: 2.193140983581543 (2.3564928891002275) + | > loss_kl: 1.8367632627487183 (1.859218072200167) + | > loss_feat: 6.618647575378418 (7.097694897997206) + | > loss_mel: 18.494150161743164 (17.653848917587943) + | > loss_duration: 1.5311447381973267 (1.4647659788961) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.67384910583496 (30.432020740232606) + | > grad_norm_1: tensor(136.5078, device='cuda:0') (tensor(245.1820, device='cuda:0')) + | > current_lr_0: 0.00019167758391749564 + | > current_lr_1: 0.00019167758391749564 + | > step_time: 0.4504 (0.38681270592454553) + | > loader_time: 0.0071 (0.006435755370319753) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4373974800109863 (2.4373974800109863) + | > loss_disc_real_0: 0.11591201275587082 (0.11591201275587082) + | > loss_disc_real_1: 0.24036361277103424 (0.24036361277103424) + | > loss_disc_real_2: 0.2674563229084015 (0.2674563229084015) + | > loss_disc_real_3: 0.24251499772071838 (0.24251499772071838) + | > loss_disc_real_4: 0.1942041665315628 (0.1942041665315628) + | > loss_disc_real_5: 0.2639424204826355 (0.2639424204826355) + | > loss_0: 2.4373974800109863 (2.4373974800109863) + | > loss_gen: 2.443058490753174 (2.443058490753174) + | > loss_kl: 2.3159403800964355 (2.3159403800964355) + | > loss_feat: 7.390232563018799 (7.390232563018799) + | > loss_mel: 19.34592056274414 (19.34592056274414) + | > loss_duration: 1.7471691370010376 (1.7471691370010376) + | > loss_1: 33.2423210144043 (33.2423210144043) + + --> STEP: 1 + | > loss_disc: 2.403393030166626 (2.403393030166626) + | > loss_disc_real_0: 0.13520537316799164 (0.13520537316799164) + | > loss_disc_real_1: 0.18752935528755188 (0.18752935528755188) + | > loss_disc_real_2: 0.22879302501678467 (0.22879302501678467) + | > loss_disc_real_3: 0.21434977650642395 (0.21434977650642395) + | > loss_disc_real_4: 0.24773895740509033 (0.24773895740509033) + | > loss_disc_real_5: 0.21388833224773407 (0.21388833224773407) + | > loss_0: 2.403393030166626 (2.403393030166626) + | > loss_gen: 2.3784139156341553 (2.3784139156341553) + | > loss_kl: 2.616480588912964 (2.616480588912964) + | > loss_feat: 7.626026630401611 (7.626026630401611) + | > loss_mel: 19.76844596862793 (19.76844596862793) + | > loss_duration: 1.7549865245819092 (1.7549865245819092) + | > loss_1: 34.144351959228516 (34.144351959228516) + + --> STEP: 2 + | > loss_disc: 2.5217435359954834 (2.4625682830810547) + | > loss_disc_real_0: 0.15152089297771454 (0.1433631330728531) + | > loss_disc_real_1: 0.20556536316871643 (0.19654735922813416) + | > loss_disc_real_2: 0.200173482298851 (0.21448325365781784) + | > loss_disc_real_3: 0.2253214567899704 (0.21983561664819717) + | > loss_disc_real_4: 0.24438239634037018 (0.24606067687273026) + | > loss_disc_real_5: 0.25726938247680664 (0.23557885736227036) + | > loss_0: 2.5217435359954834 (2.4625682830810547) + | > loss_gen: 2.205754041671753 (2.292083978652954) + | > loss_kl: 2.558814525604248 (2.587647557258606) + | > loss_feat: 7.08604621887207 (7.356036424636841) + | > loss_mel: 19.383237838745117 (19.575841903686523) + | > loss_duration: 1.7215546369552612 (1.7382705807685852) + | > loss_1: 32.955406188964844 (33.54987907409668) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031735897064208984 (-0.0002205371856689453) + | > avg_loss_disc: 2.4625682830810547 (+0.017266392707824707) + | > avg_loss_disc_real_0: 0.1433631330728531 (+0.021403566002845764) + | > avg_loss_disc_real_1: 0.19654735922813416 (+0.01641845703125) + | > avg_loss_disc_real_2: 0.21448325365781784 (-0.007312707602977753) + | > avg_loss_disc_real_3: 0.21983561664819717 (-0.01677953451871872) + | > avg_loss_disc_real_4: 0.24606067687273026 (+0.032792285084724426) + | > avg_loss_disc_real_5: 0.23557885736227036 (+0.011892654001712799) + | > avg_loss_0: 2.4625682830810547 (+0.017266392707824707) + | > avg_loss_gen: 2.292083978652954 (+0.16557621955871582) + | > avg_loss_kl: 2.587647557258606 (-0.12158358097076416) + | > avg_loss_feat: 7.356036424636841 (+0.26259326934814453) + | > avg_loss_mel: 19.575841903686523 (+1.0245904922485352) + | > avg_loss_duration: 1.7382705807685852 (-0.004726529121398926) + | > avg_loss_1: 33.54987907409668 (+1.3264484405517578) + + + > EPOCH: 341/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:35:21)  + + --> TIME: 2025-04-19 04:35:29 -- STEP: 21/142 -- GLOBAL_STEP: 427725 + | > loss_disc: 2.49114990234375 (2.4354251566387357) + | > loss_disc_real_0: 0.0941152349114418 (0.1171182475629307) + | > loss_disc_real_1: 0.22714397311210632 (0.20868317428089322) + | > loss_disc_real_2: 0.24914178252220154 (0.22417783382393064) + | > loss_disc_real_3: 0.23099613189697266 (0.2267538472300484) + | > loss_disc_real_4: 0.21487478911876678 (0.2210781446525029) + | > loss_disc_real_5: 0.22432821989059448 (0.2301607323544366) + | > loss_0: 2.49114990234375 (2.4354251566387357) + | > grad_norm_0: tensor(14.5636, device='cuda:0') (tensor(11.0038, device='cuda:0')) + | > loss_gen: 2.360582113265991 (2.4182276044573103) + | > loss_kl: 1.841881513595581 (1.816381919951666) + | > loss_feat: 6.418957233428955 (7.267492521376837) + | > loss_mel: 16.679914474487305 (17.86305245899019) + | > loss_duration: 1.423330307006836 (1.414805639357794) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.724666595458984 (30.77996035984584) + | > grad_norm_1: tensor(180.4772, device='cuda:0') (tensor(261.1033, device='cuda:0')) + | > current_lr_0: 0.00019165362421950594 + | > current_lr_1: 0.00019165362421950594 + | > step_time: 0.3412 (0.331660304750715) + | > loader_time: 0.0055 (0.004987728028070359) + + + --> TIME: 2025-04-19 04:35:38 -- STEP: 46/142 -- GLOBAL_STEP: 427750 + | > loss_disc: 2.378243923187256 (2.4736425824787314) + | > loss_disc_real_0: 0.11096727848052979 (0.12721470306101051) + | > loss_disc_real_1: 0.2085845172405243 (0.20821727293988931) + | > loss_disc_real_2: 0.2043376863002777 (0.22610240364852158) + | > loss_disc_real_3: 0.21416211128234863 (0.2287692816361137) + | > loss_disc_real_4: 0.1861627697944641 (0.2252551806361779) + | > loss_disc_real_5: 0.20174378156661987 (0.2306015682609185) + | > loss_0: 2.378243923187256 (2.4736425824787314) + | > grad_norm_0: tensor(14.4452, device='cuda:0') (tensor(10.5238, device='cuda:0')) + | > loss_gen: 2.256357192993164 (2.3645632007847666) + | > loss_kl: 1.815099835395813 (1.8415993711222773) + | > loss_feat: 7.76772403717041 (7.145689394163049) + | > loss_mel: 18.285457611083984 (17.81942977075991) + | > loss_duration: 1.436244249343872 (1.4266569044279016) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.560884475708008 (30.59793878638226) + | > grad_norm_1: tensor(395.1576, device='cuda:0') (tensor(234.4325, device='cuda:0')) + | > current_lr_0: 0.00019165362421950594 + | > current_lr_1: 0.00019165362421950594 + | > step_time: 0.367 (0.34397976813109027) + | > loader_time: 0.0059 (0.005359131356944207) + + + --> TIME: 2025-04-19 04:35:47 -- STEP: 71/142 -- GLOBAL_STEP: 427775 + | > loss_disc: 2.4833312034606934 (2.4683321798351447) + | > loss_disc_real_0: 0.10306532680988312 (0.12657560502559362) + | > loss_disc_real_1: 0.18967048823833466 (0.20915368745024776) + | > loss_disc_real_2: 0.2354767620563507 (0.2260300271527868) + | > loss_disc_real_3: 0.27300286293029785 (0.22920530236942668) + | > loss_disc_real_4: 0.2578800916671753 (0.227264501049485) + | > loss_disc_real_5: 0.28369173407554626 (0.2324357698081245) + | > loss_0: 2.4833312034606934 (2.4683321798351447) + | > grad_norm_0: tensor(13.6434, device='cuda:0') (tensor(10.7600, device='cuda:0')) + | > loss_gen: 2.3382740020751953 (2.3615200586721934) + | > loss_kl: 1.7554450035095215 (1.8487458313015146) + | > loss_feat: 7.337697505950928 (7.1205256556121395) + | > loss_mel: 18.398237228393555 (17.784202965212533) + | > loss_duration: 1.4748154878616333 (1.43906655110104) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.304468154907227 (30.554061164318675) + | > grad_norm_1: tensor(408.2121, device='cuda:0') (tensor(244.0302, device='cuda:0')) + | > current_lr_0: 0.00019165362421950594 + | > current_lr_1: 0.00019165362421950594 + | > step_time: 0.3919 (0.3562941786269067) + | > loader_time: 0.0072 (0.005687931893577037) + + + --> TIME: 2025-04-19 04:35:58 -- STEP: 96/142 -- GLOBAL_STEP: 427800 + | > loss_disc: 2.6758251190185547 (2.4735591436425852) + | > loss_disc_real_0: 0.1263163834810257 (0.12655755396311483) + | > loss_disc_real_1: 0.18478791415691376 (0.2082955421259006) + | > loss_disc_real_2: 0.19281421601772308 (0.22615661658346653) + | > loss_disc_real_3: 0.2412910908460617 (0.22906792210415006) + | > loss_disc_real_4: 0.23425886034965515 (0.22605780170609555) + | > loss_disc_real_5: 0.25797250866889954 (0.2320416575918595) + | > loss_0: 2.6758251190185547 (2.4735591436425852) + | > grad_norm_0: tensor(15.7455, device='cuda:0') (tensor(10.7976, device='cuda:0')) + | > loss_gen: 2.1415297985076904 (2.349912099540234) + | > loss_kl: 1.8688292503356934 (1.8597818315029144) + | > loss_feat: 5.991425037384033 (7.071757252017657) + | > loss_mel: 15.818842887878418 (17.738240897655476) + | > loss_duration: 1.4708998203277588 (1.44990960881114) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.29152488708496 (30.46960171063741) + | > grad_norm_1: tensor(288.5829, device='cuda:0') (tensor(256.8683, device='cuda:0')) + | > current_lr_0: 0.00019165362421950594 + | > current_lr_1: 0.00019165362421950594 + | > step_time: 0.408 (0.3680533195535342) + | > loader_time: 0.0072 (0.005994332333405811) + + + --> TIME: 2025-04-19 04:36:09 -- STEP: 121/142 -- GLOBAL_STEP: 427825 + | > loss_disc: 2.3352389335632324 (2.478065264126487) + | > loss_disc_real_0: 0.07623513787984848 (0.1292243509992095) + | > loss_disc_real_1: 0.1667027771472931 (0.20825603530426656) + | > loss_disc_real_2: 0.17733266949653625 (0.2267006319170156) + | > loss_disc_real_3: 0.21990226209163666 (0.22995097171669163) + | > loss_disc_real_4: 0.2319660186767578 (0.2281222333592817) + | > loss_disc_real_5: 0.17903149127960205 (0.23168581009896333) + | > loss_0: 2.3352389335632324 (2.478065264126487) + | > grad_norm_0: tensor(8.4644, device='cuda:0') (tensor(11.1793, device='cuda:0')) + | > loss_gen: 2.3060226440429688 (2.345669529654763) + | > loss_kl: 1.999301552772522 (1.8616994166177165) + | > loss_feat: 7.108084201812744 (7.052224454800944) + | > loss_mel: 18.468143463134766 (17.739871671377134) + | > loss_duration: 1.5272231101989746 (1.456789868922273) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.408775329589844 (30.456255037922503) + | > grad_norm_1: tensor(97.6959, device='cuda:0') (tensor(252.0081, device='cuda:0')) + | > current_lr_0: 0.00019165362421950594 + | > current_lr_1: 0.00019165362421950594 + | > step_time: 0.44 (0.3796982587861621) + | > loader_time: 0.0074 (0.0062498396093195125) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4343037605285645 (2.4343037605285645) + | > loss_disc_real_0: 0.16624872386455536 (0.16624872386455536) + | > loss_disc_real_1: 0.22726687788963318 (0.22726687788963318) + | > loss_disc_real_2: 0.23334801197052002 (0.23334801197052002) + | > loss_disc_real_3: 0.22047220170497894 (0.22047220170497894) + | > loss_disc_real_4: 0.25154179334640503 (0.25154179334640503) + | > loss_disc_real_5: 0.24335460364818573 (0.24335460364818573) + | > loss_0: 2.4343037605285645 (2.4343037605285645) + | > loss_gen: 2.430755615234375 (2.430755615234375) + | > loss_kl: 2.3825840950012207 (2.3825840950012207) + | > loss_feat: 7.701895236968994 (7.701895236968994) + | > loss_mel: 19.745332717895508 (19.745332717895508) + | > loss_duration: 1.7376419305801392 (1.7376419305801392) + | > loss_1: 33.998207092285156 (33.998207092285156) + + --> STEP: 1 + | > loss_disc: 2.5710363388061523 (2.5710363388061523) + | > loss_disc_real_0: 0.16809579730033875 (0.16809579730033875) + | > loss_disc_real_1: 0.23001857101917267 (0.23001857101917267) + | > loss_disc_real_2: 0.25210750102996826 (0.25210750102996826) + | > loss_disc_real_3: 0.22829362750053406 (0.22829362750053406) + | > loss_disc_real_4: 0.2503094971179962 (0.2503094971179962) + | > loss_disc_real_5: 0.21803274750709534 (0.21803274750709534) + | > loss_0: 2.5710363388061523 (2.5710363388061523) + | > loss_gen: 2.247990369796753 (2.247990369796753) + | > loss_kl: 2.7435102462768555 (2.7435102462768555) + | > loss_feat: 6.476151466369629 (6.476151466369629) + | > loss_mel: 16.823579788208008 (16.823579788208008) + | > loss_duration: 1.7734302282333374 (1.7734302282333374) + | > loss_1: 30.064661026000977 (30.064661026000977) + + --> STEP: 2 + | > loss_disc: 2.2938270568847656 (2.432431697845459) + | > loss_disc_real_0: 0.1538715362548828 (0.16098366677761078) + | > loss_disc_real_1: 0.2510680556297302 (0.24054331332445145) + | > loss_disc_real_2: 0.21541349589824677 (0.2337604984641075) + | > loss_disc_real_3: 0.18138599395751953 (0.2048398107290268) + | > loss_disc_real_4: 0.19453144073486328 (0.22242046892642975) + | > loss_disc_real_5: 0.19209660589694977 (0.20506467670202255) + | > loss_0: 2.2938270568847656 (2.432431697845459) + | > loss_gen: 2.5438432693481445 (2.3959168195724487) + | > loss_kl: 2.404029130935669 (2.573769688606262) + | > loss_feat: 7.425440788269043 (6.950796127319336) + | > loss_mel: 18.172176361083984 (17.497878074645996) + | > loss_duration: 1.7003204822540283 (1.7368753552436829) + | > loss_1: 32.245811462402344 (31.15523624420166) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035903453826904297 (+0.00041675567626953125) + | > avg_loss_disc: 2.432431697845459 (-0.030136585235595703) + | > avg_loss_disc_real_0: 0.16098366677761078 (+0.01762053370475769) + | > avg_loss_disc_real_1: 0.24054331332445145 (+0.04399595409631729) + | > avg_loss_disc_real_2: 0.2337604984641075 (+0.019277244806289673) + | > avg_loss_disc_real_3: 0.2048398107290268 (-0.01499580591917038) + | > avg_loss_disc_real_4: 0.22242046892642975 (-0.023640207946300507) + | > avg_loss_disc_real_5: 0.20506467670202255 (-0.030514180660247803) + | > avg_loss_0: 2.432431697845459 (-0.030136585235595703) + | > avg_loss_gen: 2.3959168195724487 (+0.10383284091949463) + | > avg_loss_kl: 2.573769688606262 (-0.01387786865234375) + | > avg_loss_feat: 6.950796127319336 (-0.4052402973175049) + | > avg_loss_mel: 17.497878074645996 (-2.0779638290405273) + | > avg_loss_duration: 1.7368753552436829 (-0.0013952255249023438) + | > avg_loss_1: 31.15523624420166 (-2.3946428298950195) + + + > EPOCH: 342/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:36:20)  + + --> TIME: 2025-04-19 04:36:22 -- STEP: 4/142 -- GLOBAL_STEP: 427850 + | > loss_disc: 2.4043221473693848 (2.4599395394325256) + | > loss_disc_real_0: 0.10632884502410889 (0.12345860153436661) + | > loss_disc_real_1: 0.17757201194763184 (0.22212110087275505) + | > loss_disc_real_2: 0.1909254640340805 (0.22809597849845886) + | > loss_disc_real_3: 0.1961466521024704 (0.21905286237597466) + | > loss_disc_real_4: 0.19784188270568848 (0.23986361920833588) + | > loss_disc_real_5: 0.20803935825824738 (0.220433309674263) + | > loss_0: 2.4043221473693848 (2.4599395394325256) + | > grad_norm_0: tensor(12.7700, device='cuda:0') (tensor(12.1612, device='cuda:0')) + | > loss_gen: 2.340736150741577 (2.3216989040374756) + | > loss_kl: 1.8459773063659668 (1.806499421596527) + | > loss_feat: 7.4819817543029785 (7.291506409645081) + | > loss_mel: 17.886884689331055 (18.295032501220703) + | > loss_duration: 1.416408658027649 (1.415980726480484) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.971988677978516 (31.130718231201172) + | > grad_norm_1: tensor(295.3625, device='cuda:0') (tensor(347.6279, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.3332 (0.32938265800476074) + | > loader_time: 0.0061 (0.004903614521026611) + + + --> TIME: 2025-04-19 04:36:30 -- STEP: 29/142 -- GLOBAL_STEP: 427875 + | > loss_disc: 2.552602767944336 (2.4674665927886963) + | > loss_disc_real_0: 0.1308193951845169 (0.123980357986072) + | > loss_disc_real_1: 0.16858461499214172 (0.20760003554409948) + | > loss_disc_real_2: 0.24328625202178955 (0.22943019353110214) + | > loss_disc_real_3: 0.22916585206985474 (0.22910217735274085) + | > loss_disc_real_4: 0.2543281614780426 (0.22994341572810864) + | > loss_disc_real_5: 0.18467016518115997 (0.23211659491062164) + | > loss_0: 2.552602767944336 (2.4674665927886963) + | > grad_norm_0: tensor(6.7006, device='cuda:0') (tensor(11.1012, device='cuda:0')) + | > loss_gen: 2.2503106594085693 (2.359896676293735) + | > loss_kl: 1.8200163841247559 (1.7807897617077004) + | > loss_feat: 6.749650478363037 (7.069357773353314) + | > loss_mel: 16.56389045715332 (17.6333055825069) + | > loss_duration: 1.4228646755218506 (1.4211143584086978) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.806732177734375 (30.264464082389043) + | > grad_norm_1: tensor(325.6381, device='cuda:0') (tensor(291.0055, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.3465 (0.33916414195093614) + | > loader_time: 0.0062 (0.005405080729517444) + + + --> TIME: 2025-04-19 04:36:40 -- STEP: 54/142 -- GLOBAL_STEP: 427900 + | > loss_disc: 2.346045732498169 (2.4432725994675257) + | > loss_disc_real_0: 0.15877531468868256 (0.11949254503404652) + | > loss_disc_real_1: 0.2134477198123932 (0.20577475842502382) + | > loss_disc_real_2: 0.22298231720924377 (0.22735205044349036) + | > loss_disc_real_3: 0.2080046683549881 (0.22860763442737084) + | > loss_disc_real_4: 0.19099335372447968 (0.22715020372911734) + | > loss_disc_real_5: 0.2105971872806549 (0.23042548337468394) + | > loss_0: 2.346045732498169 (2.4432725994675257) + | > grad_norm_0: tensor(13.4637, device='cuda:0') (tensor(11.0914, device='cuda:0')) + | > loss_gen: 2.376380681991577 (2.3831709358427258) + | > loss_kl: 1.9688444137573242 (1.8224352796872456) + | > loss_feat: 7.3942155838012695 (7.253190967771742) + | > loss_mel: 16.548633575439453 (17.76305286972611) + | > loss_duration: 1.4538192749023438 (1.4310578770107694) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.741893768310547 (30.652907971982604) + | > grad_norm_1: tensor(326.8012, device='cuda:0') (tensor(282.8827, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.3673 (0.3491619251392506) + | > loader_time: 0.0059 (0.0056759825459233035) + + + --> TIME: 2025-04-19 04:36:49 -- STEP: 79/142 -- GLOBAL_STEP: 427925 + | > loss_disc: 2.5570573806762695 (2.4514076558849487) + | > loss_disc_real_0: 0.10034266859292984 (0.12249970153162751) + | > loss_disc_real_1: 0.2577124536037445 (0.20765637955333613) + | > loss_disc_real_2: 0.22779566049575806 (0.2266927897175656) + | > loss_disc_real_3: 0.24081432819366455 (0.22795083949083014) + | > loss_disc_real_4: 0.25528860092163086 (0.22791297884681558) + | > loss_disc_real_5: 0.25688451528549194 (0.23219234713270695) + | > loss_0: 2.5570573806762695 (2.4514076558849487) + | > grad_norm_0: tensor(6.5274, device='cuda:0') (tensor(10.6671, device='cuda:0')) + | > loss_gen: 2.343158006668091 (2.3728230150440073) + | > loss_kl: 1.8240019083023071 (1.8411145300804814) + | > loss_feat: 6.707851409912109 (7.167905427232573) + | > loss_mel: 18.28951072692871 (17.642116498343555) + | > loss_duration: 1.5264887809753418 (1.4432388136658485) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.69101333618164 (30.467198408102686) + | > grad_norm_1: tensor(159.6674, device='cuda:0') (tensor(264.6236, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.392 (0.3617104910597016) + | > loader_time: 0.0068 (0.005933646914325181) + + + --> TIME: 2025-04-19 04:37:00 -- STEP: 104/142 -- GLOBAL_STEP: 427950 + | > loss_disc: 2.3890786170959473 (2.4589253388918375) + | > loss_disc_real_0: 0.09055569022893906 (0.12370533531961533) + | > loss_disc_real_1: 0.2087600976228714 (0.209105094608206) + | > loss_disc_real_2: 0.19428721070289612 (0.22714700062687582) + | > loss_disc_real_3: 0.21954455971717834 (0.22790148195165855) + | > loss_disc_real_4: 0.24829034507274628 (0.22818728235478586) + | > loss_disc_real_5: 0.25159889459609985 (0.2326420399431999) + | > loss_0: 2.3890786170959473 (2.4589253388918375) + | > grad_norm_0: tensor(6.5178, device='cuda:0') (tensor(10.4667, device='cuda:0')) + | > loss_gen: 2.4041495323181152 (2.3640510623271647) + | > loss_kl: 2.0183074474334717 (1.8567608858530338) + | > loss_feat: 7.318974018096924 (7.15168847487523) + | > loss_mel: 19.067922592163086 (17.740214256139915) + | > loss_duration: 1.500972032546997 (1.4530280564840024) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.310325622558594 (30.56574273109436) + | > grad_norm_1: tensor(240.7689, device='cuda:0') (tensor(259.8145, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.4136 (0.37277011000193083) + | > loader_time: 0.0071 (0.006185687505281886) + + + --> TIME: 2025-04-19 04:37:11 -- STEP: 129/142 -- GLOBAL_STEP: 427975 + | > loss_disc: 2.3918590545654297 (2.456480336743733) + | > loss_disc_real_0: 0.11564479023218155 (0.12352617495050726) + | > loss_disc_real_1: 0.19599895179271698 (0.2084610638923423) + | > loss_disc_real_2: 0.2121446579694748 (0.22684013346831003) + | > loss_disc_real_3: 0.2343558371067047 (0.2278533713069073) + | > loss_disc_real_4: 0.1938035488128662 (0.22777434328729793) + | > loss_disc_real_5: 0.24380354583263397 (0.23185489424901415) + | > loss_0: 2.3918590545654297 (2.456480336743733) + | > grad_norm_0: tensor(15.4260, device='cuda:0') (tensor(11.0634, device='cuda:0')) + | > loss_gen: 2.441659450531006 (2.3619116701821024) + | > loss_kl: 2.049246311187744 (1.8624383238858955) + | > loss_feat: 7.619182109832764 (7.141054563744124) + | > loss_mel: 18.067367553710938 (17.74446089692819) + | > loss_duration: 1.4820705652236938 (1.4592594185540841) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.659526824951172 (30.56912484279899) + | > grad_norm_1: tensor(397.1210, device='cuda:0') (tensor(266.9646, device='cuda:0')) + | > current_lr_0: 0.0001916296675164785 + | > current_lr_1: 0.0001916296675164785 + | > step_time: 0.4406 (0.3842922957368599) + | > loader_time: 0.0077 (0.006449161573897959) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4532582759857178 (2.4532582759857178) + | > loss_disc_real_0: 0.16700910031795502 (0.16700910031795502) + | > loss_disc_real_1: 0.23648212850093842 (0.23648212850093842) + | > loss_disc_real_2: 0.21322667598724365 (0.21322667598724365) + | > loss_disc_real_3: 0.22456926107406616 (0.22456926107406616) + | > loss_disc_real_4: 0.2092076689004898 (0.2092076689004898) + | > loss_disc_real_5: 0.27324190735816956 (0.27324190735816956) + | > loss_0: 2.4532582759857178 (2.4532582759857178) + | > loss_gen: 2.401008129119873 (2.401008129119873) + | > loss_kl: 2.5372936725616455 (2.5372936725616455) + | > loss_feat: 7.13537073135376 (7.13537073135376) + | > loss_mel: 17.87868309020996 (17.87868309020996) + | > loss_duration: 1.718953251838684 (1.718953251838684) + | > loss_1: 31.671310424804688 (31.671310424804688) + + --> STEP: 1 + | > loss_disc: 2.398893356323242 (2.398893356323242) + | > loss_disc_real_0: 0.1066475436091423 (0.1066475436091423) + | > loss_disc_real_1: 0.23076049983501434 (0.23076049983501434) + | > loss_disc_real_2: 0.21271097660064697 (0.21271097660064697) + | > loss_disc_real_3: 0.22809450328350067 (0.22809450328350067) + | > loss_disc_real_4: 0.22785642743110657 (0.22785642743110657) + | > loss_disc_real_5: 0.2244686782360077 (0.2244686782360077) + | > loss_0: 2.398893356323242 (2.398893356323242) + | > loss_gen: 2.3387324810028076 (2.3387324810028076) + | > loss_kl: 2.6010215282440186 (2.6010215282440186) + | > loss_feat: 7.344065189361572 (7.344065189361572) + | > loss_mel: 19.26972770690918 (19.26972770690918) + | > loss_duration: 1.7729967832565308 (1.7729967832565308) + | > loss_1: 33.326541900634766 (33.326541900634766) + + --> STEP: 2 + | > loss_disc: 2.3696646690368652 (2.3842790126800537) + | > loss_disc_real_0: 0.12823522090911865 (0.11744138225913048) + | > loss_disc_real_1: 0.21594640612602234 (0.22335345298051834) + | > loss_disc_real_2: 0.21857252717018127 (0.21564175188541412) + | > loss_disc_real_3: 0.255972683429718 (0.24203359335660934) + | > loss_disc_real_4: 0.20311938226222992 (0.21548790484666824) + | > loss_disc_real_5: 0.2586097717285156 (0.24153922498226166) + | > loss_0: 2.3696646690368652 (2.3842790126800537) + | > loss_gen: 2.4533374309539795 (2.3960349559783936) + | > loss_kl: 2.4182589054107666 (2.5096402168273926) + | > loss_feat: 7.9840240478515625 (7.664044618606567) + | > loss_mel: 18.673452377319336 (18.971590042114258) + | > loss_duration: 1.7006381750106812 (1.736817479133606) + | > loss_1: 33.22970962524414 (33.27812576293945) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033910274505615234 (-0.00019931793212890625) + | > avg_loss_disc: 2.3842790126800537 (-0.04815268516540527) + | > avg_loss_disc_real_0: 0.11744138225913048 (-0.0435422845184803) + | > avg_loss_disc_real_1: 0.22335345298051834 (-0.017189860343933105) + | > avg_loss_disc_real_2: 0.21564175188541412 (-0.01811874657869339) + | > avg_loss_disc_real_3: 0.24203359335660934 (+0.03719378262758255) + | > avg_loss_disc_real_4: 0.21548790484666824 (-0.006932564079761505) + | > avg_loss_disc_real_5: 0.24153922498226166 (+0.036474548280239105) + | > avg_loss_0: 2.3842790126800537 (-0.04815268516540527) + | > avg_loss_gen: 2.3960349559783936 (+0.00011813640594482422) + | > avg_loss_kl: 2.5096402168273926 (-0.06412947177886963) + | > avg_loss_feat: 7.664044618606567 (+0.7132484912872314) + | > avg_loss_mel: 18.971590042114258 (+1.4737119674682617) + | > avg_loss_duration: 1.736817479133606 (-5.78761100769043e-05) + | > avg_loss_1: 33.27812576293945 (+2.122889518737793) + + + > EPOCH: 343/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:37:18)  + + --> TIME: 2025-04-19 04:37:23 -- STEP: 12/142 -- GLOBAL_STEP: 428000 + | > loss_disc: 2.3515467643737793 (2.4313172101974487) + | > loss_disc_real_0: 0.10584943741559982 (0.12332860752940178) + | > loss_disc_real_1: 0.20261240005493164 (0.2002502717077732) + | > loss_disc_real_2: 0.19882911443710327 (0.21712836623191833) + | > loss_disc_real_3: 0.22730913758277893 (0.22615859160820642) + | > loss_disc_real_4: 0.21307440102100372 (0.2243977958957354) + | > loss_disc_real_5: 0.2466275542974472 (0.22513777762651443) + | > loss_0: 2.3515467643737793 (2.4313172101974487) + | > grad_norm_0: tensor(10.7629, device='cuda:0') (tensor(11.1728, device='cuda:0')) + | > loss_gen: 2.4439291954040527 (2.3533239563306174) + | > loss_kl: 1.9078129529953003 (1.8088232278823853) + | > loss_feat: 7.136989593505859 (7.228487213452657) + | > loss_mel: 17.960132598876953 (17.424224774042766) + | > loss_duration: 1.4226707220077515 (1.4115267992019653) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.871536254882812 (30.226386070251465) + | > grad_norm_1: tensor(363.3645, device='cuda:0') (tensor(223.1863, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.3304 (0.32641975084940594) + | > loader_time: 0.0052 (0.004734039306640625) + + + --> TIME: 2025-04-19 04:37:32 -- STEP: 37/142 -- GLOBAL_STEP: 428025 + | > loss_disc: 2.4462552070617676 (2.4391045119311356) + | > loss_disc_real_0: 0.13895802199840546 (0.1206967494374997) + | > loss_disc_real_1: 0.2557496428489685 (0.20472954938540588) + | > loss_disc_real_2: 0.22770348191261292 (0.2209790574537741) + | > loss_disc_real_3: 0.21980378031730652 (0.2242051367018674) + | > loss_disc_real_4: 0.20438766479492188 (0.225226692251257) + | > loss_disc_real_5: 0.199079692363739 (0.2303362641785596) + | > loss_0: 2.4462552070617676 (2.4391045119311356) + | > grad_norm_0: tensor(6.3173, device='cuda:0') (tensor(8.9938, device='cuda:0')) + | > loss_gen: 2.502772331237793 (2.368624448776245) + | > loss_kl: 1.8404104709625244 (1.845748530851828) + | > loss_feat: 7.739874839782715 (7.2819862236847746) + | > loss_mel: 18.795766830444336 (17.74037580232363) + | > loss_duration: 1.4514796733856201 (1.4264519085755218) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.33030700683594 (30.663186923877614) + | > grad_norm_1: tensor(218.6789, device='cuda:0') (tensor(207.6507, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.352 (0.33937505129221324) + | > loader_time: 0.0054 (0.005175294102849187) + + + --> TIME: 2025-04-19 04:37:41 -- STEP: 62/142 -- GLOBAL_STEP: 428050 + | > loss_disc: 2.4501991271972656 (2.460521571097836) + | > loss_disc_real_0: 0.20065422356128693 (0.12627421535791888) + | > loss_disc_real_1: 0.20682568848133087 (0.20550031046713552) + | > loss_disc_real_2: 0.23593097925186157 (0.22503350482833) + | > loss_disc_real_3: 0.22872613370418549 (0.22716478979395283) + | > loss_disc_real_4: 0.22389869391918182 (0.22702620202495205) + | > loss_disc_real_5: 0.22067515552043915 (0.23273997177039424) + | > loss_0: 2.4501991271972656 (2.460521571097836) + | > grad_norm_0: tensor(17.7869, device='cuda:0') (tensor(11.3443, device='cuda:0')) + | > loss_gen: 2.327422857284546 (2.3650902932690037) + | > loss_kl: 1.9471920728683472 (1.874428895211989) + | > loss_feat: 6.812997817993164 (7.20226756218941) + | > loss_mel: 17.050537109375 (17.8188950477108) + | > loss_duration: 1.4934054613113403 (1.4375954827954691) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.631555557250977 (30.69827728886758) + | > grad_norm_1: tensor(307.2416, device='cuda:0') (tensor(248.2343, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.3821 (0.35082537512625417) + | > loader_time: 0.0061 (0.0055100148723971456) + + + --> TIME: 2025-04-19 04:37:51 -- STEP: 87/142 -- GLOBAL_STEP: 428075 + | > loss_disc: 2.593000888824463 (2.468242269822921) + | > loss_disc_real_0: 0.08121153712272644 (0.12721397213894745) + | > loss_disc_real_1: 0.18785512447357178 (0.20708107434470077) + | > loss_disc_real_2: 0.23976802825927734 (0.22491084621555504) + | > loss_disc_real_3: 0.25896990299224854 (0.22720183791785403) + | > loss_disc_real_4: 0.2714153826236725 (0.22927363696454586) + | > loss_disc_real_5: 0.2583504319190979 (0.23363738823896166) + | > loss_0: 2.593000888824463 (2.468242269822921) + | > grad_norm_0: tensor(25.6048, device='cuda:0') (tensor(13.7798, device='cuda:0')) + | > loss_gen: 2.457827091217041 (2.3731302447702687) + | > loss_kl: 1.8414536714553833 (1.8828529475749225) + | > loss_feat: 7.091434478759766 (7.151491000734526) + | > loss_mel: 17.492761611938477 (17.74491122673298) + | > loss_duration: 1.489904522895813 (1.4484816310049473) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.373380661010742 (30.600867019302544) + | > grad_norm_1: tensor(613.0978, device='cuda:0') (tensor(279.3322, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.401 (0.36260674191617426) + | > loader_time: 0.0067 (0.005841918375300265) + + + --> TIME: 2025-04-19 04:38:02 -- STEP: 112/142 -- GLOBAL_STEP: 428100 + | > loss_disc: 2.5230188369750977 (2.4807635843753815) + | > loss_disc_real_0: 0.15856777131557465 (0.1290058700500854) + | > loss_disc_real_1: 0.19827134907245636 (0.20723969489336014) + | > loss_disc_real_2: 0.24079710245132446 (0.22539160307496786) + | > loss_disc_real_3: 0.24221710860729218 (0.22934076788702182) + | > loss_disc_real_4: 0.23564645648002625 (0.23122240749320813) + | > loss_disc_real_5: 0.24137811362743378 (0.23493135520922287) + | > loss_0: 2.5230188369750977 (2.4807635843753815) + | > grad_norm_0: tensor(4.9188, device='cuda:0') (tensor(13.5537, device='cuda:0')) + | > loss_gen: 2.361086130142212 (2.364320237721716) + | > loss_kl: 1.7863482236862183 (1.8893825571451868) + | > loss_feat: 6.622926235198975 (7.104545874255044) + | > loss_mel: 17.56014633178711 (17.737948477268226) + | > loss_duration: 1.4556695222854614 (1.4568574098604066) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.786176681518555 (30.553054554121836) + | > grad_norm_1: tensor(60.8906, device='cuda:0') (tensor(279.6786, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.4253 (0.37314707679407944) + | > loader_time: 0.0071 (0.006104030779429844) + + + --> TIME: 2025-04-19 04:38:13 -- STEP: 137/142 -- GLOBAL_STEP: 428125 + | > loss_disc: 2.4963557720184326 (2.4770008699737325) + | > loss_disc_real_0: 0.1278558224439621 (0.12843038364701026) + | > loss_disc_real_1: 0.18505820631980896 (0.20731652228936662) + | > loss_disc_real_2: 0.2053253948688507 (0.22557786553010453) + | > loss_disc_real_3: 0.23416893184185028 (0.2287200629493616) + | > loss_disc_real_4: 0.23325619101524353 (0.23042891549803043) + | > loss_disc_real_5: 0.26087963581085205 (0.2344643965254735) + | > loss_0: 2.4963557720184326 (2.4770008699737325) + | > grad_norm_0: tensor(7.2651, device='cuda:0') (tensor(12.5968, device='cuda:0')) + | > loss_gen: 2.3376901149749756 (2.3512037722733767) + | > loss_kl: 1.8826905488967896 (1.8919443231429496) + | > loss_feat: 6.806789875030518 (7.044432507814283) + | > loss_mel: 18.035175323486328 (17.65498577591277) + | > loss_duration: 1.4789698123931885 (1.4636343848096194) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.54131507873535 (30.406200854447636) + | > grad_norm_1: tensor(46.7213, device='cuda:0') (tensor(258.5666, device='cuda:0')) + | > current_lr_0: 0.00019160571380803893 + | > current_lr_1: 0.00019160571380803893 + | > step_time: 0.4546 (0.3851938491320088) + | > loader_time: 0.0076 (0.006381858004270679) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.439671516418457 (2.439671516418457) + | > loss_disc_real_0: 0.1571456342935562 (0.1571456342935562) + | > loss_disc_real_1: 0.2387765794992447 (0.2387765794992447) + | > loss_disc_real_2: 0.26597633957862854 (0.26597633957862854) + | > loss_disc_real_3: 0.23110394179821014 (0.23110394179821014) + | > loss_disc_real_4: 0.22353272140026093 (0.22353272140026093) + | > loss_disc_real_5: 0.2138713151216507 (0.2138713151216507) + | > loss_0: 2.439671516418457 (2.439671516418457) + | > loss_gen: 2.4646432399749756 (2.4646432399749756) + | > loss_kl: 2.231990337371826 (2.231990337371826) + | > loss_feat: 6.0142621994018555 (6.0142621994018555) + | > loss_mel: 16.860925674438477 (16.860925674438477) + | > loss_duration: 1.7425380945205688 (1.7425380945205688) + | > loss_1: 29.314359664916992 (29.314359664916992) + + --> STEP: 1 + | > loss_disc: 2.394813060760498 (2.394813060760498) + | > loss_disc_real_0: 0.1629130244255066 (0.1629130244255066) + | > loss_disc_real_1: 0.21112515032291412 (0.21112515032291412) + | > loss_disc_real_2: 0.2380014955997467 (0.2380014955997467) + | > loss_disc_real_3: 0.21234014630317688 (0.21234014630317688) + | > loss_disc_real_4: 0.22794680297374725 (0.22794680297374725) + | > loss_disc_real_5: 0.23590509593486786 (0.23590509593486786) + | > loss_0: 2.394813060760498 (2.394813060760498) + | > loss_gen: 2.39583420753479 (2.39583420753479) + | > loss_kl: 2.7613465785980225 (2.7613465785980225) + | > loss_feat: 7.18330192565918 (7.18330192565918) + | > loss_mel: 17.74994659423828 (17.74994659423828) + | > loss_duration: 1.7568576335906982 (1.7568576335906982) + | > loss_1: 31.847286224365234 (31.847286224365234) + + --> STEP: 2 + | > loss_disc: 2.4275689125061035 (2.411190986633301) + | > loss_disc_real_0: 0.22375264763832092 (0.19333283603191376) + | > loss_disc_real_1: 0.20778493583202362 (0.20945504307746887) + | > loss_disc_real_2: 0.267673134803772 (0.25283731520175934) + | > loss_disc_real_3: 0.20306646823883057 (0.20770330727100372) + | > loss_disc_real_4: 0.18754710257053375 (0.2077469527721405) + | > loss_disc_real_5: 0.2291453778743744 (0.23252523690462112) + | > loss_0: 2.4275689125061035 (2.411190986633301) + | > loss_gen: 2.4920220375061035 (2.4439281225204468) + | > loss_kl: 2.3026652336120605 (2.5320059061050415) + | > loss_feat: 7.59910774230957 (7.391204833984375) + | > loss_mel: 17.29900550842285 (17.524476051330566) + | > loss_duration: 1.7496082782745361 (1.7532329559326172) + | > loss_1: 31.44240951538086 (31.644847869873047) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033034086227416992 (-8.761882781982422e-05) + | > avg_loss_disc: 2.411190986633301 (+0.02691197395324707) + | > avg_loss_disc_real_0: 0.19333283603191376 (+0.07589145377278328) + | > avg_loss_disc_real_1: 0.20945504307746887 (-0.013898409903049469) + | > avg_loss_disc_real_2: 0.25283731520175934 (+0.037195563316345215) + | > avg_loss_disc_real_3: 0.20770330727100372 (-0.03433028608560562) + | > avg_loss_disc_real_4: 0.2077469527721405 (-0.0077409520745277405) + | > avg_loss_disc_real_5: 0.23252523690462112 (-0.009013988077640533) + | > avg_loss_0: 2.411190986633301 (+0.02691197395324707) + | > avg_loss_gen: 2.4439281225204468 (+0.04789316654205322) + | > avg_loss_kl: 2.5320059061050415 (+0.022365689277648926) + | > avg_loss_feat: 7.391204833984375 (-0.2728397846221924) + | > avg_loss_mel: 17.524476051330566 (-1.4471139907836914) + | > avg_loss_duration: 1.7532329559326172 (+0.01641547679901123) + | > avg_loss_1: 31.644847869873047 (-1.6332778930664062) + + + > EPOCH: 344/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:38:17)  + + --> TIME: 2025-04-19 04:38:24 -- STEP: 20/142 -- GLOBAL_STEP: 428150 + | > loss_disc: 2.472337484359741 (2.471540689468384) + | > loss_disc_real_0: 0.12939408421516418 (0.12132980301976204) + | > loss_disc_real_1: 0.22019191086292267 (0.20884217396378518) + | > loss_disc_real_2: 0.20881904661655426 (0.22670929431915282) + | > loss_disc_real_3: 0.2342957705259323 (0.23081404492259025) + | > loss_disc_real_4: 0.23020265996456146 (0.23190239444375038) + | > loss_disc_real_5: 0.24439725279808044 (0.2340688094496727) + | > loss_0: 2.472337484359741 (2.471540689468384) + | > grad_norm_0: tensor(7.4450, device='cuda:0') (tensor(6.9986, device='cuda:0')) + | > loss_gen: 2.5106194019317627 (2.3608704209327698) + | > loss_kl: 1.7255058288574219 (1.7554671108722686) + | > loss_feat: 7.173866271972656 (7.285878372192383) + | > loss_mel: 18.206205368041992 (17.86531352996826) + | > loss_duration: 1.4677402973175049 (1.418198162317276) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.083938598632812 (30.685727691650392) + | > grad_norm_1: tensor(222.4241, device='cuda:0') (tensor(140.4107, device='cuda:0')) + | > current_lr_0: 0.0001915817630938129 + | > current_lr_1: 0.0001915817630938129 + | > step_time: 0.3463 (0.33103965520858764) + | > loader_time: 0.006 (0.005102396011352539) + + + --> TIME: 2025-04-19 04:38:34 -- STEP: 45/142 -- GLOBAL_STEP: 428175 + | > loss_disc: 2.525984525680542 (2.455117103788588) + | > loss_disc_real_0: 0.12693201005458832 (0.12090131276183658) + | > loss_disc_real_1: 0.16809166967868805 (0.20616468720965916) + | > loss_disc_real_2: 0.254387229681015 (0.22806552317407397) + | > loss_disc_real_3: 0.2379978448152542 (0.22706008752187093) + | > loss_disc_real_4: 0.22525951266288757 (0.22851185037030114) + | > loss_disc_real_5: 0.2582574188709259 (0.23195822172694736) + | > loss_0: 2.525984525680542 (2.455117103788588) + | > grad_norm_0: tensor(7.1212, device='cuda:0') (tensor(7.9666, device='cuda:0')) + | > loss_gen: 2.2484798431396484 (2.35037947760688) + | > loss_kl: 1.8021048307418823 (1.8046526538001166) + | > loss_feat: 6.286895275115967 (7.193024900224474) + | > loss_mel: 16.859527587890625 (17.818907674153635) + | > loss_duration: 1.458401083946228 (1.4297173473570086) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.65540885925293 (30.59668189154731) + | > grad_norm_1: tensor(156.4184, device='cuda:0') (tensor(178.5520, device='cuda:0')) + | > current_lr_0: 0.0001915817630938129 + | > current_lr_1: 0.0001915817630938129 + | > step_time: 0.3767 (0.352013275358412) + | > loader_time: 0.0064 (0.0057037724388970265) + + + --> TIME: 2025-04-19 04:38:44 -- STEP: 70/142 -- GLOBAL_STEP: 428200 + | > loss_disc: 2.5039873123168945 (2.456711377416338) + | > loss_disc_real_0: 0.1752789467573166 (0.12350776216813496) + | > loss_disc_real_1: 0.19144849479198456 (0.20615451144320623) + | > loss_disc_real_2: 0.22530704736709595 (0.22713031002453396) + | > loss_disc_real_3: 0.2434622049331665 (0.22724118743624006) + | > loss_disc_real_4: 0.24762889742851257 (0.22916303638901028) + | > loss_disc_real_5: 0.26192182302474976 (0.23291829909597125) + | > loss_0: 2.5039873123168945 (2.456711377416338) + | > grad_norm_0: tensor(6.2740, device='cuda:0') (tensor(7.6170, device='cuda:0')) + | > loss_gen: 2.2804172039031982 (2.352051115036011) + | > loss_kl: 1.9947106838226318 (1.8387511696134295) + | > loss_feat: 6.521156311035156 (7.163784503936768) + | > loss_mel: 18.977500915527344 (17.78235541752406) + | > loss_duration: 1.47141695022583 (1.442051466873714) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.245201110839844 (30.578993579319544) + | > grad_norm_1: tensor(86.9617, device='cuda:0') (tensor(151.6323, device='cuda:0')) + | > current_lr_0: 0.0001915817630938129 + | > current_lr_1: 0.0001915817630938129 + | > step_time: 0.3984 (0.36516327176775254) + | > loader_time: 0.0069 (0.0059841224125453405) + + + --> TIME: 2025-04-19 04:38:54 -- STEP: 95/142 -- GLOBAL_STEP: 428225 + | > loss_disc: 2.493407726287842 (2.46739394539281) + | > loss_disc_real_0: 0.1771368384361267 (0.12664233626503696) + | > loss_disc_real_1: 0.19621334969997406 (0.20698461720817968) + | > loss_disc_real_2: 0.17728616297245026 (0.2268472671508789) + | > loss_disc_real_3: 0.1765609234571457 (0.22704926588033375) + | > loss_disc_real_4: 0.21184101700782776 (0.22912534130247017) + | > loss_disc_real_5: 0.21556496620178223 (0.2331941794407995) + | > loss_0: 2.493407726287842 (2.46739394539281) + | > grad_norm_0: tensor(23.8242, device='cuda:0') (tensor(8.2078, device='cuda:0')) + | > loss_gen: 2.1183669567108154 (2.3441068272841616) + | > loss_kl: 1.830048680305481 (1.8620889199407478) + | > loss_feat: 6.937490940093994 (7.115553429252223) + | > loss_mel: 17.679285049438477 (17.77777285324899) + | > loss_duration: 1.4898252487182617 (1.4535363837292319) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.055015563964844 (30.553058262875204) + | > grad_norm_1: tensor(147.5610, device='cuda:0') (tensor(164.8683, device='cuda:0')) + | > current_lr_0: 0.0001915817630938129 + | > current_lr_1: 0.0001915817630938129 + | > step_time: 0.4184 (0.3767275333404541) + | > loader_time: 0.0068 (0.0062240751166092725) + + + --> TIME: 2025-04-19 04:39:05 -- STEP: 120/142 -- GLOBAL_STEP: 428250 + | > loss_disc: 2.4672129154205322 (2.464478643735251) + | > loss_disc_real_0: 0.12991297245025635 (0.12626286322871846) + | > loss_disc_real_1: 0.14877942204475403 (0.20605272116760412) + | > loss_disc_real_2: 0.21158047020435333 (0.22651653153200943) + | > loss_disc_real_3: 0.2167683094739914 (0.22782641102870305) + | > loss_disc_real_4: 0.19479914009571075 (0.229026564459006) + | > loss_disc_real_5: 0.21138103306293488 (0.232916596904397) + | > loss_0: 2.4672129154205322 (2.464478643735251) + | > grad_norm_0: tensor(9.8615, device='cuda:0') (tensor(8.7182, device='cuda:0')) + | > loss_gen: 2.320666790008545 (2.3445612450440723) + | > loss_kl: 1.813213586807251 (1.8664652456839879) + | > loss_feat: 7.020937442779541 (7.0925326466560366) + | > loss_mel: 18.215007781982422 (17.75865804354349) + | > loss_duration: 1.500105619430542 (1.4601922462383905) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.869932174682617 (30.522409423192343) + | > grad_norm_1: tensor(332.3494, device='cuda:0') (tensor(179.9603, device='cuda:0')) + | > current_lr_0: 0.0001915817630938129 + | > current_lr_1: 0.0001915817630938129 + | > step_time: 0.4487 (0.38857382734616597) + | > loader_time: 0.0075 (0.006467175483703613) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4481332302093506 (2.4481332302093506) + | > loss_disc_real_0: 0.1373310536146164 (0.1373310536146164) + | > loss_disc_real_1: 0.2651459574699402 (0.2651459574699402) + | > loss_disc_real_2: 0.19914929568767548 (0.19914929568767548) + | > loss_disc_real_3: 0.228389173746109 (0.228389173746109) + | > loss_disc_real_4: 0.22946253418922424 (0.22946253418922424) + | > loss_disc_real_5: 0.2669111490249634 (0.2669111490249634) + | > loss_0: 2.4481332302093506 (2.4481332302093506) + | > loss_gen: 2.3735501766204834 (2.3735501766204834) + | > loss_kl: 2.5408084392547607 (2.5408084392547607) + | > loss_feat: 7.082959175109863 (7.082959175109863) + | > loss_mel: 18.44996452331543 (18.44996452331543) + | > loss_duration: 1.7146079540252686 (1.7146079540252686) + | > loss_1: 32.16189193725586 (32.16189193725586) + + --> STEP: 1 + | > loss_disc: 2.7289888858795166 (2.7289888858795166) + | > loss_disc_real_0: 0.15018726885318756 (0.15018726885318756) + | > loss_disc_real_1: 0.25404092669487 (0.25404092669487) + | > loss_disc_real_2: 0.21606197953224182 (0.21606197953224182) + | > loss_disc_real_3: 0.2288389652967453 (0.2288389652967453) + | > loss_disc_real_4: 0.25849366188049316 (0.25849366188049316) + | > loss_disc_real_5: 0.25351056456565857 (0.25351056456565857) + | > loss_0: 2.7289888858795166 (2.7289888858795166) + | > loss_gen: 2.027114152908325 (2.027114152908325) + | > loss_kl: 2.7191667556762695 (2.7191667556762695) + | > loss_feat: 6.091954708099365 (6.091954708099365) + | > loss_mel: 16.1075382232666 (16.1075382232666) + | > loss_duration: 1.7626674175262451 (1.7626674175262451) + | > loss_1: 28.70844078063965 (28.70844078063965) + + --> STEP: 2 + | > loss_disc: 2.4350948333740234 (2.58204185962677) + | > loss_disc_real_0: 0.17310717701911926 (0.1616472229361534) + | > loss_disc_real_1: 0.2583068609237671 (0.25617389380931854) + | > loss_disc_real_2: 0.1627325862646103 (0.18939728289842606) + | > loss_disc_real_3: 0.2108152210712433 (0.2198270931839943) + | > loss_disc_real_4: 0.21747861802577972 (0.23798613995313644) + | > loss_disc_real_5: 0.2093748301267624 (0.23144269734621048) + | > loss_0: 2.4350948333740234 (2.58204185962677) + | > loss_gen: 2.245683193206787 (2.136398673057556) + | > loss_kl: 2.3324029445648193 (2.5257848501205444) + | > loss_feat: 7.351748943328857 (6.721851825714111) + | > loss_mel: 18.261028289794922 (17.18428325653076) + | > loss_duration: 1.723383903503418 (1.7430256605148315) + | > loss_1: 31.91424560546875 (30.3113431930542) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003436446189880371 (+0.00013303756713867188) + | > avg_loss_disc: 2.58204185962677 (+0.17085087299346924) + | > avg_loss_disc_real_0: 0.1616472229361534 (-0.031685613095760345) + | > avg_loss_disc_real_1: 0.25617389380931854 (+0.04671885073184967) + | > avg_loss_disc_real_2: 0.18939728289842606 (-0.06344003230333328) + | > avg_loss_disc_real_3: 0.2198270931839943 (+0.01212378591299057) + | > avg_loss_disc_real_4: 0.23798613995313644 (+0.03023918718099594) + | > avg_loss_disc_real_5: 0.23144269734621048 (-0.0010825395584106445) + | > avg_loss_0: 2.58204185962677 (+0.17085087299346924) + | > avg_loss_gen: 2.136398673057556 (-0.3075294494628906) + | > avg_loss_kl: 2.5257848501205444 (-0.00622105598449707) + | > avg_loss_feat: 6.721851825714111 (-0.6693530082702637) + | > avg_loss_mel: 17.18428325653076 (-0.3401927947998047) + | > avg_loss_duration: 1.7430256605148315 (-0.010207295417785645) + | > avg_loss_1: 30.3113431930542 (-1.3335046768188477) + + + > EPOCH: 345/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:39:17)  + + --> TIME: 2025-04-19 04:39:19 -- STEP: 3/142 -- GLOBAL_STEP: 428275 + | > loss_disc: 2.5162487030029297 (2.491131623586019) + | > loss_disc_real_0: 0.15288309752941132 (0.10907967885335286) + | > loss_disc_real_1: 0.20452621579170227 (0.20414428412914276) + | > loss_disc_real_2: 0.23510906100273132 (0.2078717847665151) + | > loss_disc_real_3: 0.21673119068145752 (0.21320283909638724) + | > loss_disc_real_4: 0.23228450119495392 (0.22969130178292593) + | > loss_disc_real_5: 0.22850455343723297 (0.24114657441775003) + | > loss_0: 2.5162487030029297 (2.491131623586019) + | > grad_norm_0: tensor(9.4311, device='cuda:0') (tensor(15.2585, device='cuda:0')) + | > loss_gen: 2.3838753700256348 (2.345784584681193) + | > loss_kl: 1.895007848739624 (1.8430498043696086) + | > loss_feat: 6.55984354019165 (7.311999638875325) + | > loss_mel: 17.836854934692383 (17.812759399414062) + | > loss_duration: 1.424600601196289 (1.4103331565856934) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.100181579589844 (30.723926544189453) + | > grad_norm_1: tensor(79.4234, device='cuda:0') (tensor(332.4505, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.3302 (0.32925232251485187) + | > loader_time: 0.0041 (0.004364331563313802) + + + --> TIME: 2025-04-19 04:39:27 -- STEP: 28/142 -- GLOBAL_STEP: 428300 + | > loss_disc: 2.434587240219116 (2.4404756171362747) + | > loss_disc_real_0: 0.16084729135036469 (0.11792420409619808) + | > loss_disc_real_1: 0.18375949561595917 (0.20696534642151423) + | > loss_disc_real_2: 0.21878336369991302 (0.22394364486847604) + | > loss_disc_real_3: 0.21001677215099335 (0.2240952673767294) + | > loss_disc_real_4: 0.23766542971134186 (0.2287372536957264) + | > loss_disc_real_5: 0.24486090242862701 (0.23281020830784524) + | > loss_0: 2.434587240219116 (2.4404756171362747) + | > grad_norm_0: tensor(4.4733, device='cuda:0') (tensor(9.4878, device='cuda:0')) + | > loss_gen: 2.287374258041382 (2.375990799495152) + | > loss_kl: 1.8765121698379517 (1.842144821371351) + | > loss_feat: 6.2530717849731445 (7.30585823740278) + | > loss_mel: 16.49272346496582 (17.88086496080671) + | > loss_duration: 1.4587113857269287 (1.4200098855154855) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.368392944335938 (30.824868542807444) + | > grad_norm_1: tensor(180.0006, device='cuda:0') (tensor(241.6181, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.3428 (0.335510322025844) + | > loader_time: 0.0056 (0.0048596007483346125) + + + --> TIME: 2025-04-19 04:39:36 -- STEP: 53/142 -- GLOBAL_STEP: 428325 + | > loss_disc: 2.4638266563415527 (2.439900110352713) + | > loss_disc_real_0: 0.16483555734157562 (0.1209802990252117) + | > loss_disc_real_1: 0.18421299755573273 (0.2043808201011622) + | > loss_disc_real_2: 0.2046782672405243 (0.2224593935710079) + | > loss_disc_real_3: 0.22619852423667908 (0.22591473387097413) + | > loss_disc_real_4: 0.20655027031898499 (0.22638615291073638) + | > loss_disc_real_5: 0.21731635928153992 (0.23314380561405756) + | > loss_0: 2.4638266563415527 (2.439900110352713) + | > grad_norm_0: tensor(5.8869, device='cuda:0') (tensor(9.1403, device='cuda:0')) + | > loss_gen: 2.3650710582733154 (2.3662669613676255) + | > loss_kl: 1.8532356023788452 (1.850522036822337) + | > loss_feat: 7.4950761795043945 (7.215637270009743) + | > loss_mel: 16.88970184326172 (17.6297063287699) + | > loss_duration: 1.4390482902526855 (1.4309273283436614) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.042133331298828 (30.493059770116265) + | > grad_norm_1: tensor(326.0205, device='cuda:0') (tensor(236.1445, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.3684 (0.3470787057336771) + | > loader_time: 0.0053 (0.005209324494847712) + + + --> TIME: 2025-04-19 04:39:46 -- STEP: 78/142 -- GLOBAL_STEP: 428350 + | > loss_disc: 2.3737802505493164 (2.4382524276391053) + | > loss_disc_real_0: 0.10527410358190536 (0.12163913851747146) + | > loss_disc_real_1: 0.1968041956424713 (0.20469382099616223) + | > loss_disc_real_2: 0.23080869019031525 (0.22289761824485582) + | > loss_disc_real_3: 0.20962439477443695 (0.22656846027343702) + | > loss_disc_real_4: 0.24191774427890778 (0.22508456061283746) + | > loss_disc_real_5: 0.24581213295459747 (0.23255598793427149) + | > loss_0: 2.3737802505493164 (2.4382524276391053) + | > grad_norm_0: tensor(11.9954, device='cuda:0') (tensor(9.4904, device='cuda:0')) + | > loss_gen: 2.5848965644836426 (2.3689273687509385) + | > loss_kl: 1.9067248106002808 (1.8635120666944063) + | > loss_feat: 7.740505218505859 (7.24564850024688) + | > loss_mel: 18.806941986083984 (17.65137459681584) + | > loss_duration: 1.4875216484069824 (1.4429173882191002) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.526588439941406 (30.572379796932903) + | > grad_norm_1: tensor(312.3602, device='cuda:0') (tensor(240.6149, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.3904 (0.3593910290644719) + | > loader_time: 0.006 (0.00555427563496125) + + + --> TIME: 2025-04-19 04:39:57 -- STEP: 103/142 -- GLOBAL_STEP: 428375 + | > loss_disc: 2.5281505584716797 (2.4485831816219585) + | > loss_disc_real_0: 0.1748209446668625 (0.12286571167337085) + | > loss_disc_real_1: 0.1984294205904007 (0.2049555494947341) + | > loss_disc_real_2: 0.23622684180736542 (0.22398265776703657) + | > loss_disc_real_3: 0.25596123933792114 (0.22860559416048734) + | > loss_disc_real_4: 0.24060159921646118 (0.22725078015072833) + | > loss_disc_real_5: 0.2617221176624298 (0.23380167232555094) + | > loss_0: 2.5281505584716797 (2.4485831816219585) + | > grad_norm_0: tensor(9.6487, device='cuda:0') (tensor(9.8699, device='cuda:0')) + | > loss_gen: 2.299661636352539 (2.359946614330255) + | > loss_kl: 1.8602961301803589 (1.8691085426552783) + | > loss_feat: 6.793627738952637 (7.181841225300021) + | > loss_mel: 17.21809196472168 (17.67486687076902) + | > loss_duration: 1.5113444328308105 (1.4530294439167653) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.68302345275879 (30.53879261942743) + | > grad_norm_1: tensor(301.0751, device='cuda:0') (tensor(249.2668, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.4153 (0.3699625672645939) + | > loader_time: 0.0067 (0.005845250435245849) + + + --> TIME: 2025-04-19 04:40:08 -- STEP: 128/142 -- GLOBAL_STEP: 428400 + | > loss_disc: 2.4253923892974854 (2.4462686888873577) + | > loss_disc_real_0: 0.14295423030853271 (0.12355223184567876) + | > loss_disc_real_1: 0.181419238448143 (0.20506195002235472) + | > loss_disc_real_2: 0.20502790808677673 (0.22301033337134868) + | > loss_disc_real_3: 0.23710565268993378 (0.2278780611231923) + | > loss_disc_real_4: 0.2259809821844101 (0.22753063717391342) + | > loss_disc_real_5: 0.26499155163764954 (0.23320054926443845) + | > loss_0: 2.4253923892974854 (2.4462686888873577) + | > grad_norm_0: tensor(17.4063, device='cuda:0') (tensor(10.4674, device='cuda:0')) + | > loss_gen: 2.37428879737854 (2.356546428054571) + | > loss_kl: 1.8299490213394165 (1.8758826265111566) + | > loss_feat: 6.965875625610352 (7.144776452332735) + | > loss_mel: 17.47383689880371 (17.615565076470382) + | > loss_duration: 1.4952160120010376 (1.459911311045289) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.1391658782959 (30.452681839466095) + | > grad_norm_1: tensor(444.9815, device='cuda:0') (tensor(257.9069, device='cuda:0')) + | > current_lr_0: 0.00019155781537342616 + | > current_lr_1: 0.00019155781537342616 + | > step_time: 0.4346 (0.38143017143011093) + | > loader_time: 0.0074 (0.006098702549934388) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.360917568206787 (2.360917568206787) + | > loss_disc_real_0: 0.14994430541992188 (0.14994430541992188) + | > loss_disc_real_1: 0.20332856476306915 (0.20332856476306915) + | > loss_disc_real_2: 0.23633265495300293 (0.23633265495300293) + | > loss_disc_real_3: 0.24371221661567688 (0.24371221661567688) + | > loss_disc_real_4: 0.18822236359119415 (0.18822236359119415) + | > loss_disc_real_5: 0.22111660242080688 (0.22111660242080688) + | > loss_0: 2.360917568206787 (2.360917568206787) + | > loss_gen: 2.4911890029907227 (2.4911890029907227) + | > loss_kl: 2.4412169456481934 (2.4412169456481934) + | > loss_feat: 7.017576217651367 (7.017576217651367) + | > loss_mel: 18.752521514892578 (18.752521514892578) + | > loss_duration: 1.731799602508545 (1.731799602508545) + | > loss_1: 32.434303283691406 (32.434303283691406) + + --> STEP: 1 + | > loss_disc: 2.5667884349823 (2.5667884349823) + | > loss_disc_real_0: 0.19483236968517303 (0.19483236968517303) + | > loss_disc_real_1: 0.21525095403194427 (0.21525095403194427) + | > loss_disc_real_2: 0.2333332747220993 (0.2333332747220993) + | > loss_disc_real_3: 0.20962971448898315 (0.20962971448898315) + | > loss_disc_real_4: 0.19256000220775604 (0.19256000220775604) + | > loss_disc_real_5: 0.22302021086215973 (0.22302021086215973) + | > loss_0: 2.5667884349823 (2.5667884349823) + | > loss_gen: 2.3203787803649902 (2.3203787803649902) + | > loss_kl: 2.8225059509277344 (2.8225059509277344) + | > loss_feat: 6.661386489868164 (6.661386489868164) + | > loss_mel: 17.753644943237305 (17.753644943237305) + | > loss_duration: 1.782409429550171 (1.782409429550171) + | > loss_1: 31.3403263092041 (31.3403263092041) + + --> STEP: 2 + | > loss_disc: 2.3924405574798584 (2.479614496231079) + | > loss_disc_real_0: 0.20019450783729553 (0.19751343876123428) + | > loss_disc_real_1: 0.16560772061347961 (0.19042933732271194) + | > loss_disc_real_2: 0.23230613768100739 (0.23281970620155334) + | > loss_disc_real_3: 0.20191805064678192 (0.20577388256788254) + | > loss_disc_real_4: 0.20491214096546173 (0.1987360715866089) + | > loss_disc_real_5: 0.21933254599571228 (0.221176378428936) + | > loss_0: 2.3924405574798584 (2.479614496231079) + | > loss_gen: 2.37058162689209 (2.34548020362854) + | > loss_kl: 2.283632516860962 (2.553069233894348) + | > loss_feat: 7.393497943878174 (7.027442216873169) + | > loss_mel: 19.159408569335938 (18.45652675628662) + | > loss_duration: 1.6683762073516846 (1.7253928184509277) + | > loss_1: 32.8754997253418 (32.10791301727295) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035637617111206055 (+0.00012731552124023438) + | > avg_loss_disc: 2.479614496231079 (-0.10242736339569092) + | > avg_loss_disc_real_0: 0.19751343876123428 (+0.03586621582508087) + | > avg_loss_disc_real_1: 0.19042933732271194 (-0.0657445564866066) + | > avg_loss_disc_real_2: 0.23281970620155334 (+0.04342242330312729) + | > avg_loss_disc_real_3: 0.20577388256788254 (-0.014053210616111755) + | > avg_loss_disc_real_4: 0.1987360715866089 (-0.03925006836652756) + | > avg_loss_disc_real_5: 0.221176378428936 (-0.010266318917274475) + | > avg_loss_0: 2.479614496231079 (-0.10242736339569092) + | > avg_loss_gen: 2.34548020362854 (+0.2090815305709839) + | > avg_loss_kl: 2.553069233894348 (+0.02728438377380371) + | > avg_loss_feat: 7.027442216873169 (+0.3055903911590576) + | > avg_loss_mel: 18.45652675628662 (+1.2722434997558594) + | > avg_loss_duration: 1.7253928184509277 (-0.01763284206390381) + | > avg_loss_1: 32.10791301727295 (+1.79656982421875) + + + > EPOCH: 346/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:40:15)  + + --> TIME: 2025-04-19 04:40:20 -- STEP: 11/142 -- GLOBAL_STEP: 428425 + | > loss_disc: 2.5295259952545166 (2.398354638706554) + | > loss_disc_real_0: 0.1395459920167923 (0.10347753289070996) + | > loss_disc_real_1: 0.16495464742183685 (0.2005701796575026) + | > loss_disc_real_2: 0.2174791395664215 (0.22335282103581863) + | > loss_disc_real_3: 0.2343466877937317 (0.22852527959780258) + | > loss_disc_real_4: 0.20696868002414703 (0.22770738195289264) + | > loss_disc_real_5: 0.26354914903640747 (0.22463113882324912) + | > loss_0: 2.5295259952545166 (2.398354638706554) + | > grad_norm_0: tensor(4.9558, device='cuda:0') (tensor(10.4382, device='cuda:0')) + | > loss_gen: 2.2811472415924072 (2.424170212312178) + | > loss_kl: 1.9418236017227173 (1.7985153089870105) + | > loss_feat: 5.60198450088501 (7.363206733356822) + | > loss_mel: 15.6446533203125 (17.710560711947355) + | > loss_duration: 1.4071184396743774 (1.4128714258020574) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 26.876726150512695 (30.709324056451972) + | > grad_norm_1: tensor(166.9888, device='cuda:0') (tensor(293.7213, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.3297 (0.32582779364152387) + | > loader_time: 0.005 (0.004733454097401012) + + + --> TIME: 2025-04-19 04:40:29 -- STEP: 36/142 -- GLOBAL_STEP: 428450 + | > loss_disc: 2.2991323471069336 (2.397329681449466) + | > loss_disc_real_0: 0.10914285480976105 (0.10446293993542592) + | > loss_disc_real_1: 0.1978292316198349 (0.2024098030394978) + | > loss_disc_real_2: 0.2334219068288803 (0.2255090582701895) + | > loss_disc_real_3: 0.2014097422361374 (0.22207719253169167) + | > loss_disc_real_4: 0.21176359057426453 (0.2268319899837176) + | > loss_disc_real_5: 0.23343001306056976 (0.22681341651413175) + | > loss_0: 2.2991323471069336 (2.397329681449466) + | > grad_norm_0: tensor(4.6785, device='cuda:0') (tensor(10.1674, device='cuda:0')) + | > loss_gen: 2.6958916187286377 (2.429004987080892) + | > loss_kl: 1.7647385597229004 (1.8207235899236467) + | > loss_feat: 7.754893779754639 (7.455994884173076) + | > loss_mel: 18.073352813720703 (17.83012734519111) + | > loss_duration: 1.416456699371338 (1.424562070104811) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.705333709716797 (30.96041276719835) + | > grad_norm_1: tensor(484.3929, device='cuda:0') (tensor(318.1057, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.3532 (0.33883735206392074) + | > loader_time: 0.0058 (0.005178213119506836) + + + --> TIME: 2025-04-19 04:40:38 -- STEP: 61/142 -- GLOBAL_STEP: 428475 + | > loss_disc: 2.4836204051971436 (2.425142663424132) + | > loss_disc_real_0: 0.15627287328243256 (0.11206119122808098) + | > loss_disc_real_1: 0.17682531476020813 (0.20405101653982383) + | > loss_disc_real_2: 0.21036522090435028 (0.2234806008026248) + | > loss_disc_real_3: 0.1900886744260788 (0.22467128932476044) + | > loss_disc_real_4: 0.20319925248622894 (0.23042203143971865) + | > loss_disc_real_5: 0.20971377193927765 (0.2331702408731961) + | > loss_0: 2.4836204051971436 (2.425142663424132) + | > grad_norm_0: tensor(11.0214, device='cuda:0') (tensor(10.2398, device='cuda:0')) + | > loss_gen: 2.2586658000946045 (2.4053631571472667) + | > loss_kl: 1.7834289073944092 (1.8396261891380685) + | > loss_feat: 7.2480998039245605 (7.378166800639669) + | > loss_mel: 17.967931747436523 (17.782168310196674) + | > loss_duration: 1.4591225385665894 (1.4374067587930648) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.717247009277344 (30.842731131881965) + | > grad_norm_1: tensor(261.7670, device='cuda:0') (tensor(282.5419, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.3809 (0.3529483333962863) + | > loader_time: 0.0062 (0.005588723010704165) + + + --> TIME: 2025-04-19 04:40:48 -- STEP: 86/142 -- GLOBAL_STEP: 428500 + | > loss_disc: 2.5043487548828125 (2.442228965981062) + | > loss_disc_real_0: 0.13652896881103516 (0.11599002107111521) + | > loss_disc_real_1: 0.18824556469917297 (0.20657025693460954) + | > loss_disc_real_2: 0.21915502846240997 (0.22523567097824673) + | > loss_disc_real_3: 0.21923260390758514 (0.22554634442163068) + | > loss_disc_real_4: 0.24741925299167633 (0.23051896656668464) + | > loss_disc_real_5: 0.23132410645484924 (0.2334755439051362) + | > loss_0: 2.5043487548828125 (2.442228965981062) + | > grad_norm_0: tensor(11.5195, device='cuda:0') (tensor(10.4161, device='cuda:0')) + | > loss_gen: 2.188286066055298 (2.3863868547040363) + | > loss_kl: 1.863750696182251 (1.857281700123188) + | > loss_feat: 6.962148666381836 (7.283609395803407) + | > loss_mel: 16.976619720458984 (17.81586304376291) + | > loss_duration: 1.5036628246307373 (1.4492284239724624) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.49446678161621 (30.79236950985221) + | > grad_norm_1: tensor(202.2211, device='cuda:0') (tensor(274.1035, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.4114 (0.3664544044538986) + | > loader_time: 0.0066 (0.005883668744286826) + + + --> TIME: 2025-04-19 04:40:59 -- STEP: 111/142 -- GLOBAL_STEP: 428525 + | > loss_disc: 2.3877694606781006 (2.4536255501412056) + | > loss_disc_real_0: 0.11046434938907623 (0.11818734613490535) + | > loss_disc_real_1: 0.22398291528224945 (0.2069600225837381) + | > loss_disc_real_2: 0.2115848958492279 (0.22548383474349976) + | > loss_disc_real_3: 0.19256818294525146 (0.22702084991846) + | > loss_disc_real_4: 0.22048082947731018 (0.23091835186288162) + | > loss_disc_real_5: 0.22533278167247772 (0.23466846615344555) + | > loss_0: 2.3877694606781006 (2.4536255501412056) + | > grad_norm_0: tensor(17.1503, device='cuda:0') (tensor(10.8242, device='cuda:0')) + | > loss_gen: 2.3457846641540527 (2.3720197677612305) + | > loss_kl: 1.942818522453308 (1.8712443478472598) + | > loss_feat: 7.0362749099731445 (7.179849422729768) + | > loss_mel: 18.47614860534668 (17.677124117945763) + | > loss_duration: 1.4897996187210083 (1.4570420529391315) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.29082679748535 (30.557279775808524) + | > grad_norm_1: tensor(232.7620, device='cuda:0') (tensor(273.5610, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.4381 (0.37972709294911977) + | > loader_time: 0.0073 (0.006184144063038869) + + + --> TIME: 2025-04-19 04:41:11 -- STEP: 136/142 -- GLOBAL_STEP: 428550 + | > loss_disc: 2.495835065841675 (2.4575900978901806) + | > loss_disc_real_0: 0.11782822757959366 (0.11984735697179157) + | > loss_disc_real_1: 0.19082462787628174 (0.20633966464768438) + | > loss_disc_real_2: 0.2125963270664215 (0.22626067051554427) + | > loss_disc_real_3: 0.19638147950172424 (0.227278085096794) + | > loss_disc_real_4: 0.20656190812587738 (0.23130391899715452) + | > loss_disc_real_5: 0.20653581619262695 (0.23478737079045353) + | > loss_0: 2.495835065841675 (2.4575900978901806) + | > grad_norm_0: tensor(24.7432, device='cuda:0') (tensor(11.8055, device='cuda:0')) + | > loss_gen: 2.025735855102539 (2.3708473917315986) + | > loss_kl: 1.9305682182312012 (1.8749905062072418) + | > loss_feat: 6.726010799407959 (7.14881118255503) + | > loss_mel: 16.446369171142578 (17.67435904110179) + | > loss_duration: 1.501611590385437 (1.4634717737927159) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.630294799804688 (30.532479945351096) + | > grad_norm_1: tensor(372.7456, device='cuda:0') (tensor(285.6680, device='cuda:0')) + | > current_lr_0: 0.0001915338706465045 + | > current_lr_1: 0.0001915338706465045 + | > step_time: 0.4617 (0.3928609209902146) + | > loader_time: 0.0076 (0.006455356583875769) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4733362197875977 (2.4733362197875977) + | > loss_disc_real_0: 0.1487356424331665 (0.1487356424331665) + | > loss_disc_real_1: 0.18832673132419586 (0.18832673132419586) + | > loss_disc_real_2: 0.22960907220840454 (0.22960907220840454) + | > loss_disc_real_3: 0.23660340905189514 (0.23660340905189514) + | > loss_disc_real_4: 0.2587326169013977 (0.2587326169013977) + | > loss_disc_real_5: 0.2205403596162796 (0.2205403596162796) + | > loss_0: 2.4733362197875977 (2.4733362197875977) + | > loss_gen: 2.2278025150299072 (2.2278025150299072) + | > loss_kl: 2.9177801609039307 (2.9177801609039307) + | > loss_feat: 6.668126106262207 (6.668126106262207) + | > loss_mel: 17.866283416748047 (17.866283416748047) + | > loss_duration: 1.7383487224578857 (1.7383487224578857) + | > loss_1: 31.4183406829834 (31.4183406829834) + + --> STEP: 1 + | > loss_disc: 2.38309645652771 (2.38309645652771) + | > loss_disc_real_0: 0.12459816783666611 (0.12459816783666611) + | > loss_disc_real_1: 0.2035529762506485 (0.2035529762506485) + | > loss_disc_real_2: 0.20591536164283752 (0.20591536164283752) + | > loss_disc_real_3: 0.26045793294906616 (0.26045793294906616) + | > loss_disc_real_4: 0.2669644057750702 (0.2669644057750702) + | > loss_disc_real_5: 0.1940068155527115 (0.1940068155527115) + | > loss_0: 2.38309645652771 (2.38309645652771) + | > loss_gen: 2.4742581844329834 (2.4742581844329834) + | > loss_kl: 2.8190698623657227 (2.8190698623657227) + | > loss_feat: 8.260453224182129 (8.260453224182129) + | > loss_mel: 19.211811065673828 (19.211811065673828) + | > loss_duration: 1.776846170425415 (1.776846170425415) + | > loss_1: 34.54243850708008 (34.54243850708008) + + --> STEP: 2 + | > loss_disc: 2.334768772125244 (2.358932614326477) + | > loss_disc_real_0: 0.09283880144357681 (0.10871848464012146) + | > loss_disc_real_1: 0.19541127979755402 (0.19948212802410126) + | > loss_disc_real_2: 0.19450098276138306 (0.2002081722021103) + | > loss_disc_real_3: 0.20992447435855865 (0.2351912036538124) + | > loss_disc_real_4: 0.25803160667419434 (0.26249800622463226) + | > loss_disc_real_5: 0.21002255380153656 (0.20201468467712402) + | > loss_0: 2.334768772125244 (2.358932614326477) + | > loss_gen: 2.349152088165283 (2.4117051362991333) + | > loss_kl: 2.3624627590179443 (2.5907663106918335) + | > loss_feat: 6.993861198425293 (7.627157211303711) + | > loss_mel: 17.11945343017578 (18.165632247924805) + | > loss_duration: 1.7131997346878052 (1.74502295255661) + | > loss_1: 30.538127899169922 (32.540283203125) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004131317138671875 (+0.0005675554275512695) + | > avg_loss_disc: 2.358932614326477 (-0.12068188190460205) + | > avg_loss_disc_real_0: 0.10871848464012146 (-0.08879495412111282) + | > avg_loss_disc_real_1: 0.19948212802410126 (+0.009052790701389313) + | > avg_loss_disc_real_2: 0.2002081722021103 (-0.032611533999443054) + | > avg_loss_disc_real_3: 0.2351912036538124 (+0.02941732108592987) + | > avg_loss_disc_real_4: 0.26249800622463226 (+0.06376193463802338) + | > avg_loss_disc_real_5: 0.20201468467712402 (-0.01916169375181198) + | > avg_loss_0: 2.358932614326477 (-0.12068188190460205) + | > avg_loss_gen: 2.4117051362991333 (+0.06622493267059326) + | > avg_loss_kl: 2.5907663106918335 (+0.03769707679748535) + | > avg_loss_feat: 7.627157211303711 (+0.599714994430542) + | > avg_loss_mel: 18.165632247924805 (-0.2908945083618164) + | > avg_loss_duration: 1.74502295255661 (+0.019630134105682373) + | > avg_loss_1: 32.540283203125 (+0.4323701858520508) + + + > EPOCH: 347/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:41:15)  + + --> TIME: 2025-04-19 04:41:22 -- STEP: 19/142 -- GLOBAL_STEP: 428575 + | > loss_disc: 2.358842372894287 (2.386554994081196) + | > loss_disc_real_0: 0.05748027563095093 (0.11439543942871847) + | > loss_disc_real_1: 0.23582223057746887 (0.20413988907086222) + | > loss_disc_real_2: 0.2917556166648865 (0.22351892841489693) + | > loss_disc_real_3: 0.24803292751312256 (0.2248770280888206) + | > loss_disc_real_4: 0.2511434555053711 (0.22539396740888296) + | > loss_disc_real_5: 0.2708553671836853 (0.2265816668146535) + | > loss_0: 2.358842372894287 (2.386554994081196) + | > grad_norm_0: tensor(9.2052, device='cuda:0') (tensor(14.9903, device='cuda:0')) + | > loss_gen: 2.7787561416625977 (2.5229046093790153) + | > loss_kl: 1.8483240604400635 (1.8420340701153404) + | > loss_feat: 8.871282577514648 (7.541792016280325) + | > loss_mel: 18.42005157470703 (18.05668830871582) + | > loss_duration: 1.4247781038284302 (1.4160583207481785) + | > amp_scaler: 256.0 (188.6315789473684) + | > loss_1: 33.34319305419922 (31.379477049175062) + | > grad_norm_1: tensor(688.6066, device='cuda:0') (tensor(364.2813, device='cuda:0')) + | > current_lr_0: 0.00019150992891267367 + | > current_lr_1: 0.00019150992891267367 + | > step_time: 0.34 (0.3298883689077276) + | > loader_time: 0.005 (0.004717964875070672) + + + --> TIME: 2025-04-19 04:41:31 -- STEP: 44/142 -- GLOBAL_STEP: 428600 + | > loss_disc: 2.445769786834717 (2.4102311459454624) + | > loss_disc_real_0: 0.1342243254184723 (0.11143402797593312) + | > loss_disc_real_1: 0.2127254754304886 (0.20367019792849367) + | > loss_disc_real_2: 0.2160814106464386 (0.22720757363872093) + | > loss_disc_real_3: 0.20982801914215088 (0.22671804949641228) + | > loss_disc_real_4: 0.2018037885427475 (0.22314624962481586) + | > loss_disc_real_5: 0.21911099553108215 (0.23055676980452103) + | > loss_0: 2.445769786834717 (2.4102311459454624) + | > grad_norm_0: tensor(11.8566, device='cuda:0') (tensor(12.7095, device='cuda:0')) + | > loss_gen: 2.246051788330078 (2.442500878464092) + | > loss_kl: 1.8736419677734375 (1.8536767580292441) + | > loss_feat: 7.398824214935303 (7.3742861639369615) + | > loss_mel: 18.013671875 (17.90791403163563) + | > loss_duration: 1.4596590995788574 (1.4300224835222417) + | > amp_scaler: 256.0 (226.9090909090909) + | > loss_1: 30.991849899291992 (31.008400136774238) + | > grad_norm_1: tensor(346.2850, device='cuda:0') (tensor(333.4444, device='cuda:0')) + | > current_lr_0: 0.00019150992891267367 + | > current_lr_1: 0.00019150992891267367 + | > step_time: 0.364 (0.3430330157279968) + | > loader_time: 0.0053 (0.005120597102425315) + + + --> TIME: 2025-04-19 04:41:41 -- STEP: 69/142 -- GLOBAL_STEP: 428625 + | > loss_disc: 2.4608569145202637 (2.423472926236581) + | > loss_disc_real_0: 0.15519890189170837 (0.11786785522016927) + | > loss_disc_real_1: 0.25082194805145264 (0.20374142238195392) + | > loss_disc_real_2: 0.2550188899040222 (0.22608380689137225) + | > loss_disc_real_3: 0.24466493725776672 (0.22638228298097418) + | > loss_disc_real_4: 0.26007986068725586 (0.22441761713960898) + | > loss_disc_real_5: 0.23528720438480377 (0.2314616305672604) + | > loss_0: 2.4608569145202637 (2.423472926236581) + | > grad_norm_0: tensor(9.9492, device='cuda:0') (tensor(11.2375, device='cuda:0')) + | > loss_gen: 2.460413694381714 (2.407783221507417) + | > loss_kl: 1.9974067211151123 (1.8729861283647842) + | > loss_feat: 6.573338031768799 (7.258334353350211) + | > loss_mel: 16.177703857421875 (17.717331333436828) + | > loss_duration: 1.5246176719665527 (1.4405869850213977) + | > amp_scaler: 256.0 (237.44927536231884) + | > loss_1: 28.733478546142578 (30.69702176080234) + | > grad_norm_1: tensor(40.1582, device='cuda:0') (tensor(290.7962, device='cuda:0')) + | > current_lr_0: 0.00019150992891267367 + | > current_lr_1: 0.00019150992891267367 + | > step_time: 0.3994 (0.35542048924211145) + | > loader_time: 0.0061 (0.005440535752669624) + + + --> TIME: 2025-04-19 04:41:51 -- STEP: 94/142 -- GLOBAL_STEP: 428650 + | > loss_disc: 2.4653778076171875 (2.436008986006393) + | > loss_disc_real_0: 0.08744508028030396 (0.1201976673637933) + | > loss_disc_real_1: 0.19768615067005157 (0.2038315333267476) + | > loss_disc_real_2: 0.22334730625152588 (0.22607957238846638) + | > loss_disc_real_3: 0.21267804503440857 (0.2270253322860028) + | > loss_disc_real_4: 0.21859215199947357 (0.22596074228591106) + | > loss_disc_real_5: 0.22698169946670532 (0.2324553445932713) + | > loss_0: 2.4653778076171875 (2.436008986006393) + | > grad_norm_0: tensor(12.7310, device='cuda:0') (tensor(12.3725, device='cuda:0')) + | > loss_gen: 2.372490882873535 (2.3923875372460537) + | > loss_kl: 1.7969671487808228 (1.871720628535494) + | > loss_feat: 6.893009185791016 (7.200819801776968) + | > loss_mel: 16.63616180419922 (17.69029618324117) + | > loss_duration: 1.4793179035186768 (1.450349121651751) + | > amp_scaler: 256.0 (242.38297872340425) + | > loss_1: 29.177946090698242 (30.60557306573746) + | > grad_norm_1: tensor(402.6834, device='cuda:0') (tensor(299.1011, device='cuda:0')) + | > current_lr_0: 0.00019150992891267367 + | > current_lr_1: 0.00019150992891267367 + | > step_time: 0.4002 (0.36677520833116906) + | > loader_time: 0.0068 (0.005744383690205025) + + + --> TIME: 2025-04-19 04:42:02 -- STEP: 119/142 -- GLOBAL_STEP: 428675 + | > loss_disc: 2.465808629989624 (2.4435475353433316) + | > loss_disc_real_0: 0.08450192958116531 (0.12292524029602524) + | > loss_disc_real_1: 0.19267818331718445 (0.20444461427816826) + | > loss_disc_real_2: 0.21842928230762482 (0.2252563554950121) + | > loss_disc_real_3: 0.24963507056236267 (0.2281442945744811) + | > loss_disc_real_4: 0.22408168017864227 (0.22649607099905736) + | > loss_disc_real_5: 0.26691940426826477 (0.23217411677376562) + | > loss_0: 2.465808629989624 (2.4435475353433316) + | > grad_norm_0: tensor(16.4142, device='cuda:0') (tensor(13.5046, device='cuda:0')) + | > loss_gen: 2.2778356075286865 (2.3883420799960593) + | > loss_kl: 1.8374313116073608 (1.8815044645501786) + | > loss_feat: 7.3158979415893555 (7.178528601381959) + | > loss_mel: 18.115642547607422 (17.717978445421746) + | > loss_duration: 1.5355477333068848 (1.457536198511845) + | > amp_scaler: 256.0 (245.2436974789916) + | > loss_1: 31.082353591918945 (30.623889666645468) + | > grad_norm_1: tensor(378.0006, device='cuda:0') (tensor(301.3571, device='cuda:0')) + | > current_lr_0: 0.00019150992891267367 + | > current_lr_1: 0.00019150992891267367 + | > step_time: 0.4268 (0.37870642117091596) + | > loader_time: 0.0075 (0.006041743174320508) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.433995246887207 (2.433995246887207) + | > loss_disc_real_0: 0.17399351298809052 (0.17399351298809052) + | > loss_disc_real_1: 0.2571425437927246 (0.2571425437927246) + | > loss_disc_real_2: 0.23451001942157745 (0.23451001942157745) + | > loss_disc_real_3: 0.22969889640808105 (0.22969889640808105) + | > loss_disc_real_4: 0.20594705641269684 (0.20594705641269684) + | > loss_disc_real_5: 0.20078307390213013 (0.20078307390213013) + | > loss_0: 2.433995246887207 (2.433995246887207) + | > loss_gen: 2.401308536529541 (2.401308536529541) + | > loss_kl: 2.5013773441314697 (2.5013773441314697) + | > loss_feat: 7.461343765258789 (7.461343765258789) + | > loss_mel: 18.276023864746094 (18.276023864746094) + | > loss_duration: 1.7186247110366821 (1.7186247110366821) + | > loss_1: 32.35867691040039 (32.35867691040039) + + --> STEP: 1 + | > loss_disc: 2.568777561187744 (2.568777561187744) + | > loss_disc_real_0: 0.15028730034828186 (0.15028730034828186) + | > loss_disc_real_1: 0.2605787217617035 (0.2605787217617035) + | > loss_disc_real_2: 0.24765634536743164 (0.24765634536743164) + | > loss_disc_real_3: 0.26303353905677795 (0.26303353905677795) + | > loss_disc_real_4: 0.21891087293624878 (0.21891087293624878) + | > loss_disc_real_5: 0.23676474392414093 (0.23676474392414093) + | > loss_0: 2.568777561187744 (2.568777561187744) + | > loss_gen: 2.2920620441436768 (2.2920620441436768) + | > loss_kl: 2.7959818840026855 (2.7959818840026855) + | > loss_feat: 6.410820484161377 (6.410820484161377) + | > loss_mel: 16.665517807006836 (16.665517807006836) + | > loss_duration: 1.7726545333862305 (1.7726545333862305) + | > loss_1: 29.93703842163086 (29.93703842163086) + + --> STEP: 2 + | > loss_disc: 2.61348819732666 (2.591132879257202) + | > loss_disc_real_0: 0.19780929386615753 (0.1740482971072197) + | > loss_disc_real_1: 0.2364235371351242 (0.24850112944841385) + | > loss_disc_real_2: 0.2658969461917877 (0.2567766457796097) + | > loss_disc_real_3: 0.2680601179599762 (0.2655468285083771) + | > loss_disc_real_4: 0.23024089634418488 (0.22457588464021683) + | > loss_disc_real_5: 0.24052996933460236 (0.23864735662937164) + | > loss_0: 2.61348819732666 (2.591132879257202) + | > loss_gen: 2.3538904190063477 (2.322976231575012) + | > loss_kl: 2.422057628631592 (2.6090197563171387) + | > loss_feat: 6.1012282371521 (6.256024360656738) + | > loss_mel: 16.04193115234375 (16.353724479675293) + | > loss_duration: 1.6984167098999023 (1.7355356216430664) + | > loss_1: 28.617523193359375 (29.277280807495117) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00332486629486084 (-0.0008064508438110352) + | > avg_loss_disc: 2.591132879257202 (+0.2322002649307251) + | > avg_loss_disc_real_0: 0.1740482971072197 (+0.06532981246709824) + | > avg_loss_disc_real_1: 0.24850112944841385 (+0.04901900142431259) + | > avg_loss_disc_real_2: 0.2567766457796097 (+0.05656847357749939) + | > avg_loss_disc_real_3: 0.2655468285083771 (+0.030355624854564667) + | > avg_loss_disc_real_4: 0.22457588464021683 (-0.037922121584415436) + | > avg_loss_disc_real_5: 0.23864735662937164 (+0.03663267195224762) + | > avg_loss_0: 2.591132879257202 (+0.2322002649307251) + | > avg_loss_gen: 2.322976231575012 (-0.0887289047241211) + | > avg_loss_kl: 2.6090197563171387 (+0.018253445625305176) + | > avg_loss_feat: 6.256024360656738 (-1.3711328506469727) + | > avg_loss_mel: 16.353724479675293 (-1.8119077682495117) + | > avg_loss_duration: 1.7355356216430664 (-0.009487330913543701) + | > avg_loss_1: 29.277280807495117 (-3.263002395629883) + + + > EPOCH: 348/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:42:14)  + + --> TIME: 2025-04-19 04:42:15 -- STEP: 2/142 -- GLOBAL_STEP: 428700 + | > loss_disc: 2.3677103519439697 (2.3821024894714355) + | > loss_disc_real_0: 0.12748567759990692 (0.14021816104650497) + | > loss_disc_real_1: 0.20294389128684998 (0.21442117542028427) + | > loss_disc_real_2: 0.2111545205116272 (0.21556204557418823) + | > loss_disc_real_3: 0.21594513952732086 (0.2239435538649559) + | > loss_disc_real_4: 0.19542956352233887 (0.2130667045712471) + | > loss_disc_real_5: 0.1988062858581543 (0.21244629472494125) + | > loss_0: 2.3677103519439697 (2.3821024894714355) + | > grad_norm_0: tensor(14.2054, device='cuda:0') (tensor(11.0087, device='cuda:0')) + | > loss_gen: 2.3495609760284424 (2.388106942176819) + | > loss_kl: 1.9960235357284546 (1.9187089800834656) + | > loss_feat: 8.157862663269043 (7.861233711242676) + | > loss_mel: 18.98624038696289 (18.50428009033203) + | > loss_duration: 1.4439268112182617 (1.4201622605323792) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.93361282348633 (32.092491149902344) + | > grad_norm_1: tensor(491.5826, device='cuda:0') (tensor(373.9324, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.3216 (0.32467126846313477) + | > loader_time: 0.004 (0.004431962966918945) + + + --> TIME: 2025-04-19 04:42:24 -- STEP: 27/142 -- GLOBAL_STEP: 428725 + | > loss_disc: 2.475044012069702 (2.4188795707843926) + | > loss_disc_real_0: 0.19926325976848602 (0.12360883310989097) + | > loss_disc_real_1: 0.21827447414398193 (0.20045389510967113) + | > loss_disc_real_2: 0.1976589858531952 (0.22290321632667823) + | > loss_disc_real_3: 0.22488540410995483 (0.22601696848869324) + | > loss_disc_real_4: 0.23963887989521027 (0.22846428663642318) + | > loss_disc_real_5: 0.2612725496292114 (0.22678259953304572) + | > loss_0: 2.475044012069702 (2.4188795707843926) + | > grad_norm_0: tensor(8.5592, device='cuda:0') (tensor(6.7760, device='cuda:0')) + | > loss_gen: 2.3479652404785156 (2.385802304303204) + | > loss_kl: 1.8226163387298584 (1.8257162703408136) + | > loss_feat: 6.909445762634277 (7.394034014807807) + | > loss_mel: 16.550556182861328 (17.762694818002203) + | > loss_duration: 1.426658272743225 (1.4207028724529125) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.05724334716797 (30.788950037073207) + | > grad_norm_1: tensor(50.0732, device='cuda:0') (tensor(130.0099, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.3592 (0.3362871187704581) + | > loader_time: 0.0057 (0.0051634929798267505) + + + --> TIME: 2025-04-19 04:42:33 -- STEP: 52/142 -- GLOBAL_STEP: 428750 + | > loss_disc: 2.4816012382507324 (2.4335529437431918) + | > loss_disc_real_0: 0.1289375275373459 (0.12393989958442174) + | > loss_disc_real_1: 0.19901424646377563 (0.2036508175616081) + | > loss_disc_real_2: 0.24594907462596893 (0.22336570889903948) + | > loss_disc_real_3: 0.23557467758655548 (0.22736231266305998) + | > loss_disc_real_4: 0.23033888638019562 (0.2255797119667897) + | > loss_disc_real_5: 0.2450089305639267 (0.2308588598210078) + | > loss_0: 2.4816012382507324 (2.4335529437431918) + | > grad_norm_0: tensor(9.4291, device='cuda:0') (tensor(6.7182, device='cuda:0')) + | > loss_gen: 2.379547119140625 (2.3648753441297083) + | > loss_kl: 1.9048446416854858 (1.833211559515733) + | > loss_feat: 6.530434608459473 (7.222755312919617) + | > loss_mel: 17.683048248291016 (17.650045908414395) + | > loss_duration: 1.4934266805648804 (1.430718903358166) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.991302490234375 (30.50160690454336) + | > grad_norm_1: tensor(111.6167, device='cuda:0') (tensor(113.4033, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.3648 (0.3547922739615807) + | > loader_time: 0.0061 (0.005521091131063608) + + + --> TIME: 2025-04-19 04:42:43 -- STEP: 77/142 -- GLOBAL_STEP: 428775 + | > loss_disc: 2.506348133087158 (2.4467168628395375) + | > loss_disc_real_0: 0.137545645236969 (0.12746402330986867) + | > loss_disc_real_1: 0.20655348896980286 (0.20573554410562886) + | > loss_disc_real_2: 0.24820929765701294 (0.22524631371745815) + | > loss_disc_real_3: 0.256210058927536 (0.22844786593666325) + | > loss_disc_real_4: 0.20603018999099731 (0.2244888913708848) + | > loss_disc_real_5: 0.23336640000343323 (0.2322130605771944) + | > loss_0: 2.506348133087158 (2.4467168628395375) + | > grad_norm_0: tensor(6.1751, device='cuda:0') (tensor(6.5520, device='cuda:0')) + | > loss_gen: 2.375469446182251 (2.355887419217592) + | > loss_kl: 1.727789044380188 (1.8439072432456078) + | > loss_feat: 6.678475379943848 (7.163865894466252) + | > loss_mel: 16.938522338867188 (17.615295781717666) + | > loss_duration: 1.4525189399719238 (1.442948174166989) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.172775268554688 (30.4219043781231) + | > grad_norm_1: tensor(131.2982, device='cuda:0') (tensor(111.5536, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.3899 (0.365735419384845) + | > loader_time: 0.0078 (0.005808006633411755) + + + --> TIME: 2025-04-19 04:42:54 -- STEP: 102/142 -- GLOBAL_STEP: 428800 + | > loss_disc: 2.4479165077209473 (2.458969639796836) + | > loss_disc_real_0: 0.17662286758422852 (0.1290689107834124) + | > loss_disc_real_1: 0.20984934270381927 (0.20492437715623893) + | > loss_disc_real_2: 0.2486429661512375 (0.22613594622588626) + | > loss_disc_real_3: 0.264629602432251 (0.22945362124957291) + | > loss_disc_real_4: 0.2021208554506302 (0.22648065025899924) + | > loss_disc_real_5: 0.27459821105003357 (0.23502164261013853) + | > loss_0: 2.4479165077209473 (2.458969639796836) + | > grad_norm_0: tensor(9.3815, device='cuda:0') (tensor(6.6290, device='cuda:0')) + | > loss_gen: 2.4490737915039062 (2.3464010346169553) + | > loss_kl: 1.8076354265213013 (1.8493928336629681) + | > loss_feat: 7.0894975662231445 (7.073219168419931) + | > loss_mel: 18.833322525024414 (17.57419751672183) + | > loss_duration: 1.4624199867248535 (1.4537880140192365) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.641948699951172 (30.296998454075233) + | > grad_norm_1: tensor(47.4628, device='cuda:0') (tensor(110.5512, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.4195 (0.3747728946162206) + | > loader_time: 0.0071 (0.006049476417840696) + + + --> TIME: 2025-04-19 04:43:05 -- STEP: 127/142 -- GLOBAL_STEP: 428825 + | > loss_disc: 2.4949684143066406 (2.458574616064237) + | > loss_disc_real_0: 0.15252509713172913 (0.12925349850588894) + | > loss_disc_real_1: 0.1836329847574234 (0.2052003359935415) + | > loss_disc_real_2: 0.2693834602832794 (0.2259669249921333) + | > loss_disc_real_3: 0.26910051703453064 (0.22892615809215336) + | > loss_disc_real_4: 0.29168522357940674 (0.22738737337232576) + | > loss_disc_real_5: 0.23276367783546448 (0.23375964622328602) + | > loss_0: 2.4949684143066406 (2.458574616064237) + | > grad_norm_0: tensor(4.9374, device='cuda:0') (tensor(7.4982, device='cuda:0')) + | > loss_gen: 2.5342133045196533 (2.3472378854676474) + | > loss_kl: 1.9088246822357178 (1.8631729995171855) + | > loss_feat: 7.843425273895264 (7.066363544914666) + | > loss_mel: 17.908058166503906 (17.5297998743733) + | > loss_duration: 1.4809207916259766 (1.4601484452645608) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.675443649291992 (30.266722716684416) + | > grad_norm_1: tensor(452.5922, device='cuda:0') (tensor(141.1516, device='cuda:0')) + | > current_lr_0: 0.00019148599017155957 + | > current_lr_1: 0.00019148599017155957 + | > step_time: 0.4385 (0.385330187054131) + | > loader_time: 0.0076 (0.006296231052068274) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.453554391860962 (2.453554391860962) + | > loss_disc_real_0: 0.22985215485095978 (0.22985215485095978) + | > loss_disc_real_1: 0.21929286420345306 (0.21929286420345306) + | > loss_disc_real_2: 0.24215367436408997 (0.24215367436408997) + | > loss_disc_real_3: 0.22513507306575775 (0.22513507306575775) + | > loss_disc_real_4: 0.24227939546108246 (0.24227939546108246) + | > loss_disc_real_5: 0.25354498624801636 (0.25354498624801636) + | > loss_0: 2.453554391860962 (2.453554391860962) + | > loss_gen: 2.748767614364624 (2.748767614364624) + | > loss_kl: 2.5352141857147217 (2.5352141857147217) + | > loss_feat: 7.6248955726623535 (7.6248955726623535) + | > loss_mel: 19.100107192993164 (19.100107192993164) + | > loss_duration: 1.731410026550293 (1.731410026550293) + | > loss_1: 33.740394592285156 (33.740394592285156) + + --> STEP: 1 + | > loss_disc: 2.505910873413086 (2.505910873413086) + | > loss_disc_real_0: 0.16910623013973236 (0.16910623013973236) + | > loss_disc_real_1: 0.22398748993873596 (0.22398748993873596) + | > loss_disc_real_2: 0.26232394576072693 (0.26232394576072693) + | > loss_disc_real_3: 0.24840988218784332 (0.24840988218784332) + | > loss_disc_real_4: 0.2536323368549347 (0.2536323368549347) + | > loss_disc_real_5: 0.25234729051589966 (0.25234729051589966) + | > loss_0: 2.505910873413086 (2.505910873413086) + | > loss_gen: 2.4281415939331055 (2.4281415939331055) + | > loss_kl: 2.899785280227661 (2.899785280227661) + | > loss_feat: 6.374705791473389 (6.374705791473389) + | > loss_mel: 18.62929344177246 (18.62929344177246) + | > loss_duration: 1.7809711694717407 (1.7809711694717407) + | > loss_1: 32.11289978027344 (32.11289978027344) + + --> STEP: 2 + | > loss_disc: 2.557192802429199 (2.5315518379211426) + | > loss_disc_real_0: 0.1921207308769226 (0.18061348050832748) + | > loss_disc_real_1: 0.20942139625549316 (0.21670444309711456) + | > loss_disc_real_2: 0.2744850814342499 (0.2684045135974884) + | > loss_disc_real_3: 0.2244720160961151 (0.23644094914197922) + | > loss_disc_real_4: 0.24049225449562073 (0.2470622956752777) + | > loss_disc_real_5: 0.18107210099697113 (0.2167096957564354) + | > loss_0: 2.557192802429199 (2.5315518379211426) + | > loss_gen: 2.2943131923675537 (2.3612273931503296) + | > loss_kl: 2.4037528038024902 (2.6517690420150757) + | > loss_feat: 6.463207244873047 (6.418956518173218) + | > loss_mel: 18.062803268432617 (18.34604835510254) + | > loss_duration: 1.7083176374435425 (1.7446444034576416) + | > loss_1: 30.93239402770996 (31.5226469039917) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034977197647094727 (+0.0001728534698486328) + | > avg_loss_disc: 2.5315518379211426 (-0.05958104133605957) + | > avg_loss_disc_real_0: 0.18061348050832748 (+0.006565183401107788) + | > avg_loss_disc_real_1: 0.21670444309711456 (-0.031796686351299286) + | > avg_loss_disc_real_2: 0.2684045135974884 (+0.011627867817878723) + | > avg_loss_disc_real_3: 0.23644094914197922 (-0.029105879366397858) + | > avg_loss_disc_real_4: 0.2470622956752777 (+0.022486411035060883) + | > avg_loss_disc_real_5: 0.2167096957564354 (-0.02193766087293625) + | > avg_loss_0: 2.5315518379211426 (-0.05958104133605957) + | > avg_loss_gen: 2.3612273931503296 (+0.03825116157531738) + | > avg_loss_kl: 2.6517690420150757 (+0.04274928569793701) + | > avg_loss_feat: 6.418956518173218 (+0.1629321575164795) + | > avg_loss_mel: 18.34604835510254 (+1.992323875427246) + | > avg_loss_duration: 1.7446444034576416 (+0.009108781814575195) + | > avg_loss_1: 31.5226469039917 (+2.245366096496582) + + + > EPOCH: 349/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:43:13)  + + --> TIME: 2025-04-19 04:43:17 -- STEP: 10/142 -- GLOBAL_STEP: 428850 + | > loss_disc: 2.481139898300171 (2.432654309272766) + | > loss_disc_real_0: 0.1198946163058281 (0.11426126062870026) + | > loss_disc_real_1: 0.21262605488300323 (0.19763782918453215) + | > loss_disc_real_2: 0.22905349731445312 (0.22665551751852037) + | > loss_disc_real_3: 0.23076820373535156 (0.22713084071874617) + | > loss_disc_real_4: 0.25982391834259033 (0.23278308808803558) + | > loss_disc_real_5: 0.2607092261314392 (0.23282775431871414) + | > loss_0: 2.481139898300171 (2.432654309272766) + | > grad_norm_0: tensor(9.5713, device='cuda:0') (tensor(13.9685, device='cuda:0')) + | > loss_gen: 2.520308494567871 (2.4056378602981567) + | > loss_kl: 1.8133033514022827 (1.7852020859718323) + | > loss_feat: 7.558055877685547 (7.316146373748779) + | > loss_mel: 18.119718551635742 (17.887512588500975) + | > loss_duration: 1.382141351699829 (1.412536895275116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.39352798461914 (30.807036209106446) + | > grad_norm_1: tensor(370.6664, device='cuda:0') (tensor(265.5797, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.3365 (0.3260218620300293) + | > loader_time: 0.0053 (0.00473935604095459) + + + --> TIME: 2025-04-19 04:43:26 -- STEP: 35/142 -- GLOBAL_STEP: 428875 + | > loss_disc: 2.463995933532715 (2.440855932235718) + | > loss_disc_real_0: 0.13896000385284424 (0.1208592275423663) + | > loss_disc_real_1: 0.20447798073291779 (0.20293649988515036) + | > loss_disc_real_2: 0.19545422494411469 (0.22406911892550332) + | > loss_disc_real_3: 0.20666074752807617 (0.22852013409137725) + | > loss_disc_real_4: 0.2202346920967102 (0.23055459346090043) + | > loss_disc_real_5: 0.21972888708114624 (0.2295038150889533) + | > loss_0: 2.463995933532715 (2.440855932235718) + | > grad_norm_0: tensor(10.8198, device='cuda:0') (tensor(12.9431, device='cuda:0')) + | > loss_gen: 2.2616751194000244 (2.3985940252031597) + | > loss_kl: 1.946245551109314 (1.8348232678004675) + | > loss_feat: 7.111736297607422 (7.305656160627093) + | > loss_mel: 17.476539611816406 (17.76622690473284) + | > loss_duration: 1.4492934942245483 (1.4258270127432686) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.2454891204834 (30.731127166748045) + | > grad_norm_1: tensor(283.1591, device='cuda:0') (tensor(294.6549, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.3561 (0.3386711733681815) + | > loader_time: 0.0059 (0.0052690778459821425) + + + --> TIME: 2025-04-19 04:43:35 -- STEP: 60/142 -- GLOBAL_STEP: 428900 + | > loss_disc: 2.446540117263794 (2.446882732709249) + | > loss_disc_real_0: 0.16042211651802063 (0.12068224195390939) + | > loss_disc_real_1: 0.19099310040473938 (0.2041625032822291) + | > loss_disc_real_2: 0.19307081401348114 (0.2256486840546131) + | > loss_disc_real_3: 0.22307050228118896 (0.23068590213855109) + | > loss_disc_real_4: 0.2261190116405487 (0.23045965333779653) + | > loss_disc_real_5: 0.2183910757303238 (0.23171503245830535) + | > loss_0: 2.446540117263794 (2.446882732709249) + | > grad_norm_0: tensor(9.3169, device='cuda:0') (tensor(11.9186, device='cuda:0')) + | > loss_gen: 2.114647626876831 (2.3780022780100505) + | > loss_kl: 1.8084906339645386 (1.8519365727901458) + | > loss_feat: 6.117124080657959 (7.253885293006897) + | > loss_mel: 15.070815086364746 (17.64313311576844) + | > loss_duration: 1.4782843589782715 (1.4364130953947702) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 26.5893611907959 (30.56337029139201) + | > grad_norm_1: tensor(178.5948, device='cuda:0') (tensor(268.4421, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.3751 (0.3499665141105652) + | > loader_time: 0.0062 (0.005583846569061279) + + + --> TIME: 2025-04-19 04:43:45 -- STEP: 85/142 -- GLOBAL_STEP: 428925 + | > loss_disc: 2.568723201751709 (2.452354254442103) + | > loss_disc_real_0: 0.10530231148004532 (0.12273794925388168) + | > loss_disc_real_1: 0.27203693985939026 (0.20529836924637065) + | > loss_disc_real_2: 0.2684537172317505 (0.22623978292240815) + | > loss_disc_real_3: 0.259722501039505 (0.22924998367533964) + | > loss_disc_real_4: 0.26951706409454346 (0.22867013128364788) + | > loss_disc_real_5: 0.22352203726768494 (0.23140057027339936) + | > loss_0: 2.568723201751709 (2.452354254442103) + | > grad_norm_0: tensor(18.2303, device='cuda:0') (tensor(11.5674, device='cuda:0')) + | > loss_gen: 2.28989577293396 (2.369822569454417) + | > loss_kl: 1.9818344116210938 (1.8661057289908913) + | > loss_feat: 7.090214729309082 (7.200124970604391) + | > loss_mel: 17.761581420898438 (17.58636516122258) + | > loss_duration: 1.4792792797088623 (1.447662502176621) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.60280418395996 (30.4700808132396) + | > grad_norm_1: tensor(438.4015, device='cuda:0') (tensor(267.7092, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.3972 (0.3618420713088092) + | > loader_time: 0.0069 (0.0058425903320312505) + + + --> TIME: 2025-04-19 04:43:55 -- STEP: 110/142 -- GLOBAL_STEP: 428950 + | > loss_disc: 2.349436044692993 (2.457278078252619) + | > loss_disc_real_0: 0.08463376015424728 (0.12296765646473928) + | > loss_disc_real_1: 0.19559165835380554 (0.20485425117340955) + | > loss_disc_real_2: 0.18493711948394775 (0.2254245707934553) + | > loss_disc_real_3: 0.21177810430526733 (0.22964362515644593) + | > loss_disc_real_4: 0.17929470539093018 (0.22874258905649186) + | > loss_disc_real_5: 0.2047821581363678 (0.23311385810375213) + | > loss_0: 2.349436044692993 (2.457278078252619) + | > grad_norm_0: tensor(7.8032, device='cuda:0') (tensor(11.9536, device='cuda:0')) + | > loss_gen: 2.3676161766052246 (2.3621276443654824) + | > loss_kl: 1.9456740617752075 (1.8779268654910002) + | > loss_feat: 7.818411827087402 (7.168445014953614) + | > loss_mel: 19.047927856445312 (17.650465176322243) + | > loss_duration: 1.4816884994506836 (1.4564387527379117) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.661319732666016 (30.515403452786533) + | > grad_norm_1: tensor(317.0757, device='cuda:0') (tensor(279.1929, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.429 (0.3722800363193859) + | > loader_time: 0.0075 (0.006078830632296475) + + + --> TIME: 2025-04-19 04:44:07 -- STEP: 135/142 -- GLOBAL_STEP: 428975 + | > loss_disc: 2.388622283935547 (2.459621212217543) + | > loss_disc_real_0: 0.11189942806959152 (0.12435413127144178) + | > loss_disc_real_1: 0.17791512608528137 (0.2051450733785276) + | > loss_disc_real_2: 0.20532867312431335 (0.2250938457471353) + | > loss_disc_real_3: 0.21355177462100983 (0.2284681342266224) + | > loss_disc_real_4: 0.22955204546451569 (0.22935076720184747) + | > loss_disc_real_5: 0.22253820300102234 (0.23366841916684752) + | > loss_0: 2.388622283935547 (2.459621212217543) + | > grad_norm_0: tensor(11.9335, device='cuda:0') (tensor(12.2553, device='cuda:0')) + | > loss_gen: 2.265733003616333 (2.359781736797755) + | > loss_kl: 1.9299373626708984 (1.890089354691682) + | > loss_feat: 7.574480056762695 (7.1243429749100295) + | > loss_mel: 18.448333740234375 (17.636612786187065) + | > loss_duration: 1.4617856740951538 (1.4626339921244866) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.68027114868164 (30.473460812038844) + | > grad_norm_1: tensor(266.5986, device='cuda:0') (tensor(292.4980, device='cuda:0')) + | > current_lr_0: 0.00019146205442278813 + | > current_lr_1: 0.00019146205442278813 + | > step_time: 0.4546 (0.38430391064396613) + | > loader_time: 0.0079 (0.006340256443730107) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.415682554244995 (2.415682554244995) + | > loss_disc_real_0: 0.09958764165639877 (0.09958764165639877) + | > loss_disc_real_1: 0.25601726770401 (0.25601726770401) + | > loss_disc_real_2: 0.22065480053424835 (0.22065480053424835) + | > loss_disc_real_3: 0.23382146656513214 (0.23382146656513214) + | > loss_disc_real_4: 0.2756713330745697 (0.2756713330745697) + | > loss_disc_real_5: 0.2590198218822479 (0.2590198218822479) + | > loss_0: 2.415682554244995 (2.415682554244995) + | > loss_gen: 2.5097451210021973 (2.5097451210021973) + | > loss_kl: 2.5265090465545654 (2.5265090465545654) + | > loss_feat: 7.3946099281311035 (7.3946099281311035) + | > loss_mel: 18.155059814453125 (18.155059814453125) + | > loss_duration: 1.7440837621688843 (1.7440837621688843) + | > loss_1: 32.33000946044922 (32.33000946044922) + + --> STEP: 1 + | > loss_disc: 2.5345890522003174 (2.5345890522003174) + | > loss_disc_real_0: 0.16087427735328674 (0.16087427735328674) + | > loss_disc_real_1: 0.23626767098903656 (0.23626767098903656) + | > loss_disc_real_2: 0.22016219794750214 (0.22016219794750214) + | > loss_disc_real_3: 0.23915636539459229 (0.23915636539459229) + | > loss_disc_real_4: 0.3099033832550049 (0.3099033832550049) + | > loss_disc_real_5: 0.25797855854034424 (0.25797855854034424) + | > loss_0: 2.5345890522003174 (2.5345890522003174) + | > loss_gen: 2.308213949203491 (2.308213949203491) + | > loss_kl: 2.8182036876678467 (2.8182036876678467) + | > loss_feat: 5.7377495765686035 (5.7377495765686035) + | > loss_mel: 15.856319427490234 (15.856319427490234) + | > loss_duration: 1.766706943511963 (1.766706943511963) + | > loss_1: 28.487194061279297 (28.487194061279297) + + --> STEP: 2 + | > loss_disc: 2.4510152339935303 (2.492802143096924) + | > loss_disc_real_0: 0.1143408715724945 (0.13760757446289062) + | > loss_disc_real_1: 0.261201947927475 (0.24873480945825577) + | > loss_disc_real_2: 0.2606930434703827 (0.2404276207089424) + | > loss_disc_real_3: 0.22034257650375366 (0.22974947094917297) + | > loss_disc_real_4: 0.2892501950263977 (0.2995767891407013) + | > loss_disc_real_5: 0.22524753212928772 (0.24161304533481598) + | > loss_0: 2.4510152339935303 (2.492802143096924) + | > loss_gen: 2.434946060180664 (2.3715800046920776) + | > loss_kl: 2.49471116065979 (2.6564574241638184) + | > loss_feat: 7.025385856628418 (6.381567716598511) + | > loss_mel: 17.13535499572754 (16.495837211608887) + | > loss_duration: 1.7393405437469482 (1.7530237436294556) + | > loss_1: 30.82973861694336 (29.658466339111328) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036531686782836914 (+0.00015544891357421875) + | > avg_loss_disc: 2.492802143096924 (-0.03874969482421875) + | > avg_loss_disc_real_0: 0.13760757446289062 (-0.04300590604543686) + | > avg_loss_disc_real_1: 0.24873480945825577 (+0.032030366361141205) + | > avg_loss_disc_real_2: 0.2404276207089424 (-0.02797689288854599) + | > avg_loss_disc_real_3: 0.22974947094917297 (-0.006691478192806244) + | > avg_loss_disc_real_4: 0.2995767891407013 (+0.052514493465423584) + | > avg_loss_disc_real_5: 0.24161304533481598 (+0.024903349578380585) + | > avg_loss_0: 2.492802143096924 (-0.03874969482421875) + | > avg_loss_gen: 2.3715800046920776 (+0.010352611541748047) + | > avg_loss_kl: 2.6564574241638184 (+0.004688382148742676) + | > avg_loss_feat: 6.381567716598511 (-0.03738880157470703) + | > avg_loss_mel: 16.495837211608887 (-1.8502111434936523) + | > avg_loss_duration: 1.7530237436294556 (+0.008379340171813965) + | > avg_loss_1: 29.658466339111328 (-1.864180564880371) + + + > EPOCH: 350/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:44:11)  + + --> TIME: 2025-04-19 04:44:18 -- STEP: 18/142 -- GLOBAL_STEP: 429000 + | > loss_disc: 2.464776039123535 (2.4212027390797934) + | > loss_disc_real_0: 0.14042283594608307 (0.11786770924097961) + | > loss_disc_real_1: 0.2149769365787506 (0.20741992692152658) + | > loss_disc_real_2: 0.21414406597614288 (0.22899164425002205) + | > loss_disc_real_3: 0.2591123580932617 (0.2276945056186782) + | > loss_disc_real_4: 0.2414667308330536 (0.22708428319957522) + | > loss_disc_real_5: 0.26858803629875183 (0.23066471268733343) + | > loss_0: 2.464776039123535 (2.4212027390797934) + | > grad_norm_0: tensor(12.0111, device='cuda:0') (tensor(15.0415, device='cuda:0')) + | > loss_gen: 2.471320629119873 (2.429579814275106) + | > loss_kl: 1.6720999479293823 (1.7902398573027716) + | > loss_feat: 6.911015510559082 (7.435817824469672) + | > loss_mel: 17.842287063598633 (17.947623252868652) + | > loss_duration: 1.4255818128585815 (1.4150308635499742) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.32230567932129 (31.01829179128011) + | > grad_norm_1: tensor(345.0372, device='cuda:0') (tensor(309.5808, device='cuda:0')) + | > current_lr_0: 0.00019143812166598528 + | > current_lr_1: 0.00019143812166598528 + | > step_time: 0.3381 (0.33162065347035724) + | > loader_time: 0.0048 (0.00487465328640408) + + + --> TIME: 2025-04-19 04:44:27 -- STEP: 43/142 -- GLOBAL_STEP: 429025 + | > loss_disc: 2.469193696975708 (2.409199858820715) + | > loss_disc_real_0: 0.1172170415520668 (0.11401850392305574) + | > loss_disc_real_1: 0.2286527454853058 (0.20321531316568686) + | > loss_disc_real_2: 0.22425822913646698 (0.22618432689544765) + | > loss_disc_real_3: 0.24984683096408844 (0.22621416872323946) + | > loss_disc_real_4: 0.2422306090593338 (0.22627698889998502) + | > loss_disc_real_5: 0.266720712184906 (0.23200163522432016) + | > loss_0: 2.469193696975708 (2.409199858820715) + | > grad_norm_0: tensor(9.6130, device='cuda:0') (tensor(12.5493, device='cuda:0')) + | > loss_gen: 2.3498375415802 (2.407921874245932) + | > loss_kl: 1.7459839582443237 (1.8019087896790615) + | > loss_feat: 7.16880989074707 (7.37047971681107) + | > loss_mel: 18.170751571655273 (17.891808665075967) + | > loss_duration: 1.4459655284881592 (1.4277123074198879) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.88134765625 (30.899831328281138) + | > grad_norm_1: tensor(448.9889, device='cuda:0') (tensor(317.3664, device='cuda:0')) + | > current_lr_0: 0.00019143812166598528 + | > current_lr_1: 0.00019143812166598528 + | > step_time: 0.3619 (0.349795990212019) + | > loader_time: 0.0058 (0.005304436350977698) + + + --> TIME: 2025-04-19 04:44:37 -- STEP: 68/142 -- GLOBAL_STEP: 429050 + | > loss_disc: 2.5084292888641357 (2.420957113013548) + | > loss_disc_real_0: 0.12621600925922394 (0.1159883104812573) + | > loss_disc_real_1: 0.2473522275686264 (0.204532779095804) + | > loss_disc_real_2: 0.2196146696805954 (0.22498516343972264) + | > loss_disc_real_3: 0.22294585406780243 (0.22583339867346428) + | > loss_disc_real_4: 0.21340683102607727 (0.22516691991511514) + | > loss_disc_real_5: 0.22777505218982697 (0.23016088280607672) + | > loss_0: 2.5084292888641357 (2.420957113013548) + | > grad_norm_0: tensor(9.8319, device='cuda:0') (tensor(11.8046, device='cuda:0')) + | > loss_gen: 2.3352861404418945 (2.3840128358672636) + | > loss_kl: 1.8551037311553955 (1.8254125363686506) + | > loss_feat: 7.548192501068115 (7.285878777503967) + | > loss_mel: 17.7115535736084 (17.80177607255823) + | > loss_duration: 1.447211503982544 (1.4379504200290234) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.89734649658203 (30.73503065109253) + | > grad_norm_1: tensor(55.0824, device='cuda:0') (tensor(298.3436, device='cuda:0')) + | > current_lr_0: 0.00019143812166598528 + | > current_lr_1: 0.00019143812166598528 + | > step_time: 0.389 (0.35905548404244814) + | > loader_time: 0.0065 (0.0056241715655607335) + + + --> TIME: 2025-04-19 04:44:47 -- STEP: 93/142 -- GLOBAL_STEP: 429075 + | > loss_disc: 2.5698606967926025 (2.4320103199251237) + | > loss_disc_real_0: 0.17029735445976257 (0.11980449540480491) + | > loss_disc_real_1: 0.2799202799797058 (0.20518488396880452) + | > loss_disc_real_2: 0.2542591989040375 (0.22534401666733525) + | > loss_disc_real_3: 0.26240992546081543 (0.22693794388924876) + | > loss_disc_real_4: 0.26985427737236023 (0.22617887937894432) + | > loss_disc_real_5: 0.24900712072849274 (0.23223115551856258) + | > loss_0: 2.5698606967926025 (2.4320103199251237) + | > grad_norm_0: tensor(11.8708, device='cuda:0') (tensor(11.7508, device='cuda:0')) + | > loss_gen: 2.2858848571777344 (2.3839067259142475) + | > loss_kl: 1.9325640201568604 (1.8363632591821815) + | > loss_feat: 6.210693836212158 (7.255554163327781) + | > loss_mel: 15.159476280212402 (17.817332688198288) + | > loss_duration: 1.4780447483062744 (1.4489728263629382) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.06666374206543 (30.742129746303764) + | > grad_norm_1: tensor(295.6431, device='cuda:0') (tensor(284.2027, device='cuda:0')) + | > current_lr_0: 0.00019143812166598528 + | > current_lr_1: 0.00019143812166598528 + | > step_time: 0.4005 (0.3700262295302524) + | > loader_time: 0.0068 (0.0059715932415377706) + + + --> TIME: 2025-04-19 04:44:58 -- STEP: 118/142 -- GLOBAL_STEP: 429100 + | > loss_disc: 2.328068733215332 (2.4351250357547047) + | > loss_disc_real_0: 0.09931707382202148 (0.12080903386034197) + | > loss_disc_real_1: 0.19804108142852783 (0.20513423658528573) + | > loss_disc_real_2: 0.24258723855018616 (0.22546838842711206) + | > loss_disc_real_3: 0.2688523232936859 (0.22808314796726584) + | > loss_disc_real_4: 0.2348358929157257 (0.22604085719686443) + | > loss_disc_real_5: 0.24930289387702942 (0.23107977817624303) + | > loss_0: 2.328068733215332 (2.4351250357547047) + | > grad_norm_0: tensor(8.7947, device='cuda:0') (tensor(11.5694, device='cuda:0')) + | > loss_gen: 2.4324653148651123 (2.374315027463235) + | > loss_kl: 1.8096379041671753 (1.8407544594700054) + | > loss_feat: 7.155086040496826 (7.212809340428498) + | > loss_mel: 18.28125762939453 (17.776860398761297) + | > loss_duration: 1.4448435306549072 (1.4555044315629089) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.123289108276367 (30.660243680921653) + | > grad_norm_1: tensor(113.5279, device='cuda:0') (tensor(282.4545, device='cuda:0')) + | > current_lr_0: 0.00019143812166598528 + | > current_lr_1: 0.00019143812166598528 + | > step_time: 0.4408 (0.3801265449847205) + | > loader_time: 0.0074 (0.0062028755575923595) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3546090126037598 (2.3546090126037598) + | > loss_disc_real_0: 0.08808410912752151 (0.08808410912752151) + | > loss_disc_real_1: 0.188707172870636 (0.188707172870636) + | > loss_disc_real_2: 0.23675113916397095 (0.23675113916397095) + | > loss_disc_real_3: 0.23334239423274994 (0.23334239423274994) + | > loss_disc_real_4: 0.2513708770275116 (0.2513708770275116) + | > loss_disc_real_5: 0.1896740198135376 (0.1896740198135376) + | > loss_0: 2.3546090126037598 (2.3546090126037598) + | > loss_gen: 2.3540804386138916 (2.3540804386138916) + | > loss_kl: 2.459595203399658 (2.459595203399658) + | > loss_feat: 8.117520332336426 (8.117520332336426) + | > loss_mel: 19.813257217407227 (19.813257217407227) + | > loss_duration: 1.7411667108535767 (1.7411667108535767) + | > loss_1: 34.485618591308594 (34.485618591308594) + + --> STEP: 1 + | > loss_disc: 2.468057155609131 (2.468057155609131) + | > loss_disc_real_0: 0.1141718178987503 (0.1141718178987503) + | > loss_disc_real_1: 0.23242950439453125 (0.23242950439453125) + | > loss_disc_real_2: 0.23228973150253296 (0.23228973150253296) + | > loss_disc_real_3: 0.24639520049095154 (0.24639520049095154) + | > loss_disc_real_4: 0.22907914221286774 (0.22907914221286774) + | > loss_disc_real_5: 0.22802215814590454 (0.22802215814590454) + | > loss_0: 2.468057155609131 (2.468057155609131) + | > loss_gen: 2.2176027297973633 (2.2176027297973633) + | > loss_kl: 2.8145010471343994 (2.8145010471343994) + | > loss_feat: 6.930885314941406 (6.930885314941406) + | > loss_mel: 18.299814224243164 (18.299814224243164) + | > loss_duration: 1.7745016813278198 (1.7745016813278198) + | > loss_1: 32.03730392456055 (32.03730392456055) + + --> STEP: 2 + | > loss_disc: 2.4309022426605225 (2.4494796991348267) + | > loss_disc_real_0: 0.1165662407875061 (0.1153690293431282) + | > loss_disc_real_1: 0.25149664282798767 (0.24196307361125946) + | > loss_disc_real_2: 0.26279860734939575 (0.24754416942596436) + | > loss_disc_real_3: 0.2485705465078354 (0.24748287349939346) + | > loss_disc_real_4: 0.25802990794181824 (0.243554525077343) + | > loss_disc_real_5: 0.24991846084594727 (0.2389703094959259) + | > loss_0: 2.4309022426605225 (2.4494796991348267) + | > loss_gen: 2.503369092941284 (2.3604859113693237) + | > loss_kl: 2.3822689056396484 (2.598384976387024) + | > loss_feat: 8.473348617553711 (7.702116966247559) + | > loss_mel: 19.147415161132812 (18.72361469268799) + | > loss_duration: 1.7248163223266602 (1.74965900182724) + | > loss_1: 34.23121643066406 (33.134260177612305) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003615140914916992 (-3.802776336669922e-05) + | > avg_loss_disc: 2.4494796991348267 (-0.04332244396209717) + | > avg_loss_disc_real_0: 0.1153690293431282 (-0.02223854511976242) + | > avg_loss_disc_real_1: 0.24196307361125946 (-0.006771735846996307) + | > avg_loss_disc_real_2: 0.24754416942596436 (+0.007116548717021942) + | > avg_loss_disc_real_3: 0.24748287349939346 (+0.01773340255022049) + | > avg_loss_disc_real_4: 0.243554525077343 (-0.05602226406335831) + | > avg_loss_disc_real_5: 0.2389703094959259 (-0.0026427358388900757) + | > avg_loss_0: 2.4494796991348267 (-0.04332244396209717) + | > avg_loss_gen: 2.3604859113693237 (-0.011094093322753906) + | > avg_loss_kl: 2.598384976387024 (-0.058072447776794434) + | > avg_loss_feat: 7.702116966247559 (+1.3205492496490479) + | > avg_loss_mel: 18.72361469268799 (+2.2277774810791016) + | > avg_loss_duration: 1.74965900182724 (-0.003364741802215576) + | > avg_loss_1: 33.134260177612305 (+3.4757938385009766) + + + > EPOCH: 351/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:45:10)  + + --> TIME: 2025-04-19 04:45:11 -- STEP: 1/142 -- GLOBAL_STEP: 429125 + | > loss_disc: 2.3048255443573 (2.3048255443573) + | > loss_disc_real_0: 0.0929853692650795 (0.0929853692650795) + | > loss_disc_real_1: 0.20788036286830902 (0.20788036286830902) + | > loss_disc_real_2: 0.24515686929225922 (0.24515686929225922) + | > loss_disc_real_3: 0.2396218478679657 (0.2396218478679657) + | > loss_disc_real_4: 0.222867950797081 (0.222867950797081) + | > loss_disc_real_5: 0.2385062426328659 (0.2385062426328659) + | > loss_0: 2.3048255443573 (2.3048255443573) + | > grad_norm_0: tensor(11.9598, device='cuda:0') (tensor(11.9598, device='cuda:0')) + | > loss_gen: 2.6101415157318115 (2.6101415157318115) + | > loss_kl: 1.7498443126678467 (1.7498443126678467) + | > loss_feat: 8.429435729980469 (8.429435729980469) + | > loss_mel: 19.008644104003906 (19.008644104003906) + | > loss_duration: 1.3876826763153076 (1.3876826763153076) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.18574905395508 (33.18574905395508) + | > grad_norm_1: tensor(424.2760, device='cuda:0') (tensor(424.2760, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.325 (0.3250102996826172) + | > loader_time: 0.0048 (0.00479888916015625) + + + --> TIME: 2025-04-19 04:45:20 -- STEP: 26/142 -- GLOBAL_STEP: 429150 + | > loss_disc: 2.317049980163574 (2.4139295816421504) + | > loss_disc_real_0: 0.06812810897827148 (0.10782297729299618) + | > loss_disc_real_1: 0.19779984652996063 (0.20477736798616555) + | > loss_disc_real_2: 0.23839876055717468 (0.22554544588694206) + | > loss_disc_real_3: 0.2136906236410141 (0.22817558565965065) + | > loss_disc_real_4: 0.21101237833499908 (0.2269058130108393) + | > loss_disc_real_5: 0.22775892913341522 (0.23042794317007065) + | > loss_0: 2.317049980163574 (2.4139295816421504) + | > grad_norm_0: tensor(5.4942, device='cuda:0') (tensor(10.2623, device='cuda:0')) + | > loss_gen: 2.473933696746826 (2.4024250782453103) + | > loss_kl: 1.7921264171600342 (1.8268957688258245) + | > loss_feat: 7.991402626037598 (7.361570028158335) + | > loss_mel: 19.09587860107422 (17.850945766155544) + | > loss_duration: 1.441426157951355 (1.4215031633010278) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.794769287109375 (30.863340157728928) + | > grad_norm_1: tensor(126.0099, device='cuda:0') (tensor(287.9796, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.3497 (0.3335530757904053) + | > loader_time: 0.0053 (0.00490849751692552) + + + --> TIME: 2025-04-19 04:45:29 -- STEP: 51/142 -- GLOBAL_STEP: 429175 + | > loss_disc: 2.4711523056030273 (2.4304173927681108) + | > loss_disc_real_0: 0.15275242924690247 (0.11406195601996254) + | > loss_disc_real_1: 0.23107437789440155 (0.20562013194841497) + | > loss_disc_real_2: 0.2515726387500763 (0.22359511928231107) + | > loss_disc_real_3: 0.2274136245250702 (0.22734711772086574) + | > loss_disc_real_4: 0.2431686520576477 (0.22654195159089333) + | > loss_disc_real_5: 0.20147110521793365 (0.23154524816017524) + | > loss_0: 2.4711523056030273 (2.4304173927681108) + | > grad_norm_0: tensor(17.3907, device='cuda:0') (tensor(10.0852, device='cuda:0')) + | > loss_gen: 2.4878458976745605 (2.382931311925253) + | > loss_kl: 1.7361773252487183 (1.8251424747354843) + | > loss_feat: 7.584062099456787 (7.2540797626270965) + | > loss_mel: 17.558757781982422 (17.737855462466968) + | > loss_duration: 1.4494595527648926 (1.4289623428793514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.816303253173828 (30.628971660838406) + | > grad_norm_1: tensor(574.1175, device='cuda:0') (tensor(296.7403, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.3688 (0.3458944582471661) + | > loader_time: 0.0059 (0.005358307969336416) + + + --> TIME: 2025-04-19 04:45:39 -- STEP: 76/142 -- GLOBAL_STEP: 429200 + | > loss_disc: 2.438046455383301 (2.4448081851005545) + | > loss_disc_real_0: 0.12557466328144073 (0.1170613863750508) + | > loss_disc_real_1: 0.18104644119739532 (0.2065979299184523) + | > loss_disc_real_2: 0.2378467470407486 (0.2256233496101279) + | > loss_disc_real_3: 0.2407969832420349 (0.22808642646199778) + | > loss_disc_real_4: 0.20958982408046722 (0.22835025740297218) + | > loss_disc_real_5: 0.2293577939271927 (0.23316189450652977) + | > loss_0: 2.438046455383301 (2.4448081851005545) + | > grad_norm_0: tensor(9.4467, device='cuda:0') (tensor(10.0070, device='cuda:0')) + | > loss_gen: 2.3339643478393555 (2.369388865797143) + | > loss_kl: 1.7365578413009644 (1.8436249400440015) + | > loss_feat: 7.441436767578125 (7.201712796562596) + | > loss_mel: 18.442535400390625 (17.7013664120122) + | > loss_duration: 1.4909262657165527 (1.4421634234880145) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.44542121887207 (30.55825662612915) + | > grad_norm_1: tensor(164.8439, device='cuda:0') (tensor(276.4447, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.3924 (0.3579145669937134) + | > loader_time: 0.0063 (0.005670321615118729) + + + --> TIME: 2025-04-19 04:45:49 -- STEP: 101/142 -- GLOBAL_STEP: 429225 + | > loss_disc: 2.423868417739868 (2.4391671336523375) + | > loss_disc_real_0: 0.09393397718667984 (0.11669706995829497) + | > loss_disc_real_1: 0.20591333508491516 (0.2053854614496231) + | > loss_disc_real_2: 0.23303093016147614 (0.2254888660246783) + | > loss_disc_real_3: 0.2515040338039398 (0.22738749910109113) + | > loss_disc_real_4: 0.22083193063735962 (0.22694925890110507) + | > loss_disc_real_5: 0.28085705637931824 (0.23229540648436783) + | > loss_0: 2.423868417739868 (2.4391671336523375) + | > grad_norm_0: tensor(15.0317, device='cuda:0') (tensor(11.1959, device='cuda:0')) + | > loss_gen: 2.4944417476654053 (2.3763425916728402) + | > loss_kl: 1.990190029144287 (1.8577191817878496) + | > loss_feat: 7.629001617431641 (7.209049305113235) + | > loss_mel: 19.068824768066406 (17.761284459935553) + | > loss_duration: 1.4931329488754272 (1.452543717799801) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.67559051513672 (30.656939251588124) + | > grad_norm_1: tensor(461.9948, device='cuda:0') (tensor(293.7371, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.4249 (0.3687100552096225) + | > loader_time: 0.0065 (0.005884238989046305) + + + --> TIME: 2025-04-19 04:46:00 -- STEP: 126/142 -- GLOBAL_STEP: 429250 + | > loss_disc: 2.3321373462677 (2.4432593356995356) + | > loss_disc_real_0: 0.0956328734755516 (0.11863930428784991) + | > loss_disc_real_1: 0.19072365760803223 (0.20529005546418447) + | > loss_disc_real_2: 0.2358487993478775 (0.22516238441069922) + | > loss_disc_real_3: 0.2163303941488266 (0.2274324701182426) + | > loss_disc_real_4: 0.1826823502779007 (0.2275494792395168) + | > loss_disc_real_5: 0.2106049358844757 (0.2335066892324932) + | > loss_0: 2.3321373462677 (2.4432593356995356) + | > grad_norm_0: tensor(13.0841, device='cuda:0') (tensor(11.3251, device='cuda:0')) + | > loss_gen: 2.3696815967559814 (2.3707313424064997) + | > loss_kl: 1.8795689344406128 (1.8671172015250674) + | > loss_feat: 7.1699934005737305 (7.146160867479113) + | > loss_mel: 16.76875114440918 (17.680786057124056) + | > loss_duration: 1.4866893291473389 (1.4598943647884193) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.674684524536133 (30.524689795478942) + | > grad_norm_1: tensor(282.6615, device='cuda:0') (tensor(289.9369, device='cuda:0')) + | > current_lr_0: 0.00019141419190077703 + | > current_lr_1: 0.00019141419190077703 + | > step_time: 0.44 (0.3804069814227876) + | > loader_time: 0.0074 (0.0061722974928598556) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.2839322090148926 (2.2839322090148926) + | > loss_disc_real_0: 0.16471950709819794 (0.16471950709819794) + | > loss_disc_real_1: 0.22159430384635925 (0.22159430384635925) + | > loss_disc_real_2: 0.2341139316558838 (0.2341139316558838) + | > loss_disc_real_3: 0.2511524558067322 (0.2511524558067322) + | > loss_disc_real_4: 0.220146045088768 (0.220146045088768) + | > loss_disc_real_5: 0.21060185134410858 (0.21060185134410858) + | > loss_0: 2.2839322090148926 (2.2839322090148926) + | > loss_gen: 2.666332244873047 (2.666332244873047) + | > loss_kl: 2.6170785427093506 (2.6170785427093506) + | > loss_feat: 7.239008903503418 (7.239008903503418) + | > loss_mel: 18.303266525268555 (18.303266525268555) + | > loss_duration: 1.7303874492645264 (1.7303874492645264) + | > loss_1: 32.55607604980469 (32.55607604980469) + + --> STEP: 1 + | > loss_disc: 2.419041395187378 (2.419041395187378) + | > loss_disc_real_0: 0.16328933835029602 (0.16328933835029602) + | > loss_disc_real_1: 0.22463475167751312 (0.22463475167751312) + | > loss_disc_real_2: 0.23817229270935059 (0.23817229270935059) + | > loss_disc_real_3: 0.22576779127120972 (0.22576779127120972) + | > loss_disc_real_4: 0.22482098639011383 (0.22482098639011383) + | > loss_disc_real_5: 0.23870280385017395 (0.23870280385017395) + | > loss_0: 2.419041395187378 (2.419041395187378) + | > loss_gen: 2.465630531311035 (2.465630531311035) + | > loss_kl: 2.9379215240478516 (2.9379215240478516) + | > loss_feat: 6.545267105102539 (6.545267105102539) + | > loss_mel: 16.97559356689453 (16.97559356689453) + | > loss_duration: 1.7741315364837646 (1.7741315364837646) + | > loss_1: 30.698543548583984 (30.698543548583984) + + --> STEP: 2 + | > loss_disc: 2.4757394790649414 (2.4473904371261597) + | > loss_disc_real_0: 0.1717347502708435 (0.16751204431056976) + | > loss_disc_real_1: 0.25362375378608704 (0.23912925273180008) + | > loss_disc_real_2: 0.21266472339630127 (0.22541850805282593) + | > loss_disc_real_3: 0.2568594217300415 (0.2413136065006256) + | > loss_disc_real_4: 0.2508358061313629 (0.23782839626073837) + | > loss_disc_real_5: 0.23327814042568207 (0.235990472137928) + | > loss_0: 2.4757394790649414 (2.4473904371261597) + | > loss_gen: 2.5765182971954346 (2.521074414253235) + | > loss_kl: 2.4017903804779053 (2.6698559522628784) + | > loss_feat: 7.3951215744018555 (6.970194339752197) + | > loss_mel: 18.70536994934082 (17.840481758117676) + | > loss_duration: 1.7058846950531006 (1.7400081157684326) + | > loss_1: 32.78468322753906 (31.741613388061523) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003997921943664551 (+0.0003827810287475586) + | > avg_loss_disc: 2.4473904371261597 (-0.002089262008666992) + | > avg_loss_disc_real_0: 0.16751204431056976 (+0.05214301496744156) + | > avg_loss_disc_real_1: 0.23912925273180008 (-0.002833820879459381) + | > avg_loss_disc_real_2: 0.22541850805282593 (-0.022125661373138428) + | > avg_loss_disc_real_3: 0.2413136065006256 (-0.006169266998767853) + | > avg_loss_disc_real_4: 0.23782839626073837 (-0.005726128816604614) + | > avg_loss_disc_real_5: 0.235990472137928 (-0.0029798373579978943) + | > avg_loss_0: 2.4473904371261597 (-0.002089262008666992) + | > avg_loss_gen: 2.521074414253235 (+0.16058850288391113) + | > avg_loss_kl: 2.6698559522628784 (+0.07147097587585449) + | > avg_loss_feat: 6.970194339752197 (-0.7319226264953613) + | > avg_loss_mel: 17.840481758117676 (-0.8831329345703125) + | > avg_loss_duration: 1.7400081157684326 (-0.009650886058807373) + | > avg_loss_1: 31.741613388061523 (-1.3926467895507812) + + + > EPOCH: 352/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:46:09)  + + --> TIME: 2025-04-19 04:46:12 -- STEP: 9/142 -- GLOBAL_STEP: 429275 + | > loss_disc: 2.423612117767334 (2.3891989390055337) + | > loss_disc_real_0: 0.07016627490520477 (0.11053080939584309) + | > loss_disc_real_1: 0.20349544286727905 (0.2044687204890781) + | > loss_disc_real_2: 0.23996736109256744 (0.22587971223725212) + | > loss_disc_real_3: 0.21846069395542145 (0.20997440152698094) + | > loss_disc_real_4: 0.21477587521076202 (0.20882671740319994) + | > loss_disc_real_5: 0.215519517660141 (0.2197269101937612) + | > loss_0: 2.423612117767334 (2.3891989390055337) + | > grad_norm_0: tensor(14.9510, device='cuda:0') (tensor(13.8383, device='cuda:0')) + | > loss_gen: 2.584195375442505 (2.429276042514377) + | > loss_kl: 1.6946885585784912 (1.7678114573160808) + | > loss_feat: 7.805160045623779 (7.597865846421984) + | > loss_mel: 18.337661743164062 (18.289418750339085) + | > loss_duration: 1.3964142799377441 (1.4125783840815227) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.818119049072266 (31.496950573391384) + | > grad_norm_1: tensor(388.6416, device='cuda:0') (tensor(316.0198, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.3247 (0.32436111238267684) + | > loader_time: 0.0059 (0.004747417238023546) + + + --> TIME: 2025-04-19 04:46:21 -- STEP: 34/142 -- GLOBAL_STEP: 429300 + | > loss_disc: 2.436814785003662 (2.4422249303144565) + | > loss_disc_real_0: 0.10216384381055832 (0.11837461371632184) + | > loss_disc_real_1: 0.16921298205852509 (0.20706716749597998) + | > loss_disc_real_2: 0.20354942977428436 (0.22490448022589965) + | > loss_disc_real_3: 0.22771362960338593 (0.22667240482919357) + | > loss_disc_real_4: 0.22844499349594116 (0.2263796272523263) + | > loss_disc_real_5: 0.2354055792093277 (0.22726135876248865) + | > loss_0: 2.436814785003662 (2.4422249303144565) + | > grad_norm_0: tensor(15.3322, device='cuda:0') (tensor(17.2030, device='cuda:0')) + | > loss_gen: 2.364785671234131 (2.4406933433869304) + | > loss_kl: 1.9053837060928345 (1.8232834584572737) + | > loss_feat: 7.636881351470947 (7.481435747707591) + | > loss_mel: 17.723751068115234 (17.970746096442724) + | > loss_duration: 1.4455028772354126 (1.4229764026754044) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.076305389404297 (31.139135136323816) + | > grad_norm_1: tensor(285.4252, device='cuda:0') (tensor(354.0978, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.3557 (0.33771831849042105) + | > loader_time: 0.0059 (0.005240356220918543) + + + --> TIME: 2025-04-19 04:46:30 -- STEP: 59/142 -- GLOBAL_STEP: 429325 + | > loss_disc: 2.3590891361236572 (2.4327465800915737) + | > loss_disc_real_0: 0.10554758459329605 (0.11743220194416532) + | > loss_disc_real_1: 0.20832200348377228 (0.20862419726484913) + | > loss_disc_real_2: 0.19984793663024902 (0.22407852365808972) + | > loss_disc_real_3: 0.2033788114786148 (0.22659792839470555) + | > loss_disc_real_4: 0.19314251840114594 (0.22685424907732818) + | > loss_disc_real_5: 0.25648730993270874 (0.22955217603909767) + | > loss_0: 2.3590891361236572 (2.4327465800915737) + | > grad_norm_0: tensor(8.8018, device='cuda:0') (tensor(14.2807, device='cuda:0')) + | > loss_gen: 2.3592422008514404 (2.4175712213677873) + | > loss_kl: 1.8148400783538818 (1.8353845184132205) + | > loss_feat: 7.363438129425049 (7.382382312063443) + | > loss_mel: 18.35518455505371 (17.851973339662713) + | > loss_duration: 1.4658136367797852 (1.434534242597677) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.358516693115234 (30.921845743211648) + | > grad_norm_1: tensor(148.5539, device='cuda:0') (tensor(316.9327, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.3766 (0.3497226359480518) + | > loader_time: 0.0061 (0.005593174594943806) + + + --> TIME: 2025-04-19 04:46:40 -- STEP: 84/142 -- GLOBAL_STEP: 429350 + | > loss_disc: 2.381272077560425 (2.4440308780897237) + | > loss_disc_real_0: 0.1152242049574852 (0.11989511788955756) + | > loss_disc_real_1: 0.2150801718235016 (0.20816139770405634) + | > loss_disc_real_2: 0.21948540210723877 (0.22617809811518305) + | > loss_disc_real_3: 0.22236381471157074 (0.22733518055507115) + | > loss_disc_real_4: 0.218949094414711 (0.22601368260525523) + | > loss_disc_real_5: 0.23152634501457214 (0.2308770352531047) + | > loss_0: 2.381272077560425 (2.4440308780897237) + | > grad_norm_0: tensor(11.3310, device='cuda:0') (tensor(13.4095, device='cuda:0')) + | > loss_gen: 2.3616504669189453 (2.396300151234582) + | > loss_kl: 1.9463149309158325 (1.858200368427095) + | > loss_feat: 6.948647499084473 (7.29666713305882) + | > loss_mel: 17.956703186035156 (17.809135823022746) + | > loss_duration: 1.497856616973877 (1.4465415406794775) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.711172103881836 (30.80684502919515) + | > grad_norm_1: tensor(300.0025, device='cuda:0') (tensor(302.5567, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.3946 (0.3618681884947277) + | > loader_time: 0.0067 (0.005839166187104726) + + + --> TIME: 2025-04-19 04:46:51 -- STEP: 109/142 -- GLOBAL_STEP: 429375 + | > loss_disc: 2.418881416320801 (2.4546573096459063) + | > loss_disc_real_0: 0.13934381306171417 (0.12366794623913022) + | > loss_disc_real_1: 0.22113926708698273 (0.2087441861902902) + | > loss_disc_real_2: 0.2357942909002304 (0.22535684486047938) + | > loss_disc_real_3: 0.2331686168909073 (0.22781875811585592) + | > loss_disc_real_4: 0.24966026842594147 (0.2272695525797135) + | > loss_disc_real_5: 0.21970424056053162 (0.23166612419513388) + | > loss_0: 2.418881416320801 (2.4546573096459063) + | > grad_norm_0: tensor(8.6168, device='cuda:0') (tensor(12.1534, device='cuda:0')) + | > loss_gen: 2.411632537841797 (2.3877845685416403) + | > loss_kl: 1.9601048231124878 (1.8742297242540833) + | > loss_feat: 6.321088790893555 (7.231256388742989) + | > loss_mel: 18.077587127685547 (17.767630970806152) + | > loss_duration: 1.5000102519989014 (1.4549201016032367) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.270421981811523 (30.71582178238335) + | > grad_norm_1: tensor(102.8883, device='cuda:0') (tensor(258.2783, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.4114 (0.37340306798252476) + | > loader_time: 0.0067 (0.006110875978382355) + + + --> TIME: 2025-04-19 04:47:02 -- STEP: 134/142 -- GLOBAL_STEP: 429400 + | > loss_disc: 2.551412343978882 (2.4700922485607775) + | > loss_disc_real_0: 0.16718216240406036 (0.12742558994622372) + | > loss_disc_real_1: 0.22590719163417816 (0.20947468514317896) + | > loss_disc_real_2: 0.24443605542182922 (0.22704756693608727) + | > loss_disc_real_3: 0.23868508636951447 (0.22838000530627237) + | > loss_disc_real_4: 0.23481136560440063 (0.22791256533185048) + | > loss_disc_real_5: 0.22415649890899658 (0.23283991862588854) + | > loss_0: 2.551412343978882 (2.4700922485607775) + | > grad_norm_0: tensor(5.5132, device='cuda:0') (tensor(11.4164, device='cuda:0')) + | > loss_gen: 2.213435649871826 (2.372682124821108) + | > loss_kl: 2.0098001956939697 (1.8795125039655771) + | > loss_feat: 6.817376613616943 (7.177767942200846) + | > loss_mel: 18.063919067382812 (17.721644045701666) + | > loss_duration: 1.5243932008743286 (1.4619096481978004) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.628925323486328 (30.61351626666624) + | > grad_norm_1: tensor(35.6298, device='cuda:0') (tensor(242.1522, device='cuda:0')) + | > current_lr_0: 0.00019139026512678941 + | > current_lr_1: 0.00019139026512678941 + | > step_time: 0.4403 (0.3851602326578168) + | > loader_time: 0.0072 (0.00635956116576693) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4891624450683594 (2.4891624450683594) + | > loss_disc_real_0: 0.14941473305225372 (0.14941473305225372) + | > loss_disc_real_1: 0.2697918713092804 (0.2697918713092804) + | > loss_disc_real_2: 0.25158143043518066 (0.25158143043518066) + | > loss_disc_real_3: 0.2845611870288849 (0.2845611870288849) + | > loss_disc_real_4: 0.24193164706230164 (0.24193164706230164) + | > loss_disc_real_5: 0.20894430577754974 (0.20894430577754974) + | > loss_0: 2.4891624450683594 (2.4891624450683594) + | > loss_gen: 2.4077064990997314 (2.4077064990997314) + | > loss_kl: 2.4461207389831543 (2.4461207389831543) + | > loss_feat: 6.580336093902588 (6.580336093902588) + | > loss_mel: 18.046537399291992 (18.046537399291992) + | > loss_duration: 1.7426077127456665 (1.7426077127456665) + | > loss_1: 31.223308563232422 (31.223308563232422) + + --> STEP: 1 + | > loss_disc: 2.4298057556152344 (2.4298057556152344) + | > loss_disc_real_0: 0.14188139140605927 (0.14188139140605927) + | > loss_disc_real_1: 0.2819397449493408 (0.2819397449493408) + | > loss_disc_real_2: 0.23896586894989014 (0.23896586894989014) + | > loss_disc_real_3: 0.21421396732330322 (0.21421396732330322) + | > loss_disc_real_4: 0.2177133858203888 (0.2177133858203888) + | > loss_disc_real_5: 0.21834306418895721 (0.21834306418895721) + | > loss_0: 2.4298057556152344 (2.4298057556152344) + | > loss_gen: 2.474677085876465 (2.474677085876465) + | > loss_kl: 2.9028098583221436 (2.9028098583221436) + | > loss_feat: 7.082415580749512 (7.082415580749512) + | > loss_mel: 16.8062744140625 (16.8062744140625) + | > loss_duration: 1.795973300933838 (1.795973300933838) + | > loss_1: 31.062150955200195 (31.062150955200195) + + --> STEP: 2 + | > loss_disc: 2.334211826324463 (2.3820087909698486) + | > loss_disc_real_0: 0.15834695100784302 (0.15011417120695114) + | > loss_disc_real_1: 0.28718268871307373 (0.2845612168312073) + | > loss_disc_real_2: 0.20902588963508606 (0.2239958792924881) + | > loss_disc_real_3: 0.22441868484020233 (0.21931632608175278) + | > loss_disc_real_4: 0.20739325881004333 (0.21255332231521606) + | > loss_disc_real_5: 0.22157792747020721 (0.21996049582958221) + | > loss_0: 2.334211826324463 (2.3820087909698486) + | > loss_gen: 2.6401262283325195 (2.557401657104492) + | > loss_kl: 2.4693360328674316 (2.6860729455947876) + | > loss_feat: 7.2500901222229 (7.166252851486206) + | > loss_mel: 18.508018493652344 (17.657146453857422) + | > loss_duration: 1.7158267498016357 (1.7559000253677368) + | > loss_1: 32.583396911621094 (31.822773933410645) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033608675003051758 (-0.000637054443359375) + | > avg_loss_disc: 2.3820087909698486 (-0.06538164615631104) + | > avg_loss_disc_real_0: 0.15011417120695114 (-0.017397873103618622) + | > avg_loss_disc_real_1: 0.2845612168312073 (+0.045431964099407196) + | > avg_loss_disc_real_2: 0.2239958792924881 (-0.0014226287603378296) + | > avg_loss_disc_real_3: 0.21931632608175278 (-0.021997280418872833) + | > avg_loss_disc_real_4: 0.21255332231521606 (-0.02527507394552231) + | > avg_loss_disc_real_5: 0.21996049582958221 (-0.016029976308345795) + | > avg_loss_0: 2.3820087909698486 (-0.06538164615631104) + | > avg_loss_gen: 2.557401657104492 (+0.036327242851257324) + | > avg_loss_kl: 2.6860729455947876 (+0.01621699333190918) + | > avg_loss_feat: 7.166252851486206 (+0.1960585117340088) + | > avg_loss_mel: 17.657146453857422 (-0.1833353042602539) + | > avg_loss_duration: 1.7559000253677368 (+0.0158919095993042) + | > avg_loss_1: 31.822773933410645 (+0.0811605453491211) + + + > EPOCH: 353/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:47:07)  + + --> TIME: 2025-04-19 04:47:14 -- STEP: 17/142 -- GLOBAL_STEP: 429425 + | > loss_disc: 2.4946043491363525 (2.4429264489342186) + | > loss_disc_real_0: 0.1671227514743805 (0.11932704422403784) + | > loss_disc_real_1: 0.21561682224273682 (0.2020544970736784) + | > loss_disc_real_2: 0.25000667572021484 (0.22909136291812449) + | > loss_disc_real_3: 0.2131892293691635 (0.22992749249233918) + | > loss_disc_real_4: 0.2384139597415924 (0.22546524948933544) + | > loss_disc_real_5: 0.21561746299266815 (0.2324697436655269) + | > loss_0: 2.4946043491363525 (2.4429264489342186) + | > grad_norm_0: tensor(6.2670, device='cuda:0') (tensor(10.7784, device='cuda:0')) + | > loss_gen: 2.2941641807556152 (2.3915435847114113) + | > loss_kl: 1.7355586290359497 (1.8077892555909998) + | > loss_feat: 6.3000264167785645 (7.286473358378691) + | > loss_mel: 15.69287109375 (17.524241166956283) + | > loss_duration: 1.3991224765777588 (1.4104556546491736) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.421743392944336 (30.42050327974207) + | > grad_norm_1: tensor(101.5211, device='cuda:0') (tensor(269.0965, device='cuda:0')) + | > current_lr_0: 0.00019136634134364855 + | > current_lr_1: 0.00019136634134364855 + | > step_time: 0.3471 (0.3299558583427878) + | > loader_time: 0.0053 (0.004846376531264361) + + + --> TIME: 2025-04-19 04:47:23 -- STEP: 42/142 -- GLOBAL_STEP: 429450 + | > loss_disc: 2.4152135848999023 (2.4324810902277623) + | > loss_disc_real_0: 0.11669707298278809 (0.11836075090936252) + | > loss_disc_real_1: 0.20394723117351532 (0.20279143289441154) + | > loss_disc_real_2: 0.20789550244808197 (0.22702686666023164) + | > loss_disc_real_3: 0.22069591283798218 (0.22874645498536883) + | > loss_disc_real_4: 0.23800036311149597 (0.22687999513887225) + | > loss_disc_real_5: 0.24744684994220734 (0.23289892574151358) + | > loss_0: 2.4152135848999023 (2.4324810902277623) + | > grad_norm_0: tensor(4.6591, device='cuda:0') (tensor(9.4177, device='cuda:0')) + | > loss_gen: 2.3525352478027344 (2.3787236951646356) + | > loss_kl: 1.767181634902954 (1.8208172690300715) + | > loss_feat: 6.388619422912598 (7.180373441605341) + | > loss_mel: 17.60986328125 (17.631695270538323) + | > loss_duration: 1.469543695449829 (1.4256563044729686) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.58774185180664 (30.437266031901043) + | > grad_norm_1: tensor(171.9075, device='cuda:0') (tensor(221.1657, device='cuda:0')) + | > current_lr_0: 0.00019136634134364855 + | > current_lr_1: 0.00019136634134364855 + | > step_time: 0.3736 (0.3466131005968366) + | > loader_time: 0.0061 (0.005376094863528296) + + + --> TIME: 2025-04-19 04:47:33 -- STEP: 67/142 -- GLOBAL_STEP: 429475 + | > loss_disc: 2.5091209411621094 (2.439059495925902) + | > loss_disc_real_0: 0.09515299648046494 (0.12021378394383103) + | > loss_disc_real_1: 0.1882745623588562 (0.20261712127657078) + | > loss_disc_real_2: 0.22592179477214813 (0.22505610714207835) + | > loss_disc_real_3: 0.23835253715515137 (0.2289458728993117) + | > loss_disc_real_4: 0.19770623743534088 (0.22665812008416475) + | > loss_disc_real_5: 0.26024243235588074 (0.23321403229414528) + | > loss_0: 2.5091209411621094 (2.439059495925902) + | > grad_norm_0: tensor(6.4312, device='cuda:0') (tensor(8.6110, device='cuda:0')) + | > loss_gen: 2.402578830718994 (2.3674819255942734) + | > loss_kl: 1.9878920316696167 (1.8508341846181386) + | > loss_feat: 7.205405235290527 (7.167831150453482) + | > loss_mel: 17.741491317749023 (17.66151406871738) + | > loss_duration: 1.4050531387329102 (1.4358928150205472) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.742420196533203 (30.48355412839064) + | > grad_norm_1: tensor(214.5382, device='cuda:0') (tensor(191.4261, device='cuda:0')) + | > current_lr_0: 0.00019136634134364855 + | > current_lr_1: 0.00019136634134364855 + | > step_time: 0.4168 (0.3625578773555471) + | > loader_time: 0.0066 (0.005754029572899662) + + + --> TIME: 2025-04-19 04:47:43 -- STEP: 92/142 -- GLOBAL_STEP: 429500 + | > loss_disc: 2.548150062561035 (2.451246497423752) + | > loss_disc_real_0: 0.12482669204473495 (0.12329456303268671) + | > loss_disc_real_1: 0.2369878739118576 (0.2045707927773828) + | > loss_disc_real_2: 0.25948941707611084 (0.22537104315731837) + | > loss_disc_real_3: 0.22160068154335022 (0.2283956776170627) + | > loss_disc_real_4: 0.23535342514514923 (0.22821561878790025) + | > loss_disc_real_5: 0.23455427587032318 (0.2339384740461474) + | > loss_0: 2.548150062561035 (2.451246497423752) + | > grad_norm_0: tensor(5.3000, device='cuda:0') (tensor(9.0390, device='cuda:0')) + | > loss_gen: 2.2280800342559814 (2.360394187595534) + | > loss_kl: 1.8701274394989014 (1.8700603155986122) + | > loss_feat: 6.886775970458984 (7.123180244279944) + | > loss_mel: 18.14699935913086 (17.643391889074568) + | > loss_duration: 1.4252476692199707 (1.4482293945291762) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.557231903076172 (30.445255963698678) + | > grad_norm_1: tensor(200.8260, device='cuda:0') (tensor(204.2221, device='cuda:0')) + | > current_lr_0: 0.00019136634134364855 + | > current_lr_1: 0.00019136634134364855 + | > step_time: 0.4241 (0.3760387042294378) + | > loader_time: 0.0073 (0.006067532560099725) + + + --> TIME: 2025-04-19 04:47:54 -- STEP: 117/142 -- GLOBAL_STEP: 429525 + | > loss_disc: 2.530961036682129 (2.450678234426385) + | > loss_disc_real_0: 0.17608247697353363 (0.12308516210088363) + | > loss_disc_real_1: 0.22824178636074066 (0.2060522569550408) + | > loss_disc_real_2: 0.24392369389533997 (0.2255934264160629) + | > loss_disc_real_3: 0.22821663320064545 (0.22851284052062237) + | > loss_disc_real_4: 0.2575031518936157 (0.22686308100182787) + | > loss_disc_real_5: 0.2767704129219055 (0.23356582631922176) + | > loss_0: 2.530961036682129 (2.450678234426385) + | > grad_norm_0: tensor(24.9745, device='cuda:0') (tensor(10.2513, device='cuda:0')) + | > loss_gen: 2.450491428375244 (2.3621786150157975) + | > loss_kl: 1.853750228881836 (1.8760112748186812) + | > loss_feat: 7.63734769821167 (7.112047929030198) + | > loss_mel: 18.136247634887695 (17.686828499166364) + | > loss_duration: 1.4788821935653687 (1.455569944830022) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.556718826293945 (30.492636215992462) + | > grad_norm_1: tensor(446.5139, device='cuda:0') (tensor(232.9057, device='cuda:0')) + | > current_lr_0: 0.00019136634134364855 + | > current_lr_1: 0.00019136634134364855 + | > step_time: 0.4333 (0.38843046905647993) + | > loader_time: 0.0069 (0.006312398829011834) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.276151180267334 (2.276151180267334) + | > loss_disc_real_0: 0.10137052834033966 (0.10137052834033966) + | > loss_disc_real_1: 0.23252831399440765 (0.23252831399440765) + | > loss_disc_real_2: 0.25168806314468384 (0.25168806314468384) + | > loss_disc_real_3: 0.230237677693367 (0.230237677693367) + | > loss_disc_real_4: 0.18088839948177338 (0.18088839948177338) + | > loss_disc_real_5: 0.21570195257663727 (0.21570195257663727) + | > loss_0: 2.276151180267334 (2.276151180267334) + | > loss_gen: 2.4369351863861084 (2.4369351863861084) + | > loss_kl: 2.5708630084991455 (2.5708630084991455) + | > loss_feat: 7.533398151397705 (7.533398151397705) + | > loss_mel: 18.97123146057129 (18.97123146057129) + | > loss_duration: 1.7537897825241089 (1.7537897825241089) + | > loss_1: 33.26621627807617 (33.26621627807617) + + --> STEP: 1 + | > loss_disc: 2.592244863510132 (2.592244863510132) + | > loss_disc_real_0: 0.18359492719173431 (0.18359492719173431) + | > loss_disc_real_1: 0.22696951031684875 (0.22696951031684875) + | > loss_disc_real_2: 0.2218167781829834 (0.2218167781829834) + | > loss_disc_real_3: 0.23890720307826996 (0.23890720307826996) + | > loss_disc_real_4: 0.21065184473991394 (0.21065184473991394) + | > loss_disc_real_5: 0.18438392877578735 (0.18438392877578735) + | > loss_0: 2.592244863510132 (2.592244863510132) + | > loss_gen: 2.144124984741211 (2.144124984741211) + | > loss_kl: 2.9153695106506348 (2.9153695106506348) + | > loss_feat: 6.9861674308776855 (6.9861674308776855) + | > loss_mel: 17.530776977539062 (17.530776977539062) + | > loss_duration: 1.7321720123291016 (1.7321720123291016) + | > loss_1: 31.308610916137695 (31.308610916137695) + + --> STEP: 2 + | > loss_disc: 2.346566915512085 (2.4694058895111084) + | > loss_disc_real_0: 0.13835296034812927 (0.1609739437699318) + | > loss_disc_real_1: 0.24911780655384064 (0.2380436584353447) + | > loss_disc_real_2: 0.24717892706394196 (0.23449785262346268) + | > loss_disc_real_3: 0.19189834594726562 (0.2154027745127678) + | > loss_disc_real_4: 0.22082604467868805 (0.215738944709301) + | > loss_disc_real_5: 0.17294816672801971 (0.17866604775190353) + | > loss_0: 2.346566915512085 (2.4694058895111084) + | > loss_gen: 2.3694136142730713 (2.256769299507141) + | > loss_kl: 2.4947385787963867 (2.7050540447235107) + | > loss_feat: 6.83174991607666 (6.908958673477173) + | > loss_mel: 18.4043025970459 (17.96753978729248) + | > loss_duration: 1.7123886346817017 (1.7222803235054016) + | > loss_1: 31.812593460083008 (31.56060218811035) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034704208374023438 (+0.00010955333709716797) + | > avg_loss_disc: 2.4694058895111084 (+0.08739709854125977) + | > avg_loss_disc_real_0: 0.1609739437699318 (+0.010859772562980652) + | > avg_loss_disc_real_1: 0.2380436584353447 (-0.04651755839586258) + | > avg_loss_disc_real_2: 0.23449785262346268 (+0.010501973330974579) + | > avg_loss_disc_real_3: 0.2154027745127678 (-0.003913551568984985) + | > avg_loss_disc_real_4: 0.215738944709301 (+0.0031856223940849304) + | > avg_loss_disc_real_5: 0.17866604775190353 (-0.04129444807767868) + | > avg_loss_0: 2.4694058895111084 (+0.08739709854125977) + | > avg_loss_gen: 2.256769299507141 (-0.3006323575973511) + | > avg_loss_kl: 2.7050540447235107 (+0.018981099128723145) + | > avg_loss_feat: 6.908958673477173 (-0.2572941780090332) + | > avg_loss_mel: 17.96753978729248 (+0.3103933334350586) + | > avg_loss_duration: 1.7222803235054016 (-0.033619701862335205) + | > avg_loss_1: 31.56060218811035 (-0.26217174530029297) + + + > EPOCH: 354/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:48:07)  + + --> TIME: 2025-04-19 04:48:08 -- STEP: 0/142 -- GLOBAL_STEP: 429550 + | > loss_disc: 2.4850456714630127 (2.4850456714630127) + | > loss_disc_real_0: 0.09610282629728317 (0.09610282629728317) + | > loss_disc_real_1: 0.27527913451194763 (0.27527913451194763) + | > loss_disc_real_2: 0.25832945108413696 (0.25832945108413696) + | > loss_disc_real_3: 0.23854795098304749 (0.23854795098304749) + | > loss_disc_real_4: 0.231006920337677 (0.231006920337677) + | > loss_disc_real_5: 0.2092810869216919 (0.2092810869216919) + | > loss_0: 2.4850456714630127 (2.4850456714630127) + | > grad_norm_0: tensor(9.6743, device='cuda:0') (tensor(9.6743, device='cuda:0')) + | > loss_gen: 2.3270950317382812 (2.3270950317382812) + | > loss_kl: 1.6034929752349854 (1.6034929752349854) + | > loss_feat: 6.617062091827393 (6.617062091827393) + | > loss_mel: 17.562705993652344 (17.562705993652344) + | > loss_duration: 1.4327309131622314 (1.4327309131622314) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.543087005615234 (29.543087005615234) + | > grad_norm_1: tensor(69.0939, device='cuda:0') (tensor(69.0939, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.3527 (0.3527200222015381) + | > loader_time: 0.3287 (0.3287181854248047) + + + --> TIME: 2025-04-19 04:48:16 -- STEP: 25/142 -- GLOBAL_STEP: 429575 + | > loss_disc: 2.447665214538574 (2.424030361175537) + | > loss_disc_real_0: 0.08974870294332504 (0.10934311330318451) + | > loss_disc_real_1: 0.16411516070365906 (0.19905501902103423) + | > loss_disc_real_2: 0.2024620622396469 (0.2250577175617218) + | > loss_disc_real_3: 0.1983557939529419 (0.2271374100446701) + | > loss_disc_real_4: 0.20361606776714325 (0.22602633714675904) + | > loss_disc_real_5: 0.2212585210800171 (0.2343185591697693) + | > loss_0: 2.447665214538574 (2.424030361175537) + | > grad_norm_0: tensor(11.6114, device='cuda:0') (tensor(10.3258, device='cuda:0')) + | > loss_gen: 2.317660093307495 (2.4011052799224855) + | > loss_kl: 1.7649542093276978 (1.8431610107421874) + | > loss_feat: 7.277646541595459 (7.398854713439942) + | > loss_mel: 18.429656982421875 (17.9124760055542) + | > loss_duration: 1.4598255157470703 (1.4213932323455811) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.24974250793457 (30.976990585327147) + | > grad_norm_1: tensor(208.8065, device='cuda:0') (tensor(271.2022, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.3446 (0.3352498149871826) + | > loader_time: 0.0058 (0.0050389099121093735) + + + --> TIME: 2025-04-19 04:48:25 -- STEP: 50/142 -- GLOBAL_STEP: 429600 + | > loss_disc: 2.4001388549804688 (2.4357781553268434) + | > loss_disc_real_0: 0.12494196742773056 (0.11475425034761429) + | > loss_disc_real_1: 0.20714975893497467 (0.20283268243074418) + | > loss_disc_real_2: 0.24503013491630554 (0.22535363256931304) + | > loss_disc_real_3: 0.22615772485733032 (0.22823259174823762) + | > loss_disc_real_4: 0.22814925014972687 (0.22752564519643784) + | > loss_disc_real_5: 0.23622888326644897 (0.23363595128059386) + | > loss_0: 2.4001388549804688 (2.4357781553268434) + | > grad_norm_0: tensor(9.3498, device='cuda:0') (tensor(10.4252, device='cuda:0')) + | > loss_gen: 2.4001779556274414 (2.3829476928710944) + | > loss_kl: 1.9010205268859863 (1.8391322493553162) + | > loss_feat: 7.471240520477295 (7.338698682785034) + | > loss_mel: 18.00079345703125 (17.954144687652594) + | > loss_duration: 1.4721355438232422 (1.4324674510955808) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.2453670501709 (30.94739086151123) + | > grad_norm_1: tensor(220.8758, device='cuda:0') (tensor(253.8251, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.3757 (0.34723286628723143) + | > loader_time: 0.0059 (0.005378041267395018) + + + --> TIME: 2025-04-19 04:48:35 -- STEP: 75/142 -- GLOBAL_STEP: 429625 + | > loss_disc: 2.2781474590301514 (2.4374286842346193) + | > loss_disc_real_0: 0.13119475543498993 (0.11900067150592804) + | > loss_disc_real_1: 0.18974100053310394 (0.20406177679697673) + | > loss_disc_real_2: 0.17316175997257233 (0.22357295751571654) + | > loss_disc_real_3: 0.21124209463596344 (0.22788796762625377) + | > loss_disc_real_4: 0.19304445385932922 (0.22799667835235596) + | > loss_disc_real_5: 0.20149971544742584 (0.23252819180488588) + | > loss_0: 2.2781474590301514 (2.4374286842346193) + | > grad_norm_0: tensor(14.7887, device='cuda:0') (tensor(10.1993, device='cuda:0')) + | > loss_gen: 2.463655471801758 (2.3768860753377288) + | > loss_kl: 1.797116994857788 (1.8574929587046305) + | > loss_feat: 7.882971286773682 (7.303570200602214) + | > loss_mel: 18.429662704467773 (17.861209856669113) + | > loss_duration: 1.4287521839141846 (1.4443845891952514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.002159118652344 (30.84354372660319) + | > grad_norm_1: tensor(364.5286, device='cuda:0') (tensor(241.2745, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.3964 (0.3586947536468506) + | > loader_time: 0.0068 (0.005647169748942055) + + + --> TIME: 2025-04-19 04:48:45 -- STEP: 100/142 -- GLOBAL_STEP: 429650 + | > loss_disc: 2.469119071960449 (2.452437126636504) + | > loss_disc_real_0: 0.08595890551805496 (0.12294189818203449) + | > loss_disc_real_1: 0.17710308730602264 (0.20386006116867064) + | > loss_disc_real_2: 0.23004017770290375 (0.22563085585832596) + | > loss_disc_real_3: 0.19463807344436646 (0.22758983179926873) + | > loss_disc_real_4: 0.1896958351135254 (0.22717313691973687) + | > loss_disc_real_5: 0.19975286722183228 (0.23188928082585336) + | > loss_0: 2.469119071960449 (2.452437126636504) + | > grad_norm_0: tensor(18.8587, device='cuda:0') (tensor(11.5901, device='cuda:0')) + | > loss_gen: 2.422076463699341 (2.367752203941347) + | > loss_kl: 1.7822414636611938 (1.8700648999214173) + | > loss_feat: 6.928025722503662 (7.204702525138855) + | > loss_mel: 16.982223510742188 (17.832429008483892) + | > loss_duration: 1.4808692932128906 (1.45392485499382) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.595436096191406 (30.728873596191406) + | > grad_norm_1: tensor(381.5005, device='cuda:0') (tensor(266.2226, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.4156 (0.3689605259895325) + | > loader_time: 0.0072 (0.005957465171813963) + + + --> TIME: 2025-04-19 04:48:56 -- STEP: 125/142 -- GLOBAL_STEP: 429675 + | > loss_disc: 2.4690728187561035 (2.4610600910186764) + | > loss_disc_real_0: 0.08146163076162338 (0.12521420136094097) + | > loss_disc_real_1: 0.22449728846549988 (0.20548317408561706) + | > loss_disc_real_2: 0.2698552906513214 (0.22629750847816468) + | > loss_disc_real_3: 0.25698214769363403 (0.22925858092308044) + | > loss_disc_real_4: 0.24425217509269714 (0.22870517206192018) + | > loss_disc_real_5: 0.2782469093799591 (0.23275269544124602) + | > loss_0: 2.4690728187561035 (2.4610600910186764) + | > grad_norm_0: tensor(13.3070, device='cuda:0') (tensor(13.5121, device='cuda:0')) + | > loss_gen: 2.5489354133605957 (2.3777056026458747) + | > loss_kl: 1.9594897031784058 (1.8766560564041137) + | > loss_feat: 6.714224815368652 (7.2015782165527344) + | > loss_mel: 18.064836502075195 (17.80051039123536) + | > loss_duration: 1.5026135444641113 (1.4606245689392088) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.79010009765625 (30.717074935913086) + | > grad_norm_1: tensor(262.3823, device='cuda:0') (tensor(290.6031, device='cuda:0')) + | > current_lr_0: 0.0001913424205509806 + | > current_lr_1: 0.0001913424205509806 + | > step_time: 0.4409 (0.38131744766235354) + | > loader_time: 0.0072 (0.0062484092712402324) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4344708919525146 (2.4344708919525146) + | > loss_disc_real_0: 0.15455351769924164 (0.15455351769924164) + | > loss_disc_real_1: 0.22425466775894165 (0.22425466775894165) + | > loss_disc_real_2: 0.28384989500045776 (0.28384989500045776) + | > loss_disc_real_3: 0.25541695952415466 (0.25541695952415466) + | > loss_disc_real_4: 0.21574215590953827 (0.21574215590953827) + | > loss_disc_real_5: 0.26076769828796387 (0.26076769828796387) + | > loss_0: 2.4344708919525146 (2.4344708919525146) + | > loss_gen: 2.5550785064697266 (2.5550785064697266) + | > loss_kl: 2.286992311477661 (2.286992311477661) + | > loss_feat: 7.556507587432861 (7.556507587432861) + | > loss_mel: 19.033512115478516 (19.033512115478516) + | > loss_duration: 1.7470732927322388 (1.7470732927322388) + | > loss_1: 33.17916488647461 (33.17916488647461) + + --> STEP: 1 + | > loss_disc: 2.4812092781066895 (2.4812092781066895) + | > loss_disc_real_0: 0.1465579718351364 (0.1465579718351364) + | > loss_disc_real_1: 0.21071027219295502 (0.21071027219295502) + | > loss_disc_real_2: 0.3346533477306366 (0.3346533477306366) + | > loss_disc_real_3: 0.24728387594223022 (0.24728387594223022) + | > loss_disc_real_4: 0.2781922519207001 (0.2781922519207001) + | > loss_disc_real_5: 0.3173877000808716 (0.3173877000808716) + | > loss_0: 2.4812092781066895 (2.4812092781066895) + | > loss_gen: 2.6433539390563965 (2.6433539390563965) + | > loss_kl: 2.8357574939727783 (2.8357574939727783) + | > loss_feat: 7.1852827072143555 (7.1852827072143555) + | > loss_mel: 18.87355613708496 (18.87355613708496) + | > loss_duration: 1.7618372440338135 (1.7618372440338135) + | > loss_1: 33.29978942871094 (33.29978942871094) + + --> STEP: 2 + | > loss_disc: 2.379255771636963 (2.430232524871826) + | > loss_disc_real_0: 0.13618186116218567 (0.14136991649866104) + | > loss_disc_real_1: 0.24079248309135437 (0.2257513776421547) + | > loss_disc_real_2: 0.27505841851234436 (0.3048558831214905) + | > loss_disc_real_3: 0.23385733366012573 (0.24057060480117798) + | > loss_disc_real_4: 0.2455517202615738 (0.26187198609113693) + | > loss_disc_real_5: 0.22048012912273407 (0.2689339146018028) + | > loss_0: 2.379255771636963 (2.430232524871826) + | > loss_gen: 2.4886531829833984 (2.5660035610198975) + | > loss_kl: 2.5048515796661377 (2.670304536819458) + | > loss_feat: 7.870368480682373 (7.527825593948364) + | > loss_mel: 18.053743362426758 (18.46364974975586) + | > loss_duration: 1.7487003803253174 (1.7552688121795654) + | > loss_1: 32.666316986083984 (32.98305320739746) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034574270248413086 (-1.2993812561035156e-05) + | > avg_loss_disc: 2.430232524871826 (-0.03917336463928223) + | > avg_loss_disc_real_0: 0.14136991649866104 (-0.019604027271270752) + | > avg_loss_disc_real_1: 0.2257513776421547 (-0.012292280793190002) + | > avg_loss_disc_real_2: 0.3048558831214905 (+0.0703580304980278) + | > avg_loss_disc_real_3: 0.24057060480117798 (+0.025167830288410187) + | > avg_loss_disc_real_4: 0.26187198609113693 (+0.04613304138183594) + | > avg_loss_disc_real_5: 0.2689339146018028 (+0.09026786684989929) + | > avg_loss_0: 2.430232524871826 (-0.03917336463928223) + | > avg_loss_gen: 2.5660035610198975 (+0.30923426151275635) + | > avg_loss_kl: 2.670304536819458 (-0.034749507904052734) + | > avg_loss_feat: 7.527825593948364 (+0.6188669204711914) + | > avg_loss_mel: 18.46364974975586 (+0.4961099624633789) + | > avg_loss_duration: 1.7552688121795654 (+0.03298848867416382) + | > avg_loss_1: 32.98305320739746 (+1.4224510192871094) + + + > EPOCH: 355/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:49:06)  + + --> TIME: 2025-04-19 04:49:09 -- STEP: 8/142 -- GLOBAL_STEP: 429700 + | > loss_disc: 2.390613079071045 (2.4175431728363037) + | > loss_disc_real_0: 0.12227927893400192 (0.10196703299880028) + | > loss_disc_real_1: 0.20511746406555176 (0.20366722717881203) + | > loss_disc_real_2: 0.23638148605823517 (0.22847355157136917) + | > loss_disc_real_3: 0.21344108879566193 (0.22541457414627075) + | > loss_disc_real_4: 0.19810929894447327 (0.22449969314038754) + | > loss_disc_real_5: 0.1901828795671463 (0.2303977720439434) + | > loss_0: 2.390613079071045 (2.4175431728363037) + | > grad_norm_0: tensor(4.5650, device='cuda:0') (tensor(9.3625, device='cuda:0')) + | > loss_gen: 2.3007376194000244 (2.3371754586696625) + | > loss_kl: 1.7796361446380615 (1.770050972700119) + | > loss_feat: 7.394068717956543 (7.3268813490867615) + | > loss_mel: 17.586198806762695 (17.441062927246094) + | > loss_duration: 1.4268479347229004 (1.4153512716293335) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.48748779296875 (30.29052209854126) + | > grad_norm_1: tensor(80.6038, device='cuda:0') (tensor(224.5288, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.3243 (0.32475510239601135) + | > loader_time: 0.0051 (0.004554450511932373) + + + --> TIME: 2025-04-19 04:49:18 -- STEP: 33/142 -- GLOBAL_STEP: 429725 + | > loss_disc: 2.375953435897827 (2.4220723672346636) + | > loss_disc_real_0: 0.11351944506168365 (0.11549901375264833) + | > loss_disc_real_1: 0.19760732352733612 (0.20191140653509082) + | > loss_disc_real_2: 0.21253733336925507 (0.2258514120723262) + | > loss_disc_real_3: 0.24263319373130798 (0.22600737846258914) + | > loss_disc_real_4: 0.2436235100030899 (0.2293387601772944) + | > loss_disc_real_5: 0.2549869418144226 (0.22837123274803162) + | > loss_0: 2.375953435897827 (2.4220723672346636) + | > grad_norm_0: tensor(4.1525, device='cuda:0') (tensor(8.3966, device='cuda:0')) + | > loss_gen: 2.3995161056518555 (2.3520410638867) + | > loss_kl: 1.9540780782699585 (1.82278480313041) + | > loss_feat: 7.033956527709961 (7.2659648837465225) + | > loss_mel: 17.431406021118164 (17.612991679798473) + | > loss_duration: 1.4396390914916992 (1.4214805328484736) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.25859832763672 (30.47526336438728) + | > grad_norm_1: tensor(101.2307, device='cuda:0') (tensor(178.4599, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.3582 (0.3377566626577666) + | > loader_time: 0.0053 (0.005172274329445579) + + + --> TIME: 2025-04-19 04:49:27 -- STEP: 58/142 -- GLOBAL_STEP: 429750 + | > loss_disc: 2.4326865673065186 (2.4243799489119953) + | > loss_disc_real_0: 0.10800661146640778 (0.11714905333416216) + | > loss_disc_real_1: 0.2116369754076004 (0.20213040555345602) + | > loss_disc_real_2: 0.230264350771904 (0.22706844631967874) + | > loss_disc_real_3: 0.2251768559217453 (0.22776894502598663) + | > loss_disc_real_4: 0.2948824465274811 (0.2293348356053747) + | > loss_disc_real_5: 0.23428846895694733 (0.22880447655916214) + | > loss_0: 2.4326865673065186 (2.4243799489119953) + | > grad_norm_0: tensor(17.3425, device='cuda:0') (tensor(8.4582, device='cuda:0')) + | > loss_gen: 2.383734703063965 (2.368888772767166) + | > loss_kl: 2.0655295848846436 (1.8507012827643032) + | > loss_feat: 6.8635149002075195 (7.260595017466052) + | > loss_mel: 17.630422592163086 (17.610537167253163) + | > loss_duration: 1.4672242403030396 (1.43401981633285) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.41042709350586 (30.524742422432734) + | > grad_norm_1: tensor(163.2152, device='cuda:0') (tensor(193.2932, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.3772 (0.3494352061173012) + | > loader_time: 0.0061 (0.005466592722925647) + + + --> TIME: 2025-04-19 04:49:37 -- STEP: 83/142 -- GLOBAL_STEP: 429775 + | > loss_disc: 2.4354567527770996 (2.43175948958799) + | > loss_disc_real_0: 0.09253580868244171 (0.11868802718369358) + | > loss_disc_real_1: 0.22134870290756226 (0.2023255169750696) + | > loss_disc_real_2: 0.2353677600622177 (0.22571819446173058) + | > loss_disc_real_3: 0.21345160901546478 (0.22685789631073733) + | > loss_disc_real_4: 0.20220008492469788 (0.22724176512425204) + | > loss_disc_real_5: 0.21735826134681702 (0.22927529230175248) + | > loss_0: 2.4354567527770996 (2.43175948958799) + | > grad_norm_0: tensor(7.1214, device='cuda:0') (tensor(8.8004, device='cuda:0')) + | > loss_gen: 2.4915595054626465 (2.366120812404587) + | > loss_kl: 1.9891774654388428 (1.874178277440818) + | > loss_feat: 7.371232032775879 (7.256883724626288) + | > loss_mel: 18.744647979736328 (17.669161624219043) + | > loss_duration: 1.5099856853485107 (1.444712582840977) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.10660171508789 (30.61105716659362) + | > grad_norm_1: tensor(177.3251, device='cuda:0') (tensor(200.1901, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.4001 (0.3623227556067778) + | > loader_time: 0.0069 (0.005774124559149684) + + + --> TIME: 2025-04-19 04:49:48 -- STEP: 108/142 -- GLOBAL_STEP: 429800 + | > loss_disc: 2.444431781768799 (2.4432084935682785) + | > loss_disc_real_0: 0.1346953958272934 (0.12116634576684898) + | > loss_disc_real_1: 0.2046593725681305 (0.204630730052789) + | > loss_disc_real_2: 0.21284641325473785 (0.22642003675853764) + | > loss_disc_real_3: 0.21815121173858643 (0.22754659296737778) + | > loss_disc_real_4: 0.24127493798732758 (0.227755483791784) + | > loss_disc_real_5: 0.23422369360923767 (0.22946420215346194) + | > loss_0: 2.444431781768799 (2.4432084935682785) + | > grad_norm_0: tensor(11.1866, device='cuda:0') (tensor(9.2757, device='cuda:0')) + | > loss_gen: 2.454510450363159 (2.360861455952681) + | > loss_kl: 1.8813802003860474 (1.884046494960785) + | > loss_feat: 7.8346147537231445 (7.202851644268742) + | > loss_mel: 18.347410202026367 (17.703942705083787) + | > loss_duration: 1.4863203763961792 (1.4538665877448187) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.004234313964844 (30.605569044748943) + | > grad_norm_1: tensor(202.0247, device='cuda:0') (tensor(200.9444, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.4227 (0.37363969617419773) + | > loader_time: 0.0071 (0.00605745448006524) + + + --> TIME: 2025-04-19 04:49:59 -- STEP: 133/142 -- GLOBAL_STEP: 429825 + | > loss_disc: 2.447591781616211 (2.444102305218689) + | > loss_disc_real_0: 0.1536117047071457 (0.12256196589398205) + | > loss_disc_real_1: 0.25973454117774963 (0.20525615948035303) + | > loss_disc_real_2: 0.2579021453857422 (0.22627668430034378) + | > loss_disc_real_3: 0.22171185910701752 (0.22790138445850602) + | > loss_disc_real_4: 0.20757396519184113 (0.22753950617367164) + | > loss_disc_real_5: 0.24016085267066956 (0.22913086660822532) + | > loss_0: 2.447591781616211 (2.444102305218689) + | > grad_norm_0: tensor(6.7850, device='cuda:0') (tensor(9.5102, device='cuda:0')) + | > loss_gen: 2.585339069366455 (2.3613550788477857) + | > loss_kl: 1.9266815185546875 (1.8882360019181903) + | > loss_feat: 7.567191123962402 (7.178620471093888) + | > loss_mel: 17.83926010131836 (17.685481286586683) + | > loss_duration: 1.533273696899414 (1.4600877708062194) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.451745986938477 (30.573780748180877) + | > grad_norm_1: tensor(118.3258, device='cuda:0') (tensor(209.3641, device='cuda:0')) + | > current_lr_0: 0.00019131850274841172 + | > current_lr_1: 0.00019131850274841172 + | > step_time: 0.4378 (0.38582224953443484) + | > loader_time: 0.0075 (0.006328993273856945) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4748027324676514 (2.4748027324676514) + | > loss_disc_real_0: 0.16274766623973846 (0.16274766623973846) + | > loss_disc_real_1: 0.24548329412937164 (0.24548329412937164) + | > loss_disc_real_2: 0.24487757682800293 (0.24487757682800293) + | > loss_disc_real_3: 0.22122110426425934 (0.22122110426425934) + | > loss_disc_real_4: 0.19219176471233368 (0.19219176471233368) + | > loss_disc_real_5: 0.2265775203704834 (0.2265775203704834) + | > loss_0: 2.4748027324676514 (2.4748027324676514) + | > loss_gen: 2.4774601459503174 (2.4774601459503174) + | > loss_kl: 2.643558979034424 (2.643558979034424) + | > loss_feat: 8.074724197387695 (8.074724197387695) + | > loss_mel: 19.35330581665039 (19.35330581665039) + | > loss_duration: 1.7202951908111572 (1.7202951908111572) + | > loss_1: 34.269344329833984 (34.269344329833984) + + --> STEP: 1 + | > loss_disc: 2.684709310531616 (2.684709310531616) + | > loss_disc_real_0: 0.21814461052417755 (0.21814461052417755) + | > loss_disc_real_1: 0.29556772112846375 (0.29556772112846375) + | > loss_disc_real_2: 0.2600790560245514 (0.2600790560245514) + | > loss_disc_real_3: 0.19914445281028748 (0.19914445281028748) + | > loss_disc_real_4: 0.19434335827827454 (0.19434335827827454) + | > loss_disc_real_5: 0.2199995517730713 (0.2199995517730713) + | > loss_0: 2.684709310531616 (2.684709310531616) + | > loss_gen: 2.08829927444458 (2.08829927444458) + | > loss_kl: 2.9279417991638184 (2.9279417991638184) + | > loss_feat: 4.929286956787109 (4.929286956787109) + | > loss_mel: 13.574572563171387 (13.574572563171387) + | > loss_duration: 1.76768159866333 (1.76768159866333) + | > loss_1: 25.28778076171875 (25.28778076171875) + + --> STEP: 2 + | > loss_disc: 2.430030107498169 (2.5573697090148926) + | > loss_disc_real_0: 0.20086680352687836 (0.20950570702552795) + | > loss_disc_real_1: 0.2270725965499878 (0.26132015883922577) + | > loss_disc_real_2: 0.21628354489803314 (0.23818130046129227) + | > loss_disc_real_3: 0.21783292293548584 (0.20848868787288666) + | > loss_disc_real_4: 0.19912920892238617 (0.19673628360033035) + | > loss_disc_real_5: 0.22880598902702332 (0.2244027704000473) + | > loss_0: 2.430030107498169 (2.5573697090148926) + | > loss_gen: 2.414534330368042 (2.251416802406311) + | > loss_kl: 2.3719277381896973 (2.649934768676758) + | > loss_feat: 7.976452350616455 (6.452869653701782) + | > loss_mel: 18.867507934570312 (16.22104024887085) + | > loss_duration: 1.7187318801879883 (1.7432067394256592) + | > loss_1: 33.34915542602539 (29.31846809387207) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037288665771484375 (+0.0002714395523071289) + | > avg_loss_disc: 2.5573697090148926 (+0.1271371841430664) + | > avg_loss_disc_real_0: 0.20950570702552795 (+0.06813579052686691) + | > avg_loss_disc_real_1: 0.26132015883922577 (+0.035568781197071075) + | > avg_loss_disc_real_2: 0.23818130046129227 (-0.06667458266019821) + | > avg_loss_disc_real_3: 0.20848868787288666 (-0.03208191692829132) + | > avg_loss_disc_real_4: 0.19673628360033035 (-0.06513570249080658) + | > avg_loss_disc_real_5: 0.2244027704000473 (-0.044531144201755524) + | > avg_loss_0: 2.5573697090148926 (+0.1271371841430664) + | > avg_loss_gen: 2.251416802406311 (-0.3145867586135864) + | > avg_loss_kl: 2.649934768676758 (-0.020369768142700195) + | > avg_loss_feat: 6.452869653701782 (-1.074955940246582) + | > avg_loss_mel: 16.22104024887085 (-2.2426095008850098) + | > avg_loss_duration: 1.7432067394256592 (-0.01206207275390625) + | > avg_loss_1: 29.31846809387207 (-3.6645851135253906) + + + > EPOCH: 356/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:50:05)  + + --> TIME: 2025-04-19 04:50:11 -- STEP: 16/142 -- GLOBAL_STEP: 429850 + | > loss_disc: 2.5689690113067627 (2.444646805524826) + | > loss_disc_real_0: 0.11412010341882706 (0.11888543702661991) + | > loss_disc_real_1: 0.20207317173480988 (0.2139410525560379) + | > loss_disc_real_2: 0.25686413049697876 (0.23419070709496737) + | > loss_disc_real_3: 0.2349437028169632 (0.22417308297008276) + | > loss_disc_real_4: 0.2640056908130646 (0.22832524590194225) + | > loss_disc_real_5: 0.21121209859848022 (0.22264000121504068) + | > loss_0: 2.5689690113067627 (2.444646805524826) + | > grad_norm_0: tensor(8.6182, device='cuda:0') (tensor(7.2787, device='cuda:0')) + | > loss_gen: 2.2779695987701416 (2.376264989376068) + | > loss_kl: 1.81138277053833 (1.822465293109417) + | > loss_feat: 6.6696858406066895 (7.355735629796982) + | > loss_mel: 17.451740264892578 (17.78264945745468) + | > loss_duration: 1.4037045240402222 (1.4123356267809868) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.61448097229004 (30.74945068359375) + | > grad_norm_1: tensor(102.1242, device='cuda:0') (tensor(136.3057, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.3386 (0.32837799191474915) + | > loader_time: 0.0051 (0.00472855567932129) + + + --> TIME: 2025-04-19 04:50:20 -- STEP: 41/142 -- GLOBAL_STEP: 429875 + | > loss_disc: 2.467313289642334 (2.4337242405589032) + | > loss_disc_real_0: 0.09754376858472824 (0.11700547141272848) + | > loss_disc_real_1: 0.14974677562713623 (0.20586746131501546) + | > loss_disc_real_2: 0.23130333423614502 (0.22937462933179809) + | > loss_disc_real_3: 0.2530980110168457 (0.22803627427031353) + | > loss_disc_real_4: 0.23188775777816772 (0.22982155513472674) + | > loss_disc_real_5: 0.2516978681087494 (0.23147341463623977) + | > loss_0: 2.467313289642334 (2.4337242405589032) + | > grad_norm_0: tensor(5.3297, device='cuda:0') (tensor(6.8206, device='cuda:0')) + | > loss_gen: 2.369096040725708 (2.3878075262395346) + | > loss_kl: 1.912104845046997 (1.8376945344413198) + | > loss_feat: 6.723768711090088 (7.3001899021427805) + | > loss_mel: 16.826753616333008 (17.78866016574022) + | > loss_duration: 1.4125272035598755 (1.4254704045086373) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.244251251220703 (30.739822387695312) + | > grad_norm_1: tensor(153.9155, device='cuda:0') (tensor(148.9589, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.3608 (0.34283717667184227) + | > loader_time: 0.0059 (0.005220430653269698) + + + --> TIME: 2025-04-19 04:50:29 -- STEP: 66/142 -- GLOBAL_STEP: 429900 + | > loss_disc: 2.312819719314575 (2.4421199524041377) + | > loss_disc_real_0: 0.0964338555932045 (0.12113105686325015) + | > loss_disc_real_1: 0.18206070363521576 (0.2047580683773214) + | > loss_disc_real_2: 0.19823457300662994 (0.22886242220799127) + | > loss_disc_real_3: 0.2157655954360962 (0.22767332738096063) + | > loss_disc_real_4: 0.21311722695827484 (0.23151928547656897) + | > loss_disc_real_5: 0.20438404381275177 (0.23088121978622494) + | > loss_0: 2.312819719314575 (2.4421199524041377) + | > grad_norm_0: tensor(17.9231, device='cuda:0') (tensor(8.5226, device='cuda:0')) + | > loss_gen: 2.3379452228546143 (2.3735052455555317) + | > loss_kl: 1.9605175256729126 (1.858023155819286) + | > loss_feat: 6.860254287719727 (7.182047114227757) + | > loss_mel: 18.247623443603516 (17.66505582404859) + | > loss_duration: 1.432478427886963 (1.4365209630041411) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.83881950378418 (30.51515209313595) + | > grad_norm_1: tensor(339.2753, device='cuda:0') (tensor(180.1284, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.3862 (0.3565140384616274) + | > loader_time: 0.0067 (0.005567193031311033) + + + --> TIME: 2025-04-19 04:50:40 -- STEP: 91/142 -- GLOBAL_STEP: 429925 + | > loss_disc: 2.4860243797302246 (2.44471291919331) + | > loss_disc_real_0: 0.11544499546289444 (0.1218324594281532) + | > loss_disc_real_1: 0.22096005082130432 (0.20502716499370532) + | > loss_disc_real_2: 0.226094052195549 (0.22805236017966007) + | > loss_disc_real_3: 0.20803962647914886 (0.22766431920476013) + | > loss_disc_real_4: 0.21822787821292877 (0.23014020870675098) + | > loss_disc_real_5: 0.2160428911447525 (0.23083168743075905) + | > loss_0: 2.4860243797302246 (2.44471291919331) + | > grad_norm_0: tensor(6.0961, device='cuda:0') (tensor(9.1994, device='cuda:0')) + | > loss_gen: 2.312791347503662 (2.370309808752039) + | > loss_kl: 1.966827154159546 (1.8736346829068529) + | > loss_feat: 7.229870319366455 (7.165803290985441) + | > loss_mel: 17.381046295166016 (17.640944847693806) + | > loss_duration: 1.4842894077301025 (1.44846924844679) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.37482452392578 (30.499161709795942) + | > grad_norm_1: tensor(239.9011, device='cuda:0') (tensor(191.0892, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.4062 (0.3680895082243196) + | > loader_time: 0.0066 (0.005867358092423321) + + + --> TIME: 2025-04-19 04:50:50 -- STEP: 116/142 -- GLOBAL_STEP: 429950 + | > loss_disc: 2.413759708404541 (2.451714012129554) + | > loss_disc_real_0: 0.1283458024263382 (0.12308321963867237) + | > loss_disc_real_1: 0.22978606820106506 (0.20472148150719446) + | > loss_disc_real_2: 0.25874635577201843 (0.2287626519542316) + | > loss_disc_real_3: 0.2193576544523239 (0.22784662426545702) + | > loss_disc_real_4: 0.23887626826763153 (0.23104977569189564) + | > loss_disc_real_5: 0.19783493876457214 (0.229972738901089) + | > loss_0: 2.413759708404541 (2.451714012129554) + | > grad_norm_0: tensor(17.1534, device='cuda:0') (tensor(9.3272, device='cuda:0')) + | > loss_gen: 2.5447139739990234 (2.3616154563838037) + | > loss_kl: 1.8301658630371094 (1.8741390931195225) + | > loss_feat: 7.271824836730957 (7.112331875439347) + | > loss_mel: 18.140766143798828 (17.566873188676507) + | > loss_duration: 1.4719130992889404 (1.4557026223889717) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.259382247924805 (30.37066204794522) + | > grad_norm_1: tensor(454.1234, device='cuda:0') (tensor(201.8003, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.4326 (0.37888603991475595) + | > loader_time: 0.0069 (0.006119401290498929) + + + --> TIME: 2025-04-19 04:51:01 -- STEP: 141/142 -- GLOBAL_STEP: 429975 + | > loss_disc: 2.1002092361450195 (2.449735560315721) + | > loss_disc_real_0: 0.04504065960645676 (0.1240817278306535) + | > loss_disc_real_1: 0.16855274140834808 (0.20414327782519318) + | > loss_disc_real_2: 0.159496009349823 (0.22810259545948489) + | > loss_disc_real_3: 0.21069641411304474 (0.22826167412683468) + | > loss_disc_real_4: 0.18239636719226837 (0.2305094690610331) + | > loss_disc_real_5: 0.16215907037258148 (0.22981033849378005) + | > loss_0: 2.1002092361450195 (2.449735560315721) + | > grad_norm_0: tensor(7.1093, device='cuda:0') (tensor(9.5857, device='cuda:0')) + | > loss_gen: 2.6270647048950195 (2.3634699726781103) + | > loss_kl: 1.6102533340454102 (1.875177451059328) + | > loss_feat: 9.129002571105957 (7.1114102796459875) + | > loss_mel: 18.668027877807617 (17.57540368695631) + | > loss_duration: 1.3156219720840454 (1.4612611396938353) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 33.34996795654297 (30.38672241589702) + | > grad_norm_1: tensor(291.7695, device='cuda:0') (tensor(212.3448, device='cuda:0')) + | > current_lr_0: 0.00019129458793556815 + | > current_lr_1: 0.00019129458793556815 + | > step_time: 0.2012 (0.38949202645754977) + | > loader_time: 0.0029 (0.006432215372721352) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4124503135681152 (2.4124503135681152) + | > loss_disc_real_0: 0.08604583144187927 (0.08604583144187927) + | > loss_disc_real_1: 0.1589474380016327 (0.1589474380016327) + | > loss_disc_real_2: 0.22956900298595428 (0.22956900298595428) + | > loss_disc_real_3: 0.20631806552410126 (0.20631806552410126) + | > loss_disc_real_4: 0.23093339800834656 (0.23093339800834656) + | > loss_disc_real_5: 0.24231822788715363 (0.24231822788715363) + | > loss_0: 2.4124503135681152 (2.4124503135681152) + | > loss_gen: 2.1725873947143555 (2.1725873947143555) + | > loss_kl: 2.4623334407806396 (2.4623334407806396) + | > loss_feat: 7.772858142852783 (7.772858142852783) + | > loss_mel: 19.400423049926758 (19.400423049926758) + | > loss_duration: 1.7505154609680176 (1.7505154609680176) + | > loss_1: 33.5587158203125 (33.5587158203125) + + --> STEP: 1 + | > loss_disc: 2.561814069747925 (2.561814069747925) + | > loss_disc_real_0: 0.09964599460363388 (0.09964599460363388) + | > loss_disc_real_1: 0.1902337223291397 (0.1902337223291397) + | > loss_disc_real_2: 0.22623005509376526 (0.22623005509376526) + | > loss_disc_real_3: 0.22711913287639618 (0.22711913287639618) + | > loss_disc_real_4: 0.2473037838935852 (0.2473037838935852) + | > loss_disc_real_5: 0.2093603014945984 (0.2093603014945984) + | > loss_0: 2.561814069747925 (2.561814069747925) + | > loss_gen: 1.9359489679336548 (1.9359489679336548) + | > loss_kl: 2.732719898223877 (2.732719898223877) + | > loss_feat: 7.093891143798828 (7.093891143798828) + | > loss_mel: 18.967058181762695 (18.967058181762695) + | > loss_duration: 1.7935748100280762 (1.7935748100280762) + | > loss_1: 32.523193359375 (32.523193359375) + + --> STEP: 2 + | > loss_disc: 2.555478096008301 (2.558646082878113) + | > loss_disc_real_0: 0.07460153102874756 (0.08712376281619072) + | > loss_disc_real_1: 0.19394852221012115 (0.19209112226963043) + | > loss_disc_real_2: 0.2695408761501312 (0.24788546562194824) + | > loss_disc_real_3: 0.24339589476585388 (0.23525751382112503) + | > loss_disc_real_4: 0.20165273547172546 (0.22447825968265533) + | > loss_disc_real_5: 0.24348708987236023 (0.2264236956834793) + | > loss_0: 2.555478096008301 (2.558646082878113) + | > loss_gen: 2.059098243713379 (1.9975236058235168) + | > loss_kl: 2.380603790283203 (2.55666184425354) + | > loss_feat: 5.872835636138916 (6.483363389968872) + | > loss_mel: 15.157306671142578 (17.062182426452637) + | > loss_duration: 1.716793179512024 (1.75518399477005) + | > loss_1: 27.18663787841797 (29.854915618896484) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032634735107421875 (-0.00046539306640625) + | > avg_loss_disc: 2.558646082878113 (+0.0012763738632202148) + | > avg_loss_disc_real_0: 0.08712376281619072 (-0.12238194420933723) + | > avg_loss_disc_real_1: 0.19209112226963043 (-0.06922903656959534) + | > avg_loss_disc_real_2: 0.24788546562194824 (+0.009704165160655975) + | > avg_loss_disc_real_3: 0.23525751382112503 (+0.026768825948238373) + | > avg_loss_disc_real_4: 0.22447825968265533 (+0.02774197608232498) + | > avg_loss_disc_real_5: 0.2264236956834793 (+0.002020925283432007) + | > avg_loss_0: 2.558646082878113 (+0.0012763738632202148) + | > avg_loss_gen: 1.9975236058235168 (-0.2538931965827942) + | > avg_loss_kl: 2.55666184425354 (-0.09327292442321777) + | > avg_loss_feat: 6.483363389968872 (+0.030493736267089844) + | > avg_loss_mel: 17.062182426452637 (+0.8411421775817871) + | > avg_loss_duration: 1.75518399477005 (+0.01197725534439087) + | > avg_loss_1: 29.854915618896484 (+0.5364475250244141) + + + > EPOCH: 357/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:51:04)  + + --> TIME: 2025-04-19 04:51:13 -- STEP: 24/142 -- GLOBAL_STEP: 430000 + | > loss_disc: 2.4692437648773193 (2.455504318078359) + | > loss_disc_real_0: 0.12161918729543686 (0.11698487152655919) + | > loss_disc_real_1: 0.1758720725774765 (0.20847942431767783) + | > loss_disc_real_2: 0.24575379490852356 (0.23151700695355734) + | > loss_disc_real_3: 0.26285162568092346 (0.228194460272789) + | > loss_disc_real_4: 0.24212516844272614 (0.23031597149868807) + | > loss_disc_real_5: 0.2793121337890625 (0.2325375204284986) + | > loss_0: 2.4692437648773193 (2.455504318078359) + | > grad_norm_0: tensor(19.8762, device='cuda:0') (tensor(13.5540, device='cuda:0')) + | > loss_gen: 2.5559537410736084 (2.4293878078460693) + | > loss_kl: 1.890956163406372 (1.8094160159428914) + | > loss_feat: 7.650807857513428 (7.331793824831645) + | > loss_mel: 18.855161666870117 (17.91494774818421) + | > loss_duration: 1.446103811264038 (1.416530619064967) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.398983001708984 (30.90207600593567) + | > grad_norm_1: tensor(269.2424, device='cuda:0') (tensor(319.6062, device='cuda:0')) + | > current_lr_0: 0.0001912706761120762 + | > current_lr_1: 0.0001912706761120762 + | > step_time: 0.3428 (0.333048274119695) + | > loader_time: 0.0055 (0.0049463311831156416) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_430000.pth + + --> TIME: 2025-04-19 04:51:23 -- STEP: 49/142 -- GLOBAL_STEP: 430025 + | > loss_disc: 2.377156972885132 (2.439048786552585) + | > loss_disc_real_0: 0.13177065551280975 (0.12468471081585301) + | > loss_disc_real_1: 0.2256251573562622 (0.20550757707381734) + | > loss_disc_real_2: 0.23633582890033722 (0.2255075476607498) + | > loss_disc_real_3: 0.2121761441230774 (0.22673377790013138) + | > loss_disc_real_4: 0.21053564548492432 (0.2258160223766249) + | > loss_disc_real_5: 0.2329968512058258 (0.22885774410500817) + | > loss_0: 2.377156972885132 (2.439048786552585) + | > grad_norm_0: tensor(28.2108, device='cuda:0') (tensor(16.3884, device='cuda:0')) + | > loss_gen: 2.333686590194702 (2.4266465488745244) + | > loss_kl: 1.8204752206802368 (1.8344071154691735) + | > loss_feat: 7.39524507522583 (7.317908053495446) + | > loss_mel: 18.161352157592773 (17.80237544312769) + | > loss_duration: 1.445889949798584 (1.4261029715440712) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.156648635864258 (30.80744015440649) + | > grad_norm_1: tensor(335.8985, device='cuda:0') (tensor(328.8250, device='cuda:0')) + | > current_lr_0: 0.0001912706761120762 + | > current_lr_1: 0.0001912706761120762 + | > step_time: 0.3699 (0.34515107407861834) + | > loader_time: 0.0059 (0.005398326990555745) + + + --> TIME: 2025-04-19 04:51:33 -- STEP: 74/142 -- GLOBAL_STEP: 430050 + | > loss_disc: 2.485468864440918 (2.4546809003159806) + | > loss_disc_real_0: 0.1281270980834961 (0.12325824777016768) + | > loss_disc_real_1: 0.22979889810085297 (0.2062219469933896) + | > loss_disc_real_2: 0.241681307554245 (0.22748936390554583) + | > loss_disc_real_3: 0.22363612055778503 (0.2278011175426277) + | > loss_disc_real_4: 0.22294215857982635 (0.22953804703177633) + | > loss_disc_real_5: 0.2433953881263733 (0.23298450096233472) + | > loss_0: 2.485468864440918 (2.4546809003159806) + | > grad_norm_0: tensor(4.9032, device='cuda:0') (tensor(15.4894, device='cuda:0')) + | > loss_gen: 2.2380008697509766 (2.406035252519557) + | > loss_kl: 1.884769320487976 (1.8520962273752368) + | > loss_feat: 6.010590553283691 (7.198247696902301) + | > loss_mel: 16.34783935546875 (17.72223173605429) + | > loss_duration: 1.4546020030975342 (1.4398881841350246) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.935800552368164 (30.618499008384912) + | > grad_norm_1: tensor(127.9396, device='cuda:0') (tensor(321.7468, device='cuda:0')) + | > current_lr_0: 0.0001912706761120762 + | > current_lr_1: 0.0001912706761120762 + | > step_time: 0.3882 (0.35726818200704225) + | > loader_time: 0.0064 (0.005657315254211427) + + + --> TIME: 2025-04-19 04:51:43 -- STEP: 99/142 -- GLOBAL_STEP: 430075 + | > loss_disc: 2.388258218765259 (2.4557750080571035) + | > loss_disc_real_0: 0.10480866581201553 (0.1230643286247446) + | > loss_disc_real_1: 0.19858522713184357 (0.2063021686944094) + | > loss_disc_real_2: 0.22398310899734497 (0.22676149161175044) + | > loss_disc_real_3: 0.2004656344652176 (0.22732611557449958) + | > loss_disc_real_4: 0.22207292914390564 (0.22996453669938174) + | > loss_disc_real_5: 0.22263319790363312 (0.23268880582217014) + | > loss_0: 2.388258218765259 (2.4557750080571035) + | > grad_norm_0: tensor(5.0587, device='cuda:0') (tensor(13.5659, device='cuda:0')) + | > loss_gen: 2.3902993202209473 (2.381836929706613) + | > loss_kl: 1.9304323196411133 (1.8610536582542188) + | > loss_feat: 7.288414478302002 (7.133232709133264) + | > loss_mel: 18.478544235229492 (17.6461980800436) + | > loss_duration: 1.5034489631652832 (1.450084609214706) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.59113883972168 (30.47240588640926) + | > grad_norm_1: tensor(83.8721, device='cuda:0') (tensor(294.3251, device='cuda:0')) + | > current_lr_0: 0.0001912706761120762 + | > current_lr_1: 0.0001912706761120762 + | > step_time: 0.405 (0.3679404282810714) + | > loader_time: 0.0066 (0.005921257866753472) + + + --> TIME: 2025-04-19 04:51:54 -- STEP: 124/142 -- GLOBAL_STEP: 430100 + | > loss_disc: 2.364591598510742 (2.458141394199863) + | > loss_disc_real_0: 0.11348166316747665 (0.12431565654133597) + | > loss_disc_real_1: 0.1914704442024231 (0.20703323582007033) + | > loss_disc_real_2: 0.25067418813705444 (0.22668555234709092) + | > loss_disc_real_3: 0.21156153082847595 (0.2278151985618376) + | > loss_disc_real_4: 0.2294158935546875 (0.2297067613370957) + | > loss_disc_real_5: 0.20978066325187683 (0.23165265650999162) + | > loss_0: 2.364591598510742 (2.458141394199863) + | > grad_norm_0: tensor(6.3804, device='cuda:0') (tensor(12.1247, device='cuda:0')) + | > loss_gen: 2.564546585083008 (2.3742248896629587) + | > loss_kl: 2.001307964324951 (1.8668991404194986) + | > loss_feat: 7.633415222167969 (7.128576097949859) + | > loss_mel: 18.606361389160156 (17.679691353151878) + | > loss_duration: 1.5130314826965332 (1.4572678237192094) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.31866455078125 (30.506659215496434) + | > grad_norm_1: tensor(247.0990, device='cuda:0') (tensor(260.8239, device='cuda:0')) + | > current_lr_0: 0.0001912706761120762 + | > current_lr_1: 0.0001912706761120762 + | > step_time: 0.4397 (0.3796945541135728) + | > loader_time: 0.0078 (0.006178036812813052) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5898189544677734 (2.5898189544677734) + | > loss_disc_real_0: 0.21110689640045166 (0.21110689640045166) + | > loss_disc_real_1: 0.20673291385173798 (0.20673291385173798) + | > loss_disc_real_2: 0.23312923312187195 (0.23312923312187195) + | > loss_disc_real_3: 0.25627025961875916 (0.25627025961875916) + | > loss_disc_real_4: 0.257215678691864 (0.257215678691864) + | > loss_disc_real_5: 0.22646665573120117 (0.22646665573120117) + | > loss_0: 2.5898189544677734 (2.5898189544677734) + | > loss_gen: 2.3164124488830566 (2.3164124488830566) + | > loss_kl: 2.4638473987579346 (2.4638473987579346) + | > loss_feat: 6.377223491668701 (6.377223491668701) + | > loss_mel: 17.67125129699707 (17.67125129699707) + | > loss_duration: 1.747603178024292 (1.747603178024292) + | > loss_1: 30.576335906982422 (30.576335906982422) + + --> STEP: 1 + | > loss_disc: 2.5595569610595703 (2.5595569610595703) + | > loss_disc_real_0: 0.24779529869556427 (0.24779529869556427) + | > loss_disc_real_1: 0.20607490837574005 (0.20607490837574005) + | > loss_disc_real_2: 0.21062394976615906 (0.21062394976615906) + | > loss_disc_real_3: 0.21588341891765594 (0.21588341891765594) + | > loss_disc_real_4: 0.24242277443408966 (0.24242277443408966) + | > loss_disc_real_5: 0.26931118965148926 (0.26931118965148926) + | > loss_0: 2.5595569610595703 (2.5595569610595703) + | > loss_gen: 2.3442673683166504 (2.3442673683166504) + | > loss_kl: 2.6690845489501953 (2.6690845489501953) + | > loss_feat: 6.422097682952881 (6.422097682952881) + | > loss_mel: 16.457300186157227 (16.457300186157227) + | > loss_duration: 1.7731441259384155 (1.7731441259384155) + | > loss_1: 29.6658935546875 (29.6658935546875) + + --> STEP: 2 + | > loss_disc: 2.501833200454712 (2.530695080757141) + | > loss_disc_real_0: 0.25009918212890625 (0.24894724041223526) + | > loss_disc_real_1: 0.174166738986969 (0.19012082368135452) + | > loss_disc_real_2: 0.22925332188606262 (0.21993863582611084) + | > loss_disc_real_3: 0.21803361177444458 (0.21695851534605026) + | > loss_disc_real_4: 0.2696974277496338 (0.2560601010918617) + | > loss_disc_real_5: 0.18369828164577484 (0.22650473564863205) + | > loss_0: 2.501833200454712 (2.530695080757141) + | > loss_gen: 2.4439704418182373 (2.394118905067444) + | > loss_kl: 2.3537137508392334 (2.5113991498947144) + | > loss_feat: 6.426257610321045 (6.424177646636963) + | > loss_mel: 17.5083065032959 (16.982803344726562) + | > loss_duration: 1.726203203201294 (1.7496736645698547) + | > loss_1: 30.458450317382812 (30.062171936035156) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003228902816772461 (-3.457069396972656e-05) + | > avg_loss_disc: 2.530695080757141 (-0.02795100212097168) + | > avg_loss_disc_real_0: 0.24894724041223526 (+0.16182347759604454) + | > avg_loss_disc_real_1: 0.19012082368135452 (-0.0019702985882759094) + | > avg_loss_disc_real_2: 0.21993863582611084 (-0.027946829795837402) + | > avg_loss_disc_real_3: 0.21695851534605026 (-0.018298998475074768) + | > avg_loss_disc_real_4: 0.2560601010918617 (+0.03158184140920639) + | > avg_loss_disc_real_5: 0.22650473564863205 (+8.103996515274048e-05) + | > avg_loss_0: 2.530695080757141 (-0.02795100212097168) + | > avg_loss_gen: 2.394118905067444 (+0.396595299243927) + | > avg_loss_kl: 2.5113991498947144 (-0.045262694358825684) + | > avg_loss_feat: 6.424177646636963 (-0.05918574333190918) + | > avg_loss_mel: 16.982803344726562 (-0.07937908172607422) + | > avg_loss_duration: 1.7496736645698547 (-0.0055103302001953125) + | > avg_loss_1: 30.062171936035156 (+0.20725631713867188) + + + > EPOCH: 358/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:52:03)  + + --> TIME: 2025-04-19 04:52:06 -- STEP: 7/142 -- GLOBAL_STEP: 430125 + | > loss_disc: 2.353571891784668 (2.4422618321010043) + | > loss_disc_real_0: 0.10622692108154297 (0.13267113055501664) + | > loss_disc_real_1: 0.2286408692598343 (0.21902242515768325) + | > loss_disc_real_2: 0.20059657096862793 (0.23174412974289485) + | > loss_disc_real_3: 0.21082623302936554 (0.22925423085689545) + | > loss_disc_real_4: 0.21041269600391388 (0.2232097259589604) + | > loss_disc_real_5: 0.2159605771303177 (0.22184079246861593) + | > loss_0: 2.353571891784668 (2.4422618321010043) + | > grad_norm_0: tensor(11.1730, device='cuda:0') (tensor(7.6732, device='cuda:0')) + | > loss_gen: 2.327836751937866 (2.34534021786281) + | > loss_kl: 1.801773190498352 (1.8012887580054147) + | > loss_feat: 7.970849990844727 (7.222284248897007) + | > loss_mel: 17.9793758392334 (17.37402479989188) + | > loss_duration: 1.408665657043457 (1.4124928712844849) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.488502502441406 (30.15543120247977) + | > grad_norm_1: tensor(239.0844, device='cuda:0') (tensor(163.4613, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.3217 (0.32416997637067524) + | > loader_time: 0.0044 (0.004507371357509068) + + + --> TIME: 2025-04-19 04:52:15 -- STEP: 32/142 -- GLOBAL_STEP: 430150 + | > loss_disc: 2.296241283416748 (2.431002028286457) + | > loss_disc_real_0: 0.11185941845178604 (0.1204170873388648) + | > loss_disc_real_1: 0.21504338085651398 (0.20953551633283496) + | > loss_disc_real_2: 0.218027725815773 (0.22718575457111) + | > loss_disc_real_3: 0.24203771352767944 (0.22287064930424094) + | > loss_disc_real_4: 0.2373170405626297 (0.22566190687939525) + | > loss_disc_real_5: 0.20330213010311127 (0.22409535245969892) + | > loss_0: 2.296241283416748 (2.431002028286457) + | > grad_norm_0: tensor(4.3994, device='cuda:0') (tensor(6.9320, device='cuda:0')) + | > loss_gen: 2.538444995880127 (2.3717757388949394) + | > loss_kl: 1.8487508296966553 (1.8300532512366772) + | > loss_feat: 7.736282825469971 (7.277026548981667) + | > loss_mel: 17.307743072509766 (17.65118658542633) + | > loss_duration: 1.4196228981018066 (1.41924636811018) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.85084342956543 (30.549288630485535) + | > grad_norm_1: tensor(160.9945, device='cuda:0') (tensor(154.3817, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.3564 (0.3434712365269661) + | > loader_time: 0.0053 (0.005170933902263641) + + + --> TIME: 2025-04-19 04:52:25 -- STEP: 57/142 -- GLOBAL_STEP: 430175 + | > loss_disc: 2.535288095474243 (2.4289134552604277) + | > loss_disc_real_0: 0.1441943496465683 (0.12090040429642326) + | > loss_disc_real_1: 0.22732248902320862 (0.2073407201913365) + | > loss_disc_real_2: 0.2614879012107849 (0.2250965355258239) + | > loss_disc_real_3: 0.1969337910413742 (0.22540966904999918) + | > loss_disc_real_4: 0.23834708333015442 (0.2259941582094159) + | > loss_disc_real_5: 0.25480780005455017 (0.22852342510432527) + | > loss_0: 2.535288095474243 (2.4289134552604277) + | > grad_norm_0: tensor(3.9550, device='cuda:0') (tensor(7.1261, device='cuda:0')) + | > loss_gen: 2.2812843322753906 (2.385787060386257) + | > loss_kl: 1.9471620321273804 (1.8359530955030208) + | > loss_feat: 6.635192394256592 (7.290329054782265) + | > loss_mel: 17.07711410522461 (17.7022351716694) + | > loss_duration: 1.485181450843811 (1.4317980774661954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.425935745239258 (30.64610257065087) + | > grad_norm_1: tensor(97.4546, device='cuda:0') (tensor(150.0294, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.3731 (0.3527295003857529) + | > loader_time: 0.0059 (0.005554993947347005) + + + --> TIME: 2025-04-19 04:52:35 -- STEP: 82/142 -- GLOBAL_STEP: 430200 + | > loss_disc: 2.4106500148773193 (2.4424007404141315) + | > loss_disc_real_0: 0.12365055084228516 (0.123506678586326) + | > loss_disc_real_1: 0.22418469190597534 (0.20680825593994884) + | > loss_disc_real_2: 0.24077197909355164 (0.2243247026713883) + | > loss_disc_real_3: 0.18969571590423584 (0.22586867485831424) + | > loss_disc_real_4: 0.16759748756885529 (0.2258566494395093) + | > loss_disc_real_5: 0.2244805246591568 (0.23155663253330602) + | > loss_0: 2.4106500148773193 (2.4424007404141315) + | > grad_norm_0: tensor(10.7530, device='cuda:0') (tensor(6.8151, device='cuda:0')) + | > loss_gen: 2.22953200340271 (2.3758482293384837) + | > loss_kl: 1.935869812965393 (1.842414050567441) + | > loss_feat: 6.831432342529297 (7.237416924499884) + | > loss_mel: 17.2171630859375 (17.7074164064919) + | > loss_duration: 1.524749994277954 (1.443390775017622) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.73874855041504 (30.606486367016302) + | > grad_norm_1: tensor(100.6089, device='cuda:0') (tensor(134.4031, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.3935 (0.3636723117130559) + | > loader_time: 0.0063 (0.005798034551667003) + + + --> TIME: 2025-04-19 04:52:45 -- STEP: 107/142 -- GLOBAL_STEP: 430225 + | > loss_disc: 2.4469428062438965 (2.453970519181725) + | > loss_disc_real_0: 0.13205735385417938 (0.12507202274331417) + | > loss_disc_real_1: 0.17845965921878815 (0.20644087810939718) + | > loss_disc_real_2: 0.22620485723018646 (0.22590934116149616) + | > loss_disc_real_3: 0.22945182025432587 (0.2261513539842356) + | > loss_disc_real_4: 0.20750898122787476 (0.22848930634627831) + | > loss_disc_real_5: 0.2081233710050583 (0.2324621081352234) + | > loss_0: 2.4469428062438965 (2.453970519181725) + | > grad_norm_0: tensor(4.5155, device='cuda:0') (tensor(6.8177, device='cuda:0')) + | > loss_gen: 2.2451844215393066 (2.36631810330899) + | > loss_kl: 1.8558390140533447 (1.8501482834325773) + | > loss_feat: 7.418851852416992 (7.201877580624875) + | > loss_mel: 18.136638641357422 (17.74509121547235) + | > loss_duration: 1.4587185382843018 (1.452370831899554) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.115232467651367 (30.615805991342135) + | > grad_norm_1: tensor(77.8347, device='cuda:0') (tensor(126.2770, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.4158 (0.3735060803244047) + | > loader_time: 0.0068 (0.006030452585665977) + + + --> TIME: 2025-04-19 04:52:56 -- STEP: 132/142 -- GLOBAL_STEP: 430250 + | > loss_disc: 2.4047932624816895 (2.4575809944759723) + | > loss_disc_real_0: 0.12031304091215134 (0.12627824040299107) + | > loss_disc_real_1: 0.21655964851379395 (0.2072997081911925) + | > loss_disc_real_2: 0.21683350205421448 (0.22602649913592773) + | > loss_disc_real_3: 0.20362260937690735 (0.22620322555303574) + | > loss_disc_real_4: 0.20310485363006592 (0.22799056542642188) + | > loss_disc_real_5: 0.18621380627155304 (0.23250730335712433) + | > loss_0: 2.4047932624816895 (2.4575809944759723) + | > grad_norm_0: tensor(14.4729, device='cuda:0') (tensor(7.4023, device='cuda:0')) + | > loss_gen: 2.2929773330688477 (2.3623638767184634) + | > loss_kl: 1.8212438821792603 (1.8488100532329443) + | > loss_feat: 7.389251232147217 (7.1497325066364175) + | > loss_mel: 16.724103927612305 (17.7074963685238) + | > loss_duration: 1.4674826860427856 (1.4590788664239829) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.69506072998047 (30.52748174378366) + | > grad_norm_1: tensor(449.9673, device='cuda:0') (tensor(145.7872, device='cuda:0')) + | > current_lr_0: 0.00019124676727756216 + | > current_lr_1: 0.00019124676727756216 + | > step_time: 0.447 (0.38529526645487006) + | > loader_time: 0.0078 (0.006293231790715996) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5762815475463867 (2.5762815475463867) + | > loss_disc_real_0: 0.17146553099155426 (0.17146553099155426) + | > loss_disc_real_1: 0.27941304445266724 (0.27941304445266724) + | > loss_disc_real_2: 0.2885085344314575 (0.2885085344314575) + | > loss_disc_real_3: 0.23863188922405243 (0.23863188922405243) + | > loss_disc_real_4: 0.29662656784057617 (0.29662656784057617) + | > loss_disc_real_5: 0.2738577723503113 (0.2738577723503113) + | > loss_0: 2.5762815475463867 (2.5762815475463867) + | > loss_gen: 2.611046314239502 (2.611046314239502) + | > loss_kl: 2.341081142425537 (2.341081142425537) + | > loss_feat: 6.886373519897461 (6.886373519897461) + | > loss_mel: 18.680259704589844 (18.680259704589844) + | > loss_duration: 1.7384122610092163 (1.7384122610092163) + | > loss_1: 32.257171630859375 (32.257171630859375) + + --> STEP: 1 + | > loss_disc: 2.589815855026245 (2.589815855026245) + | > loss_disc_real_0: 0.17923010885715485 (0.17923010885715485) + | > loss_disc_real_1: 0.2528224587440491 (0.2528224587440491) + | > loss_disc_real_2: 0.29621803760528564 (0.29621803760528564) + | > loss_disc_real_3: 0.24468189477920532 (0.24468189477920532) + | > loss_disc_real_4: 0.22415800392627716 (0.22415800392627716) + | > loss_disc_real_5: 0.26214101910591125 (0.26214101910591125) + | > loss_0: 2.589815855026245 (2.589815855026245) + | > loss_gen: 2.405557155609131 (2.405557155609131) + | > loss_kl: 2.634350538253784 (2.634350538253784) + | > loss_feat: 7.165035247802734 (7.165035247802734) + | > loss_mel: 19.02507209777832 (19.02507209777832) + | > loss_duration: 1.7689855098724365 (1.7689855098724365) + | > loss_1: 32.999000549316406 (32.999000549316406) + + --> STEP: 2 + | > loss_disc: 2.387099504470825 (2.488457679748535) + | > loss_disc_real_0: 0.11569587886333466 (0.14746299386024475) + | > loss_disc_real_1: 0.22612833976745605 (0.23947539925575256) + | > loss_disc_real_2: 0.2990536391735077 (0.29763583838939667) + | > loss_disc_real_3: 0.27533039450645447 (0.2600061446428299) + | > loss_disc_real_4: 0.2557646632194519 (0.23996133357286453) + | > loss_disc_real_5: 0.26935893297195435 (0.2657499760389328) + | > loss_0: 2.387099504470825 (2.488457679748535) + | > loss_gen: 2.5710256099700928 (2.488291382789612) + | > loss_kl: 2.0605194568634033 (2.3474349975585938) + | > loss_feat: 7.527914047241211 (7.346474647521973) + | > loss_mel: 18.99875259399414 (19.01191234588623) + | > loss_duration: 1.7107470035552979 (1.7398662567138672) + | > loss_1: 32.86895751953125 (32.93397903442383) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00349271297454834 (+0.0002638101577758789) + | > avg_loss_disc: 2.488457679748535 (-0.04223740100860596) + | > avg_loss_disc_real_0: 0.14746299386024475 (-0.10148424655199051) + | > avg_loss_disc_real_1: 0.23947539925575256 (+0.04935457557439804) + | > avg_loss_disc_real_2: 0.29763583838939667 (+0.07769720256328583) + | > avg_loss_disc_real_3: 0.2600061446428299 (+0.04304762929677963) + | > avg_loss_disc_real_4: 0.23996133357286453 (-0.016098767518997192) + | > avg_loss_disc_real_5: 0.2657499760389328 (+0.03924524039030075) + | > avg_loss_0: 2.488457679748535 (-0.04223740100860596) + | > avg_loss_gen: 2.488291382789612 (+0.09417247772216797) + | > avg_loss_kl: 2.3474349975585938 (-0.1639641523361206) + | > avg_loss_feat: 7.346474647521973 (+0.9222970008850098) + | > avg_loss_mel: 19.01191234588623 (+2.029109001159668) + | > avg_loss_duration: 1.7398662567138672 (-0.009807407855987549) + | > avg_loss_1: 32.93397903442383 (+2.871807098388672) + + + > EPOCH: 359/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:53:02)  + + --> TIME: 2025-04-19 04:53:08 -- STEP: 15/142 -- GLOBAL_STEP: 430275 + | > loss_disc: 2.3498642444610596 (2.41969100634257) + | > loss_disc_real_0: 0.08115893602371216 (0.11393603533506394) + | > loss_disc_real_1: 0.1971750408411026 (0.20156932870546976) + | > loss_disc_real_2: 0.23663876950740814 (0.2222797950108846) + | > loss_disc_real_3: 0.24605271220207214 (0.2282514085372289) + | > loss_disc_real_4: 0.21266265213489532 (0.23708652158578236) + | > loss_disc_real_5: 0.23404741287231445 (0.2256954739491145) + | > loss_0: 2.3498642444610596 (2.41969100634257) + | > grad_norm_0: tensor(7.5862, device='cuda:0') (tensor(10.6136, device='cuda:0')) + | > loss_gen: 2.5643506050109863 (2.4089353879292807) + | > loss_kl: 1.799206256866455 (1.8240115960439047) + | > loss_feat: 7.894808769226074 (7.294861888885498) + | > loss_mel: 19.246997833251953 (17.958976364135744) + | > loss_duration: 1.4229756593704224 (1.414189354578654) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.928340911865234 (30.90097452799479) + | > grad_norm_1: tensor(315.2585, device='cuda:0') (tensor(245.2888, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.3309 (0.33679766654968263) + | > loader_time: 0.0057 (0.004942019780476889) + + + --> TIME: 2025-04-19 04:53:17 -- STEP: 40/142 -- GLOBAL_STEP: 430300 + | > loss_disc: 2.4596753120422363 (2.449942582845689) + | > loss_disc_real_0: 0.1137622743844986 (0.12080437019467354) + | > loss_disc_real_1: 0.1693309247493744 (0.20560012571513653) + | > loss_disc_real_2: 0.2022685557603836 (0.22738995514810084) + | > loss_disc_real_3: 0.25298845767974854 (0.22962801679968833) + | > loss_disc_real_4: 0.24240447580814362 (0.23252934217453003) + | > loss_disc_real_5: 0.26480725407600403 (0.23208552971482277) + | > loss_0: 2.4596753120422363 (2.449942582845689) + | > grad_norm_0: tensor(12.3210, device='cuda:0') (tensor(9.8239, device='cuda:0')) + | > loss_gen: 2.26930570602417 (2.3829194724559786) + | > loss_kl: 1.8693137168884277 (1.8338013261556625) + | > loss_feat: 7.6726789474487305 (7.287945055961609) + | > loss_mel: 18.10285186767578 (17.88748621940613) + | > loss_duration: 1.4115986824035645 (1.42566859126091) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.325748443603516 (30.817820453643797) + | > grad_norm_1: tensor(350.2152, device='cuda:0') (tensor(244.3799, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.3616 (0.34470742940902716) + | > loader_time: 0.006 (0.005436551570892333) + + + --> TIME: 2025-04-19 04:53:26 -- STEP: 65/142 -- GLOBAL_STEP: 430325 + | > loss_disc: 2.525413751602173 (2.4540169569162225) + | > loss_disc_real_0: 0.11747486889362335 (0.1221574074946917) + | > loss_disc_real_1: 0.18094274401664734 (0.20723909460581266) + | > loss_disc_real_2: 0.22783713042736053 (0.22569427559008964) + | > loss_disc_real_3: 0.23760537803173065 (0.22736073136329651) + | > loss_disc_real_4: 0.22246341407299042 (0.22996791211458353) + | > loss_disc_real_5: 0.2595165967941284 (0.2316100741808231) + | > loss_0: 2.525413751602173 (2.4540169569162225) + | > grad_norm_0: tensor(7.1002, device='cuda:0') (tensor(8.5649, device='cuda:0')) + | > loss_gen: 2.2761125564575195 (2.3709389686584474) + | > loss_kl: 1.805023193359375 (1.8377315612939689) + | > loss_feat: 6.851088047027588 (7.258526530632606) + | > loss_mel: 17.661561965942383 (17.909655996469358) + | > loss_duration: 1.4593780040740967 (1.4380767547167264) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05316162109375 (30.81492966871995) + | > grad_norm_1: tensor(138.7422, device='cuda:0') (tensor(192.8497, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.3911 (0.3551220637101394) + | > loader_time: 0.0078 (0.005749808824979341) + + + --> TIME: 2025-04-19 04:53:37 -- STEP: 90/142 -- GLOBAL_STEP: 430350 + | > loss_disc: 2.4718973636627197 (2.4590537495083287) + | > loss_disc_real_0: 0.13775692880153656 (0.12355444472697046) + | > loss_disc_real_1: 0.21946382522583008 (0.20744172400898403) + | > loss_disc_real_2: 0.21819818019866943 (0.22567595061328677) + | > loss_disc_real_3: 0.22200828790664673 (0.2280857922302352) + | > loss_disc_real_4: 0.19097109138965607 (0.2302553160323037) + | > loss_disc_real_5: 0.1874028444290161 (0.23166111558675767) + | > loss_0: 2.4718973636627197 (2.4590537495083287) + | > grad_norm_0: tensor(4.8423, device='cuda:0') (tensor(8.1445, device='cuda:0')) + | > loss_gen: 2.2973434925079346 (2.361539154582553) + | > loss_kl: 1.711266040802002 (1.8516096207830641) + | > loss_feat: 7.527743816375732 (7.168569225735134) + | > loss_mel: 18.49310874938965 (17.80906742943658) + | > loss_duration: 1.4958076477050781 (1.4478845556577047) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.525270462036133 (30.63866990407308) + | > grad_norm_1: tensor(173.2349, device='cuda:0') (tensor(180.5639, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.4015 (0.36592243777381056) + | > loader_time: 0.0067 (0.0059777922100491) + + + --> TIME: 2025-04-19 04:53:47 -- STEP: 115/142 -- GLOBAL_STEP: 430375 + | > loss_disc: 2.372699499130249 (2.4625200478926947) + | > loss_disc_real_0: 0.11621391773223877 (0.125208738575811) + | > loss_disc_real_1: 0.21950991451740265 (0.2076551099186358) + | > loss_disc_real_2: 0.2127828747034073 (0.2264862370231877) + | > loss_disc_real_3: 0.2498198002576828 (0.2276239089343859) + | > loss_disc_real_4: 0.2215779572725296 (0.22997227544369905) + | > loss_disc_real_5: 0.2127504050731659 (0.23130561657573867) + | > loss_0: 2.372699499130249 (2.4625200478926947) + | > grad_norm_0: tensor(7.9113, device='cuda:0') (tensor(8.1883, device='cuda:0')) + | > loss_gen: 2.5162193775177 (2.350930201488992) + | > loss_kl: 1.9501372575759888 (1.8591152522874914) + | > loss_feat: 7.489958763122559 (7.121088591865871) + | > loss_mel: 19.489227294921875 (17.790856311632233) + | > loss_duration: 1.5036847591400146 (1.4549583569816922) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.94922637939453 (30.576948663462762) + | > grad_norm_1: tensor(125.9335, device='cuda:0') (tensor(183.3593, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.4234 (0.3762590574181599) + | > loader_time: 0.0079 (0.006224671654079271) + + + --> TIME: 2025-04-19 04:53:58 -- STEP: 140/142 -- GLOBAL_STEP: 430400 + | > loss_disc: 2.4875574111938477 (2.460118067264557) + | > loss_disc_real_0: 0.15680690109729767 (0.12589863283293587) + | > loss_disc_real_1: 0.2073742151260376 (0.20695774778723716) + | > loss_disc_real_2: 0.19942882657051086 (0.22636646351643971) + | > loss_disc_real_3: 0.20944933593273163 (0.2277314562882696) + | > loss_disc_real_4: 0.23048612475395203 (0.22997115903667043) + | > loss_disc_real_5: 0.22765794396400452 (0.23068500118596214) + | > loss_0: 2.4875574111938477 (2.460118067264557) + | > grad_norm_0: tensor(10.0843, device='cuda:0') (tensor(8.3491, device='cuda:0')) + | > loss_gen: 2.467139482498169 (2.349609856946128) + | > loss_kl: 1.7279962301254272 (1.8596004452024186) + | > loss_feat: 7.524211883544922 (7.094642795835223) + | > loss_mel: 18.053159713745117 (17.700912645884916) + | > loss_duration: 1.5250592231750488 (1.4615854884896962) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.297565460205078 (30.466351195744107) + | > grad_norm_1: tensor(421.3564, device='cuda:0') (tensor(184.8452, device='cuda:0')) + | > current_lr_0: 0.00019122286143165244 + | > current_lr_1: 0.00019122286143165244 + | > step_time: 0.4576 (0.3880615523883275) + | > loader_time: 0.0079 (0.006486899512154715) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4548797607421875 (2.4548797607421875) + | > loss_disc_real_0: 0.1426001489162445 (0.1426001489162445) + | > loss_disc_real_1: 0.20135971903800964 (0.20135971903800964) + | > loss_disc_real_2: 0.22760848701000214 (0.22760848701000214) + | > loss_disc_real_3: 0.25549769401550293 (0.25549769401550293) + | > loss_disc_real_4: 0.25605159997940063 (0.25605159997940063) + | > loss_disc_real_5: 0.3070276081562042 (0.3070276081562042) + | > loss_0: 2.4548797607421875 (2.4548797607421875) + | > loss_gen: 2.48239803314209 (2.48239803314209) + | > loss_kl: 2.3457367420196533 (2.3457367420196533) + | > loss_feat: 7.314389705657959 (7.314389705657959) + | > loss_mel: 18.677597045898438 (18.677597045898438) + | > loss_duration: 1.7449052333831787 (1.7449052333831787) + | > loss_1: 32.565025329589844 (32.565025329589844) + + --> STEP: 1 + | > loss_disc: 2.5611729621887207 (2.5611729621887207) + | > loss_disc_real_0: 0.17402110993862152 (0.17402110993862152) + | > loss_disc_real_1: 0.24484966695308685 (0.24484966695308685) + | > loss_disc_real_2: 0.22403933107852936 (0.22403933107852936) + | > loss_disc_real_3: 0.2125001698732376 (0.2125001698732376) + | > loss_disc_real_4: 0.19188246130943298 (0.19188246130943298) + | > loss_disc_real_5: 0.2556447386741638 (0.2556447386741638) + | > loss_0: 2.5611729621887207 (2.5611729621887207) + | > loss_gen: 2.2963945865631104 (2.2963945865631104) + | > loss_kl: 2.8131301403045654 (2.8131301403045654) + | > loss_feat: 6.364319324493408 (6.364319324493408) + | > loss_mel: 17.552759170532227 (17.552759170532227) + | > loss_duration: 1.7658326625823975 (1.7658326625823975) + | > loss_1: 30.792436599731445 (30.792436599731445) + + --> STEP: 2 + | > loss_disc: 2.4318888187408447 (2.4965308904647827) + | > loss_disc_real_0: 0.16826753318309784 (0.17114432156085968) + | > loss_disc_real_1: 0.227702796459198 (0.23627623170614243) + | > loss_disc_real_2: 0.23997323215007782 (0.2320062816143036) + | > loss_disc_real_3: 0.2333214432001114 (0.2229108065366745) + | > loss_disc_real_4: 0.22672584652900696 (0.20930415391921997) + | > loss_disc_real_5: 0.2613351047039032 (0.2584899216890335) + | > loss_0: 2.4318888187408447 (2.4965308904647827) + | > loss_gen: 2.4954261779785156 (2.395910382270813) + | > loss_kl: 2.5528087615966797 (2.6829694509506226) + | > loss_feat: 7.309512138366699 (6.836915731430054) + | > loss_mel: 17.855365753173828 (17.704062461853027) + | > loss_duration: 1.718163013458252 (1.7419978380203247) + | > loss_1: 31.9312744140625 (31.361855506896973) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00401914119720459 (+0.00052642822265625) + | > avg_loss_disc: 2.4965308904647827 (+0.008073210716247559) + | > avg_loss_disc_real_0: 0.17114432156085968 (+0.02368132770061493) + | > avg_loss_disc_real_1: 0.23627623170614243 (-0.003199167549610138) + | > avg_loss_disc_real_2: 0.2320062816143036 (-0.06562955677509308) + | > avg_loss_disc_real_3: 0.2229108065366745 (-0.037095338106155396) + | > avg_loss_disc_real_4: 0.20930415391921997 (-0.030657179653644562) + | > avg_loss_disc_real_5: 0.2584899216890335 (-0.007260054349899292) + | > avg_loss_0: 2.4965308904647827 (+0.008073210716247559) + | > avg_loss_gen: 2.395910382270813 (-0.09238100051879883) + | > avg_loss_kl: 2.6829694509506226 (+0.3355344533920288) + | > avg_loss_feat: 6.836915731430054 (-0.509558916091919) + | > avg_loss_mel: 17.704062461853027 (-1.3078498840332031) + | > avg_loss_duration: 1.7419978380203247 (+0.0021315813064575195) + | > avg_loss_1: 31.361855506896973 (-1.5721235275268555) + + + > EPOCH: 360/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:54:01)  + + --> TIME: 2025-04-19 04:54:09 -- STEP: 23/142 -- GLOBAL_STEP: 430425 + | > loss_disc: 2.6227219104766846 (2.410604622053063) + | > loss_disc_real_0: 0.11170034855604172 (0.11185914537181026) + | > loss_disc_real_1: 0.2392781674861908 (0.20273017753725467) + | > loss_disc_real_2: 0.2684638202190399 (0.2234709813542988) + | > loss_disc_real_3: 0.25262370705604553 (0.22799369239288828) + | > loss_disc_real_4: 0.2649385631084442 (0.2277787068615789) + | > loss_disc_real_5: 0.2926606237888336 (0.2367485962484194) + | > loss_0: 2.6227219104766846 (2.410604622053063) + | > grad_norm_0: tensor(9.0572, device='cuda:0') (tensor(10.4849, device='cuda:0')) + | > loss_gen: 2.2391204833984375 (2.4398425143697984) + | > loss_kl: 1.8389945030212402 (1.7525997265525486) + | > loss_feat: 6.885368824005127 (7.350536989129108) + | > loss_mel: 17.147071838378906 (17.51355279010275) + | > loss_duration: 1.429875135421753 (1.4155699532964956) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.540430068969727 (30.472102621327277) + | > grad_norm_1: tensor(331.4266, device='cuda:0') (tensor(294.1498, device='cuda:0')) + | > current_lr_0: 0.0001911989585739735 + | > current_lr_1: 0.0001911989585739735 + | > step_time: 0.3448 (0.3332953556724217) + | > loader_time: 0.0053 (0.005029823469079059) + + + --> TIME: 2025-04-19 04:54:19 -- STEP: 48/142 -- GLOBAL_STEP: 430450 + | > loss_disc: 2.4565224647521973 (2.434093837936719) + | > loss_disc_real_0: 0.21615977585315704 (0.11611329500253002) + | > loss_disc_real_1: 0.22064141929149628 (0.20422895004351935) + | > loss_disc_real_2: 0.2398245483636856 (0.22529996217538914) + | > loss_disc_real_3: 0.2537780702114105 (0.22791197368254265) + | > loss_disc_real_4: 0.2563764452934265 (0.2272561335315307) + | > loss_disc_real_5: 0.23679254949092865 (0.23284874794383845) + | > loss_0: 2.4565224647521973 (2.434093837936719) + | > grad_norm_0: tensor(16.2893, device='cuda:0') (tensor(13.3965, device='cuda:0')) + | > loss_gen: 2.712137222290039 (2.45301487048467) + | > loss_kl: 1.8447586297988892 (1.786690170566241) + | > loss_feat: 6.942016124725342 (7.324070632457733) + | > loss_mel: 18.336950302124023 (17.64654602607091) + | > loss_duration: 1.461104393005371 (1.4291556775569916) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.296966552734375 (30.639477769533794) + | > grad_norm_1: tensor(165.2295, device='cuda:0') (tensor(308.3139, device='cuda:0')) + | > current_lr_0: 0.0001911989585739735 + | > current_lr_1: 0.0001911989585739735 + | > step_time: 0.3652 (0.34538876016934705) + | > loader_time: 0.0056 (0.005341296394666036) + + + --> TIME: 2025-04-19 04:54:28 -- STEP: 73/142 -- GLOBAL_STEP: 430475 + | > loss_disc: 2.4938361644744873 (2.4372688515545566) + | > loss_disc_real_0: 0.11045711487531662 (0.11744786278433995) + | > loss_disc_real_1: 0.2206423431634903 (0.20428805134884298) + | > loss_disc_real_2: 0.22300805151462555 (0.2263272372010636) + | > loss_disc_real_3: 0.22629234194755554 (0.22697942387567807) + | > loss_disc_real_4: 0.2684459984302521 (0.2282053911930894) + | > loss_disc_real_5: 0.25340747833251953 (0.2328090986160383) + | > loss_0: 2.4938361644744873 (2.4372688515545566) + | > grad_norm_0: tensor(8.0386, device='cuda:0') (tensor(13.9045, device='cuda:0')) + | > loss_gen: 2.4201416969299316 (2.4309569548254144) + | > loss_kl: 1.8799062967300415 (1.8218783042202258) + | > loss_feat: 6.877727508544922 (7.307555845339004) + | > loss_mel: 17.04486083984375 (17.73738483533468) + | > loss_duration: 1.507162094116211 (1.4421068593247297) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.72979736328125 (30.739883187698993) + | > grad_norm_1: tensor(75.4170, device='cuda:0') (tensor(304.2981, device='cuda:0')) + | > current_lr_0: 0.0001911989585739735 + | > current_lr_1: 0.0001911989585739735 + | > step_time: 0.3965 (0.35786453338518537) + | > loader_time: 0.0067 (0.005715961325658511) + + + --> TIME: 2025-04-19 04:54:39 -- STEP: 98/142 -- GLOBAL_STEP: 430500 + | > loss_disc: 2.474165201187134 (2.436068109103612) + | > loss_disc_real_0: 0.12305953353643417 (0.11810222602620417) + | > loss_disc_real_1: 0.20811155438423157 (0.20441122170613737) + | > loss_disc_real_2: 0.22208243608474731 (0.22657250172021437) + | > loss_disc_real_3: 0.2595160901546478 (0.226161429164361) + | > loss_disc_real_4: 0.2400885671377182 (0.22806044561522348) + | > loss_disc_real_5: 0.2768993675708771 (0.23165446899983347) + | > loss_0: 2.474165201187134 (2.436068109103612) + | > grad_norm_0: tensor(5.9040, device='cuda:0') (tensor(13.0601, device='cuda:0')) + | > loss_gen: 2.417908191680908 (2.4176976169858655) + | > loss_kl: 1.8493481874465942 (1.8407306160245622) + | > loss_feat: 7.634451389312744 (7.278419713584745) + | > loss_mel: 17.20340919494629 (17.77595937495329) + | > loss_duration: 1.4486746788024902 (1.4523261870656698) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.55379295349121 (30.765133799338827) + | > grad_norm_1: tensor(88.2881, device='cuda:0') (tensor(291.1784, device='cuda:0')) + | > current_lr_0: 0.0001911989585739735 + | > current_lr_1: 0.0001911989585739735 + | > step_time: 0.4148 (0.3692255555366984) + | > loader_time: 0.0072 (0.005946538886245416) + + + --> TIME: 2025-04-19 04:54:50 -- STEP: 123/142 -- GLOBAL_STEP: 430525 + | > loss_disc: 2.5332415103912354 (2.4418197608575594) + | > loss_disc_real_0: 0.12322578579187393 (0.12008613735679688) + | > loss_disc_real_1: 0.20928546786308289 (0.20469440180596296) + | > loss_disc_real_2: 0.2229255884885788 (0.2264390561154218) + | > loss_disc_real_3: 0.24963709712028503 (0.22663837392640307) + | > loss_disc_real_4: 0.22133874893188477 (0.22959974747363146) + | > loss_disc_real_5: 0.2383846491575241 (0.23175316358484874) + | > loss_0: 2.5332415103912354 (2.4418197608575594) + | > grad_norm_0: tensor(6.0475, device='cuda:0') (tensor(12.0864, device='cuda:0')) + | > loss_gen: 2.188505172729492 (2.3979889261044134) + | > loss_kl: 1.9408446550369263 (1.8497535193838723) + | > loss_feat: 6.340540409088135 (7.202612458205804) + | > loss_mel: 16.36132049560547 (17.710632766165386) + | > loss_duration: 1.474593162536621 (1.4592410519840273) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.305805206298828 (30.62022897673816) + | > grad_norm_1: tensor(169.4264, device='cuda:0') (tensor(265.5302, device='cuda:0')) + | > current_lr_0: 0.0001911989585739735 + | > current_lr_1: 0.0001911989585739735 + | > step_time: 0.4369 (0.38111321906733325) + | > loader_time: 0.0074 (0.0061981018965806415) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4490506649017334 (2.4490506649017334) + | > loss_disc_real_0: 0.2538255751132965 (0.2538255751132965) + | > loss_disc_real_1: 0.18570221960544586 (0.18570221960544586) + | > loss_disc_real_2: 0.23568768799304962 (0.23568768799304962) + | > loss_disc_real_3: 0.22762531042099 (0.22762531042099) + | > loss_disc_real_4: 0.2262008786201477 (0.2262008786201477) + | > loss_disc_real_5: 0.2421097606420517 (0.2421097606420517) + | > loss_0: 2.4490506649017334 (2.4490506649017334) + | > loss_gen: 2.590339183807373 (2.590339183807373) + | > loss_kl: 2.608442783355713 (2.608442783355713) + | > loss_feat: 7.917637348175049 (7.917637348175049) + | > loss_mel: 18.63786506652832 (18.63786506652832) + | > loss_duration: 1.7503758668899536 (1.7503758668899536) + | > loss_1: 33.504661560058594 (33.504661560058594) + + --> STEP: 1 + | > loss_disc: 2.5481905937194824 (2.5481905937194824) + | > loss_disc_real_0: 0.20464959740638733 (0.20464959740638733) + | > loss_disc_real_1: 0.21600691974163055 (0.21600691974163055) + | > loss_disc_real_2: 0.2394685447216034 (0.2394685447216034) + | > loss_disc_real_3: 0.25223323702812195 (0.25223323702812195) + | > loss_disc_real_4: 0.25033268332481384 (0.25033268332481384) + | > loss_disc_real_5: 0.23000960052013397 (0.23000960052013397) + | > loss_0: 2.5481905937194824 (2.5481905937194824) + | > loss_gen: 2.40792179107666 (2.40792179107666) + | > loss_kl: 2.788557529449463 (2.788557529449463) + | > loss_feat: 7.045873641967773 (7.045873641967773) + | > loss_mel: 18.452232360839844 (18.452232360839844) + | > loss_duration: 1.7823855876922607 (1.7823855876922607) + | > loss_1: 32.47697067260742 (32.47697067260742) + + --> STEP: 2 + | > loss_disc: 2.3923163414001465 (2.4702534675598145) + | > loss_disc_real_0: 0.17538076639175415 (0.19001518189907074) + | > loss_disc_real_1: 0.22229456901550293 (0.21915074437856674) + | > loss_disc_real_2: 0.23197399079799652 (0.23572126775979996) + | > loss_disc_real_3: 0.2075212299823761 (0.22987723350524902) + | > loss_disc_real_4: 0.25161775946617126 (0.25097522139549255) + | > loss_disc_real_5: 0.25601765513420105 (0.2430136278271675) + | > loss_0: 2.3923163414001465 (2.4702534675598145) + | > loss_gen: 2.6056299209594727 (2.5067758560180664) + | > loss_kl: 2.226416826248169 (2.507487177848816) + | > loss_feat: 8.075183868408203 (7.560528755187988) + | > loss_mel: 18.21625518798828 (18.334243774414062) + | > loss_duration: 1.7119505405426025 (1.7471680641174316) + | > loss_1: 32.8354377746582 (32.65620422363281) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032919645309448242 (-0.0007271766662597656) + | > avg_loss_disc: 2.4702534675598145 (-0.02627742290496826) + | > avg_loss_disc_real_0: 0.19001518189907074 (+0.01887086033821106) + | > avg_loss_disc_real_1: 0.21915074437856674 (-0.017125487327575684) + | > avg_loss_disc_real_2: 0.23572126775979996 (+0.0037149861454963684) + | > avg_loss_disc_real_3: 0.22987723350524902 (+0.006966426968574524) + | > avg_loss_disc_real_4: 0.25097522139549255 (+0.04167106747627258) + | > avg_loss_disc_real_5: 0.2430136278271675 (-0.015476293861865997) + | > avg_loss_0: 2.4702534675598145 (-0.02627742290496826) + | > avg_loss_gen: 2.5067758560180664 (+0.11086547374725342) + | > avg_loss_kl: 2.507487177848816 (-0.17548227310180664) + | > avg_loss_feat: 7.560528755187988 (+0.7236130237579346) + | > avg_loss_mel: 18.334243774414062 (+0.6301813125610352) + | > avg_loss_duration: 1.7471680641174316 (+0.005170226097106934) + | > avg_loss_1: 32.65620422363281 (+1.2943487167358398) + + + > EPOCH: 361/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:55:00)  + + --> TIME: 2025-04-19 04:55:02 -- STEP: 6/142 -- GLOBAL_STEP: 430550 + | > loss_disc: 2.4451942443847656 (2.412479519844055) + | > loss_disc_real_0: 0.10299860686063766 (0.10970677932103474) + | > loss_disc_real_1: 0.2082066386938095 (0.20984502136707306) + | > loss_disc_real_2: 0.22447066009044647 (0.22572175165017447) + | > loss_disc_real_3: 0.2117992341518402 (0.228225144247214) + | > loss_disc_real_4: 0.18658147752285004 (0.2197693387667338) + | > loss_disc_real_5: 0.1825139820575714 (0.23892523596684137) + | > loss_0: 2.4451942443847656 (2.412479519844055) + | > grad_norm_0: tensor(6.4549, device='cuda:0') (tensor(7.6596, device='cuda:0')) + | > loss_gen: 2.3211183547973633 (2.37894070148468) + | > loss_kl: 1.714016079902649 (1.7097900907198589) + | > loss_feat: 7.359058856964111 (7.488429307937622) + | > loss_mel: 17.89291000366211 (17.917588233947754) + | > loss_duration: 1.3956403732299805 (1.4119566082954407) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.682743072509766 (30.906704584757488) + | > grad_norm_1: tensor(147.0485, device='cuda:0') (tensor(144.9584, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.3287 (0.3238248825073242) + | > loader_time: 0.0042 (0.004279653231302897) + + + --> TIME: 2025-04-19 04:55:11 -- STEP: 31/142 -- GLOBAL_STEP: 430575 + | > loss_disc: 2.5332794189453125 (2.4150592434790825) + | > loss_disc_real_0: 0.11860513687133789 (0.11575773478515687) + | > loss_disc_real_1: 0.20035523176193237 (0.20695455756879622) + | > loss_disc_real_2: 0.1904883086681366 (0.2243737237107369) + | > loss_disc_real_3: 0.22541485726833344 (0.22740916763582536) + | > loss_disc_real_4: 0.23534715175628662 (0.2253078919264578) + | > loss_disc_real_5: 0.2484889030456543 (0.23344281508076575) + | > loss_0: 2.5332794189453125 (2.4150592434790825) + | > grad_norm_0: tensor(5.9016, device='cuda:0') (tensor(8.0520, device='cuda:0')) + | > loss_gen: 2.239267110824585 (2.426145461297805) + | > loss_kl: 1.7496211528778076 (1.7985604347721222) + | > loss_feat: 7.245387554168701 (7.51108850971345) + | > loss_mel: 16.804521560668945 (17.93278604938138) + | > loss_duration: 1.4147104024887085 (1.4229848538675616) + | > amp_scaler: 512.0 (330.3225806451613) + | > loss_1: 29.453508377075195 (31.091565347486927) + | > grad_norm_1: tensor(131.3561, device='cuda:0') (tensor(211.0032, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.3803 (0.3381672982246645) + | > loader_time: 0.0064 (0.005077554333594537) + + + --> TIME: 2025-04-19 04:55:20 -- STEP: 56/142 -- GLOBAL_STEP: 430600 + | > loss_disc: 2.426142930984497 (2.4284153240067616) + | > loss_disc_real_0: 0.07927080243825912 (0.11926285669739757) + | > loss_disc_real_1: 0.26997488737106323 (0.20764486465070928) + | > loss_disc_real_2: 0.25429850816726685 (0.2245773851339306) + | > loss_disc_real_3: 0.22598424553871155 (0.22429877492998326) + | > loss_disc_real_4: 0.23217052221298218 (0.2243419728640999) + | > loss_disc_real_5: 0.2310667335987091 (0.2282823169870036) + | > loss_0: 2.426142930984497 (2.4284153240067616) + | > grad_norm_0: tensor(9.0102, device='cuda:0') (tensor(7.4873, device='cuda:0')) + | > loss_gen: 2.427765130996704 (2.394862473011017) + | > loss_kl: 1.8759695291519165 (1.8122290287699019) + | > loss_feat: 7.1897430419921875 (7.340300840990884) + | > loss_mel: 17.539642333984375 (17.802948083196373) + | > loss_duration: 1.450057029724121 (1.4318824197564806) + | > amp_scaler: 512.0 (411.42857142857144) + | > loss_1: 30.483177185058594 (30.78222291810172) + | > grad_norm_1: tensor(273.3523, device='cuda:0') (tensor(177.5129, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.3739 (0.3510053668703352) + | > loader_time: 0.0062 (0.005515928779329575) + + + --> TIME: 2025-04-19 04:55:30 -- STEP: 81/142 -- GLOBAL_STEP: 430625 + | > loss_disc: 2.4335107803344727 (2.431917293572131) + | > loss_disc_real_0: 0.11592470854520798 (0.12222100030860783) + | > loss_disc_real_1: 0.2099558264017105 (0.2055246725126549) + | > loss_disc_real_2: 0.21300651133060455 (0.22450833096180434) + | > loss_disc_real_3: 0.2225923389196396 (0.22488691556600876) + | > loss_disc_real_4: 0.2234930396080017 (0.22583045065402985) + | > loss_disc_real_5: 0.20918014645576477 (0.22934581799271667) + | > loss_0: 2.4335107803344727 (2.431917293572131) + | > grad_norm_0: tensor(6.5467, device='cuda:0') (tensor(7.8718, device='cuda:0')) + | > loss_gen: 2.454317092895508 (2.3923614790410177) + | > loss_kl: 1.8984475135803223 (1.8260771816159471) + | > loss_feat: 6.990509986877441 (7.2744179949348355) + | > loss_mel: 17.397090911865234 (17.777866846249438) + | > loss_duration: 1.4513304233551025 (1.4440974070702073) + | > amp_scaler: 512.0 (442.4691358024691) + | > loss_1: 30.191696166992188 (30.714820956006463) + | > grad_norm_1: tensor(105.0866, device='cuda:0') (tensor(178.6050, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.3942 (0.36306157818547) + | > loader_time: 0.0067 (0.005840698877970379) + + + --> TIME: 2025-04-19 04:55:41 -- STEP: 106/142 -- GLOBAL_STEP: 430650 + | > loss_disc: 2.3869259357452393 (2.4438438527988926) + | > loss_disc_real_0: 0.12869806587696075 (0.12566480666117852) + | > loss_disc_real_1: 0.18437433242797852 (0.20718276894317483) + | > loss_disc_real_2: 0.18099169433116913 (0.22524728611955103) + | > loss_disc_real_3: 0.20583851635456085 (0.22555672953713615) + | > loss_disc_real_4: 0.19138628244400024 (0.2267533543537248) + | > loss_disc_real_5: 0.18859151005744934 (0.2296178877915976) + | > loss_0: 2.3869259357452393 (2.4438438527988926) + | > grad_norm_0: tensor(7.0440, device='cuda:0') (tensor(8.0183, device='cuda:0')) + | > loss_gen: 2.3312342166900635 (2.37900238217048) + | > loss_kl: 1.870273470878601 (1.8316266199327864) + | > loss_feat: 7.090649604797363 (7.224870947172057) + | > loss_mel: 18.105968475341797 (17.825288475684406) + | > loss_duration: 1.4776530265808105 (1.4538072460102585) + | > amp_scaler: 512.0 (458.8679245283019) + | > loss_1: 30.875778198242188 (30.71459574069617) + | > grad_norm_1: tensor(207.0711, device='cuda:0') (tensor(175.5263, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.4139 (0.37406678244752706) + | > loader_time: 0.0076 (0.00612103264286833) + + + --> TIME: 2025-04-19 04:55:52 -- STEP: 131/142 -- GLOBAL_STEP: 430675 + | > loss_disc: 2.46219801902771 (2.447200709626874) + | > loss_disc_real_0: 0.15418292582035065 (0.1263531246258102) + | > loss_disc_real_1: 0.20461438596248627 (0.2066246615793869) + | > loss_disc_real_2: 0.2334045022726059 (0.22501388681298903) + | > loss_disc_real_3: 0.2067006379365921 (0.2265507375238506) + | > loss_disc_real_4: 0.2739216387271881 (0.2279917613695596) + | > loss_disc_real_5: 0.22309492528438568 (0.2295904528093702) + | > loss_0: 2.46219801902771 (2.447200709626874) + | > grad_norm_0: tensor(11.3272, device='cuda:0') (tensor(8.0946, device='cuda:0')) + | > loss_gen: 2.487809181213379 (2.37608487551449) + | > loss_kl: 1.9759323596954346 (1.8419840563344592) + | > loss_feat: 7.381406784057617 (7.20927785553095) + | > loss_mel: 18.616640090942383 (17.80384206407853) + | > loss_duration: 1.493676781654358 (1.4599095519262415) + | > amp_scaler: 512.0 (469.00763358778624) + | > loss_1: 31.95546531677246 (30.691098453434368) + | > grad_norm_1: tensor(290.3600, device='cuda:0') (tensor(177.1186, device='cuda:0')) + | > current_lr_0: 0.00019117505870415173 + | > current_lr_1: 0.00019117505870415173 + | > step_time: 0.4467 (0.3857093712755742) + | > loader_time: 0.0073 (0.0064042196928999805) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.527392864227295 (2.527392864227295) + | > loss_disc_real_0: 0.2149546891450882 (0.2149546891450882) + | > loss_disc_real_1: 0.2669251561164856 (0.2669251561164856) + | > loss_disc_real_2: 0.255689412355423 (0.255689412355423) + | > loss_disc_real_3: 0.25553178787231445 (0.25553178787231445) + | > loss_disc_real_4: 0.22585585713386536 (0.22585585713386536) + | > loss_disc_real_5: 0.24727517366409302 (0.24727517366409302) + | > loss_0: 2.527392864227295 (2.527392864227295) + | > loss_gen: 2.4960904121398926 (2.4960904121398926) + | > loss_kl: 2.4166159629821777 (2.4166159629821777) + | > loss_feat: 5.968101501464844 (5.968101501464844) + | > loss_mel: 14.77777099609375 (14.77777099609375) + | > loss_duration: 1.7165329456329346 (1.7165329456329346) + | > loss_1: 27.375112533569336 (27.375112533569336) + + --> STEP: 1 + | > loss_disc: 2.4957470893859863 (2.4957470893859863) + | > loss_disc_real_0: 0.17954038083553314 (0.17954038083553314) + | > loss_disc_real_1: 0.273334264755249 (0.273334264755249) + | > loss_disc_real_2: 0.2798040807247162 (0.2798040807247162) + | > loss_disc_real_3: 0.24598486721515656 (0.24598486721515656) + | > loss_disc_real_4: 0.23465773463249207 (0.23465773463249207) + | > loss_disc_real_5: 0.23600824177265167 (0.23600824177265167) + | > loss_0: 2.4957470893859863 (2.4957470893859863) + | > loss_gen: 2.520108222961426 (2.520108222961426) + | > loss_kl: 2.774254322052002 (2.774254322052002) + | > loss_feat: 7.3244948387146 (7.3244948387146) + | > loss_mel: 19.144561767578125 (19.144561767578125) + | > loss_duration: 1.7843562364578247 (1.7843562364578247) + | > loss_1: 33.54777526855469 (33.54777526855469) + + --> STEP: 2 + | > loss_disc: 2.4187445640563965 (2.4572458267211914) + | > loss_disc_real_0: 0.1629430502653122 (0.17124171555042267) + | > loss_disc_real_1: 0.3068016469478607 (0.29006795585155487) + | > loss_disc_real_2: 0.22407296299934387 (0.25193852186203003) + | > loss_disc_real_3: 0.23134493827819824 (0.2386649027466774) + | > loss_disc_real_4: 0.20693901181221008 (0.22079837322235107) + | > loss_disc_real_5: 0.22980229556560516 (0.23290526866912842) + | > loss_0: 2.4187445640563965 (2.4572458267211914) + | > loss_gen: 2.5807852745056152 (2.5504467487335205) + | > loss_kl: 2.3608477115631104 (2.567551016807556) + | > loss_feat: 7.539356231689453 (7.431925535202026) + | > loss_mel: 18.481603622436523 (18.813082695007324) + | > loss_duration: 1.7160533666610718 (1.7502048015594482) + | > loss_1: 32.678646087646484 (33.113210678100586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034890174865722656 (+0.0001970529556274414) + | > avg_loss_disc: 2.4572458267211914 (-0.013007640838623047) + | > avg_loss_disc_real_0: 0.17124171555042267 (-0.01877346634864807) + | > avg_loss_disc_real_1: 0.29006795585155487 (+0.07091721147298813) + | > avg_loss_disc_real_2: 0.25193852186203003 (+0.016217254102230072) + | > avg_loss_disc_real_3: 0.2386649027466774 (+0.008787669241428375) + | > avg_loss_disc_real_4: 0.22079837322235107 (-0.03017684817314148) + | > avg_loss_disc_real_5: 0.23290526866912842 (-0.010108359158039093) + | > avg_loss_0: 2.4572458267211914 (-0.013007640838623047) + | > avg_loss_gen: 2.5504467487335205 (+0.0436708927154541) + | > avg_loss_kl: 2.567551016807556 (+0.060063838958740234) + | > avg_loss_feat: 7.431925535202026 (-0.12860321998596191) + | > avg_loss_mel: 18.813082695007324 (+0.4788389205932617) + | > avg_loss_duration: 1.7502048015594482 (+0.0030367374420166016) + | > avg_loss_1: 33.113210678100586 (+0.45700645446777344) + + + > EPOCH: 362/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:55:59)  + + --> TIME: 2025-04-19 04:56:04 -- STEP: 14/142 -- GLOBAL_STEP: 430700 + | > loss_disc: 2.3900089263916016 (2.392109291894095) + | > loss_disc_real_0: 0.1329628974199295 (0.11383028009108134) + | > loss_disc_real_1: 0.20225131511688232 (0.20058022333042963) + | > loss_disc_real_2: 0.2402988225221634 (0.21759437663214548) + | > loss_disc_real_3: 0.22557857632637024 (0.22346674650907516) + | > loss_disc_real_4: 0.22853372991085052 (0.22410437996898377) + | > loss_disc_real_5: 0.19937758147716522 (0.22525679852281297) + | > loss_0: 2.3900089263916016 (2.392109291894095) + | > grad_norm_0: tensor(19.7983, device='cuda:0') (tensor(13.0296, device='cuda:0')) + | > loss_gen: 2.6183829307556152 (2.5443101099559238) + | > loss_kl: 1.8160324096679688 (1.7686872737748283) + | > loss_feat: 7.449583530426025 (7.779546635491507) + | > loss_mel: 18.623380661010742 (18.055696896144322) + | > loss_duration: 1.420459270477295 (1.4065480487687247) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.927839279174805 (31.55478913443429) + | > grad_norm_1: tensor(192.7098, device='cuda:0') (tensor(298.6872, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.3332 (0.327955961227417) + | > loader_time: 0.0045 (0.004354613167898995) + + + --> TIME: 2025-04-19 04:56:13 -- STEP: 39/142 -- GLOBAL_STEP: 430725 + | > loss_disc: 2.3662009239196777 (2.4223451553246913) + | > loss_disc_real_0: 0.12437987327575684 (0.11893341279564759) + | > loss_disc_real_1: 0.19673295319080353 (0.20582124552665612) + | > loss_disc_real_2: 0.24826841056346893 (0.22461620431679946) + | > loss_disc_real_3: 0.23019254207611084 (0.22613455890080866) + | > loss_disc_real_4: 0.23204068839550018 (0.22567187058619964) + | > loss_disc_real_5: 0.24114291369915009 (0.22828696171442667) + | > loss_0: 2.3662009239196777 (2.4223451553246913) + | > grad_norm_0: tensor(7.0568, device='cuda:0') (tensor(15.2373, device='cuda:0')) + | > loss_gen: 2.4069674015045166 (2.4742050170898433) + | > loss_kl: 2.011665105819702 (1.8193250436049242) + | > loss_feat: 7.208388328552246 (7.505854215377417) + | > loss_mel: 17.250797271728516 (17.875989327063923) + | > loss_duration: 1.4630879163742065 (1.4233260521521938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.340906143188477 (31.098699520795773) + | > grad_norm_1: tensor(136.0155, device='cuda:0') (tensor(333.1704, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.3564 (0.3404622567005647) + | > loader_time: 0.0053 (0.004854630201290814) + + + --> TIME: 2025-04-19 04:56:22 -- STEP: 64/142 -- GLOBAL_STEP: 430750 + | > loss_disc: 2.443983793258667 (2.427773762494326) + | > loss_disc_real_0: 0.08994410187005997 (0.11872882879106328) + | > loss_disc_real_1: 0.20724491775035858 (0.206165820825845) + | > loss_disc_real_2: 0.19887639582157135 (0.22352580749429762) + | > loss_disc_real_3: 0.21825602650642395 (0.2254315889440477) + | > loss_disc_real_4: 0.2088337540626526 (0.22643751977011561) + | > loss_disc_real_5: 0.23429159820079803 (0.22886253171600401) + | > loss_0: 2.443983793258667 (2.427773762494326) + | > grad_norm_0: tensor(6.0490, device='cuda:0') (tensor(12.9609, device='cuda:0')) + | > loss_gen: 2.3163797855377197 (2.4323118925094604) + | > loss_kl: 1.9648720026016235 (1.828312262892723) + | > loss_feat: 7.290104866027832 (7.36867331713438) + | > loss_mel: 18.30607032775879 (17.815369725227356) + | > loss_duration: 1.4816138744354248 (1.4358251858502629) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.359039306640625 (30.88049226999283) + | > grad_norm_1: tensor(181.5323, device='cuda:0') (tensor(287.3191, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.3815 (0.3523565009236336) + | > loader_time: 0.0059 (0.005250774323940277) + + + --> TIME: 2025-04-19 04:56:32 -- STEP: 89/142 -- GLOBAL_STEP: 430775 + | > loss_disc: 2.5137481689453125 (2.4329620130946137) + | > loss_disc_real_0: 0.09519551694393158 (0.12298007087593668) + | > loss_disc_real_1: 0.236650750041008 (0.20628348193811566) + | > loss_disc_real_2: 0.20997019112110138 (0.2236873298213723) + | > loss_disc_real_3: 0.2504386305809021 (0.2257209045163701) + | > loss_disc_real_4: 0.25106334686279297 (0.22809054309062743) + | > loss_disc_real_5: 0.2624310553073883 (0.22861694050638864) + | > loss_0: 2.5137481689453125 (2.4329620130946137) + | > grad_norm_0: tensor(7.4746, device='cuda:0') (tensor(11.5020, device='cuda:0')) + | > loss_gen: 2.291060209274292 (2.4142045572902378) + | > loss_kl: 2.001932382583618 (1.8368746805726812) + | > loss_feat: 7.10029411315918 (7.329161665412817) + | > loss_mel: 17.76699447631836 (17.846598893069153) + | > loss_duration: 1.4541712999343872 (1.4467167023862346) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.614452362060547 (30.873556479979097) + | > grad_norm_1: tensor(102.6158, device='cuda:0') (tensor(245.7625, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.3998 (0.3643462845448698) + | > loader_time: 0.0063 (0.005524396896362305) + + + --> TIME: 2025-04-19 04:56:43 -- STEP: 114/142 -- GLOBAL_STEP: 430800 + | > loss_disc: 2.520792245864868 (2.4358948009055967) + | > loss_disc_real_0: 0.12961064279079437 (0.12410192809214718) + | > loss_disc_real_1: 0.19035837054252625 (0.20538397616984552) + | > loss_disc_real_2: 0.2193276584148407 (0.22454017544524713) + | > loss_disc_real_3: 0.1973392516374588 (0.22616451403550936) + | > loss_disc_real_4: 0.20522446930408478 (0.227274457875051) + | > loss_disc_real_5: 0.24864308536052704 (0.2283338844253306) + | > loss_0: 2.520792245864868 (2.4358948009055967) + | > grad_norm_0: tensor(4.5589, device='cuda:0') (tensor(10.3049, device='cuda:0')) + | > loss_gen: 2.224203586578369 (2.3988949625115636) + | > loss_kl: 1.9529962539672852 (1.840269660740568) + | > loss_feat: 7.34244966506958 (7.27762456525836) + | > loss_mel: 17.78721809387207 (17.807270585444936) + | > loss_duration: 1.4859544038772583 (1.4541585068953664) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.792821884155273 (30.778218353003787) + | > grad_norm_1: tensor(172.5338, device='cuda:0') (tensor(211.8847, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.4216 (0.3749130190464488) + | > loader_time: 0.0068 (0.005776179464239823) + + + --> TIME: 2025-04-19 04:56:54 -- STEP: 139/142 -- GLOBAL_STEP: 430825 + | > loss_disc: 2.4776997566223145 (2.443788574753906) + | > loss_disc_real_0: 0.08724849671125412 (0.12607762466767708) + | > loss_disc_real_1: 0.19792993366718292 (0.20655810650732878) + | > loss_disc_real_2: 0.21929830312728882 (0.22538124903929319) + | > loss_disc_real_3: 0.23415233194828033 (0.22670798411043427) + | > loss_disc_real_4: 0.22288937866687775 (0.22786749363374367) + | > loss_disc_real_5: 0.2342691272497177 (0.22860966976598013) + | > loss_0: 2.4776997566223145 (2.443788574753906) + | > grad_norm_0: tensor(17.8902, device='cuda:0') (tensor(10.0201, device='cuda:0')) + | > loss_gen: 2.269205331802368 (2.384818589944633) + | > loss_kl: 1.8141629695892334 (1.852426249346287) + | > loss_feat: 7.371114730834961 (7.2132358996988195) + | > loss_mel: 17.45870590209961 (17.73449385938027) + | > loss_duration: 1.4809765815734863 (1.461072143033254) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.3941650390625 (30.646046851178724) + | > grad_norm_1: tensor(256.7763, device='cuda:0') (tensor(197.8671, device='cuda:0')) + | > current_lr_0: 0.0001911511618218137 + | > current_lr_1: 0.0001911511618218137 + | > step_time: 0.4539 (0.38696346523092806) + | > loader_time: 0.0072 (0.006040746359516391) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.343538999557495 (2.343538999557495) + | > loss_disc_real_0: 0.19174164533615112 (0.19174164533615112) + | > loss_disc_real_1: 0.23081792891025543 (0.23081792891025543) + | > loss_disc_real_2: 0.2272948920726776 (0.2272948920726776) + | > loss_disc_real_3: 0.20701810717582703 (0.20701810717582703) + | > loss_disc_real_4: 0.2189323455095291 (0.2189323455095291) + | > loss_disc_real_5: 0.14877919852733612 (0.14877919852733612) + | > loss_0: 2.343538999557495 (2.343538999557495) + | > loss_gen: 2.397252082824707 (2.397252082824707) + | > loss_kl: 2.5025811195373535 (2.5025811195373535) + | > loss_feat: 6.811121463775635 (6.811121463775635) + | > loss_mel: 17.529951095581055 (17.529951095581055) + | > loss_duration: 1.7330925464630127 (1.7330925464630127) + | > loss_1: 30.9739990234375 (30.9739990234375) + + --> STEP: 1 + | > loss_disc: 2.35903000831604 (2.35903000831604) + | > loss_disc_real_0: 0.18192513287067413 (0.18192513287067413) + | > loss_disc_real_1: 0.20822341740131378 (0.20822341740131378) + | > loss_disc_real_2: 0.24181415140628815 (0.24181415140628815) + | > loss_disc_real_3: 0.21181602776050568 (0.21181602776050568) + | > loss_disc_real_4: 0.25738799571990967 (0.25738799571990967) + | > loss_disc_real_5: 0.17371761798858643 (0.17371761798858643) + | > loss_0: 2.35903000831604 (2.35903000831604) + | > loss_gen: 2.446258544921875 (2.446258544921875) + | > loss_kl: 2.9635047912597656 (2.9635047912597656) + | > loss_feat: 6.804858207702637 (6.804858207702637) + | > loss_mel: 18.330543518066406 (18.330543518066406) + | > loss_duration: 1.7938694953918457 (1.7938694953918457) + | > loss_1: 32.33903503417969 (32.33903503417969) + + --> STEP: 2 + | > loss_disc: 2.300942897796631 (2.3299864530563354) + | > loss_disc_real_0: 0.18609458208084106 (0.1840098574757576) + | > loss_disc_real_1: 0.2583794891834259 (0.23330145329236984) + | > loss_disc_real_2: 0.20146086812019348 (0.22163750976324081) + | > loss_disc_real_3: 0.22815167903900146 (0.21998385339975357) + | > loss_disc_real_4: 0.21995317935943604 (0.23867058753967285) + | > loss_disc_real_5: 0.19569049775600433 (0.18470405787229538) + | > loss_0: 2.300942897796631 (2.3299864530563354) + | > loss_gen: 2.819639205932617 (2.632948875427246) + | > loss_kl: 2.287184953689575 (2.6253448724746704) + | > loss_feat: 7.530116558074951 (7.167487382888794) + | > loss_mel: 17.602664947509766 (17.966604232788086) + | > loss_duration: 1.70801842212677 (1.7509439587593079) + | > loss_1: 31.9476261138916 (32.143330574035645) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003797173500061035 (+0.00030815601348876953) + | > avg_loss_disc: 2.3299864530563354 (-0.12725937366485596) + | > avg_loss_disc_real_0: 0.1840098574757576 (+0.01276814192533493) + | > avg_loss_disc_real_1: 0.23330145329236984 (-0.05676650255918503) + | > avg_loss_disc_real_2: 0.22163750976324081 (-0.030301012098789215) + | > avg_loss_disc_real_3: 0.21998385339975357 (-0.018681049346923828) + | > avg_loss_disc_real_4: 0.23867058753967285 (+0.017872214317321777) + | > avg_loss_disc_real_5: 0.18470405787229538 (-0.04820121079683304) + | > avg_loss_0: 2.3299864530563354 (-0.12725937366485596) + | > avg_loss_gen: 2.632948875427246 (+0.08250212669372559) + | > avg_loss_kl: 2.6253448724746704 (+0.05779385566711426) + | > avg_loss_feat: 7.167487382888794 (-0.2644381523132324) + | > avg_loss_mel: 17.966604232788086 (-0.8464784622192383) + | > avg_loss_duration: 1.7509439587593079 (+0.0007391571998596191) + | > avg_loss_1: 32.143330574035645 (-0.9698801040649414) + + + > EPOCH: 363/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:56:57)  + + --> TIME: 2025-04-19 04:57:05 -- STEP: 22/142 -- GLOBAL_STEP: 430850 + | > loss_disc: 2.3540353775024414 (2.3894519155675713) + | > loss_disc_real_0: 0.09418205171823502 (0.11509330502965233) + | > loss_disc_real_1: 0.18679557740688324 (0.2046466977758841) + | > loss_disc_real_2: 0.1953226625919342 (0.21839980239217932) + | > loss_disc_real_3: 0.21907269954681396 (0.2220246114514091) + | > loss_disc_real_4: 0.19698038697242737 (0.22141556238586252) + | > loss_disc_real_5: 0.22454477846622467 (0.2272254221818664) + | > loss_0: 2.3540353775024414 (2.3894519155675713) + | > grad_norm_0: tensor(8.1982, device='cuda:0') (tensor(8.7547, device='cuda:0')) + | > loss_gen: 2.4547486305236816 (2.4241239699450405) + | > loss_kl: 1.9328118562698364 (1.781389664519917) + | > loss_feat: 7.323897361755371 (7.451902562921697) + | > loss_mel: 17.90691375732422 (17.58856838399714) + | > loss_duration: 1.3942800760269165 (1.4141904440793125) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.012651443481445 (30.660175236788664) + | > grad_norm_1: tensor(163.5012, device='cuda:0') (tensor(221.4568, device='cuda:0')) + | > current_lr_0: 0.00019112726792658598 + | > current_lr_1: 0.00019112726792658598 + | > step_time: 0.3511 (0.331604231487621) + | > loader_time: 0.0054 (0.004923126914284446) + + + --> TIME: 2025-04-19 04:57:14 -- STEP: 47/142 -- GLOBAL_STEP: 430875 + | > loss_disc: 2.362980604171753 (2.40013249376987) + | > loss_disc_real_0: 0.14830167591571808 (0.11686803685857895) + | > loss_disc_real_1: 0.2039276510477066 (0.20442325130422065) + | > loss_disc_real_2: 0.23095616698265076 (0.22123231342498292) + | > loss_disc_real_3: 0.21001937985420227 (0.2244715582817159) + | > loss_disc_real_4: 0.2110711634159088 (0.22279269422622436) + | > loss_disc_real_5: 0.24825672805309296 (0.22708946244513734) + | > loss_0: 2.362980604171753 (2.40013249376987) + | > grad_norm_0: tensor(10.4061, device='cuda:0') (tensor(12.1299, device='cuda:0')) + | > loss_gen: 2.534928798675537 (2.4164686304457645) + | > loss_kl: 1.9701766967773438 (1.8267813997065767) + | > loss_feat: 7.746429920196533 (7.421870951956891) + | > loss_mel: 17.576452255249023 (17.74748775806832) + | > loss_duration: 1.417085886001587 (1.4259469813488899) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.245073318481445 (30.838555843272108) + | > grad_norm_1: tensor(54.9796, device='cuda:0') (tensor(266.3155, device='cuda:0')) + | > current_lr_0: 0.00019112726792658598 + | > current_lr_1: 0.00019112726792658598 + | > step_time: 0.3742 (0.34854768692178945) + | > loader_time: 0.0059 (0.0053323431217924076) + + + --> TIME: 2025-04-19 04:57:24 -- STEP: 72/142 -- GLOBAL_STEP: 430900 + | > loss_disc: 2.4749600887298584 (2.4103095995055313) + | > loss_disc_real_0: 0.09328623861074448 (0.11805865877411431) + | > loss_disc_real_1: 0.19347436726093292 (0.204726603296068) + | > loss_disc_real_2: 0.19223131239414215 (0.2209095118774308) + | > loss_disc_real_3: 0.21931268274784088 (0.22614341187808248) + | > loss_disc_real_4: 0.20797564089298248 (0.22400775075786644) + | > loss_disc_real_5: 0.23174476623535156 (0.22671139178176722) + | > loss_0: 2.4749600887298584 (2.4103095995055313) + | > grad_norm_0: tensor(14.4136, device='cuda:0') (tensor(13.3319, device='cuda:0')) + | > loss_gen: 2.489158868789673 (2.4232553210523395) + | > loss_kl: 1.8819828033447266 (1.8400250971317291) + | > loss_feat: 7.718007564544678 (7.409114460150401) + | > loss_mel: 16.459218978881836 (17.73203059037526) + | > loss_duration: 1.4511189460754395 (1.4391938265826967) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.999486923217773 (30.843619293636745) + | > grad_norm_1: tensor(465.4667, device='cuda:0') (tensor(293.0279, device='cuda:0')) + | > current_lr_0: 0.00019112726792658598 + | > current_lr_1: 0.00019112726792658598 + | > step_time: 0.4037 (0.3643228146764968) + | > loader_time: 0.0068 (0.005675531095928616) + + + --> TIME: 2025-04-19 04:57:35 -- STEP: 97/142 -- GLOBAL_STEP: 430925 + | > loss_disc: 2.4237730503082275 (2.418031218125649) + | > loss_disc_real_0: 0.11901260912418365 (0.11940983402513966) + | > loss_disc_real_1: 0.2269410938024521 (0.20531540847930715) + | > loss_disc_real_2: 0.22911310195922852 (0.22132625530675515) + | > loss_disc_real_3: 0.24111570417881012 (0.226653178788952) + | > loss_disc_real_4: 0.21532657742500305 (0.2262882246184595) + | > loss_disc_real_5: 0.22050730884075165 (0.22662303389347704) + | > loss_0: 2.4237730503082275 (2.418031218125649) + | > grad_norm_0: tensor(11.9003, device='cuda:0') (tensor(12.7657, device='cuda:0')) + | > loss_gen: 2.484311103820801 (2.4122584800130307) + | > loss_kl: 1.8380181789398193 (1.8548531298784865) + | > loss_feat: 7.62554407119751 (7.300658796251435) + | > loss_mel: 18.085527420043945 (17.680743905686832) + | > loss_duration: 1.4845967292785645 (1.4511199919218873) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.51799774169922 (30.699634276714523) + | > grad_norm_1: tensor(401.3408, device='cuda:0') (tensor(292.5218, device='cuda:0')) + | > current_lr_0: 0.00019112726792658598 + | > current_lr_1: 0.00019112726792658598 + | > step_time: 0.4244 (0.3763261873697497) + | > loader_time: 0.0073 (0.005998638487353767) + + + --> TIME: 2025-04-19 04:57:46 -- STEP: 122/142 -- GLOBAL_STEP: 430950 + | > loss_disc: 2.392392158508301 (2.4262021916811594) + | > loss_disc_real_0: 0.12472131103277206 (0.12063536447946166) + | > loss_disc_real_1: 0.17973646521568298 (0.20597406550020472) + | > loss_disc_real_2: 0.18770872056484222 (0.22312115743512012) + | > loss_disc_real_3: 0.1800190508365631 (0.22604562391023167) + | > loss_disc_real_4: 0.1824081838130951 (0.2261341833433167) + | > loss_disc_real_5: 0.2034902125597 (0.2284976284279198) + | > loss_0: 2.392392158508301 (2.4262021916811594) + | > grad_norm_0: tensor(20.5698, device='cuda:0') (tensor(12.3526, device='cuda:0')) + | > loss_gen: 2.3520755767822266 (2.4040224219931927) + | > loss_kl: 1.8078012466430664 (1.8641221601454938) + | > loss_feat: 7.1071271896362305 (7.260261973396677) + | > loss_mel: 18.109426498413086 (17.671151434788936) + | > loss_duration: 1.483290672302246 (1.4583031838057474) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.859722137451172 (30.657861131136535) + | > grad_norm_1: tensor(252.8480, device='cuda:0') (tensor(284.2159, device='cuda:0')) + | > current_lr_0: 0.00019112726792658598 + | > current_lr_1: 0.00019112726792658598 + | > step_time: 0.4654 (0.3886295088001939) + | > loader_time: 0.0074 (0.006279875020511815) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.486236095428467 (2.486236095428467) + | > loss_disc_real_0: 0.2137967348098755 (0.2137967348098755) + | > loss_disc_real_1: 0.2543320059776306 (0.2543320059776306) + | > loss_disc_real_2: 0.24660724401474 (0.24660724401474) + | > loss_disc_real_3: 0.21174117922782898 (0.21174117922782898) + | > loss_disc_real_4: 0.2519657611846924 (0.2519657611846924) + | > loss_disc_real_5: 0.2777664363384247 (0.2777664363384247) + | > loss_0: 2.486236095428467 (2.486236095428467) + | > loss_gen: 2.6521294116973877 (2.6521294116973877) + | > loss_kl: 2.639989137649536 (2.639989137649536) + | > loss_feat: 7.474119186401367 (7.474119186401367) + | > loss_mel: 19.976747512817383 (19.976747512817383) + | > loss_duration: 1.7313804626464844 (1.7313804626464844) + | > loss_1: 34.474365234375 (34.474365234375) + + --> STEP: 1 + | > loss_disc: 2.4550118446350098 (2.4550118446350098) + | > loss_disc_real_0: 0.20510275661945343 (0.20510275661945343) + | > loss_disc_real_1: 0.22541046142578125 (0.22541046142578125) + | > loss_disc_real_2: 0.23461641371250153 (0.23461641371250153) + | > loss_disc_real_3: 0.20665250718593597 (0.20665250718593597) + | > loss_disc_real_4: 0.23733477294445038 (0.23733477294445038) + | > loss_disc_real_5: 0.2131510227918625 (0.2131510227918625) + | > loss_0: 2.4550118446350098 (2.4550118446350098) + | > loss_gen: 2.371044635772705 (2.371044635772705) + | > loss_kl: 2.773664951324463 (2.773664951324463) + | > loss_feat: 7.167446136474609 (7.167446136474609) + | > loss_mel: 17.90757179260254 (17.90757179260254) + | > loss_duration: 1.7899739742279053 (1.7899739742279053) + | > loss_1: 32.009700775146484 (32.009700775146484) + + --> STEP: 2 + | > loss_disc: 2.552252769470215 (2.5036323070526123) + | > loss_disc_real_0: 0.2685961127281189 (0.23684943467378616) + | > loss_disc_real_1: 0.202558696269989 (0.21398457884788513) + | > loss_disc_real_2: 0.19718891382217407 (0.2159026637673378) + | > loss_disc_real_3: 0.1946599930524826 (0.2006562501192093) + | > loss_disc_real_4: 0.2182702273130417 (0.22780250012874603) + | > loss_disc_real_5: 0.21676981449127197 (0.21496041864156723) + | > loss_0: 2.552252769470215 (2.5036323070526123) + | > loss_gen: 2.413600444793701 (2.392322540283203) + | > loss_kl: 2.418729066848755 (2.596197009086609) + | > loss_feat: 7.611511707305908 (7.389478921890259) + | > loss_mel: 19.02995491027832 (18.46876335144043) + | > loss_duration: 1.7258553504943848 (1.757914662361145) + | > loss_1: 33.19965362548828 (32.60467720031738) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036575794219970703 (-0.00013959407806396484) + | > avg_loss_disc: 2.5036323070526123 (+0.17364585399627686) + | > avg_loss_disc_real_0: 0.23684943467378616 (+0.052839577198028564) + | > avg_loss_disc_real_1: 0.21398457884788513 (-0.01931687444448471) + | > avg_loss_disc_real_2: 0.2159026637673378 (-0.005734845995903015) + | > avg_loss_disc_real_3: 0.2006562501192093 (-0.01932760328054428) + | > avg_loss_disc_real_4: 0.22780250012874603 (-0.010868087410926819) + | > avg_loss_disc_real_5: 0.21496041864156723 (+0.03025636076927185) + | > avg_loss_0: 2.5036323070526123 (+0.17364585399627686) + | > avg_loss_gen: 2.392322540283203 (-0.24062633514404297) + | > avg_loss_kl: 2.596197009086609 (-0.029147863388061523) + | > avg_loss_feat: 7.389478921890259 (+0.22199153900146484) + | > avg_loss_mel: 18.46876335144043 (+0.5021591186523438) + | > avg_loss_duration: 1.757914662361145 (+0.006970703601837158) + | > avg_loss_1: 32.60467720031738 (+0.4613466262817383) + + + > EPOCH: 364/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:57:57)  + + --> TIME: 2025-04-19 04:57:59 -- STEP: 5/142 -- GLOBAL_STEP: 430975 + | > loss_disc: 2.5398736000061035 (2.461331844329834) + | > loss_disc_real_0: 0.17517270147800446 (0.1201935201883316) + | > loss_disc_real_1: 0.23744699358940125 (0.22014290392398833) + | > loss_disc_real_2: 0.2046087086200714 (0.23041017949581147) + | > loss_disc_real_3: 0.2739626467227936 (0.2336478590965271) + | > loss_disc_real_4: 0.2236855924129486 (0.23482440412044525) + | > loss_disc_real_5: 0.23351043462753296 (0.2618685275316238) + | > loss_0: 2.5398736000061035 (2.461331844329834) + | > grad_norm_0: tensor(10.3091, device='cuda:0') (tensor(11.2837, device='cuda:0')) + | > loss_gen: 2.4445061683654785 (2.4415740966796875) + | > loss_kl: 1.7931442260742188 (1.8167245626449584) + | > loss_feat: 7.702939510345459 (7.427018356323242) + | > loss_mel: 17.898601531982422 (17.502630615234374) + | > loss_duration: 1.4251697063446045 (1.4181551218032837) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.264360427856445 (30.606103134155273) + | > grad_norm_1: tensor(335.4557, device='cuda:0') (tensor(264.1559, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.325 (0.32457637786865234) + | > loader_time: 0.0044 (0.004847002029418945) + + + --> TIME: 2025-04-19 04:58:08 -- STEP: 30/142 -- GLOBAL_STEP: 431000 + | > loss_disc: 2.5545501708984375 (2.443957225481669) + | > loss_disc_real_0: 0.1160249263048172 (0.11821054841081301) + | > loss_disc_real_1: 0.21133077144622803 (0.20858522405227026) + | > loss_disc_real_2: 0.2517605423927307 (0.22458647638559343) + | > loss_disc_real_3: 0.2752993106842041 (0.23133727709452312) + | > loss_disc_real_4: 0.28952160477638245 (0.23129274398088456) + | > loss_disc_real_5: 0.2775219976902008 (0.2345183437069257) + | > loss_0: 2.5545501708984375 (2.443957225481669) + | > grad_norm_0: tensor(11.0055, device='cuda:0') (tensor(9.0485, device='cuda:0')) + | > loss_gen: 2.3338606357574463 (2.3904462019602457) + | > loss_kl: 1.7767952680587769 (1.796377178033193) + | > loss_feat: 7.504077911376953 (7.43898442586263) + | > loss_mel: 18.217147827148438 (18.225985463460287) + | > loss_duration: 1.4313795566558838 (1.4205451528231303) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.263259887695312 (31.27233861287435) + | > grad_norm_1: tensor(274.3582, device='cuda:0') (tensor(215.8607, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.3487 (0.336825426419576) + | > loader_time: 0.006 (0.005195379257202149) + + + --> TIME: 2025-04-19 04:58:17 -- STEP: 55/142 -- GLOBAL_STEP: 431025 + | > loss_disc: 2.44460391998291 (2.44079513983293) + | > loss_disc_real_0: 0.14355099201202393 (0.118952430378307) + | > loss_disc_real_1: 0.25969335436820984 (0.20691382722421123) + | > loss_disc_real_2: 0.25597938895225525 (0.22457328628409992) + | > loss_disc_real_3: 0.250337690114975 (0.22797564213926141) + | > loss_disc_real_4: 0.2832893431186676 (0.23075339577414772) + | > loss_disc_real_5: 0.26580294966697693 (0.23161977258595554) + | > loss_0: 2.44460391998291 (2.44079513983293) + | > grad_norm_0: tensor(6.3656, device='cuda:0') (tensor(9.3080, device='cuda:0')) + | > loss_gen: 2.595431327819824 (2.383902623436668) + | > loss_kl: 1.9314063787460327 (1.8354030392386698) + | > loss_feat: 7.491505146026611 (7.3124512845819645) + | > loss_mel: 18.756959915161133 (17.907143540815863) + | > loss_duration: 1.497901201248169 (1.430107849294489) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.2732048034668 (30.869008567116477) + | > grad_norm_1: tensor(187.3936, device='cuda:0') (tensor(232.6468, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.3734 (0.3480861533771862) + | > loader_time: 0.0062 (0.005477792566472835) + + + --> TIME: 2025-04-19 04:58:27 -- STEP: 80/142 -- GLOBAL_STEP: 431050 + | > loss_disc: 2.5437722206115723 (2.4506209045648575) + | > loss_disc_real_0: 0.16548310220241547 (0.12111545195803046) + | > loss_disc_real_1: 0.18222486972808838 (0.20602311845868826) + | > loss_disc_real_2: 0.19365645945072174 (0.22541837338358164) + | > loss_disc_real_3: 0.2281472384929657 (0.2288074078038335) + | > loss_disc_real_4: 0.24763061106204987 (0.23070884682238102) + | > loss_disc_real_5: 0.23971663415431976 (0.23272514902055264) + | > loss_0: 2.5437722206115723 (2.4506209045648575) + | > grad_norm_0: tensor(14.1218, device='cuda:0') (tensor(11.0273, device='cuda:0')) + | > loss_gen: 2.2946488857269287 (2.3829480886459358) + | > loss_kl: 1.9696789979934692 (1.8547780126333238) + | > loss_feat: 6.3996734619140625 (7.267188286781311) + | > loss_mel: 16.725296020507812 (17.835215914249417) + | > loss_duration: 1.4549442529678345 (1.440406359732151) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.844242095947266 (30.78053674697876) + | > grad_norm_1: tensor(139.0724, device='cuda:0') (tensor(254.5179, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.4108 (0.3600345104932786) + | > loader_time: 0.0067 (0.0057172954082489015) + + + --> TIME: 2025-04-19 04:58:37 -- STEP: 105/142 -- GLOBAL_STEP: 431075 + | > loss_disc: 2.421825647354126 (2.4537734394981747) + | > loss_disc_real_0: 0.11726780235767365 (0.12133591849179495) + | > loss_disc_real_1: 0.2286372035741806 (0.20768902117297763) + | > loss_disc_real_2: 0.23853279650211334 (0.22594824660392035) + | > loss_disc_real_3: 0.26189395785331726 (0.22895371814568838) + | > loss_disc_real_4: 0.2484087496995926 (0.2303780101594471) + | > loss_disc_real_5: 0.22107641398906708 (0.23102823694547017) + | > loss_0: 2.421825647354126 (2.4537734394981747) + | > grad_norm_0: tensor(21.6557, device='cuda:0') (tensor(12.1882, device='cuda:0')) + | > loss_gen: 2.300971746444702 (2.380648519879296) + | > loss_kl: 2.007697582244873 (1.8667160465603783) + | > loss_feat: 7.9101243019104 (7.2519146419706795) + | > loss_mel: 18.89266586303711 (17.837151536487394) + | > loss_duration: 1.489214539527893 (1.450155426207043) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.60067367553711 (30.786586252848306) + | > grad_norm_1: tensor(391.7856, device='cuda:0') (tensor(277.6871, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.4171 (0.3703369685581754) + | > loader_time: 0.0067 (0.00593674069359189) + + + --> TIME: 2025-04-19 04:58:48 -- STEP: 130/142 -- GLOBAL_STEP: 431100 + | > loss_disc: 2.4848766326904297 (2.455135785616361) + | > loss_disc_real_0: 0.09198635816574097 (0.12251504161036932) + | > loss_disc_real_1: 0.19737762212753296 (0.20802427587600855) + | > loss_disc_real_2: 0.2216402292251587 (0.225620336830616) + | > loss_disc_real_3: 0.24752725660800934 (0.22820066752342077) + | > loss_disc_real_4: 0.22692488133907318 (0.2300700278236316) + | > loss_disc_real_5: 0.2576415538787842 (0.23015706206743533) + | > loss_0: 2.4848766326904297 (2.455135785616361) + | > grad_norm_0: tensor(13.1364, device='cuda:0') (tensor(12.8116, device='cuda:0')) + | > loss_gen: 2.414016008377075 (2.3724578288885265) + | > loss_kl: 1.988973617553711 (1.8700242821986859) + | > loss_feat: 7.772367000579834 (7.221907182840201) + | > loss_mel: 18.541913986206055 (17.794768098684457) + | > loss_duration: 1.4802303314208984 (1.4570689614002523) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.19750213623047 (30.71622641636775) + | > grad_norm_1: tensor(432.4131, device='cuda:0') (tensor(289.7156, device='cuda:0')) + | > current_lr_0: 0.00019110337701809513 + | > current_lr_1: 0.00019110337701809513 + | > step_time: 0.447 (0.3820185459577121) + | > loader_time: 0.0075 (0.006200029299809382) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.392036199569702 (2.392036199569702) + | > loss_disc_real_0: 0.2711421549320221 (0.2711421549320221) + | > loss_disc_real_1: 0.23348292708396912 (0.23348292708396912) + | > loss_disc_real_2: 0.21863314509391785 (0.21863314509391785) + | > loss_disc_real_3: 0.2786768674850464 (0.2786768674850464) + | > loss_disc_real_4: 0.26323026418685913 (0.26323026418685913) + | > loss_disc_real_5: 0.26063570380210876 (0.26063570380210876) + | > loss_0: 2.392036199569702 (2.392036199569702) + | > loss_gen: 2.9965219497680664 (2.9965219497680664) + | > loss_kl: 2.7172305583953857 (2.7172305583953857) + | > loss_feat: 7.9866623878479 (7.9866623878479) + | > loss_mel: 19.08562660217285 (19.08562660217285) + | > loss_duration: 1.7486554384231567 (1.7486554384231567) + | > loss_1: 34.534698486328125 (34.534698486328125) + + --> STEP: 1 + | > loss_disc: 2.423049211502075 (2.423049211502075) + | > loss_disc_real_0: 0.2412402182817459 (0.2412402182817459) + | > loss_disc_real_1: 0.25852492451667786 (0.25852492451667786) + | > loss_disc_real_2: 0.23122482001781464 (0.23122482001781464) + | > loss_disc_real_3: 0.24790820479393005 (0.24790820479393005) + | > loss_disc_real_4: 0.22751183807849884 (0.22751183807849884) + | > loss_disc_real_5: 0.19401422142982483 (0.19401422142982483) + | > loss_0: 2.423049211502075 (2.423049211502075) + | > loss_gen: 2.866391658782959 (2.866391658782959) + | > loss_kl: 2.9442577362060547 (2.9442577362060547) + | > loss_feat: 7.414794445037842 (7.414794445037842) + | > loss_mel: 19.028823852539062 (19.028823852539062) + | > loss_duration: 1.791640043258667 (1.791640043258667) + | > loss_1: 34.0459098815918 (34.0459098815918) + + --> STEP: 2 + | > loss_disc: 2.4126687049865723 (2.4178589582443237) + | > loss_disc_real_0: 0.28630709648132324 (0.2637736573815346) + | > loss_disc_real_1: 0.23415087163448334 (0.2463378980755806) + | > loss_disc_real_2: 0.20210936665534973 (0.21666709333658218) + | > loss_disc_real_3: 0.2350057065486908 (0.24145695567131042) + | > loss_disc_real_4: 0.1670394241809845 (0.19727563112974167) + | > loss_disc_real_5: 0.21998777985572815 (0.2070010006427765) + | > loss_0: 2.4126687049865723 (2.4178589582443237) + | > loss_gen: 2.6652472019195557 (2.7658194303512573) + | > loss_kl: 2.4466261863708496 (2.695441961288452) + | > loss_feat: 7.778524398803711 (7.596659421920776) + | > loss_mel: 18.35024642944336 (18.68953514099121) + | > loss_duration: 1.7255253791809082 (1.7585827112197876) + | > loss_1: 32.96617126464844 (33.50604057312012) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037925243377685547 (+0.00013494491577148438) + | > avg_loss_disc: 2.4178589582443237 (-0.08577334880828857) + | > avg_loss_disc_real_0: 0.2637736573815346 (+0.026924222707748413) + | > avg_loss_disc_real_1: 0.2463378980755806 (+0.032353319227695465) + | > avg_loss_disc_real_2: 0.21666709333658218 (+0.0007644295692443848) + | > avg_loss_disc_real_3: 0.24145695567131042 (+0.040800705552101135) + | > avg_loss_disc_real_4: 0.19727563112974167 (-0.030526868999004364) + | > avg_loss_disc_real_5: 0.2070010006427765 (-0.007959417998790741) + | > avg_loss_0: 2.4178589582443237 (-0.08577334880828857) + | > avg_loss_gen: 2.7658194303512573 (+0.3734968900680542) + | > avg_loss_kl: 2.695441961288452 (+0.09924495220184326) + | > avg_loss_feat: 7.596659421920776 (+0.20718050003051758) + | > avg_loss_mel: 18.68953514099121 (+0.22077178955078125) + | > avg_loss_duration: 1.7585827112197876 (+0.0006680488586425781) + | > avg_loss_1: 33.50604057312012 (+0.9013633728027344) + + + > EPOCH: 365/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:58:55)  + + --> TIME: 2025-04-19 04:59:00 -- STEP: 13/142 -- GLOBAL_STEP: 431125 + | > loss_disc: 2.426553726196289 (2.400876081906832) + | > loss_disc_real_0: 0.13576112687587738 (0.1194848378117268) + | > loss_disc_real_1: 0.21306350827217102 (0.19899013409247765) + | > loss_disc_real_2: 0.18496690690517426 (0.21977356878610757) + | > loss_disc_real_3: 0.21292465925216675 (0.22603312134742737) + | > loss_disc_real_4: 0.21745909750461578 (0.22201262987576997) + | > loss_disc_real_5: 0.17280252277851105 (0.22929403415093055) + | > loss_0: 2.426553726196289 (2.400876081906832) + | > grad_norm_0: tensor(7.3181, device='cuda:0') (tensor(8.6685, device='cuda:0')) + | > loss_gen: 2.309187173843384 (2.3994367856245775) + | > loss_kl: 1.818333387374878 (1.8178118192232573) + | > loss_feat: 7.939337730407715 (7.571818425105168) + | > loss_mel: 17.27085304260254 (17.847967001108024) + | > loss_duration: 1.4157779216766357 (1.410950935803927) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.753488540649414 (31.047985076904297) + | > grad_norm_1: tensor(137.0398, device='cuda:0') (tensor(213.4651, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.3288 (0.32991653222304124) + | > loader_time: 0.0051 (0.005067146741426908) + + + --> TIME: 2025-04-19 04:59:09 -- STEP: 38/142 -- GLOBAL_STEP: 431150 + | > loss_disc: 2.353226900100708 (2.439586695871855) + | > loss_disc_real_0: 0.11876704543828964 (0.11851224342459127) + | > loss_disc_real_1: 0.20307466387748718 (0.2017245418147037) + | > loss_disc_real_2: 0.24910077452659607 (0.22573394994986684) + | > loss_disc_real_3: 0.21805310249328613 (0.22985082354984784) + | > loss_disc_real_4: 0.22810107469558716 (0.22892640805558154) + | > loss_disc_real_5: 0.21474097669124603 (0.2322089127019832) + | > loss_0: 2.353226900100708 (2.439586695871855) + | > grad_norm_0: tensor(8.5674, device='cuda:0') (tensor(7.6178, device='cuda:0')) + | > loss_gen: 2.4573373794555664 (2.3633043640538265) + | > loss_kl: 1.7493754625320435 (1.8166586537110179) + | > loss_feat: 7.730628967285156 (7.28381243504976) + | > loss_mel: 17.015106201171875 (17.647951025711862) + | > loss_duration: 1.4317625761032104 (1.4219876496415387) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.38420867919922 (30.533714294433594) + | > grad_norm_1: tensor(211.0991, device='cuda:0') (tensor(159.4283, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.3541 (0.3415772852144744) + | > loader_time: 0.0057 (0.005280105691207082) + + + --> TIME: 2025-04-19 04:59:19 -- STEP: 63/142 -- GLOBAL_STEP: 431175 + | > loss_disc: 2.3638968467712402 (2.418994771109687) + | > loss_disc_real_0: 0.08200576901435852 (0.11808713525533676) + | > loss_disc_real_1: 0.19005900621414185 (0.2018172409799364) + | > loss_disc_real_2: 0.22803695499897003 (0.2238652907193653) + | > loss_disc_real_3: 0.24235540628433228 (0.2256817744364814) + | > loss_disc_real_4: 0.21274425089359283 (0.2256557418713494) + | > loss_disc_real_5: 0.22348004579544067 (0.22907813106264388) + | > loss_0: 2.3638968467712402 (2.418994771109687) + | > grad_norm_0: tensor(6.9002, device='cuda:0') (tensor(7.1997, device='cuda:0')) + | > loss_gen: 2.5132839679718018 (2.383729631938631) + | > loss_kl: 1.7848715782165527 (1.8212524425415766) + | > loss_feat: 7.05147123336792 (7.3341175185309515) + | > loss_mel: 18.926366806030273 (17.820646573626814) + | > loss_duration: 1.4846405982971191 (1.4332262731733771) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.760635375976562 (30.79297253442189) + | > grad_norm_1: tensor(81.7952, device='cuda:0') (tensor(161.5750, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.3906 (0.35308032944088896) + | > loader_time: 0.0059 (0.005549900115482389) + + + --> TIME: 2025-04-19 04:59:29 -- STEP: 88/142 -- GLOBAL_STEP: 431200 + | > loss_disc: 2.5682177543640137 (2.432991599494761) + | > loss_disc_real_0: 0.18684940040111542 (0.12147223754701289) + | > loss_disc_real_1: 0.2298283725976944 (0.20359387739815496) + | > loss_disc_real_2: 0.1840689331293106 (0.22426002015444366) + | > loss_disc_real_3: 0.21839173138141632 (0.22739289599386128) + | > loss_disc_real_4: 0.2449096292257309 (0.22607614828104322) + | > loss_disc_real_5: 0.1907578706741333 (0.23001935841007667) + | > loss_0: 2.5682177543640137 (2.432991599494761) + | > grad_norm_0: tensor(12.4572, device='cuda:0') (tensor(7.8318, device='cuda:0')) + | > loss_gen: 2.331760883331299 (2.3822704526511114) + | > loss_kl: 2.0034632682800293 (1.8449329178441654) + | > loss_feat: 7.306771755218506 (7.310840368270874) + | > loss_mel: 17.928800582885742 (17.818532824516296) + | > loss_duration: 1.5154128074645996 (1.4447501762346786) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.086210250854492 (30.801326990127563) + | > grad_norm_1: tensor(423.7325, device='cuda:0') (tensor(169.2003, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.4079 (0.36600941961461847) + | > loader_time: 0.0073 (0.005810249935496937) + + + --> TIME: 2025-04-19 04:59:40 -- STEP: 113/142 -- GLOBAL_STEP: 431225 + | > loss_disc: 2.524554491043091 (2.44600881306471) + | > loss_disc_real_0: 0.16476397216320038 (0.12278386343896916) + | > loss_disc_real_1: 0.2360680103302002 (0.20552269613320848) + | > loss_disc_real_2: 0.2099030464887619 (0.22520801570035714) + | > loss_disc_real_3: 0.26252779364585876 (0.22866320755101938) + | > loss_disc_real_4: 0.2226015329360962 (0.22650434577887038) + | > loss_disc_real_5: 0.20807422697544098 (0.2301456292646121) + | > loss_0: 2.524554491043091 (2.44600881306471) + | > grad_norm_0: tensor(17.3972, device='cuda:0') (tensor(8.4053, device='cuda:0')) + | > loss_gen: 2.3637821674346924 (2.3663551617512666) + | > loss_kl: 1.7705063819885254 (1.8528217741873412) + | > loss_feat: 6.960726737976074 (7.210956809795008) + | > loss_mel: 18.06951904296875 (17.732139410170834) + | > loss_duration: 1.4881534576416016 (1.4522849996533014) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.65268898010254 (30.614558380261986) + | > grad_norm_1: tensor(280.5980, device='cuda:0') (tensor(190.7930, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.4311 (0.37833801413004375) + | > loader_time: 0.0067 (0.006033696959503984) + + + --> TIME: 2025-04-19 04:59:51 -- STEP: 138/142 -- GLOBAL_STEP: 431250 + | > loss_disc: 2.449191093444824 (2.4521248910738076) + | > loss_disc_real_0: 0.1029873639345169 (0.12460968762204266) + | > loss_disc_real_1: 0.19882574677467346 (0.2062899027613626) + | > loss_disc_real_2: 0.22695975005626678 (0.22673442344302713) + | > loss_disc_real_3: 0.2550472617149353 (0.22909708558649255) + | > loss_disc_real_4: 0.24427297711372375 (0.22728459429049838) + | > loss_disc_real_5: 0.22532914578914642 (0.22957077417252716) + | > loss_0: 2.449191093444824 (2.4521248910738076) + | > grad_norm_0: tensor(4.2124, device='cuda:0') (tensor(8.7475, device='cuda:0')) + | > loss_gen: 2.36555552482605 (2.359486571256665) + | > loss_kl: 2.0067660808563232 (1.858942525110383) + | > loss_feat: 7.039501667022705 (7.137472080147785) + | > loss_mel: 18.913148880004883 (17.628825339718137) + | > loss_duration: 1.5126433372497559 (1.4597045545992644) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.837614059448242 (30.444431222003438) + | > grad_norm_1: tensor(73.9832, device='cuda:0') (tensor(201.0728, device='cuda:0')) + | > current_lr_0: 0.00019107948909596786 + | > current_lr_1: 0.00019107948909596786 + | > step_time: 0.4528 (0.3899673569029656) + | > loader_time: 0.0081 (0.006315725437109021) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.443020820617676 (2.443020820617676) + | > loss_disc_real_0: 0.15736544132232666 (0.15736544132232666) + | > loss_disc_real_1: 0.21153558790683746 (0.21153558790683746) + | > loss_disc_real_2: 0.2700762450695038 (0.2700762450695038) + | > loss_disc_real_3: 0.2307276874780655 (0.2307276874780655) + | > loss_disc_real_4: 0.264797180891037 (0.264797180891037) + | > loss_disc_real_5: 0.24917353689670563 (0.24917353689670563) + | > loss_0: 2.443020820617676 (2.443020820617676) + | > loss_gen: 2.524294853210449 (2.524294853210449) + | > loss_kl: 2.69976806640625 (2.69976806640625) + | > loss_feat: 6.808345794677734 (6.808345794677734) + | > loss_mel: 17.242692947387695 (17.242692947387695) + | > loss_duration: 1.7204707860946655 (1.7204707860946655) + | > loss_1: 30.99557113647461 (30.99557113647461) + + --> STEP: 1 + | > loss_disc: 2.559901237487793 (2.559901237487793) + | > loss_disc_real_0: 0.25346341729164124 (0.25346341729164124) + | > loss_disc_real_1: 0.2359052300453186 (0.2359052300453186) + | > loss_disc_real_2: 0.2555440068244934 (0.2555440068244934) + | > loss_disc_real_3: 0.24649657309055328 (0.24649657309055328) + | > loss_disc_real_4: 0.26127228140830994 (0.26127228140830994) + | > loss_disc_real_5: 0.22558119893074036 (0.22558119893074036) + | > loss_0: 2.559901237487793 (2.559901237487793) + | > loss_gen: 2.5855374336242676 (2.5855374336242676) + | > loss_kl: 2.9292898178100586 (2.9292898178100586) + | > loss_feat: 5.987428188323975 (5.987428188323975) + | > loss_mel: 16.719783782958984 (16.719783782958984) + | > loss_duration: 1.8040626049041748 (1.8040626049041748) + | > loss_1: 30.026100158691406 (30.026100158691406) + + --> STEP: 2 + | > loss_disc: 2.4947612285614014 (2.527331233024597) + | > loss_disc_real_0: 0.17595000565052032 (0.21470671147108078) + | > loss_disc_real_1: 0.2912261486053467 (0.26356568932533264) + | > loss_disc_real_2: 0.23657861351966858 (0.246061310172081) + | > loss_disc_real_3: 0.21761180460453033 (0.2320541888475418) + | > loss_disc_real_4: 0.25780820846557617 (0.25954024493694305) + | > loss_disc_real_5: 0.24949805438518524 (0.2375396266579628) + | > loss_0: 2.4947612285614014 (2.527331233024597) + | > loss_gen: 2.6154191493988037 (2.6004782915115356) + | > loss_kl: 2.4250106811523438 (2.677150249481201) + | > loss_feat: 7.337184906005859 (6.662306547164917) + | > loss_mel: 18.448135375976562 (17.583959579467773) + | > loss_duration: 1.702507734298706 (1.7532851696014404) + | > loss_1: 32.52825927734375 (31.277179718017578) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038949251174926758 (+0.0001024007797241211) + | > avg_loss_disc: 2.527331233024597 (+0.10947227478027344) + | > avg_loss_disc_real_0: 0.21470671147108078 (-0.049066945910453796) + | > avg_loss_disc_real_1: 0.26356568932533264 (+0.017227791249752045) + | > avg_loss_disc_real_2: 0.246061310172081 (+0.02939421683549881) + | > avg_loss_disc_real_3: 0.2320541888475418 (-0.009402766823768616) + | > avg_loss_disc_real_4: 0.25954024493694305 (+0.062264613807201385) + | > avg_loss_disc_real_5: 0.2375396266579628 (+0.03053862601518631) + | > avg_loss_0: 2.527331233024597 (+0.10947227478027344) + | > avg_loss_gen: 2.6004782915115356 (-0.16534113883972168) + | > avg_loss_kl: 2.677150249481201 (-0.018291711807250977) + | > avg_loss_feat: 6.662306547164917 (-0.9343528747558594) + | > avg_loss_mel: 17.583959579467773 (-1.1055755615234375) + | > avg_loss_duration: 1.7532851696014404 (-0.005297541618347168) + | > avg_loss_1: 31.277179718017578 (-2.228860855102539) + + + > EPOCH: 366/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 04:59:54)  + + --> TIME: 2025-04-19 05:00:02 -- STEP: 21/142 -- GLOBAL_STEP: 431275 + | > loss_disc: 2.431605815887451 (2.388249942234584) + | > loss_disc_real_0: 0.1390003114938736 (0.11429281071538017) + | > loss_disc_real_1: 0.2112366110086441 (0.20220098963805608) + | > loss_disc_real_2: 0.24447812139987946 (0.22379763211522782) + | > loss_disc_real_3: 0.1991054117679596 (0.22625756547564552) + | > loss_disc_real_4: 0.2115810513496399 (0.22350531887440456) + | > loss_disc_real_5: 0.21547409892082214 (0.2249194248801186) + | > loss_0: 2.431605815887451 (2.388249942234584) + | > grad_norm_0: tensor(9.9801, device='cuda:0') (tensor(10.6531, device='cuda:0')) + | > loss_gen: 2.4038360118865967 (2.4318569274175736) + | > loss_kl: 1.8137871026992798 (1.798623794601077) + | > loss_feat: 6.61604642868042 (7.592208499000186) + | > loss_mel: 16.767417907714844 (17.689425195966447) + | > loss_duration: 1.4122390747070312 (1.4132239761806669) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.013324737548828 (30.92533829098656) + | > grad_norm_1: tensor(194.0895, device='cuda:0') (tensor(272.5908, device='cuda:0')) + | > current_lr_0: 0.00019105560415983087 + | > current_lr_1: 0.00019105560415983087 + | > step_time: 0.3446 (0.3318000180380685) + | > loader_time: 0.0053 (0.005003361474900019) + + + --> TIME: 2025-04-19 05:00:11 -- STEP: 46/142 -- GLOBAL_STEP: 431300 + | > loss_disc: 2.4721832275390625 (2.4204922396203745) + | > loss_disc_real_0: 0.15571892261505127 (0.11823771651024403) + | > loss_disc_real_1: 0.21903403103351593 (0.20465159578167874) + | > loss_disc_real_2: 0.23047788441181183 (0.22504812997320425) + | > loss_disc_real_3: 0.2425948977470398 (0.22752729803323746) + | > loss_disc_real_4: 0.22336600720882416 (0.22891774287690286) + | > loss_disc_real_5: 0.182627871632576 (0.22637013216381488) + | > loss_0: 2.4721832275390625 (2.4204922396203745) + | > grad_norm_0: tensor(6.5158, device='cuda:0') (tensor(9.7222, device='cuda:0')) + | > loss_gen: 2.2294085025787354 (2.3974904858547705) + | > loss_kl: 1.7305946350097656 (1.8112171603285747) + | > loss_feat: 6.712467193603516 (7.3147140171216884) + | > loss_mel: 17.13234519958496 (17.56647644872251) + | > loss_duration: 1.4237887859344482 (1.4263948901839882) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.22860336303711 (30.516293111054793) + | > grad_norm_1: tensor(235.7690, device='cuda:0') (tensor(251.2975, device='cuda:0')) + | > current_lr_0: 0.00019105560415983087 + | > current_lr_1: 0.00019105560415983087 + | > step_time: 0.3637 (0.34381599011628533) + | > loader_time: 0.0059 (0.005360250887663468) + + + --> TIME: 2025-04-19 05:00:21 -- STEP: 71/142 -- GLOBAL_STEP: 431325 + | > loss_disc: 2.5718367099761963 (2.426670605028179) + | > loss_disc_real_0: 0.11963119357824326 (0.1188624134995568) + | > loss_disc_real_1: 0.2313242107629776 (0.20479936457016099) + | > loss_disc_real_2: 0.2575549781322479 (0.22613262815374724) + | > loss_disc_real_3: 0.22605165839195251 (0.2271913025580662) + | > loss_disc_real_4: 0.2801331579685211 (0.22822978987660206) + | > loss_disc_real_5: 0.2796115279197693 (0.22637016042857103) + | > loss_0: 2.5718367099761963 (2.426670605028179) + | > grad_norm_0: tensor(6.6407, device='cuda:0') (tensor(9.3100, device='cuda:0')) + | > loss_gen: 2.254420042037964 (2.3945499843275035) + | > loss_kl: 1.8790866136550903 (1.8323748968016933) + | > loss_feat: 6.742494583129883 (7.285672597482171) + | > loss_mel: 17.284709930419922 (17.638939467953968) + | > loss_duration: 1.4769500494003296 (1.4382353114410191) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63766098022461 (30.589772157266108) + | > grad_norm_1: tensor(189.4557, device='cuda:0') (tensor(242.9177, device='cuda:0')) + | > current_lr_0: 0.00019105560415983087 + | > current_lr_1: 0.00019105560415983087 + | > step_time: 0.3912 (0.35589048560236547) + | > loader_time: 0.0068 (0.005674741637538856) + + + --> TIME: 2025-04-19 05:00:31 -- STEP: 96/142 -- GLOBAL_STEP: 431350 + | > loss_disc: 2.4555866718292236 (2.43306980530421) + | > loss_disc_real_0: 0.09242042154073715 (0.11932368452350299) + | > loss_disc_real_1: 0.18296794593334198 (0.2054027154420813) + | > loss_disc_real_2: 0.230623260140419 (0.22667070214326182) + | > loss_disc_real_3: 0.23734551668167114 (0.22764083448176586) + | > loss_disc_real_4: 0.2259933352470398 (0.22852091118693352) + | > loss_disc_real_5: 0.19978249073028564 (0.22643394752716026) + | > loss_0: 2.4555866718292236 (2.43306980530421) + | > grad_norm_0: tensor(20.3780, device='cuda:0') (tensor(10.4723, device='cuda:0')) + | > loss_gen: 2.334477663040161 (2.3943027853965764) + | > loss_kl: 1.8807646036148071 (1.8397906981408596) + | > loss_feat: 7.700201034545898 (7.289165193835895) + | > loss_mel: 17.417728424072266 (17.753811955451972) + | > loss_duration: 1.4901883602142334 (1.448341420541207) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.823362350463867 (30.7254119515419) + | > grad_norm_1: tensor(291.7647, device='cuda:0') (tensor(261.9657, device='cuda:0')) + | > current_lr_0: 0.00019105560415983087 + | > current_lr_1: 0.00019105560415983087 + | > step_time: 0.4076 (0.3676666915416719) + | > loader_time: 0.0071 (0.005941631893316905) + + + --> TIME: 2025-04-19 05:00:42 -- STEP: 121/142 -- GLOBAL_STEP: 431375 + | > loss_disc: 2.423917293548584 (2.4365890380764794) + | > loss_disc_real_0: 0.08827264606952667 (0.11990738167496752) + | > loss_disc_real_1: 0.21503369510173798 (0.2051941732729762) + | > loss_disc_real_2: 0.1861242949962616 (0.22628359398073403) + | > loss_disc_real_3: 0.21526651084423065 (0.2276882174586462) + | > loss_disc_real_4: 0.2353082150220871 (0.2305447972756772) + | > loss_disc_real_5: 0.18121883273124695 (0.2266519394787875) + | > loss_0: 2.423917293548584 (2.4365890380764794) + | > grad_norm_0: tensor(22.8041, device='cuda:0') (tensor(11.9798, device='cuda:0')) + | > loss_gen: 2.4224047660827637 (2.406184627990092) + | > loss_kl: 2.004377603530884 (1.8500834240401087) + | > loss_feat: 7.484185218811035 (7.286110472088017) + | > loss_mel: 17.542219161987305 (17.70727097692569) + | > loss_duration: 1.5136754512786865 (1.4561014530087306) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.966861724853516 (30.705750820065333) + | > grad_norm_1: tensor(404.7743, device='cuda:0') (tensor(287.3900, device='cuda:0')) + | > current_lr_0: 0.00019105560415983087 + | > current_lr_1: 0.00019105560415983087 + | > step_time: 0.4438 (0.37931987470831763) + | > loader_time: 0.0075 (0.006221915079542428) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5514631271362305 (2.5514631271362305) + | > loss_disc_real_0: 0.21542072296142578 (0.21542072296142578) + | > loss_disc_real_1: 0.29927608370780945 (0.29927608370780945) + | > loss_disc_real_2: 0.2525108754634857 (0.2525108754634857) + | > loss_disc_real_3: 0.24191997945308685 (0.24191997945308685) + | > loss_disc_real_4: 0.2301570624113083 (0.2301570624113083) + | > loss_disc_real_5: 0.23046138882637024 (0.23046138882637024) + | > loss_0: 2.5514631271362305 (2.5514631271362305) + | > loss_gen: 2.350529670715332 (2.350529670715332) + | > loss_kl: 2.517347812652588 (2.517347812652588) + | > loss_feat: 5.206752777099609 (5.206752777099609) + | > loss_mel: 16.09119415283203 (16.09119415283203) + | > loss_duration: 1.7585651874542236 (1.7585651874542236) + | > loss_1: 27.92439079284668 (27.92439079284668) + + --> STEP: 1 + | > loss_disc: 2.5509238243103027 (2.5509238243103027) + | > loss_disc_real_0: 0.1755814254283905 (0.1755814254283905) + | > loss_disc_real_1: 0.3372194170951843 (0.3372194170951843) + | > loss_disc_real_2: 0.23785822093486786 (0.23785822093486786) + | > loss_disc_real_3: 0.2499655783176422 (0.2499655783176422) + | > loss_disc_real_4: 0.25470706820487976 (0.25470706820487976) + | > loss_disc_real_5: 0.2367510199546814 (0.2367510199546814) + | > loss_0: 2.5509238243103027 (2.5509238243103027) + | > loss_gen: 2.497732639312744 (2.497732639312744) + | > loss_kl: 2.821964740753174 (2.821964740753174) + | > loss_feat: 6.233030319213867 (6.233030319213867) + | > loss_mel: 17.26528549194336 (17.26528549194336) + | > loss_duration: 1.7795138359069824 (1.7795138359069824) + | > loss_1: 30.59752655029297 (30.59752655029297) + + --> STEP: 2 + | > loss_disc: 2.520043134689331 (2.535483479499817) + | > loss_disc_real_0: 0.16416612267494202 (0.16987377405166626) + | > loss_disc_real_1: 0.28574398159980774 (0.31148169934749603) + | > loss_disc_real_2: 0.260067343711853 (0.24896278232336044) + | > loss_disc_real_3: 0.22299502789974213 (0.23648030310869217) + | > loss_disc_real_4: 0.2597793936729431 (0.25724323093891144) + | > loss_disc_real_5: 0.16571691632270813 (0.20123396813869476) + | > loss_0: 2.520043134689331 (2.535483479499817) + | > loss_gen: 2.40187406539917 (2.449803352355957) + | > loss_kl: 2.310012102127075 (2.5659884214401245) + | > loss_feat: 8.375269889831543 (7.304150104522705) + | > loss_mel: 18.919490814208984 (18.092388153076172) + | > loss_duration: 1.7308454513549805 (1.7551796436309814) + | > loss_1: 33.737491607666016 (32.16750907897949) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032806396484375 (-0.0006142854690551758) + | > avg_loss_disc: 2.535483479499817 (+0.008152246475219727) + | > avg_loss_disc_real_0: 0.16987377405166626 (-0.04483293741941452) + | > avg_loss_disc_real_1: 0.31148169934749603 (+0.04791601002216339) + | > avg_loss_disc_real_2: 0.24896278232336044 (+0.0029014721512794495) + | > avg_loss_disc_real_3: 0.23648030310869217 (+0.00442611426115036) + | > avg_loss_disc_real_4: 0.25724323093891144 (-0.002297013998031616) + | > avg_loss_disc_real_5: 0.20123396813869476 (-0.036305658519268036) + | > avg_loss_0: 2.535483479499817 (+0.008152246475219727) + | > avg_loss_gen: 2.449803352355957 (-0.1506749391555786) + | > avg_loss_kl: 2.5659884214401245 (-0.11116182804107666) + | > avg_loss_feat: 7.304150104522705 (+0.6418435573577881) + | > avg_loss_mel: 18.092388153076172 (+0.5084285736083984) + | > avg_loss_duration: 1.7551796436309814 (+0.0018944740295410156) + | > avg_loss_1: 32.16750907897949 (+0.8903293609619141) + + + > EPOCH: 367/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:00:53)  + + --> TIME: 2025-04-19 05:00:55 -- STEP: 4/142 -- GLOBAL_STEP: 431400 + | > loss_disc: 2.3279614448547363 (2.3750099539756775) + | > loss_disc_real_0: 0.12336990982294083 (0.10997780971229076) + | > loss_disc_real_1: 0.1885261982679367 (0.19484354183077812) + | > loss_disc_real_2: 0.19078882038593292 (0.21416037529706955) + | > loss_disc_real_3: 0.2173682153224945 (0.2285602167248726) + | > loss_disc_real_4: 0.2261066734790802 (0.22697202116250992) + | > loss_disc_real_5: 0.2182977795600891 (0.22239431366324425) + | > loss_0: 2.3279614448547363 (2.3750099539756775) + | > grad_norm_0: tensor(7.3376, device='cuda:0') (tensor(8.4037, device='cuda:0')) + | > loss_gen: 2.548999547958374 (2.4281633496284485) + | > loss_kl: 1.7819533348083496 (1.8021917343139648) + | > loss_feat: 7.810284614562988 (7.2107155323028564) + | > loss_mel: 18.656240463256836 (17.645897388458252) + | > loss_duration: 1.434180736541748 (1.4125896692276) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.231658935546875 (30.499558448791504) + | > grad_norm_1: tensor(206.6558, device='cuda:0') (tensor(182.6450, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.3198 (0.3233732581138611) + | > loader_time: 0.005 (0.004742801189422607) + + + --> TIME: 2025-04-19 05:01:04 -- STEP: 29/142 -- GLOBAL_STEP: 431425 + | > loss_disc: 2.470606803894043 (2.425276410990748) + | > loss_disc_real_0: 0.13370953500270844 (0.11573662377637008) + | > loss_disc_real_1: 0.21905416250228882 (0.2070501251467343) + | > loss_disc_real_2: 0.21122461557388306 (0.22356969477801486) + | > loss_disc_real_3: 0.2406458705663681 (0.22827080303225025) + | > loss_disc_real_4: 0.23757846653461456 (0.22503554101648002) + | > loss_disc_real_5: 0.20471186935901642 (0.22949882519656215) + | > loss_0: 2.470606803894043 (2.425276410990748) + | > grad_norm_0: tensor(9.9199, device='cuda:0') (tensor(8.8337, device='cuda:0')) + | > loss_gen: 2.26590633392334 (2.3920432288071205) + | > loss_kl: 1.8150707483291626 (1.7901480814506268) + | > loss_feat: 6.318552494049072 (7.233014271177095) + | > loss_mel: 16.548828125 (17.495856515292463) + | > loss_duration: 1.4265964031219482 (1.4178046481362705) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.37495231628418 (30.328866629764953) + | > grad_norm_1: tensor(260.2608, device='cuda:0') (tensor(210.2216, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.3461 (0.33591863204692973) + | > loader_time: 0.0053 (0.005061059162534518) + + + --> TIME: 2025-04-19 05:01:13 -- STEP: 54/142 -- GLOBAL_STEP: 431450 + | > loss_disc: 2.4945976734161377 (2.4290366790912765) + | > loss_disc_real_0: 0.15410399436950684 (0.11535352758235401) + | > loss_disc_real_1: 0.21619746088981628 (0.20943839958420507) + | > loss_disc_real_2: 0.21971110999584198 (0.22262747558178725) + | > loss_disc_real_3: 0.21484772861003876 (0.22740541370930495) + | > loss_disc_real_4: 0.19923743605613708 (0.22392739052021945) + | > loss_disc_real_5: 0.23436106741428375 (0.233886677633833) + | > loss_0: 2.4945976734161377 (2.4290366790912765) + | > grad_norm_0: tensor(5.8114, device='cuda:0') (tensor(9.1927, device='cuda:0')) + | > loss_gen: 2.183321714401245 (2.3914677699406948) + | > loss_kl: 1.8275760412216187 (1.807508850539172) + | > loss_feat: 6.302613258361816 (7.256798594086258) + | > loss_mel: 15.887322425842285 (17.481991185082332) + | > loss_duration: 1.4559217691421509 (1.4273934651304174) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.656753540039062 (30.365159741154425) + | > grad_norm_1: tensor(114.7727, device='cuda:0') (tensor(248.2228, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.3716 (0.3474210456565575) + | > loader_time: 0.0058 (0.005432848577146178) + + + --> TIME: 2025-04-19 05:01:23 -- STEP: 79/142 -- GLOBAL_STEP: 431475 + | > loss_disc: 2.4319353103637695 (2.4328353495537467) + | > loss_disc_real_0: 0.0989302322268486 (0.1184570855354961) + | > loss_disc_real_1: 0.22401654720306396 (0.20879347301736662) + | > loss_disc_real_2: 0.24454627931118011 (0.22310854117327097) + | > loss_disc_real_3: 0.23277442157268524 (0.2265029811783682) + | > loss_disc_real_4: 0.2683004140853882 (0.2247412664980828) + | > loss_disc_real_5: 0.21754978597164154 (0.23104334726363798) + | > loss_0: 2.4319353103637695 (2.4328353495537467) + | > grad_norm_0: tensor(3.6743, device='cuda:0') (tensor(10.1477, device='cuda:0')) + | > loss_gen: 2.4992945194244385 (2.3902723608137695) + | > loss_kl: 1.9082876443862915 (1.8360690044451364) + | > loss_feat: 7.29193639755249 (7.256311971930009) + | > loss_mel: 18.874237060546875 (17.582034569752366) + | > loss_duration: 1.500709056854248 (1.4394509113287621) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.074462890625 (30.504138680953012) + | > grad_norm_1: tensor(217.2500, device='cuda:0') (tensor(262.6588, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.3871 (0.36083509348615816) + | > loader_time: 0.0069 (0.005713363237018828) + + + --> TIME: 2025-04-19 05:01:33 -- STEP: 104/142 -- GLOBAL_STEP: 431500 + | > loss_disc: 2.4159388542175293 (2.435124947474552) + | > loss_disc_real_0: 0.10852878540754318 (0.11906324463108411) + | > loss_disc_real_1: 0.20900392532348633 (0.2082735510686269) + | > loss_disc_real_2: 0.20687277615070343 (0.2233322559354397) + | > loss_disc_real_3: 0.2103634476661682 (0.2276380888831157) + | > loss_disc_real_4: 0.23023071885108948 (0.22648693893391353) + | > loss_disc_real_5: 0.23749412596225739 (0.22906661635408035) + | > loss_0: 2.4159388542175293 (2.435124947474552) + | > grad_norm_0: tensor(7.3744, device='cuda:0') (tensor(10.4762, device='cuda:0')) + | > loss_gen: 2.4580764770507812 (2.3863178216494036) + | > loss_kl: 1.914351224899292 (1.8422646178649023) + | > loss_feat: 6.8786115646362305 (7.213126691488119) + | > loss_mel: 17.65460968017578 (17.696105746122512) + | > loss_duration: 1.5027313232421875 (1.450631394982338) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.40838050842285 (30.588446140289307) + | > grad_norm_1: tensor(39.1468, device='cuda:0') (tensor(264.2231, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.4076 (0.3712930679321288) + | > loader_time: 0.007 (0.005964850003902729) + + + --> TIME: 2025-04-19 05:01:44 -- STEP: 129/142 -- GLOBAL_STEP: 431525 + | > loss_disc: 2.4399900436401367 (2.4439972204755436) + | > loss_disc_real_0: 0.11711756139993668 (0.1218468341832013) + | > loss_disc_real_1: 0.21871495246887207 (0.20800732676835024) + | > loss_disc_real_2: 0.21782971918582916 (0.22487217673035556) + | > loss_disc_real_3: 0.22733336687088013 (0.22733969778515573) + | > loss_disc_real_4: 0.22102601826190948 (0.2269902101089788) + | > loss_disc_real_5: 0.24680852890014648 (0.22964753834314125) + | > loss_0: 2.4399900436401367 (2.4439972204755436) + | > grad_norm_0: tensor(5.6278, device='cuda:0') (tensor(11.1782, device='cuda:0')) + | > loss_gen: 2.3520069122314453 (2.3784259788749744) + | > loss_kl: 2.000225782394409 (1.8515644858973894) + | > loss_feat: 7.1927809715271 (7.187749773957009) + | > loss_mel: 17.060338973999023 (17.672087698943855) + | > loss_duration: 1.4804151058197021 (1.4580328852631326) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.08576774597168 (30.547860744387606) + | > grad_norm_1: tensor(129.1839, device='cuda:0') (tensor(279.6765, device='cuda:0')) + | > current_lr_0: 0.0001910317222093109 + | > current_lr_1: 0.0001910317222093109 + | > step_time: 0.449 (0.3834517667459886) + | > loader_time: 0.0073 (0.006261243376620981) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4212660789489746 (2.4212660789489746) + | > loss_disc_real_0: 0.11219976097345352 (0.11219976097345352) + | > loss_disc_real_1: 0.2952696979045868 (0.2952696979045868) + | > loss_disc_real_2: 0.25818854570388794 (0.25818854570388794) + | > loss_disc_real_3: 0.26508545875549316 (0.26508545875549316) + | > loss_disc_real_4: 0.23847587406635284 (0.23847587406635284) + | > loss_disc_real_5: 0.28367987275123596 (0.28367987275123596) + | > loss_0: 2.4212660789489746 (2.4212660789489746) + | > loss_gen: 2.7470693588256836 (2.7470693588256836) + | > loss_kl: 2.3722870349884033 (2.3722870349884033) + | > loss_feat: 7.567069053649902 (7.567069053649902) + | > loss_mel: 19.563474655151367 (19.563474655151367) + | > loss_duration: 1.7192317247390747 (1.7192317247390747) + | > loss_1: 33.96913146972656 (33.96913146972656) + + --> STEP: 1 + | > loss_disc: 2.515486240386963 (2.515486240386963) + | > loss_disc_real_0: 0.11560400575399399 (0.11560400575399399) + | > loss_disc_real_1: 0.3079029321670532 (0.3079029321670532) + | > loss_disc_real_2: 0.25379326939582825 (0.25379326939582825) + | > loss_disc_real_3: 0.22467267513275146 (0.22467267513275146) + | > loss_disc_real_4: 0.21213532984256744 (0.21213532984256744) + | > loss_disc_real_5: 0.25702282786369324 (0.25702282786369324) + | > loss_0: 2.515486240386963 (2.515486240386963) + | > loss_gen: 2.421334981918335 (2.421334981918335) + | > loss_kl: 2.7213761806488037 (2.7213761806488037) + | > loss_feat: 7.793745040893555 (7.793745040893555) + | > loss_mel: 18.860271453857422 (18.860271453857422) + | > loss_duration: 1.7718799114227295 (1.7718799114227295) + | > loss_1: 33.568607330322266 (33.568607330322266) + + --> STEP: 2 + | > loss_disc: 2.465578556060791 (2.490532398223877) + | > loss_disc_real_0: 0.15607166290283203 (0.135837834328413) + | > loss_disc_real_1: 0.2924667298793793 (0.30018483102321625) + | > loss_disc_real_2: 0.24739693105220795 (0.2505951002240181) + | > loss_disc_real_3: 0.2667417526245117 (0.2457072138786316) + | > loss_disc_real_4: 0.2160545140504837 (0.21409492194652557) + | > loss_disc_real_5: 0.27321189641952515 (0.2651173621416092) + | > loss_0: 2.465578556060791 (2.490532398223877) + | > loss_gen: 2.6204798221588135 (2.520907402038574) + | > loss_kl: 2.318026542663574 (2.519701361656189) + | > loss_feat: 8.061331748962402 (7.9275383949279785) + | > loss_mel: 18.717056274414062 (18.788663864135742) + | > loss_duration: 1.6971075534820557 (1.7344937324523926) + | > loss_1: 33.41400146484375 (33.49130439758301) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036368370056152344 (+0.0003561973571777344) + | > avg_loss_disc: 2.490532398223877 (-0.04495108127593994) + | > avg_loss_disc_real_0: 0.135837834328413 (-0.03403593972325325) + | > avg_loss_disc_real_1: 0.30018483102321625 (-0.011296868324279785) + | > avg_loss_disc_real_2: 0.2505951002240181 (+0.0016323179006576538) + | > avg_loss_disc_real_3: 0.2457072138786316 (+0.009226910769939423) + | > avg_loss_disc_real_4: 0.21409492194652557 (-0.043148308992385864) + | > avg_loss_disc_real_5: 0.2651173621416092 (+0.06388339400291443) + | > avg_loss_0: 2.490532398223877 (-0.04495108127593994) + | > avg_loss_gen: 2.520907402038574 (+0.07110404968261719) + | > avg_loss_kl: 2.519701361656189 (-0.04628705978393555) + | > avg_loss_feat: 7.9275383949279785 (+0.6233882904052734) + | > avg_loss_mel: 18.788663864135742 (+0.6962757110595703) + | > avg_loss_duration: 1.7344937324523926 (-0.020685911178588867) + | > avg_loss_1: 33.49130439758301 (+1.3237953186035156) + + + > EPOCH: 368/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:01:52)  + + --> TIME: 2025-04-19 05:01:57 -- STEP: 12/142 -- GLOBAL_STEP: 431550 + | > loss_disc: 2.4167370796203613 (2.3920505046844482) + | > loss_disc_real_0: 0.16187551617622375 (0.1181523979951938) + | > loss_disc_real_1: 0.22189828753471375 (0.2105994994441668) + | > loss_disc_real_2: 0.256166934967041 (0.22613722706834474) + | > loss_disc_real_3: 0.21911165118217468 (0.22054913764198622) + | > loss_disc_real_4: 0.23902259767055511 (0.2217810613413652) + | > loss_disc_real_5: 0.26509377360343933 (0.22772679602106413) + | > loss_0: 2.4167370796203613 (2.3920505046844482) + | > grad_norm_0: tensor(4.2510, device='cuda:0') (tensor(6.6937, device='cuda:0')) + | > loss_gen: 2.5077619552612305 (2.4149836699167886) + | > loss_kl: 2.023862361907959 (1.8044112424055736) + | > loss_feat: 7.23894739151001 (7.559075911839803) + | > loss_mel: 18.065473556518555 (17.988966306050617) + | > loss_duration: 1.4439387321472168 (1.4182880123456318) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.279983520507812 (31.185725053151447) + | > grad_norm_1: tensor(204.5916, device='cuda:0') (tensor(175.8285, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.3297 (0.3460039695103963) + | > loader_time: 0.0047 (0.004603664080301921) + + + --> TIME: 2025-04-19 05:02:06 -- STEP: 37/142 -- GLOBAL_STEP: 431575 + | > loss_disc: 2.525730609893799 (2.4092164490673995) + | > loss_disc_real_0: 0.15056157112121582 (0.1170210576540715) + | > loss_disc_real_1: 0.19693298637866974 (0.20651074559301943) + | > loss_disc_real_2: 0.20935536921024323 (0.22258776506862124) + | > loss_disc_real_3: 0.22196128964424133 (0.22355990836749207) + | > loss_disc_real_4: 0.21903331577777863 (0.22583796003380338) + | > loss_disc_real_5: 0.2365874946117401 (0.22605384201616854) + | > loss_0: 2.525730609893799 (2.4092164490673995) + | > grad_norm_0: tensor(5.9155, device='cuda:0') (tensor(7.1870, device='cuda:0')) + | > loss_gen: 2.2973036766052246 (2.405676139367593) + | > loss_kl: 1.9149196147918701 (1.7877928147444855) + | > loss_feat: 7.107365608215332 (7.4161448994198365) + | > loss_mel: 17.2838134765625 (17.592025602186048) + | > loss_duration: 1.4502604007720947 (1.4270839852255743) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05366325378418 (30.628723608480918) + | > grad_norm_1: tensor(82.3180, device='cuda:0') (tensor(160.0743, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.3534 (0.346994296924488) + | > loader_time: 0.0057 (0.005020669988683752) + + + --> TIME: 2025-04-19 05:02:15 -- STEP: 62/142 -- GLOBAL_STEP: 431600 + | > loss_disc: 2.516131639480591 (2.433982691457194) + | > loss_disc_real_0: 0.1347171515226364 (0.11983987773137708) + | > loss_disc_real_1: 0.22158606350421906 (0.20739643540113203) + | > loss_disc_real_2: 0.22426897287368774 (0.22343467848916207) + | > loss_disc_real_3: 0.2106853425502777 (0.22608454285129423) + | > loss_disc_real_4: 0.2181408703327179 (0.2276133063339418) + | > loss_disc_real_5: 0.20395545661449432 (0.23076922446489334) + | > loss_0: 2.516131639480591 (2.433982691457194) + | > grad_norm_0: tensor(13.2213, device='cuda:0') (tensor(7.4555, device='cuda:0')) + | > loss_gen: 2.1853463649749756 (2.373932277002642) + | > loss_kl: 1.6521623134613037 (1.8076003289991809) + | > loss_feat: 6.3394455909729 (7.248219036286877) + | > loss_mel: 16.762649536132812 (17.501508435895367) + | > loss_duration: 1.4730721712112427 (1.436798178380536) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.412675857543945 (30.368058389233006) + | > grad_norm_1: tensor(105.0381, device='cuda:0') (tensor(157.2639, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.3842 (0.3555776419178133) + | > loader_time: 0.006 (0.005324129135377945) + + + --> TIME: 2025-04-19 05:02:25 -- STEP: 87/142 -- GLOBAL_STEP: 431625 + | > loss_disc: 2.448085069656372 (2.440311607273145) + | > loss_disc_real_0: 0.09940450638532639 (0.122501817585408) + | > loss_disc_real_1: 0.19380219280719757 (0.2064457367891553) + | > loss_disc_real_2: 0.227276012301445 (0.22472345332304636) + | > loss_disc_real_3: 0.20407456159591675 (0.22619131995343614) + | > loss_disc_real_4: 0.21444463729858398 (0.22822282882942552) + | > loss_disc_real_5: 0.2539973855018616 (0.23014847266262975) + | > loss_0: 2.448085069656372 (2.440311607273145) + | > grad_norm_0: tensor(6.4355, device='cuda:0') (tensor(8.1213, device='cuda:0')) + | > loss_gen: 2.318265676498413 (2.3707484787908095) + | > loss_kl: 1.7396836280822754 (1.8254167279977909) + | > loss_feat: 7.863422393798828 (7.215294388518936) + | > loss_mel: 19.310195922851562 (17.57188156829483) + | > loss_duration: 1.4878363609313965 (1.4472716512351205) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.71940231323242 (30.43061284909303) + | > grad_norm_1: tensor(123.7043, device='cuda:0') (tensor(177.1331, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.4017 (0.36603227154961954) + | > loader_time: 0.0066 (0.0056460643636769255) + + + --> TIME: 2025-04-19 05:02:36 -- STEP: 112/142 -- GLOBAL_STEP: 431650 + | > loss_disc: 2.4590706825256348 (2.4438491633960178) + | > loss_disc_real_0: 0.19805771112442017 (0.12418076455859202) + | > loss_disc_real_1: 0.2024976760149002 (0.20679187907704286) + | > loss_disc_real_2: 0.2466125339269638 (0.225012273793774) + | > loss_disc_real_3: 0.21918323636054993 (0.2266697912876095) + | > loss_disc_real_4: 0.2384466528892517 (0.22829237221074955) + | > loss_disc_real_5: 0.17649969458580017 (0.2305459094100765) + | > loss_0: 2.4590706825256348 (2.4438491633960178) + | > grad_norm_0: tensor(18.8299, device='cuda:0') (tensor(9.3835, device='cuda:0')) + | > loss_gen: 2.330828905105591 (2.372857274753706) + | > loss_kl: 1.791076898574829 (1.8357660004070826) + | > loss_feat: 7.781754016876221 (7.193591858659472) + | > loss_mel: 18.357234954833984 (17.57460279124123) + | > loss_duration: 1.4588631391525269 (1.4547201682414332) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.719757080078125 (30.431538122040884) + | > grad_norm_1: tensor(390.0268, device='cuda:0') (tensor(204.9897, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.4262 (0.3761695453098842) + | > loader_time: 0.0064 (0.005911758967808314) + + + --> TIME: 2025-04-19 05:02:47 -- STEP: 137/142 -- GLOBAL_STEP: 431675 + | > loss_disc: 2.5071418285369873 (2.4482461066141616) + | > loss_disc_real_0: 0.141794815659523 (0.12450826711898302) + | > loss_disc_real_1: 0.2531633675098419 (0.20746918305428358) + | > loss_disc_real_2: 0.22769653797149658 (0.22489916603930676) + | > loss_disc_real_3: 0.2610734701156616 (0.2272370157450655) + | > loss_disc_real_4: 0.2775951325893402 (0.22911437084204958) + | > loss_disc_real_5: 0.3015865683555603 (0.23078712396813136) + | > loss_0: 2.5071418285369873 (2.4482461066141616) + | > grad_norm_0: tensor(13.4461, device='cuda:0') (tensor(10.2425, device='cuda:0')) + | > loss_gen: 2.4206807613372803 (2.3727276447045536) + | > loss_kl: 1.7742409706115723 (1.8386918332454931) + | > loss_feat: 6.606921672821045 (7.1736937891827885) + | > loss_mel: 18.97187042236328 (17.606063160583044) + | > loss_duration: 1.480210781097412 (1.4604185167020258) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.253925323486328 (30.451595014029177) + | > grad_norm_1: tensor(124.8993, device='cuda:0') (tensor(223.9888, device='cuda:0')) + | > current_lr_0: 0.00019100784324403472 + | > current_lr_1: 0.00019100784324403472 + | > step_time: 0.4558 (0.3876866215336932) + | > loader_time: 0.0071 (0.006202072992812108) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4683351516723633 (2.4683351516723633) + | > loss_disc_real_0: 0.1874416321516037 (0.1874416321516037) + | > loss_disc_real_1: 0.16640953719615936 (0.16640953719615936) + | > loss_disc_real_2: 0.1855715960264206 (0.1855715960264206) + | > loss_disc_real_3: 0.22949324548244476 (0.22949324548244476) + | > loss_disc_real_4: 0.23169037699699402 (0.23169037699699402) + | > loss_disc_real_5: 0.22947221994400024 (0.22947221994400024) + | > loss_0: 2.4683351516723633 (2.4683351516723633) + | > loss_gen: 2.3240950107574463 (2.3240950107574463) + | > loss_kl: 2.2213239669799805 (2.2213239669799805) + | > loss_feat: 7.186460971832275 (7.186460971832275) + | > loss_mel: 19.281877517700195 (19.281877517700195) + | > loss_duration: 1.7308430671691895 (1.7308430671691895) + | > loss_1: 32.74460220336914 (32.74460220336914) + + --> STEP: 1 + | > loss_disc: 2.4675192832946777 (2.4675192832946777) + | > loss_disc_real_0: 0.18167634308338165 (0.18167634308338165) + | > loss_disc_real_1: 0.18971265852451324 (0.18971265852451324) + | > loss_disc_real_2: 0.21148915588855743 (0.21148915588855743) + | > loss_disc_real_3: 0.21514764428138733 (0.21514764428138733) + | > loss_disc_real_4: 0.22979383170604706 (0.22979383170604706) + | > loss_disc_real_5: 0.23905374109745026 (0.23905374109745026) + | > loss_0: 2.4675192832946777 (2.4675192832946777) + | > loss_gen: 2.4172894954681396 (2.4172894954681396) + | > loss_kl: 2.894538640975952 (2.894538640975952) + | > loss_feat: 6.361380577087402 (6.361380577087402) + | > loss_mel: 16.923492431640625 (16.923492431640625) + | > loss_duration: 1.7883656024932861 (1.7883656024932861) + | > loss_1: 30.385068893432617 (30.385068893432617) + + --> STEP: 2 + | > loss_disc: 2.4475841522216797 (2.4575517177581787) + | > loss_disc_real_0: 0.18483418226242065 (0.18325526267290115) + | > loss_disc_real_1: 0.19189667701721191 (0.19080466777086258) + | > loss_disc_real_2: 0.21170736849308014 (0.2115982621908188) + | > loss_disc_real_3: 0.2063058465719223 (0.21072674542665482) + | > loss_disc_real_4: 0.23600983619689941 (0.23290183395147324) + | > loss_disc_real_5: 0.22940152883529663 (0.23422763496637344) + | > loss_0: 2.4475841522216797 (2.4575517177581787) + | > loss_gen: 2.5011868476867676 (2.4592381715774536) + | > loss_kl: 2.335789203643799 (2.6151639223098755) + | > loss_feat: 7.323550224304199 (6.842465400695801) + | > loss_mel: 17.910661697387695 (17.41707706451416) + | > loss_duration: 1.702703595161438 (1.745534598827362) + | > loss_1: 31.77389144897461 (31.079480171203613) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003982424736022949 (+0.00034558773040771484) + | > avg_loss_disc: 2.4575517177581787 (-0.03298068046569824) + | > avg_loss_disc_real_0: 0.18325526267290115 (+0.047417428344488144) + | > avg_loss_disc_real_1: 0.19080466777086258 (-0.10938016325235367) + | > avg_loss_disc_real_2: 0.2115982621908188 (-0.03899683803319931) + | > avg_loss_disc_real_3: 0.21072674542665482 (-0.034980468451976776) + | > avg_loss_disc_real_4: 0.23290183395147324 (+0.018806912004947662) + | > avg_loss_disc_real_5: 0.23422763496637344 (-0.03088972717523575) + | > avg_loss_0: 2.4575517177581787 (-0.03298068046569824) + | > avg_loss_gen: 2.4592381715774536 (-0.061669230461120605) + | > avg_loss_kl: 2.6151639223098755 (+0.09546256065368652) + | > avg_loss_feat: 6.842465400695801 (-1.0850729942321777) + | > avg_loss_mel: 17.41707706451416 (-1.371586799621582) + | > avg_loss_duration: 1.745534598827362 (+0.011040866374969482) + | > avg_loss_1: 31.079480171203613 (-2.4118242263793945) + + + > EPOCH: 369/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:02:50)  + + --> TIME: 2025-04-19 05:02:58 -- STEP: 20/142 -- GLOBAL_STEP: 431700 + | > loss_disc: 2.3301637172698975 (2.394357943534851) + | > loss_disc_real_0: 0.1090940460562706 (0.11203084401786327) + | > loss_disc_real_1: 0.1892857402563095 (0.20973706170916556) + | > loss_disc_real_2: 0.22711414098739624 (0.22424805611371995) + | > loss_disc_real_3: 0.22299741208553314 (0.2238783746957779) + | > loss_disc_real_4: 0.23270924389362335 (0.2223029337823391) + | > loss_disc_real_5: 0.21583572030067444 (0.21938229501247405) + | > loss_0: 2.3301637172698975 (2.394357943534851) + | > grad_norm_0: tensor(6.1038, device='cuda:0') (tensor(9.9157, device='cuda:0')) + | > loss_gen: 2.5112833976745605 (2.4060060501098635) + | > loss_kl: 1.7411139011383057 (1.7815787613391876) + | > loss_feat: 7.452748775482178 (7.2652768850326535) + | > loss_mel: 18.502357482910156 (17.466883277893068) + | > loss_duration: 1.4905732870101929 (1.4152454018592835) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.698078155517578 (30.334990787506104) + | > grad_norm_1: tensor(218.0326, device='cuda:0') (tensor(229.9957, device='cuda:0')) + | > current_lr_0: 0.0001909839672636292 + | > current_lr_1: 0.0001909839672636292 + | > step_time: 0.3457 (0.3301667332649231) + | > loader_time: 0.0061 (0.004943466186523438) + + + --> TIME: 2025-04-19 05:03:07 -- STEP: 45/142 -- GLOBAL_STEP: 431725 + | > loss_disc: 2.443688154220581 (2.412978235880534) + | > loss_disc_real_0: 0.14183318614959717 (0.11684904777341419) + | > loss_disc_real_1: 0.20734691619873047 (0.20736101037926144) + | > loss_disc_real_2: 0.2282683551311493 (0.22385137379169465) + | > loss_disc_real_3: 0.2668144404888153 (0.22711422873867881) + | > loss_disc_real_4: 0.2400108426809311 (0.22536363436116114) + | > loss_disc_real_5: 0.21101315319538116 (0.22402575214703876) + | > loss_0: 2.443688154220581 (2.412978235880534) + | > grad_norm_0: tensor(4.2270, device='cuda:0') (tensor(8.5380, device='cuda:0')) + | > loss_gen: 2.255544900894165 (2.3981856452094186) + | > loss_kl: 2.018075704574585 (1.794081695874532) + | > loss_feat: 6.060514450073242 (7.229707251654731) + | > loss_mel: 17.332923889160156 (17.63287281460232) + | > loss_duration: 1.4430606365203857 (1.4268064843283765) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.110118865966797 (30.48165414598253) + | > grad_norm_1: tensor(78.4184, device='cuda:0') (tensor(169.6444, device='cuda:0')) + | > current_lr_0: 0.0001909839672636292 + | > current_lr_1: 0.0001909839672636292 + | > step_time: 0.3668 (0.34406828880310064) + | > loader_time: 0.0058 (0.005355760786268446) + + + --> TIME: 2025-04-19 05:03:17 -- STEP: 70/142 -- GLOBAL_STEP: 431750 + | > loss_disc: 2.2962019443511963 (2.4252835852759227) + | > loss_disc_real_0: 0.09145565330982208 (0.11947739134941782) + | > loss_disc_real_1: 0.1991795152425766 (0.20763700348990305) + | > loss_disc_real_2: 0.24908794462680817 (0.22475318908691405) + | > loss_disc_real_3: 0.19925373792648315 (0.22700886598655157) + | > loss_disc_real_4: 0.22485722601413727 (0.22466573417186736) + | > loss_disc_real_5: 0.221516415476799 (0.22576904616185597) + | > loss_0: 2.2962019443511963 (2.4252835852759227) + | > grad_norm_0: tensor(9.8095, device='cuda:0') (tensor(8.7059, device='cuda:0')) + | > loss_gen: 2.429723024368286 (2.3952155555997576) + | > loss_kl: 1.8150368928909302 (1.810497828892299) + | > loss_feat: 7.631589412689209 (7.289776740755354) + | > loss_mel: 18.49834632873535 (17.743756294250485) + | > loss_duration: 1.4666357040405273 (1.4369294473103118) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.841331481933594 (30.676176098414828) + | > grad_norm_1: tensor(164.7010, device='cuda:0') (tensor(188.3069, device='cuda:0')) + | > current_lr_0: 0.0001909839672636292 + | > current_lr_1: 0.0001909839672636292 + | > step_time: 0.3878 (0.3557525941303798) + | > loader_time: 0.0074 (0.0056256702968052455) + + + --> TIME: 2025-04-19 05:03:27 -- STEP: 95/142 -- GLOBAL_STEP: 431775 + | > loss_disc: 2.446012020111084 (2.432496156190572) + | > loss_disc_real_0: 0.11008347570896149 (0.12174999447245347) + | > loss_disc_real_1: 0.20907501876354218 (0.20702240435700667) + | > loss_disc_real_2: 0.194249227643013 (0.22451021137990448) + | > loss_disc_real_3: 0.22598469257354736 (0.22805226664794118) + | > loss_disc_real_4: 0.20834240317344666 (0.22655889705607765) + | > loss_disc_real_5: 0.2438371479511261 (0.2251144747985037) + | > loss_0: 2.446012020111084 (2.432496156190572) + | > grad_norm_0: tensor(14.2334, device='cuda:0') (tensor(10.0228, device='cuda:0')) + | > loss_gen: 2.3358051776885986 (2.395221348812707) + | > loss_kl: 1.7866706848144531 (1.8225613004282901) + | > loss_feat: 6.976833343505859 (7.264633233923661) + | > loss_mel: 18.488344192504883 (17.772643059178392) + | > loss_duration: 1.4958696365356445 (1.448198738851046) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.08352279663086 (30.703257851851614) + | > grad_norm_1: tensor(380.1214, device='cuda:0') (tensor(227.5405, device='cuda:0')) + | > current_lr_0: 0.0001909839672636292 + | > current_lr_1: 0.0001909839672636292 + | > step_time: 0.4031 (0.36641396974262436) + | > loader_time: 0.0069 (0.005878802349692897) + + + --> TIME: 2025-04-19 05:03:37 -- STEP: 120/142 -- GLOBAL_STEP: 431800 + | > loss_disc: 2.5066001415252686 (2.4392996311187742) + | > loss_disc_real_0: 0.1018884927034378 (0.12423030603677035) + | > loss_disc_real_1: 0.19256694614887238 (0.20697628036141397) + | > loss_disc_real_2: 0.1973770260810852 (0.22512987752755484) + | > loss_disc_real_3: 0.2172040343284607 (0.2280986109127601) + | > loss_disc_real_4: 0.213016539812088 (0.22766571665803592) + | > loss_disc_real_5: 0.22005020081996918 (0.22500980297724407) + | > loss_0: 2.5066001415252686 (2.4392996311187742) + | > grad_norm_0: tensor(15.1206, device='cuda:0') (tensor(10.5610, device='cuda:0')) + | > loss_gen: 2.153956413269043 (2.382123057047527) + | > loss_kl: 1.8855905532836914 (1.8307979186375936) + | > loss_feat: 6.335234642028809 (7.206065920988719) + | > loss_mel: 15.931520462036133 (17.70656406879425) + | > loss_duration: 1.496324896812439 (1.4552695473035182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.80262565612793 (30.580820655822755) + | > grad_norm_1: tensor(360.9942, device='cuda:0') (tensor(236.9268, device='cuda:0')) + | > current_lr_0: 0.0001909839672636292 + | > current_lr_1: 0.0001909839672636292 + | > step_time: 0.4347 (0.3773943324883779) + | > loader_time: 0.007 (0.006108347574869792) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5318870544433594 (2.5318870544433594) + | > loss_disc_real_0: 0.18245334923267365 (0.18245334923267365) + | > loss_disc_real_1: 0.25271075963974 (0.25271075963974) + | > loss_disc_real_2: 0.2762775719165802 (0.2762775719165802) + | > loss_disc_real_3: 0.2485310286283493 (0.2485310286283493) + | > loss_disc_real_4: 0.28038734197616577 (0.28038734197616577) + | > loss_disc_real_5: 0.20460648834705353 (0.20460648834705353) + | > loss_0: 2.5318870544433594 (2.5318870544433594) + | > loss_gen: 2.512847900390625 (2.512847900390625) + | > loss_kl: 2.534888505935669 (2.534888505935669) + | > loss_feat: 6.906272888183594 (6.906272888183594) + | > loss_mel: 18.04352569580078 (18.04352569580078) + | > loss_duration: 1.739715576171875 (1.739715576171875) + | > loss_1: 31.73725128173828 (31.73725128173828) + + --> STEP: 1 + | > loss_disc: 2.534902334213257 (2.534902334213257) + | > loss_disc_real_0: 0.1602858006954193 (0.1602858006954193) + | > loss_disc_real_1: 0.2817635238170624 (0.2817635238170624) + | > loss_disc_real_2: 0.2787993252277374 (0.2787993252277374) + | > loss_disc_real_3: 0.2486656755208969 (0.2486656755208969) + | > loss_disc_real_4: 0.2896811068058014 (0.2896811068058014) + | > loss_disc_real_5: 0.26537764072418213 (0.26537764072418213) + | > loss_0: 2.534902334213257 (2.534902334213257) + | > loss_gen: 2.5528273582458496 (2.5528273582458496) + | > loss_kl: 3.035905361175537 (3.035905361175537) + | > loss_feat: 7.402930736541748 (7.402930736541748) + | > loss_mel: 19.12848472595215 (19.12848472595215) + | > loss_duration: 1.771103858947754 (1.771103858947754) + | > loss_1: 33.89125061035156 (33.89125061035156) + + --> STEP: 2 + | > loss_disc: 2.5409603118896484 (2.5379313230514526) + | > loss_disc_real_0: 0.1613401174545288 (0.16081295907497406) + | > loss_disc_real_1: 0.33864399790763855 (0.31020376086235046) + | > loss_disc_real_2: 0.2854713797569275 (0.28213535249233246) + | > loss_disc_real_3: 0.25788450241088867 (0.2532750889658928) + | > loss_disc_real_4: 0.3004879355430603 (0.29508452117443085) + | > loss_disc_real_5: 0.150813490152359 (0.20809556543827057) + | > loss_0: 2.5409603118896484 (2.5379313230514526) + | > loss_gen: 2.672724485397339 (2.6127759218215942) + | > loss_kl: 2.5584702491760254 (2.7971878051757812) + | > loss_feat: 8.845701217651367 (8.124315977096558) + | > loss_mel: 18.589914321899414 (18.85919952392578) + | > loss_duration: 1.7084002494812012 (1.7397520542144775) + | > loss_1: 34.37520980834961 (34.133230209350586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033522844314575195 (-0.0006301403045654297) + | > avg_loss_disc: 2.5379313230514526 (+0.08037960529327393) + | > avg_loss_disc_real_0: 0.16081295907497406 (-0.022442303597927094) + | > avg_loss_disc_real_1: 0.31020376086235046 (+0.11939909309148788) + | > avg_loss_disc_real_2: 0.28213535249233246 (+0.07053709030151367) + | > avg_loss_disc_real_3: 0.2532750889658928 (+0.042548343539237976) + | > avg_loss_disc_real_4: 0.29508452117443085 (+0.06218268722295761) + | > avg_loss_disc_real_5: 0.20809556543827057 (-0.026132069528102875) + | > avg_loss_0: 2.5379313230514526 (+0.08037960529327393) + | > avg_loss_gen: 2.6127759218215942 (+0.15353775024414062) + | > avg_loss_kl: 2.7971878051757812 (+0.18202388286590576) + | > avg_loss_feat: 8.124315977096558 (+1.2818505764007568) + | > avg_loss_mel: 18.85919952392578 (+1.442122459411621) + | > avg_loss_duration: 1.7397520542144775 (-0.0057825446128845215) + | > avg_loss_1: 34.133230209350586 (+3.0537500381469727) + + + > EPOCH: 370/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:03:49)  + + --> TIME: 2025-04-19 05:03:51 -- STEP: 3/142 -- GLOBAL_STEP: 431825 + | > loss_disc: 2.483088970184326 (2.5103488763173423) + | > loss_disc_real_0: 0.10059608519077301 (0.11713388313849767) + | > loss_disc_real_1: 0.17032361030578613 (0.22859432796637216) + | > loss_disc_real_2: 0.23201319575309753 (0.2558354338010152) + | > loss_disc_real_3: 0.25294941663742065 (0.26752596100171405) + | > loss_disc_real_4: 0.2886083424091339 (0.25877852241198224) + | > loss_disc_real_5: 0.2649840712547302 (0.2448422759771347) + | > loss_0: 2.483088970184326 (2.5103488763173423) + | > grad_norm_0: tensor(6.8957, device='cuda:0') (tensor(13.3940, device='cuda:0')) + | > loss_gen: 2.3506054878234863 (2.3413912455240884) + | > loss_kl: 1.8425027132034302 (1.8123795588811238) + | > loss_feat: 6.844818115234375 (7.444414774576823) + | > loss_mel: 18.848955154418945 (18.19919713338216) + | > loss_duration: 1.4229425191879272 (1.4147435426712036) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.309823989868164 (31.21212641398112) + | > grad_norm_1: tensor(289.5025, device='cuda:0') (tensor(375.6397, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.3239 (0.3281959692637126) + | > loader_time: 0.004 (0.004767576853434245) + + + --> TIME: 2025-04-19 05:04:00 -- STEP: 28/142 -- GLOBAL_STEP: 431850 + | > loss_disc: 2.4139766693115234 (2.4280770421028137) + | > loss_disc_real_0: 0.11087198555469513 (0.11216427997819015) + | > loss_disc_real_1: 0.16960389912128448 (0.2084545764539923) + | > loss_disc_real_2: 0.21825739741325378 (0.22508221492171288) + | > loss_disc_real_3: 0.21229465305805206 (0.22721173986792564) + | > loss_disc_real_4: 0.21491046249866486 (0.226301799927439) + | > loss_disc_real_5: 0.24404044449329376 (0.23297325149178505) + | > loss_0: 2.4139766693115234 (2.4280770421028137) + | > grad_norm_0: tensor(4.1193, device='cuda:0') (tensor(9.8391, device='cuda:0')) + | > loss_gen: 2.3448541164398193 (2.383675966944012) + | > loss_kl: 1.7292959690093994 (1.8167745641299657) + | > loss_feat: 7.249856472015381 (7.248959268842425) + | > loss_mel: 19.372802734375 (17.781629596437732) + | > loss_duration: 1.4466056823730469 (1.4232145760740553) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.14341735839844 (30.654253823416575) + | > grad_norm_1: tensor(221.0047, device='cuda:0') (tensor(263.2235, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.3472 (0.336088444505419) + | > loader_time: 0.0059 (0.005140100206647601) + + + --> TIME: 2025-04-19 05:04:09 -- STEP: 53/142 -- GLOBAL_STEP: 431875 + | > loss_disc: 2.2846784591674805 (2.4212488093466127) + | > loss_disc_real_0: 0.08664946258068085 (0.11498952136849458) + | > loss_disc_real_1: 0.1764964461326599 (0.20609134829269266) + | > loss_disc_real_2: 0.2147083878517151 (0.22508108981375424) + | > loss_disc_real_3: 0.23467537760734558 (0.2274399146718799) + | > loss_disc_real_4: 0.2500167191028595 (0.2268923138672451) + | > loss_disc_real_5: 0.17102359235286713 (0.22989843588954997) + | > loss_0: 2.2846784591674805 (2.4212488093466127) + | > grad_norm_0: tensor(7.2567, device='cuda:0') (tensor(10.6194, device='cuda:0')) + | > loss_gen: 2.4758641719818115 (2.397671838976302) + | > loss_kl: 1.8162753582000732 (1.8284923535472941) + | > loss_feat: 8.222103118896484 (7.292848497066858) + | > loss_mel: 18.277788162231445 (17.75579864573929) + | > loss_duration: 1.423294186592102 (1.43035507876918) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.21532440185547 (30.70516633087734) + | > grad_norm_1: tensor(303.2009, device='cuda:0') (tensor(287.4720, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.368 (0.34763479232788086) + | > loader_time: 0.006 (0.005461126003625259) + + + --> TIME: 2025-04-19 05:04:19 -- STEP: 78/142 -- GLOBAL_STEP: 431900 + | > loss_disc: 2.4476850032806396 (2.4370103921645727) + | > loss_disc_real_0: 0.1254703253507614 (0.11722352011845662) + | > loss_disc_real_1: 0.20669341087341309 (0.20706747758847016) + | > loss_disc_real_2: 0.2317221313714981 (0.22545477021963167) + | > loss_disc_real_3: 0.2264043539762497 (0.22907829341980127) + | > loss_disc_real_4: 0.22556018829345703 (0.2278082168255097) + | > loss_disc_real_5: 0.25943684577941895 (0.23214381436506906) + | > loss_0: 2.4476850032806396 (2.4370103921645727) + | > grad_norm_0: tensor(6.0674, device='cuda:0') (tensor(10.0856, device='cuda:0')) + | > loss_gen: 2.3236682415008545 (2.3915659342056665) + | > loss_kl: 1.9036651849746704 (1.852109996172098) + | > loss_feat: 7.311107635498047 (7.2739667647924175) + | > loss_mel: 17.698781967163086 (17.688261191050213) + | > loss_duration: 1.4860866069793701 (1.4422882214570656) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.723308563232422 (30.648192063356056) + | > grad_norm_1: tensor(353.1938, device='cuda:0') (tensor(273.3711, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.3936 (0.35967936576941073) + | > loader_time: 0.0061 (0.005705555280049642) + + + --> TIME: 2025-04-19 05:04:29 -- STEP: 103/142 -- GLOBAL_STEP: 431925 + | > loss_disc: 2.528489112854004 (2.454423534059987) + | > loss_disc_real_0: 0.21034860610961914 (0.12020847338785245) + | > loss_disc_real_1: 0.1920764148235321 (0.2067908160026791) + | > loss_disc_real_2: 0.21431556344032288 (0.2276820245298367) + | > loss_disc_real_3: 0.20813001692295074 (0.229578142316596) + | > loss_disc_real_4: 0.2314169555902481 (0.23011580948690766) + | > loss_disc_real_5: 0.24682150781154633 (0.2321698555668581) + | > loss_0: 2.528489112854004 (2.454423534059987) + | > grad_norm_0: tensor(21.5679, device='cuda:0') (tensor(10.3653, device='cuda:0')) + | > loss_gen: 2.2988672256469727 (2.376593462471824) + | > loss_kl: 1.8229693174362183 (1.8655172574867322) + | > loss_feat: 6.915304660797119 (7.177900277295159) + | > loss_mel: 17.71674156188965 (17.654797683641746) + | > loss_duration: 1.4945671558380127 (1.4521879050338153) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.248451232910156 (30.526996519959088) + | > grad_norm_1: tensor(257.5651, device='cuda:0') (tensor(270.2799, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.4149 (0.370419127269856) + | > loader_time: 0.0066 (0.005930027915436087) + + + --> TIME: 2025-04-19 05:04:40 -- STEP: 128/142 -- GLOBAL_STEP: 431950 + | > loss_disc: 2.4927234649658203 (2.4555787406861778) + | > loss_disc_real_0: 0.0898442193865776 (0.12326449449756183) + | > loss_disc_real_1: 0.2634200155735016 (0.2067631734535098) + | > loss_disc_real_2: 0.2421073019504547 (0.2264630978461355) + | > loss_disc_real_3: 0.25599369406700134 (0.22961612755898386) + | > loss_disc_real_4: 0.24254083633422852 (0.22995331708807498) + | > loss_disc_real_5: 0.24778476357460022 (0.23196230561006814) + | > loss_0: 2.4927234649658203 (2.4555787406861778) + | > grad_norm_0: tensor(9.6834, device='cuda:0') (tensor(12.0670, device='cuda:0')) + | > loss_gen: 2.4021835327148438 (2.3871977012604484) + | > loss_kl: 1.836328387260437 (1.8733961321413517) + | > loss_feat: 8.027758598327637 (7.190844908356667) + | > loss_mel: 18.04762077331543 (17.66414047777652) + | > loss_duration: 1.5124120712280273 (1.4595955098047853) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.826305389404297 (30.575174644589424) + | > grad_norm_1: tensor(426.3456, device='cuda:0') (tensor(291.5951, device='cuda:0')) + | > current_lr_0: 0.00019096009426772123 + | > current_lr_1: 0.00019096009426772123 + | > step_time: 0.4365 (0.38180745020508766) + | > loader_time: 0.0073 (0.006189947947859764) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4271421432495117 (2.4271421432495117) + | > loss_disc_real_0: 0.1399325579404831 (0.1399325579404831) + | > loss_disc_real_1: 0.24540089070796967 (0.24540089070796967) + | > loss_disc_real_2: 0.24901464581489563 (0.24901464581489563) + | > loss_disc_real_3: 0.23156270384788513 (0.23156270384788513) + | > loss_disc_real_4: 0.21018743515014648 (0.21018743515014648) + | > loss_disc_real_5: 0.24474118649959564 (0.24474118649959564) + | > loss_0: 2.4271421432495117 (2.4271421432495117) + | > loss_gen: 2.419969320297241 (2.419969320297241) + | > loss_kl: 2.3060383796691895 (2.3060383796691895) + | > loss_feat: 7.24197244644165 (7.24197244644165) + | > loss_mel: 19.839683532714844 (19.839683532714844) + | > loss_duration: 1.7575479745864868 (1.7575479745864868) + | > loss_1: 33.56521224975586 (33.56521224975586) + + --> STEP: 1 + | > loss_disc: 2.5946459770202637 (2.5946459770202637) + | > loss_disc_real_0: 0.1370706558227539 (0.1370706558227539) + | > loss_disc_real_1: 0.2602628767490387 (0.2602628767490387) + | > loss_disc_real_2: 0.21235042810440063 (0.21235042810440063) + | > loss_disc_real_3: 0.21691732108592987 (0.21691732108592987) + | > loss_disc_real_4: 0.2236873060464859 (0.2236873060464859) + | > loss_disc_real_5: 0.2105114758014679 (0.2105114758014679) + | > loss_0: 2.5946459770202637 (2.5946459770202637) + | > loss_gen: 2.0448312759399414 (2.0448312759399414) + | > loss_kl: 2.4550631046295166 (2.4550631046295166) + | > loss_feat: 5.910777568817139 (5.910777568817139) + | > loss_mel: 16.190814971923828 (16.190814971923828) + | > loss_duration: 1.769172191619873 (1.769172191619873) + | > loss_1: 28.37065887451172 (28.37065887451172) + + --> STEP: 2 + | > loss_disc: 2.662649154663086 (2.628647565841675) + | > loss_disc_real_0: 0.1622750461101532 (0.14967285096645355) + | > loss_disc_real_1: 0.2329520285129547 (0.2466074526309967) + | > loss_disc_real_2: 0.23011595010757446 (0.22123318910598755) + | > loss_disc_real_3: 0.2750254273414612 (0.24597137421369553) + | > loss_disc_real_4: 0.2629948556423187 (0.2433410808444023) + | > loss_disc_real_5: 0.2912899851799011 (0.2509007304906845) + | > loss_0: 2.662649154663086 (2.628647565841675) + | > loss_gen: 2.254196882247925 (2.149514079093933) + | > loss_kl: 2.556323289871216 (2.505693197250366) + | > loss_feat: 6.4226837158203125 (6.166730642318726) + | > loss_mel: 18.09062957763672 (17.140722274780273) + | > loss_duration: 1.7019410133361816 (1.7355566024780273) + | > loss_1: 31.025774002075195 (29.698216438293457) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033953189849853516 (+4.303455352783203e-05) + | > avg_loss_disc: 2.628647565841675 (+0.09071624279022217) + | > avg_loss_disc_real_0: 0.14967285096645355 (-0.011140108108520508) + | > avg_loss_disc_real_1: 0.2466074526309967 (-0.06359630823135376) + | > avg_loss_disc_real_2: 0.22123318910598755 (-0.06090216338634491) + | > avg_loss_disc_real_3: 0.24597137421369553 (-0.007303714752197266) + | > avg_loss_disc_real_4: 0.2433410808444023 (-0.051743440330028534) + | > avg_loss_disc_real_5: 0.2509007304906845 (+0.04280516505241394) + | > avg_loss_0: 2.628647565841675 (+0.09071624279022217) + | > avg_loss_gen: 2.149514079093933 (-0.46326184272766113) + | > avg_loss_kl: 2.505693197250366 (-0.29149460792541504) + | > avg_loss_feat: 6.166730642318726 (-1.957585334777832) + | > avg_loss_mel: 17.140722274780273 (-1.7184772491455078) + | > avg_loss_duration: 1.7355566024780273 (-0.004195451736450195) + | > avg_loss_1: 29.698216438293457 (-4.435013771057129) + + + > EPOCH: 371/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:04:48)  + + --> TIME: 2025-04-19 05:04:52 -- STEP: 11/142 -- GLOBAL_STEP: 431975 + | > loss_disc: 2.3487465381622314 (2.4300739765167236) + | > loss_disc_real_0: 0.09553059190511703 (0.12211506610566919) + | > loss_disc_real_1: 0.20171551406383514 (0.20406748218969864) + | > loss_disc_real_2: 0.2090204507112503 (0.22441589425910602) + | > loss_disc_real_3: 0.21940667927265167 (0.22622549669309097) + | > loss_disc_real_4: 0.22480040788650513 (0.22470550780946558) + | > loss_disc_real_5: 0.2427358776330948 (0.22974190386858853) + | > loss_0: 2.3487465381622314 (2.4300739765167236) + | > grad_norm_0: tensor(4.1800, device='cuda:0') (tensor(11.4376, device='cuda:0')) + | > loss_gen: 2.467102289199829 (2.3856650482524526) + | > loss_kl: 1.8145711421966553 (1.8162784576416016) + | > loss_feat: 7.3768534660339355 (7.492857282811945) + | > loss_mel: 17.992422103881836 (17.943468614058062) + | > loss_duration: 1.4108848571777344 (1.4114461161873557) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.061832427978516 (31.04971521550959) + | > grad_norm_1: tensor(196.1475, device='cuda:0') (tensor(264.2265, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.3291 (0.3254978656768799) + | > loader_time: 0.0053 (0.0047061876817183065) + + + --> TIME: 2025-04-19 05:05:01 -- STEP: 36/142 -- GLOBAL_STEP: 432000 + | > loss_disc: 2.378920793533325 (2.450461096233792) + | > loss_disc_real_0: 0.0908953920006752 (0.11861184984445572) + | > loss_disc_real_1: 0.2207547426223755 (0.207960428049167) + | > loss_disc_real_2: 0.21227450668811798 (0.22687789218293297) + | > loss_disc_real_3: 0.2432766556739807 (0.22842175058192676) + | > loss_disc_real_4: 0.21229973435401917 (0.22746597975492477) + | > loss_disc_real_5: 0.23168329894542694 (0.23017021434174645) + | > loss_0: 2.378920793533325 (2.450461096233792) + | > grad_norm_0: tensor(5.0637, device='cuda:0') (tensor(8.9033, device='cuda:0')) + | > loss_gen: 2.464390277862549 (2.3564963142077127) + | > loss_kl: 2.000969409942627 (1.8236141635311975) + | > loss_feat: 7.467772483825684 (7.297116465038723) + | > loss_mel: 17.47339630126953 (17.732894049750435) + | > loss_duration: 1.4245902299880981 (1.4252495533890197) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.831119537353516 (30.63537046644423) + | > grad_norm_1: tensor(85.6438, device='cuda:0') (tensor(177.1127, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.3617 (0.34243571095996433) + | > loader_time: 0.0059 (0.005205081568823921) + + + --> TIME: 2025-04-19 05:05:10 -- STEP: 61/142 -- GLOBAL_STEP: 432025 + | > loss_disc: 2.4446890354156494 (2.4443537758999185) + | > loss_disc_real_0: 0.10559028387069702 (0.122060841468514) + | > loss_disc_real_1: 0.2303314357995987 (0.20742880907215056) + | > loss_disc_real_2: 0.2304389923810959 (0.22318407648899516) + | > loss_disc_real_3: 0.2397792786359787 (0.2276535305331965) + | > loss_disc_real_4: 0.23605728149414062 (0.22665746583313237) + | > loss_disc_real_5: 0.22339344024658203 (0.23065518795466813) + | > loss_0: 2.4446890354156494 (2.4443537758999185) + | > grad_norm_0: tensor(4.3524, device='cuda:0') (tensor(8.0841, device='cuda:0')) + | > loss_gen: 2.2776095867156982 (2.3585851309729415) + | > loss_kl: 1.8017030954360962 (1.8410400308546473) + | > loss_feat: 6.250051498413086 (7.212203502655029) + | > loss_mel: 15.930373191833496 (17.59791338248331) + | > loss_duration: 1.4441838264465332 (1.4364755622676166) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 27.703920364379883 (30.446217490024253) + | > grad_norm_1: tensor(98.1219, device='cuda:0') (tensor(152.6893, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.3928 (0.35630385602106807) + | > loader_time: 0.0067 (0.005585666562690109) + + + --> TIME: 2025-04-19 05:05:21 -- STEP: 86/142 -- GLOBAL_STEP: 432050 + | > loss_disc: 2.478166103363037 (2.4461108834244487) + | > loss_disc_real_0: 0.11006343364715576 (0.1232083574977032) + | > loss_disc_real_1: 0.21720024943351746 (0.20705021865839182) + | > loss_disc_real_2: 0.20523501932621002 (0.2231491390009259) + | > loss_disc_real_3: 0.25149089097976685 (0.22750911133926968) + | > loss_disc_real_4: 0.2842138409614563 (0.22772528456394062) + | > loss_disc_real_5: 0.251665323972702 (0.23079699225897013) + | > loss_0: 2.478166103363037 (2.4461108834244487) + | > grad_norm_0: tensor(4.0154, device='cuda:0') (tensor(7.6919, device='cuda:0')) + | > loss_gen: 2.3467857837677 (2.3749272767887555) + | > loss_kl: 1.8715318441390991 (1.8595997899077659) + | > loss_feat: 7.252815246582031 (7.267857235531475) + | > loss_mel: 17.356082916259766 (17.73632745964582) + | > loss_duration: 1.5020971298217773 (1.4474511770314953) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.329315185546875 (30.686162815537564) + | > grad_norm_1: tensor(45.1705, device='cuda:0') (tensor(141.2915, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.3981 (0.37207311253214964) + | > loader_time: 0.0066 (0.005869740663572801) + + + --> TIME: 2025-04-19 05:05:31 -- STEP: 111/142 -- GLOBAL_STEP: 432075 + | > loss_disc: 2.454111099243164 (2.45648988087972) + | > loss_disc_real_0: 0.10727938264608383 (0.1251268198227023) + | > loss_disc_real_1: 0.21276861429214478 (0.20887808313777856) + | > loss_disc_real_2: 0.23899602890014648 (0.22621927672141307) + | > loss_disc_real_3: 0.23424172401428223 (0.22780494069730914) + | > loss_disc_real_4: 0.2660585939884186 (0.22867562091565347) + | > loss_disc_real_5: 0.24754580855369568 (0.23066918578770784) + | > loss_0: 2.454111099243164 (2.45648988087972) + | > grad_norm_0: tensor(5.8948, device='cuda:0') (tensor(8.1817, device='cuda:0')) + | > loss_gen: 2.5030736923217773 (2.3707979691995167) + | > loss_kl: 2.000917673110962 (1.8626216424478066) + | > loss_feat: 7.059426307678223 (7.218299784101881) + | > loss_mel: 19.075729370117188 (17.702645997743335) + | > loss_duration: 1.4989467859268188 (1.4548853270642395) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.13809585571289 (30.609250644305806) + | > grad_norm_1: tensor(88.6821, device='cuda:0') (tensor(167.6470, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.4377 (0.38097274840415035) + | > loader_time: 0.0065 (0.006044697117161107) + + + --> TIME: 2025-04-19 05:05:43 -- STEP: 136/142 -- GLOBAL_STEP: 432100 + | > loss_disc: 2.527198553085327 (2.4602243234129513) + | > loss_disc_real_0: 0.10928242653608322 (0.12729247265002303) + | > loss_disc_real_1: 0.2085149735212326 (0.2100354568923221) + | > loss_disc_real_2: 0.2105652540922165 (0.22483580110265927) + | > loss_disc_real_3: 0.2371620386838913 (0.22745119385859547) + | > loss_disc_real_4: 0.23219382762908936 (0.2283211087479311) + | > loss_disc_real_5: 0.2019791454076767 (0.22997628371505177) + | > loss_0: 2.527198553085327 (2.4602243234129513) + | > grad_norm_0: tensor(10.5169, device='cuda:0') (tensor(9.1272, device='cuda:0')) + | > loss_gen: 2.18231463432312 (2.3607274293899545) + | > loss_kl: 2.0200135707855225 (1.8669054052409004) + | > loss_feat: 7.61112117767334 (7.160227729993708) + | > loss_mel: 18.136613845825195 (17.598266328082353) + | > loss_duration: 1.5005292892456055 (1.4611662179231644) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.450592041015625 (30.44729300106273) + | > grad_norm_1: tensor(255.0083, device='cuda:0') (tensor(191.9501, device='cuda:0')) + | > current_lr_0: 0.00019093622425593777 + | > current_lr_1: 0.00019093622425593777 + | > step_time: 0.4488 (0.39165367624338915) + | > loader_time: 0.0077 (0.006303303381975959) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.510533332824707 (2.510533332824707) + | > loss_disc_real_0: 0.15939079225063324 (0.15939079225063324) + | > loss_disc_real_1: 0.1806788444519043 (0.1806788444519043) + | > loss_disc_real_2: 0.21997229754924774 (0.21997229754924774) + | > loss_disc_real_3: 0.2572726011276245 (0.2572726011276245) + | > loss_disc_real_4: 0.2624376118183136 (0.2624376118183136) + | > loss_disc_real_5: 0.2517016530036926 (0.2517016530036926) + | > loss_0: 2.510533332824707 (2.510533332824707) + | > loss_gen: 2.2438364028930664 (2.2438364028930664) + | > loss_kl: 2.409031391143799 (2.409031391143799) + | > loss_feat: 6.654285430908203 (6.654285430908203) + | > loss_mel: 18.09385871887207 (18.09385871887207) + | > loss_duration: 1.716187596321106 (1.716187596321106) + | > loss_1: 31.11720085144043 (31.11720085144043) + + --> STEP: 1 + | > loss_disc: 2.6611826419830322 (2.6611826419830322) + | > loss_disc_real_0: 0.154470294713974 (0.154470294713974) + | > loss_disc_real_1: 0.21240729093551636 (0.21240729093551636) + | > loss_disc_real_2: 0.2514200210571289 (0.2514200210571289) + | > loss_disc_real_3: 0.2924114763736725 (0.2924114763736725) + | > loss_disc_real_4: 0.2904835045337677 (0.2904835045337677) + | > loss_disc_real_5: 0.28087857365608215 (0.28087857365608215) + | > loss_0: 2.6611826419830322 (2.6611826419830322) + | > loss_gen: 2.2598037719726562 (2.2598037719726562) + | > loss_kl: 2.8075411319732666 (2.8075411319732666) + | > loss_feat: 6.649695873260498 (6.649695873260498) + | > loss_mel: 17.335960388183594 (17.335960388183594) + | > loss_duration: 1.7836107015609741 (1.7836107015609741) + | > loss_1: 30.836612701416016 (30.836612701416016) + + --> STEP: 2 + | > loss_disc: 2.4545085430145264 (2.5578455924987793) + | > loss_disc_real_0: 0.10739389061927795 (0.13093209266662598) + | > loss_disc_real_1: 0.19875231385231018 (0.20557980239391327) + | > loss_disc_real_2: 0.21081842482089996 (0.23111922293901443) + | > loss_disc_real_3: 0.25100210309028625 (0.27170678973197937) + | > loss_disc_real_4: 0.2133268117904663 (0.251905158162117) + | > loss_disc_real_5: 0.22563917934894562 (0.2532588765025139) + | > loss_0: 2.4545085430145264 (2.5578455924987793) + | > loss_gen: 1.9943217039108276 (2.127062737941742) + | > loss_kl: 2.2443418502807617 (2.525941491127014) + | > loss_feat: 5.972835540771484 (6.311265707015991) + | > loss_mel: 16.250253677368164 (16.79310703277588) + | > loss_duration: 1.7179057598114014 (1.7507582306861877) + | > loss_1: 28.179656982421875 (29.508134841918945) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035355091094970703 (+0.00014019012451171875) + | > avg_loss_disc: 2.5578455924987793 (-0.07080197334289551) + | > avg_loss_disc_real_0: 0.13093209266662598 (-0.018740758299827576) + | > avg_loss_disc_real_1: 0.20557980239391327 (-0.041027650237083435) + | > avg_loss_disc_real_2: 0.23111922293901443 (+0.009886033833026886) + | > avg_loss_disc_real_3: 0.27170678973197937 (+0.025735415518283844) + | > avg_loss_disc_real_4: 0.251905158162117 (+0.008564077317714691) + | > avg_loss_disc_real_5: 0.2532588765025139 (+0.0023581460118293762) + | > avg_loss_0: 2.5578455924987793 (-0.07080197334289551) + | > avg_loss_gen: 2.127062737941742 (-0.022451341152191162) + | > avg_loss_kl: 2.525941491127014 (+0.02024829387664795) + | > avg_loss_feat: 6.311265707015991 (+0.14453506469726562) + | > avg_loss_mel: 16.79310703277588 (-0.34761524200439453) + | > avg_loss_duration: 1.7507582306861877 (+0.0152016282081604) + | > avg_loss_1: 29.508134841918945 (-0.19008159637451172) + + + > EPOCH: 372/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:05:47)  + + --> TIME: 2025-04-19 05:05:54 -- STEP: 19/142 -- GLOBAL_STEP: 432125 + | > loss_disc: 2.454522132873535 (2.402150028630307) + | > loss_disc_real_0: 0.11751764267683029 (0.11731234937906265) + | > loss_disc_real_1: 0.18373852968215942 (0.20049001198065908) + | > loss_disc_real_2: 0.231370747089386 (0.22590911545251546) + | > loss_disc_real_3: 0.25402042269706726 (0.2241859859541843) + | > loss_disc_real_4: 0.2357243299484253 (0.22072169812102066) + | > loss_disc_real_5: 0.2622821033000946 (0.22708743810653687) + | > loss_0: 2.454522132873535 (2.402150028630307) + | > grad_norm_0: tensor(32.6468, device='cuda:0') (tensor(14.6757, device='cuda:0')) + | > loss_gen: 2.5851364135742188 (2.431919574737549) + | > loss_kl: 1.8937917947769165 (1.8095149680187828) + | > loss_feat: 8.27807331085205 (7.315701183519866) + | > loss_mel: 17.785602569580078 (17.61573354821456) + | > loss_duration: 1.412909984588623 (1.4097480836667513) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.955514907836914 (30.582617609124433) + | > grad_norm_1: tensor(558.2468, device='cuda:0') (tensor(351.3787, device='cuda:0')) + | > current_lr_0: 0.00019091235722790576 + | > current_lr_1: 0.00019091235722790576 + | > step_time: 0.3625 (0.3365887591713353) + | > loader_time: 0.005 (0.0049453660061484885) + + + --> TIME: 2025-04-19 05:06:03 -- STEP: 44/142 -- GLOBAL_STEP: 432150 + | > loss_disc: 2.496861696243286 (2.4215818481011824) + | > loss_disc_real_0: 0.12999336421489716 (0.11620438378304243) + | > loss_disc_real_1: 0.20278222858905792 (0.20258602635426956) + | > loss_disc_real_2: 0.2405502200126648 (0.22584875097328966) + | > loss_disc_real_3: 0.23550191521644592 (0.2281401292844252) + | > loss_disc_real_4: 0.22051379084587097 (0.2269089821387421) + | > loss_disc_real_5: 0.22828400135040283 (0.22983413223515858) + | > loss_0: 2.496861696243286 (2.4215818481011824) + | > grad_norm_0: tensor(12.8637, device='cuda:0') (tensor(14.3274, device='cuda:0')) + | > loss_gen: 2.2972400188446045 (2.4243087064136155) + | > loss_kl: 1.760351300239563 (1.8247106075286865) + | > loss_feat: 7.04145622253418 (7.296459967439825) + | > loss_mel: 17.20157814025879 (17.709782188588925) + | > loss_duration: 1.4617658853530884 (1.4238746220415288) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.76239013671875 (30.679136189547453) + | > grad_norm_1: tensor(352.0280, device='cuda:0') (tensor(342.8380, device='cuda:0')) + | > current_lr_0: 0.00019091235722790576 + | > current_lr_1: 0.00019091235722790576 + | > step_time: 0.3578 (0.34742538495497266) + | > loader_time: 0.0061 (0.00551173361864957) + + + --> TIME: 2025-04-19 05:06:13 -- STEP: 69/142 -- GLOBAL_STEP: 432175 + | > loss_disc: 2.451233386993408 (2.4259668329487676) + | > loss_disc_real_0: 0.10232050716876984 (0.11600624980485957) + | > loss_disc_real_1: 0.24155567586421967 (0.2047819182060767) + | > loss_disc_real_2: 0.2334061712026596 (0.22494680056537408) + | > loss_disc_real_3: 0.2476431429386139 (0.22802924332411392) + | > loss_disc_real_4: 0.23428431153297424 (0.22625762656115098) + | > loss_disc_real_5: 0.2583126425743103 (0.22901972523633984) + | > loss_0: 2.451233386993408 (2.4259668329487676) + | > grad_norm_0: tensor(7.9050, device='cuda:0') (tensor(13.7804, device='cuda:0')) + | > loss_gen: 2.399764060974121 (2.4163932973060054) + | > loss_kl: 1.8090381622314453 (1.8332808104114255) + | > loss_feat: 7.266784191131592 (7.339196972225023) + | > loss_mel: 17.930805206298828 (17.816106893014222) + | > loss_duration: 1.506861925125122 (1.4359033245971238) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.913253784179688 (30.840881237085316) + | > grad_norm_1: tensor(385.2827, device='cuda:0') (tensor(345.7170, device='cuda:0')) + | > current_lr_0: 0.00019091235722790576 + | > current_lr_1: 0.00019091235722790576 + | > step_time: 0.3938 (0.35772475297900214) + | > loader_time: 0.006 (0.005747214607570483) + + + --> TIME: 2025-04-19 05:06:23 -- STEP: 94/142 -- GLOBAL_STEP: 432200 + | > loss_disc: 2.4903807640075684 (2.434596320416065) + | > loss_disc_real_0: 0.14424912631511688 (0.11881771806548251) + | > loss_disc_real_1: 0.22907277941703796 (0.20516870091570186) + | > loss_disc_real_2: 0.22719742357730865 (0.22588211583330273) + | > loss_disc_real_3: 0.2684817612171173 (0.2279407467613829) + | > loss_disc_real_4: 0.2977072596549988 (0.22817095591032754) + | > loss_disc_real_5: 0.25060978531837463 (0.2297463336206497) + | > loss_0: 2.4903807640075684 (2.434596320416065) + | > grad_norm_0: tensor(13.0734, device='cuda:0') (tensor(13.6291, device='cuda:0')) + | > loss_gen: 2.4305388927459717 (2.403780919440251) + | > loss_kl: 1.6925545930862427 (1.8476637094578845) + | > loss_feat: 7.025073051452637 (7.2997100251786255) + | > loss_mel: 18.48935890197754 (17.851893080041773) + | > loss_duration: 1.4947421550750732 (1.4474614604990532) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.132265090942383 (30.850509176863) + | > grad_norm_1: tensor(118.9378, device='cuda:0') (tensor(335.3276, device='cuda:0')) + | > current_lr_0: 0.00019091235722790576 + | > current_lr_1: 0.00019091235722790576 + | > step_time: 0.3963 (0.36793157902169743) + | > loader_time: 0.0062 (0.005986142665781876) + + + --> TIME: 2025-04-19 05:06:34 -- STEP: 119/142 -- GLOBAL_STEP: 432225 + | > loss_disc: 2.4915714263916016 (2.440279912547905) + | > loss_disc_real_0: 0.1258162260055542 (0.12122667364838745) + | > loss_disc_real_1: 0.2502470314502716 (0.2066408255520989) + | > loss_disc_real_2: 0.20561476051807404 (0.2253360099652234) + | > loss_disc_real_3: 0.23453252017498016 (0.2279895660506577) + | > loss_disc_real_4: 0.24996337294578552 (0.22785049777071012) + | > loss_disc_real_5: 0.24985001981258392 (0.22853918223320938) + | > loss_0: 2.4915714263916016 (2.440279912547905) + | > grad_norm_0: tensor(10.0625, device='cuda:0') (tensor(13.4593, device='cuda:0')) + | > loss_gen: 2.2500381469726562 (2.393877708611369) + | > loss_kl: 1.847980260848999 (1.8519577228722452) + | > loss_feat: 6.906492710113525 (7.249867387178566) + | > loss_mel: 17.90021324157715 (17.836514681327248) + | > loss_duration: 1.5181716680526733 (1.4554511759461475) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.422895431518555 (30.787668660909187) + | > grad_norm_1: tensor(103.0704, device='cuda:0') (tensor(329.2324, device='cuda:0')) + | > current_lr_0: 0.00019091235722790576 + | > current_lr_1: 0.00019091235722790576 + | > step_time: 0.4237 (0.3784771586666589) + | > loader_time: 0.0072 (0.006206047635118502) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3929598331451416 (2.3929598331451416) + | > loss_disc_real_0: 0.14734256267547607 (0.14734256267547607) + | > loss_disc_real_1: 0.2259819507598877 (0.2259819507598877) + | > loss_disc_real_2: 0.22300991415977478 (0.22300991415977478) + | > loss_disc_real_3: 0.27137261629104614 (0.27137261629104614) + | > loss_disc_real_4: 0.20353922247886658 (0.20353922247886658) + | > loss_disc_real_5: 0.18772278726100922 (0.18772278726100922) + | > loss_0: 2.3929598331451416 (2.3929598331451416) + | > loss_gen: 2.635145425796509 (2.635145425796509) + | > loss_kl: 2.562767267227173 (2.562767267227173) + | > loss_feat: 8.071502685546875 (8.071502685546875) + | > loss_mel: 19.446823120117188 (19.446823120117188) + | > loss_duration: 1.7599973678588867 (1.7599973678588867) + | > loss_1: 34.47623825073242 (34.47623825073242) + + --> STEP: 1 + | > loss_disc: 2.5154318809509277 (2.5154318809509277) + | > loss_disc_real_0: 0.22510209679603577 (0.22510209679603577) + | > loss_disc_real_1: 0.22607499361038208 (0.22607499361038208) + | > loss_disc_real_2: 0.22426722943782806 (0.22426722943782806) + | > loss_disc_real_3: 0.24683448672294617 (0.24683448672294617) + | > loss_disc_real_4: 0.23460893332958221 (0.23460893332958221) + | > loss_disc_real_5: 0.24523687362670898 (0.24523687362670898) + | > loss_0: 2.5154318809509277 (2.5154318809509277) + | > loss_gen: 2.5310025215148926 (2.5310025215148926) + | > loss_kl: 2.8070991039276123 (2.8070991039276123) + | > loss_feat: 7.1512298583984375 (7.1512298583984375) + | > loss_mel: 17.744815826416016 (17.744815826416016) + | > loss_duration: 1.7839350700378418 (1.7839350700378418) + | > loss_1: 32.01808166503906 (32.01808166503906) + + --> STEP: 2 + | > loss_disc: 2.337068557739258 (2.4262502193450928) + | > loss_disc_real_0: 0.17496654391288757 (0.20003432035446167) + | > loss_disc_real_1: 0.24050436913967133 (0.2332896813750267) + | > loss_disc_real_2: 0.1831607073545456 (0.20371396839618683) + | > loss_disc_real_3: 0.24806463718414307 (0.24744956195354462) + | > loss_disc_real_4: 0.22102440893650055 (0.22781667113304138) + | > loss_disc_real_5: 0.19679062068462372 (0.22101374715566635) + | > loss_0: 2.337068557739258 (2.4262502193450928) + | > loss_gen: 2.5394959449768066 (2.5352492332458496) + | > loss_kl: 2.2705812454223633 (2.538840174674988) + | > loss_feat: 8.678533554077148 (7.914881706237793) + | > loss_mel: 19.03980255126953 (18.392309188842773) + | > loss_duration: 1.7148014307022095 (1.7493682503700256) + | > loss_1: 34.24321746826172 (33.13064956665039) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003441929817199707 (-9.357929229736328e-05) + | > avg_loss_disc: 2.4262502193450928 (-0.13159537315368652) + | > avg_loss_disc_real_0: 0.20003432035446167 (+0.0691022276878357) + | > avg_loss_disc_real_1: 0.2332896813750267 (+0.027709878981113434) + | > avg_loss_disc_real_2: 0.20371396839618683 (-0.027405254542827606) + | > avg_loss_disc_real_3: 0.24744956195354462 (-0.024257227778434753) + | > avg_loss_disc_real_4: 0.22781667113304138 (-0.024088487029075623) + | > avg_loss_disc_real_5: 0.22101374715566635 (-0.032245129346847534) + | > avg_loss_0: 2.4262502193450928 (-0.13159537315368652) + | > avg_loss_gen: 2.5352492332458496 (+0.40818649530410767) + | > avg_loss_kl: 2.538840174674988 (+0.012898683547973633) + | > avg_loss_feat: 7.914881706237793 (+1.6036159992218018) + | > avg_loss_mel: 18.392309188842773 (+1.5992021560668945) + | > avg_loss_duration: 1.7493682503700256 (-0.0013899803161621094) + | > avg_loss_1: 33.13064956665039 (+3.6225147247314453) + + + > EPOCH: 373/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:06:46)  + + --> TIME: 2025-04-19 05:06:47 -- STEP: 2/142 -- GLOBAL_STEP: 432250 + | > loss_disc: 2.3734183311462402 (2.3481847047805786) + | > loss_disc_real_0: 0.0838746652007103 (0.10658476874232292) + | > loss_disc_real_1: 0.17310190200805664 (0.18615169823169708) + | > loss_disc_real_2: 0.22135749459266663 (0.23151078820228577) + | > loss_disc_real_3: 0.2044292390346527 (0.21029724925756454) + | > loss_disc_real_4: 0.18437154591083527 (0.21131416410207748) + | > loss_disc_real_5: 0.20899252593517303 (0.20605488121509552) + | > loss_0: 2.3734183311462402 (2.3481847047805786) + | > grad_norm_0: tensor(21.6060, device='cuda:0') (tensor(18.3371, device='cuda:0')) + | > loss_gen: 2.2611424922943115 (2.3270291090011597) + | > loss_kl: 1.918886423110962 (1.7617173790931702) + | > loss_feat: 7.7082672119140625 (7.8175950050354) + | > loss_mel: 16.751510620117188 (17.519375801086426) + | > loss_duration: 1.4320526123046875 (1.4042027592658997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.07185935974121 (30.829919815063477) + | > grad_norm_1: tensor(554.6754, device='cuda:0') (tensor(569.7894, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.3215 (0.3233991861343384) + | > loader_time: 0.004 (0.003997802734375) + + + --> TIME: 2025-04-19 05:06:56 -- STEP: 27/142 -- GLOBAL_STEP: 432275 + | > loss_disc: 2.3577303886413574 (2.411712266780712) + | > loss_disc_real_0: 0.0914020761847496 (0.11053407744125084) + | > loss_disc_real_1: 0.18113265931606293 (0.2018841008345286) + | > loss_disc_real_2: 0.2188020497560501 (0.22525957171563749) + | > loss_disc_real_3: 0.19308769702911377 (0.22404442451618337) + | > loss_disc_real_4: 0.18518182635307312 (0.22397695040261303) + | > loss_disc_real_5: 0.20654447376728058 (0.22814510762691498) + | > loss_0: 2.3577303886413574 (2.411712266780712) + | > grad_norm_0: tensor(5.2286, device='cuda:0') (tensor(10.1231, device='cuda:0')) + | > loss_gen: 2.3698267936706543 (2.411541956442374) + | > loss_kl: 1.7103850841522217 (1.7972255194628681) + | > loss_feat: 7.2915873527526855 (7.387273435239439) + | > loss_mel: 16.840608596801758 (17.7088644946063) + | > loss_duration: 1.4244433641433716 (1.4182359642452664) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.636852264404297 (30.723141140407986) + | > grad_norm_1: tensor(121.2480, device='cuda:0') (tensor(242.5520, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.3486 (0.3343382234926576) + | > loader_time: 0.0054 (0.0050313737657335066) + + + --> TIME: 2025-04-19 05:07:05 -- STEP: 52/142 -- GLOBAL_STEP: 432300 + | > loss_disc: 2.4914135932922363 (2.4275789490112905) + | > loss_disc_real_0: 0.13260988891124725 (0.11713890301493499) + | > loss_disc_real_1: 0.19607634842395782 (0.20526674590431726) + | > loss_disc_real_2: 0.267337441444397 (0.22762581734703138) + | > loss_disc_real_3: 0.2500619888305664 (0.22745997687945) + | > loss_disc_real_4: 0.3067033886909485 (0.22566664362182984) + | > loss_disc_real_5: 0.24244460463523865 (0.22772498428821564) + | > loss_0: 2.4914135932922363 (2.4275789490112905) + | > grad_norm_0: tensor(25.1167, device='cuda:0') (tensor(9.8052, device='cuda:0')) + | > loss_gen: 2.482198715209961 (2.389999981109913) + | > loss_kl: 1.8322348594665527 (1.8008365906201875) + | > loss_feat: 7.060473442077637 (7.25621903859652) + | > loss_mel: 19.4757080078125 (17.708971170278694) + | > loss_duration: 1.4939589500427246 (1.4285090267658234) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 32.344573974609375 (30.58453585551335) + | > grad_norm_1: tensor(408.1150, device='cuda:0') (tensor(196.7462, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.3663 (0.34608688262792736) + | > loader_time: 0.0061 (0.0054019506161029525) + + + --> TIME: 2025-04-19 05:07:15 -- STEP: 77/142 -- GLOBAL_STEP: 432325 + | > loss_disc: 2.3991780281066895 (2.436175408301415) + | > loss_disc_real_0: 0.1166994497179985 (0.12129081282522772) + | > loss_disc_real_1: 0.16820308566093445 (0.20566474249610656) + | > loss_disc_real_2: 0.23353862762451172 (0.22615852681073276) + | > loss_disc_real_3: 0.22437557578086853 (0.2268972878719305) + | > loss_disc_real_4: 0.18955940008163452 (0.22478674371521196) + | > loss_disc_real_5: 0.20186465978622437 (0.2278144301532151) + | > loss_0: 2.3991780281066895 (2.436175408301415) + | > grad_norm_0: tensor(6.5818, device='cuda:0') (tensor(9.9207, device='cuda:0')) + | > loss_gen: 2.381765604019165 (2.3826114852707105) + | > loss_kl: 1.7929210662841797 (1.82592811677363) + | > loss_feat: 6.913883209228516 (7.2476071939839946) + | > loss_mel: 16.74644660949707 (17.616331137620016) + | > loss_duration: 1.4552547931671143 (1.4394278557269604) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.290271759033203 (30.511905868332107) + | > grad_norm_1: tensor(254.9743, device='cuda:0') (tensor(214.5254, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.3888 (0.3582755621377524) + | > loader_time: 0.0062 (0.005705232744093065) + + + --> TIME: 2025-04-19 05:07:25 -- STEP: 102/142 -- GLOBAL_STEP: 432350 + | > loss_disc: 2.6250076293945312 (2.436065346586937) + | > loss_disc_real_0: 0.13078421354293823 (0.12152362855918267) + | > loss_disc_real_1: 0.24695879220962524 (0.2066115944408903) + | > loss_disc_real_2: 0.19276005029678345 (0.2264353372887069) + | > loss_disc_real_3: 0.2458798587322235 (0.22677920627243378) + | > loss_disc_real_4: 0.23302261531352997 (0.22537172132847355) + | > loss_disc_real_5: 0.22626356780529022 (0.22708095317962124) + | > loss_0: 2.6250076293945312 (2.436065346586937) + | > grad_norm_0: tensor(12.0262, device='cuda:0') (tensor(10.0254, device='cuda:0')) + | > loss_gen: 2.2275896072387695 (2.3845538064545257) + | > loss_kl: 1.7703511714935303 (1.836871968764885) + | > loss_feat: 6.918537616729736 (7.228350073683496) + | > loss_mel: 17.59454345703125 (17.664734457053395) + | > loss_duration: 1.4545326232910156 (1.4492613509589551) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.965553283691406 (30.56377177144967) + | > grad_norm_1: tensor(188.0176, device='cuda:0') (tensor(224.7101, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.4193 (0.369068000830856) + | > loader_time: 0.0067 (0.00593999554129208) + + + --> TIME: 2025-04-19 05:07:36 -- STEP: 127/142 -- GLOBAL_STEP: 432375 + | > loss_disc: 2.505645513534546 (2.442534296531376) + | > loss_disc_real_0: 0.1863403022289276 (0.12240568049780028) + | > loss_disc_real_1: 0.2578907310962677 (0.20674783460737214) + | > loss_disc_real_2: 0.20538781583309174 (0.22738261790726128) + | > loss_disc_real_3: 0.2400730550289154 (0.22752671976258435) + | > loss_disc_real_4: 0.25402432680130005 (0.22658104021248854) + | > loss_disc_real_5: 0.1903795599937439 (0.22735180523921186) + | > loss_0: 2.505645513534546 (2.442534296531376) + | > grad_norm_0: tensor(11.6079, device='cuda:0') (tensor(10.4417, device='cuda:0')) + | > loss_gen: 2.351961374282837 (2.3824484986583063) + | > loss_kl: 1.8503211736679077 (1.8452756029414379) + | > loss_feat: 6.863135814666748 (7.1823286259268215) + | > loss_mel: 16.776142120361328 (17.604590821453904) + | > loss_duration: 1.4753674268722534 (1.456003788887985) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.316926956176758 (30.470647541556772) + | > grad_norm_1: tensor(73.2265, device='cuda:0') (tensor(238.4714, device='cuda:0')) + | > current_lr_0: 0.00019088849318325227 + | > current_lr_1: 0.00019088849318325227 + | > step_time: 0.4382 (0.3807617247573973) + | > loader_time: 0.0072 (0.006180858987522877) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.355337381362915 (2.355337381362915) + | > loss_disc_real_0: 0.1723238229751587 (0.1723238229751587) + | > loss_disc_real_1: 0.25000694394111633 (0.25000694394111633) + | > loss_disc_real_2: 0.2441100776195526 (0.2441100776195526) + | > loss_disc_real_3: 0.22063979506492615 (0.22063979506492615) + | > loss_disc_real_4: 0.2202989012002945 (0.2202989012002945) + | > loss_disc_real_5: 0.233025461435318 (0.233025461435318) + | > loss_0: 2.355337381362915 (2.355337381362915) + | > loss_gen: 2.588716745376587 (2.588716745376587) + | > loss_kl: 2.551381826400757 (2.551381826400757) + | > loss_feat: 7.439711093902588 (7.439711093902588) + | > loss_mel: 18.534046173095703 (18.534046173095703) + | > loss_duration: 1.7280356884002686 (1.7280356884002686) + | > loss_1: 32.84189224243164 (32.84189224243164) + + --> STEP: 1 + | > loss_disc: 2.4150032997131348 (2.4150032997131348) + | > loss_disc_real_0: 0.1754637211561203 (0.1754637211561203) + | > loss_disc_real_1: 0.2859351933002472 (0.2859351933002472) + | > loss_disc_real_2: 0.24275349080562592 (0.24275349080562592) + | > loss_disc_real_3: 0.21030345559120178 (0.21030345559120178) + | > loss_disc_real_4: 0.21967537701129913 (0.21967537701129913) + | > loss_disc_real_5: 0.26273614168167114 (0.26273614168167114) + | > loss_0: 2.4150032997131348 (2.4150032997131348) + | > loss_gen: 2.6525635719299316 (2.6525635719299316) + | > loss_kl: 2.5862138271331787 (2.5862138271331787) + | > loss_feat: 7.17106819152832 (7.17106819152832) + | > loss_mel: 17.8964786529541 (17.8964786529541) + | > loss_duration: 1.7652771472930908 (1.7652771472930908) + | > loss_1: 32.07160186767578 (32.07160186767578) + + --> STEP: 2 + | > loss_disc: 2.506531238555908 (2.4607672691345215) + | > loss_disc_real_0: 0.22272981703281403 (0.19909676909446716) + | > loss_disc_real_1: 0.2793542444705963 (0.28264471888542175) + | > loss_disc_real_2: 0.2788580358028412 (0.26080576330423355) + | > loss_disc_real_3: 0.2231183797121048 (0.2167109176516533) + | > loss_disc_real_4: 0.22630979120731354 (0.22299258410930634) + | > loss_disc_real_5: 0.2720295488834381 (0.2673828452825546) + | > loss_0: 2.506531238555908 (2.4607672691345215) + | > loss_gen: 2.708742618560791 (2.6806530952453613) + | > loss_kl: 2.202266216278076 (2.3942400217056274) + | > loss_feat: 7.909200668334961 (7.540134429931641) + | > loss_mel: 18.473777770996094 (18.185128211975098) + | > loss_duration: 1.7067725658416748 (1.7360248565673828) + | > loss_1: 33.00075912475586 (32.53618049621582) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003929853439331055 (+0.00048792362213134766) + | > avg_loss_disc: 2.4607672691345215 (+0.03451704978942871) + | > avg_loss_disc_real_0: 0.19909676909446716 (-0.0009375512599945068) + | > avg_loss_disc_real_1: 0.28264471888542175 (+0.04935503751039505) + | > avg_loss_disc_real_2: 0.26080576330423355 (+0.05709179490804672) + | > avg_loss_disc_real_3: 0.2167109176516533 (-0.030738644301891327) + | > avg_loss_disc_real_4: 0.22299258410930634 (-0.004824087023735046) + | > avg_loss_disc_real_5: 0.2673828452825546 (+0.046369098126888275) + | > avg_loss_0: 2.4607672691345215 (+0.03451704978942871) + | > avg_loss_gen: 2.6806530952453613 (+0.14540386199951172) + | > avg_loss_kl: 2.3942400217056274 (-0.14460015296936035) + | > avg_loss_feat: 7.540134429931641 (-0.37474727630615234) + | > avg_loss_mel: 18.185128211975098 (-0.20718097686767578) + | > avg_loss_duration: 1.7360248565673828 (-0.013343393802642822) + | > avg_loss_1: 32.53618049621582 (-0.5944690704345703) + + + > EPOCH: 374/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:07:44)  + + --> TIME: 2025-04-19 05:07:48 -- STEP: 10/142 -- GLOBAL_STEP: 432400 + | > loss_disc: 2.4450695514678955 (2.3668933153152465) + | > loss_disc_real_0: 0.11208496242761612 (0.10896174311637878) + | > loss_disc_real_1: 0.1681603640317917 (0.19569317549467086) + | > loss_disc_real_2: 0.28324198722839355 (0.23035443276166917) + | > loss_disc_real_3: 0.23908717930316925 (0.22221452742815018) + | > loss_disc_real_4: 0.2361374944448471 (0.21441515535116196) + | > loss_disc_real_5: 0.19282503426074982 (0.2269066423177719) + | > loss_0: 2.4450695514678955 (2.3668933153152465) + | > grad_norm_0: tensor(12.6324, device='cuda:0') (tensor(9.4874, device='cuda:0')) + | > loss_gen: 2.483328342437744 (2.4170448064804075) + | > loss_kl: 1.9164320230484009 (1.749399983882904) + | > loss_feat: 6.7806596755981445 (7.221006250381469) + | > loss_mel: 17.37004280090332 (17.37144556045532) + | > loss_duration: 1.3832597732543945 (1.4044018983840942) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.93372344970703 (30.16329860687256) + | > grad_norm_1: tensor(283.4219, device='cuda:0') (tensor(264.6320, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.3358 (0.32533414363861085) + | > loader_time: 0.0052 (0.004593205451965332) + + + --> TIME: 2025-04-19 05:07:57 -- STEP: 35/142 -- GLOBAL_STEP: 432425 + | > loss_disc: 2.4694178104400635 (2.437300082615444) + | > loss_disc_real_0: 0.12413215637207031 (0.11820166707038879) + | > loss_disc_real_1: 0.19135776162147522 (0.20827009294714247) + | > loss_disc_real_2: 0.2169528603553772 (0.22810484681810653) + | > loss_disc_real_3: 0.22733236849308014 (0.226564684510231) + | > loss_disc_real_4: 0.21927101910114288 (0.22585072559969765) + | > loss_disc_real_5: 0.24955838918685913 (0.22700130939483643) + | > loss_0: 2.4694178104400635 (2.437300082615444) + | > grad_norm_0: tensor(19.3572, device='cuda:0') (tensor(11.4856, device='cuda:0')) + | > loss_gen: 2.2709031105041504 (2.396698420388358) + | > loss_kl: 1.784868597984314 (1.7921963146754674) + | > loss_feat: 7.0542826652526855 (7.239413642883301) + | > loss_mel: 16.984838485717773 (17.570167786734444) + | > loss_duration: 1.4321558475494385 (1.4186108895710536) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.527048110961914 (30.417086955479213) + | > grad_norm_1: tensor(398.0006, device='cuda:0') (tensor(319.2077, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.3681 (0.3418304443359375) + | > loader_time: 0.0059 (0.00521045412336077) + + + --> TIME: 2025-04-19 05:08:07 -- STEP: 60/142 -- GLOBAL_STEP: 432450 + | > loss_disc: 2.469276189804077 (2.4260751048723863) + | > loss_disc_real_0: 0.16684524714946747 (0.11875061976412932) + | > loss_disc_real_1: 0.22641916573047638 (0.2062371792892615) + | > loss_disc_real_2: 0.221257284283638 (0.2263545739154021) + | > loss_disc_real_3: 0.1966802030801773 (0.22621972585717837) + | > loss_disc_real_4: 0.2491183876991272 (0.22694851085543633) + | > loss_disc_real_5: 0.21412906050682068 (0.22665323341886204) + | > loss_0: 2.469276189804077 (2.4260751048723863) + | > grad_norm_0: tensor(14.0827, device='cuda:0') (tensor(13.7735, device='cuda:0')) + | > loss_gen: 2.453622341156006 (2.4226132233937574) + | > loss_kl: 1.6744530200958252 (1.811019359032313) + | > loss_feat: 6.5700883865356445 (7.311425503094991) + | > loss_mel: 16.041297912597656 (17.696408128738412) + | > loss_duration: 1.4637656211853027 (1.4295232971509297) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.203227996826172 (30.670989322662354) + | > grad_norm_1: tensor(149.7362, device='cuda:0') (tensor(330.6930, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.3831 (0.3577822287877401) + | > loader_time: 0.0059 (0.0056498964627583826) + + + --> TIME: 2025-04-19 05:08:17 -- STEP: 85/142 -- GLOBAL_STEP: 432475 + | > loss_disc: 2.5566391944885254 (2.43025998508229) + | > loss_disc_real_0: 0.10417389124631882 (0.12185046699117212) + | > loss_disc_real_1: 0.19770000874996185 (0.20647080523126266) + | > loss_disc_real_2: 0.24871964752674103 (0.22619422078132628) + | > loss_disc_real_3: 0.24528256058692932 (0.22657472649041344) + | > loss_disc_real_4: 0.23233556747436523 (0.22712118292556088) + | > loss_disc_real_5: 0.2982630431652069 (0.227224364175516) + | > loss_0: 2.5566391944885254 (2.43025998508229) + | > grad_norm_0: tensor(22.2288, device='cuda:0') (tensor(16.0286, device='cuda:0')) + | > loss_gen: 2.251704216003418 (2.423639325534596) + | > loss_kl: 1.7814654111862183 (1.8177921239067527) + | > loss_feat: 6.440240859985352 (7.287944995655733) + | > loss_mel: 16.38133430480957 (17.76771058475271) + | > loss_duration: 1.4827463626861572 (1.4427143433514764) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.33749008178711 (30.73980124978458) + | > grad_norm_1: tensor(518.3851, device='cuda:0') (tensor(341.5696, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.4074 (0.3712733689476462) + | > loader_time: 0.0067 (0.005917148029103) + + + --> TIME: 2025-04-19 05:08:28 -- STEP: 110/142 -- GLOBAL_STEP: 432500 + | > loss_disc: 2.361781597137451 (2.4386487115513193) + | > loss_disc_real_0: 0.11388399451971054 (0.125928770005703) + | > loss_disc_real_1: 0.18126465380191803 (0.20576917800036343) + | > loss_disc_real_2: 0.22011032700538635 (0.22639375884424556) + | > loss_disc_real_3: 0.21223194897174835 (0.22604824873534116) + | > loss_disc_real_4: 0.19081173837184906 (0.2276794182983312) + | > loss_disc_real_5: 0.19684188067913055 (0.22744007354432885) + | > loss_0: 2.361781597137451 (2.4386487115513193) + | > grad_norm_0: tensor(19.2746, device='cuda:0') (tensor(17.4168, device='cuda:0')) + | > loss_gen: 2.2478179931640625 (2.4199981581081045) + | > loss_kl: 1.9569872617721558 (1.8308661937713624) + | > loss_feat: 7.44058895111084 (7.241009317744862) + | > loss_mel: 17.883073806762695 (17.713597141612652) + | > loss_duration: 1.471524715423584 (1.451520025730133) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.9999942779541 (30.656990692832252) + | > grad_norm_1: tensor(451.8485, device='cuda:0') (tensor(358.7231, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.4398 (0.3826842979951338) + | > loader_time: 0.0074 (0.006181383132934572) + + + --> TIME: 2025-04-19 05:08:39 -- STEP: 135/142 -- GLOBAL_STEP: 432525 + | > loss_disc: 2.445708751678467 (2.439755581043385) + | > loss_disc_real_0: 0.16929911077022552 (0.1265024951210728) + | > loss_disc_real_1: 0.1991153359413147 (0.20622261078269394) + | > loss_disc_real_2: 0.22900433838367462 (0.22579717680259986) + | > loss_disc_real_3: 0.19706875085830688 (0.2267414864566591) + | > loss_disc_real_4: 0.25864389538764954 (0.22799372728224154) + | > loss_disc_real_5: 0.23933616280555725 (0.22715102665954165) + | > loss_0: 2.445708751678467 (2.439755581043385) + | > grad_norm_0: tensor(8.6848, device='cuda:0') (tensor(16.3703, device='cuda:0')) + | > loss_gen: 2.298682928085327 (2.4071061540532996) + | > loss_kl: 1.8799737691879272 (1.8394918830306441) + | > loss_feat: 6.346346855163574 (7.1993085649278425) + | > loss_mel: 16.812946319580078 (17.670104514227965) + | > loss_duration: 1.4624700546264648 (1.458195639539648) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.800418853759766 (30.574206641868308) + | > grad_norm_1: tensor(129.1094, device='cuda:0') (tensor(335.9850, device='cuda:0')) + | > current_lr_0: 0.00019086463212160436 + | > current_lr_1: 0.00019086463212160436 + | > step_time: 0.4676 (0.39539299187836824) + | > loader_time: 0.008 (0.006433365080091689) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4549193382263184 (2.4549193382263184) + | > loss_disc_real_0: 0.19725266098976135 (0.19725266098976135) + | > loss_disc_real_1: 0.2678632140159607 (0.2678632140159607) + | > loss_disc_real_2: 0.2121277153491974 (0.2121277153491974) + | > loss_disc_real_3: 0.2411811351776123 (0.2411811351776123) + | > loss_disc_real_4: 0.2325066179037094 (0.2325066179037094) + | > loss_disc_real_5: 0.21961209177970886 (0.21961209177970886) + | > loss_0: 2.4549193382263184 (2.4549193382263184) + | > loss_gen: 2.4540326595306396 (2.4540326595306396) + | > loss_kl: 2.5271685123443604 (2.5271685123443604) + | > loss_feat: 6.646020412445068 (6.646020412445068) + | > loss_mel: 18.08543586730957 (18.08543586730957) + | > loss_duration: 1.743323802947998 (1.743323802947998) + | > loss_1: 31.455982208251953 (31.455982208251953) + + --> STEP: 1 + | > loss_disc: 2.4308362007141113 (2.4308362007141113) + | > loss_disc_real_0: 0.15417633950710297 (0.15417633950710297) + | > loss_disc_real_1: 0.26433631777763367 (0.26433631777763367) + | > loss_disc_real_2: 0.240679070353508 (0.240679070353508) + | > loss_disc_real_3: 0.2731207609176636 (0.2731207609176636) + | > loss_disc_real_4: 0.23502309620380402 (0.23502309620380402) + | > loss_disc_real_5: 0.20967011153697968 (0.20967011153697968) + | > loss_0: 2.4308362007141113 (2.4308362007141113) + | > loss_gen: 2.606961488723755 (2.606961488723755) + | > loss_kl: 2.9900500774383545 (2.9900500774383545) + | > loss_feat: 7.406203269958496 (7.406203269958496) + | > loss_mel: 18.36663055419922 (18.36663055419922) + | > loss_duration: 1.7816362380981445 (1.7816362380981445) + | > loss_1: 33.15148162841797 (33.15148162841797) + + --> STEP: 2 + | > loss_disc: 2.357685089111328 (2.3942606449127197) + | > loss_disc_real_0: 0.1544424295425415 (0.15430938452482224) + | > loss_disc_real_1: 0.2451690137386322 (0.25475266575813293) + | > loss_disc_real_2: 0.2228851020336151 (0.23178208619356155) + | > loss_disc_real_3: 0.21476000547409058 (0.24394038319587708) + | > loss_disc_real_4: 0.22463685274124146 (0.22982997447252274) + | > loss_disc_real_5: 0.2620721161365509 (0.2358711138367653) + | > loss_0: 2.357685089111328 (2.3942606449127197) + | > loss_gen: 2.591175079345703 (2.599068284034729) + | > loss_kl: 2.325051784515381 (2.6575509309768677) + | > loss_feat: 6.9245171546936035 (7.16536021232605) + | > loss_mel: 18.610231399536133 (18.488430976867676) + | > loss_duration: 1.7088189125061035 (1.745227575302124) + | > loss_1: 32.159793853759766 (32.65563774108887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036056041717529297 (-0.000324249267578125) + | > avg_loss_disc: 2.3942606449127197 (-0.06650662422180176) + | > avg_loss_disc_real_0: 0.15430938452482224 (-0.04478738456964493) + | > avg_loss_disc_real_1: 0.25475266575813293 (-0.02789205312728882) + | > avg_loss_disc_real_2: 0.23178208619356155 (-0.029023677110671997) + | > avg_loss_disc_real_3: 0.24394038319587708 (+0.027229465544223785) + | > avg_loss_disc_real_4: 0.22982997447252274 (+0.0068373903632164) + | > avg_loss_disc_real_5: 0.2358711138367653 (-0.03151173144578934) + | > avg_loss_0: 2.3942606449127197 (-0.06650662422180176) + | > avg_loss_gen: 2.599068284034729 (-0.08158481121063232) + | > avg_loss_kl: 2.6575509309768677 (+0.26331090927124023) + | > avg_loss_feat: 7.16536021232605 (-0.3747742176055908) + | > avg_loss_mel: 18.488430976867676 (+0.3033027648925781) + | > avg_loss_duration: 1.745227575302124 (+0.009202718734741211) + | > avg_loss_1: 32.65563774108887 (+0.11945724487304688) + + + > EPOCH: 375/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:08:44)  + + --> TIME: 2025-04-19 05:08:51 -- STEP: 18/142 -- GLOBAL_STEP: 432550 + | > loss_disc: 2.4096996784210205 (2.4017489353815713) + | > loss_disc_real_0: 0.12101515382528305 (0.11813010027011235) + | > loss_disc_real_1: 0.19587969779968262 (0.21087295396460426) + | > loss_disc_real_2: 0.21426662802696228 (0.21766791078779432) + | > loss_disc_real_3: 0.20895801484584808 (0.2229180989993943) + | > loss_disc_real_4: 0.2291286289691925 (0.21843502091036904) + | > loss_disc_real_5: 0.17436523735523224 (0.21792230920659172) + | > loss_0: 2.4096996784210205 (2.4017489353815713) + | > grad_norm_0: tensor(6.9483, device='cuda:0') (tensor(6.2652, device='cuda:0')) + | > loss_gen: 2.4592063426971436 (2.3947221305635242) + | > loss_kl: 1.5705476999282837 (1.7785760296715631) + | > loss_feat: 7.7018938064575195 (7.445182508892483) + | > loss_mel: 17.841651916503906 (17.9392958217197) + | > loss_duration: 1.4168577194213867 (1.4149869018130832) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.990158081054688 (30.972763273451065) + | > grad_norm_1: tensor(80.5121, device='cuda:0') (tensor(83.6916, device='cuda:0')) + | > current_lr_0: 0.00019084077404258915 + | > current_lr_1: 0.00019084077404258915 + | > step_time: 0.3402 (0.33099614249335396) + | > loader_time: 0.0052 (0.004839883910285102) + + + --> TIME: 2025-04-19 05:09:00 -- STEP: 43/142 -- GLOBAL_STEP: 432575 + | > loss_disc: 2.507805824279785 (2.4271583390790363) + | > loss_disc_real_0: 0.15689635276794434 (0.12049842400606288) + | > loss_disc_real_1: 0.2407129406929016 (0.20903412131376045) + | > loss_disc_real_2: 0.24555151164531708 (0.22287227074767268) + | > loss_disc_real_3: 0.23215392231941223 (0.2283430466818255) + | > loss_disc_real_4: 0.21753554046154022 (0.2244231146435405) + | > loss_disc_real_5: 0.24370433390140533 (0.2278826829305915) + | > loss_0: 2.507805824279785 (2.4271583390790363) + | > grad_norm_0: tensor(6.1627, device='cuda:0') (tensor(6.5787, device='cuda:0')) + | > loss_gen: 2.3605990409851074 (2.379711306372354) + | > loss_kl: 1.7376505136489868 (1.8142055772071661) + | > loss_feat: 6.472883701324463 (7.351860112922136) + | > loss_mel: 18.104236602783203 (17.811639697052716) + | > loss_duration: 1.4441263675689697 (1.4292510747909546) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.119495391845703 (30.786667757256087) + | > grad_norm_1: tensor(34.8842, device='cuda:0') (tensor(85.3372, device='cuda:0')) + | > current_lr_0: 0.00019084077404258915 + | > current_lr_1: 0.00019084077404258915 + | > step_time: 0.3645 (0.3427995581959569) + | > loader_time: 0.0062 (0.0052221486734789465) + + + --> TIME: 2025-04-19 05:09:09 -- STEP: 68/142 -- GLOBAL_STEP: 432600 + | > loss_disc: 2.42543888092041 (2.4320166601854214) + | > loss_disc_real_0: 0.14437976479530334 (0.12227615363457624) + | > loss_disc_real_1: 0.21407705545425415 (0.20888126992127476) + | > loss_disc_real_2: 0.23672892153263092 (0.2233825220781214) + | > loss_disc_real_3: 0.22097034752368927 (0.22683968000552235) + | > loss_disc_real_4: 0.2545924186706543 (0.22578203678131104) + | > loss_disc_real_5: 0.2111680656671524 (0.22755929279853315) + | > loss_0: 2.42543888092041 (2.4320166601854214) + | > grad_norm_0: tensor(9.3250, device='cuda:0') (tensor(6.9977, device='cuda:0')) + | > loss_gen: 2.2590084075927734 (2.3718769094523267) + | > loss_kl: 1.8764623403549194 (1.8275164260583765) + | > loss_feat: 7.064460754394531 (7.299527932615841) + | > loss_mel: 17.79524803161621 (17.812387213987463) + | > loss_duration: 1.435511827468872 (1.4385989851811354) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.430692672729492 (30.74990757773904) + | > grad_norm_1: tensor(85.0732, device='cuda:0') (tensor(96.3856, device='cuda:0')) + | > current_lr_0: 0.00019084077404258915 + | > current_lr_1: 0.00019084077404258915 + | > step_time: 0.3866 (0.3543872412513284) + | > loader_time: 0.006 (0.005504380254184499) + + + --> TIME: 2025-04-19 05:09:20 -- STEP: 93/142 -- GLOBAL_STEP: 432625 + | > loss_disc: 2.5800931453704834 (2.444848850209226) + | > loss_disc_real_0: 0.12430033832788467 (0.12413253474940536) + | > loss_disc_real_1: 0.22177499532699585 (0.20936260428479922) + | > loss_disc_real_2: 0.24172058701515198 (0.22511586306556577) + | > loss_disc_real_3: 0.2618016302585602 (0.2277747849302907) + | > loss_disc_real_4: 0.25060030817985535 (0.2265233641029686) + | > loss_disc_real_5: 0.23954154551029205 (0.2280894128545638) + | > loss_0: 2.5800931453704834 (2.444848850209226) + | > grad_norm_0: tensor(5.4325, device='cuda:0') (tensor(7.1212, device='cuda:0')) + | > loss_gen: 2.259394645690918 (2.356162617283483) + | > loss_kl: 1.7570384740829468 (1.8380258211525538) + | > loss_feat: 5.982881546020508 (7.212882308549778) + | > loss_mel: 16.891794204711914 (17.740825581294235) + | > loss_duration: 1.4624629020690918 (1.448493056399848) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.353572845458984 (30.596389462870935) + | > grad_norm_1: tensor(111.8289, device='cuda:0') (tensor(102.8139, device='cuda:0')) + | > current_lr_0: 0.00019084077404258915 + | > current_lr_1: 0.00019084077404258915 + | > step_time: 0.3995 (0.36615590639011836) + | > loader_time: 0.0064 (0.005745946720082274) + + + --> TIME: 2025-04-19 05:09:30 -- STEP: 118/142 -- GLOBAL_STEP: 432650 + | > loss_disc: 2.5686140060424805 (2.454640632968839) + | > loss_disc_real_0: 0.10596052557229996 (0.126722981111478) + | > loss_disc_real_1: 0.19660699367523193 (0.20832985058679418) + | > loss_disc_real_2: 0.25817155838012695 (0.22467237357365882) + | > loss_disc_real_3: 0.25649088621139526 (0.22836131665666223) + | > loss_disc_real_4: 0.2387346774339676 (0.22786672703795513) + | > loss_disc_real_5: 0.2926941514015198 (0.2296352290501029) + | > loss_0: 2.5686140060424805 (2.454640632968839) + | > grad_norm_0: tensor(8.5443, device='cuda:0') (tensor(7.2045, device='cuda:0')) + | > loss_gen: 2.3328075408935547 (2.353302636389005) + | > loss_kl: 1.8188567161560059 (1.8473390290292644) + | > loss_feat: 6.4991936683654785 (7.1920322604098565) + | > loss_mel: 16.46731948852539 (17.719209792250293) + | > loss_duration: 1.449167013168335 (1.4553138583393421) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.567344665527344 (30.56719768653482) + | > grad_norm_1: tensor(366.1017, device='cuda:0') (tensor(113.7643, device='cuda:0')) + | > current_lr_0: 0.00019084077404258915 + | > current_lr_1: 0.00019084077404258915 + | > step_time: 0.4441 (0.37701541488453494) + | > loader_time: 0.008 (0.006034255027770997) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5584025382995605 (2.5584025382995605) + | > loss_disc_real_0: 0.14949481189250946 (0.14949481189250946) + | > loss_disc_real_1: 0.22937265038490295 (0.22937265038490295) + | > loss_disc_real_2: 0.2888655662536621 (0.2888655662536621) + | > loss_disc_real_3: 0.2574615776538849 (0.2574615776538849) + | > loss_disc_real_4: 0.2363375723361969 (0.2363375723361969) + | > loss_disc_real_5: 0.24879799783229828 (0.24879799783229828) + | > loss_0: 2.5584025382995605 (2.5584025382995605) + | > loss_gen: 2.2928273677825928 (2.2928273677825928) + | > loss_kl: 2.408205509185791 (2.408205509185791) + | > loss_feat: 6.580515384674072 (6.580515384674072) + | > loss_mel: 17.721736907958984 (17.721736907958984) + | > loss_duration: 1.742293357849121 (1.742293357849121) + | > loss_1: 30.74557876586914 (30.74557876586914) + + --> STEP: 1 + | > loss_disc: 2.6738266944885254 (2.6738266944885254) + | > loss_disc_real_0: 0.18122820556163788 (0.18122820556163788) + | > loss_disc_real_1: 0.25225409865379333 (0.25225409865379333) + | > loss_disc_real_2: 0.3215295076370239 (0.3215295076370239) + | > loss_disc_real_3: 0.27952897548675537 (0.27952897548675537) + | > loss_disc_real_4: 0.2922517955303192 (0.2922517955303192) + | > loss_disc_real_5: 0.27683767676353455 (0.27683767676353455) + | > loss_0: 2.6738266944885254 (2.6738266944885254) + | > loss_gen: 2.4034152030944824 (2.4034152030944824) + | > loss_kl: 3.1549603939056396 (3.1549603939056396) + | > loss_feat: 6.500240325927734 (6.500240325927734) + | > loss_mel: 17.62635040283203 (17.62635040283203) + | > loss_duration: 1.769325852394104 (1.769325852394104) + | > loss_1: 31.45429229736328 (31.45429229736328) + + --> STEP: 2 + | > loss_disc: 2.6367077827453613 (2.6552672386169434) + | > loss_disc_real_0: 0.1457083523273468 (0.16346827894449234) + | > loss_disc_real_1: 0.26744720339775085 (0.2598506510257721) + | > loss_disc_real_2: 0.26641014218330383 (0.2939698249101639) + | > loss_disc_real_3: 0.2745807468891144 (0.2770548611879349) + | > loss_disc_real_4: 0.29323330521583557 (0.2927425503730774) + | > loss_disc_real_5: 0.28408268094062805 (0.2804601788520813) + | > loss_0: 2.6367077827453613 (2.6552672386169434) + | > loss_gen: 2.376107931137085 (2.3897615671157837) + | > loss_kl: 2.2738914489746094 (2.7144259214401245) + | > loss_feat: 6.7878522872924805 (6.644046306610107) + | > loss_mel: 16.85032844543457 (17.2383394241333) + | > loss_duration: 1.7146823406219482 (1.7420040965080261) + | > loss_1: 30.00286102294922 (30.72857666015625) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003500223159790039 (-0.00010538101196289062) + | > avg_loss_disc: 2.6552672386169434 (+0.26100659370422363) + | > avg_loss_disc_real_0: 0.16346827894449234 (+0.009158894419670105) + | > avg_loss_disc_real_1: 0.2598506510257721 (+0.00509798526763916) + | > avg_loss_disc_real_2: 0.2939698249101639 (+0.062187738716602325) + | > avg_loss_disc_real_3: 0.2770548611879349 (+0.0331144779920578) + | > avg_loss_disc_real_4: 0.2927425503730774 (+0.06291257590055466) + | > avg_loss_disc_real_5: 0.2804601788520813 (+0.04458906501531601) + | > avg_loss_0: 2.6552672386169434 (+0.26100659370422363) + | > avg_loss_gen: 2.3897615671157837 (-0.2093067169189453) + | > avg_loss_kl: 2.7144259214401245 (+0.056874990463256836) + | > avg_loss_feat: 6.644046306610107 (-0.5213139057159424) + | > avg_loss_mel: 17.2383394241333 (-1.250091552734375) + | > avg_loss_duration: 1.7420040965080261 (-0.0032234787940979004) + | > avg_loss_1: 30.72857666015625 (-1.9270610809326172) + + + > EPOCH: 376/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:09:43)  + + --> TIME: 2025-04-19 05:09:44 -- STEP: 1/142 -- GLOBAL_STEP: 432675 + | > loss_disc: 2.4746639728546143 (2.4746639728546143) + | > loss_disc_real_0: 0.13096550107002258 (0.13096550107002258) + | > loss_disc_real_1: 0.22598779201507568 (0.22598779201507568) + | > loss_disc_real_2: 0.263869047164917 (0.263869047164917) + | > loss_disc_real_3: 0.2491312026977539 (0.2491312026977539) + | > loss_disc_real_4: 0.26567161083221436 (0.26567161083221436) + | > loss_disc_real_5: 0.24292699992656708 (0.24292699992656708) + | > loss_0: 2.4746639728546143 (2.4746639728546143) + | > grad_norm_0: tensor(11.1567, device='cuda:0') (tensor(11.1567, device='cuda:0')) + | > loss_gen: 2.3521320819854736 (2.3521320819854736) + | > loss_kl: 1.7703502178192139 (1.7703502178192139) + | > loss_feat: 7.100306987762451 (7.100306987762451) + | > loss_mel: 17.96759033203125 (17.96759033203125) + | > loss_duration: 1.3677937984466553 (1.3677937984466553) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.55817413330078 (30.55817413330078) + | > grad_norm_1: tensor(329.2944, device='cuda:0') (tensor(329.2944, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.3277 (0.32771921157836914) + | > loader_time: 0.004 (0.00401759147644043) + + + --> TIME: 2025-04-19 05:09:52 -- STEP: 26/142 -- GLOBAL_STEP: 432700 + | > loss_disc: 2.4339563846588135 (2.431534290313721) + | > loss_disc_real_0: 0.12827114760875702 (0.1159637882732428) + | > loss_disc_real_1: 0.19650305807590485 (0.20369289414240763) + | > loss_disc_real_2: 0.19159702956676483 (0.22901220963551447) + | > loss_disc_real_3: 0.21306264400482178 (0.22965659373081648) + | > loss_disc_real_4: 0.22643065452575684 (0.23692302004649088) + | > loss_disc_real_5: 0.2052772045135498 (0.23049286466378432) + | > loss_0: 2.4339563846588135 (2.431534290313721) + | > grad_norm_0: tensor(4.8215, device='cuda:0') (tensor(7.2080, device='cuda:0')) + | > loss_gen: 2.279670476913452 (2.3870775974713836) + | > loss_kl: 1.845416784286499 (1.8258413213949938) + | > loss_feat: 6.590348243713379 (7.385839278881367) + | > loss_mel: 17.676328659057617 (17.99918772624089) + | > loss_duration: 1.4430646896362305 (1.4176716208457947) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.83483123779297 (31.01561766404372) + | > grad_norm_1: tensor(63.9548, device='cuda:0') (tensor(98.2179, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.3498 (0.33398280693934523) + | > loader_time: 0.0049 (0.004951926378103403) + + + --> TIME: 2025-04-19 05:10:02 -- STEP: 51/142 -- GLOBAL_STEP: 432725 + | > loss_disc: 2.5898780822753906 (2.436294275171617) + | > loss_disc_real_0: 0.1289995014667511 (0.11965032783793468) + | > loss_disc_real_1: 0.15296882390975952 (0.2048253045946944) + | > loss_disc_real_2: 0.21578505635261536 (0.226678313870056) + | > loss_disc_real_3: 0.24251987040042877 (0.22714520757104836) + | > loss_disc_real_4: 0.21267136931419373 (0.2302412089763903) + | > loss_disc_real_5: 0.20256254076957703 (0.226046126554994) + | > loss_0: 2.5898780822753906 (2.436294275171617) + | > grad_norm_0: tensor(8.3543, device='cuda:0') (tensor(6.3362, device='cuda:0')) + | > loss_gen: 2.1595914363861084 (2.3607779904907824) + | > loss_kl: 1.7185070514678955 (1.8218671410691505) + | > loss_feat: 7.1547393798828125 (7.27608819101371) + | > loss_mel: 16.100507736206055 (17.805648485819496) + | > loss_duration: 1.4191646575927734 (1.428170421544243) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.55251121520996 (30.69255245433134) + | > grad_norm_1: tensor(226.9801, device='cuda:0') (tensor(101.4248, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.368 (0.3497920503803328) + | > loader_time: 0.006 (0.005294098573572496) + + + --> TIME: 2025-04-19 05:10:11 -- STEP: 76/142 -- GLOBAL_STEP: 432750 + | > loss_disc: 2.4365105628967285 (2.4251399228447363) + | > loss_disc_real_0: 0.11337220668792725 (0.12058998889436848) + | > loss_disc_real_1: 0.18267293274402618 (0.20421902737335149) + | > loss_disc_real_2: 0.24748659133911133 (0.22558154793162094) + | > loss_disc_real_3: 0.22911520302295685 (0.22658548582541316) + | > loss_disc_real_4: 0.24813580513000488 (0.22738238875018923) + | > loss_disc_real_5: 0.246384397149086 (0.22671470516606382) + | > loss_0: 2.4365105628967285 (2.4251399228447363) + | > grad_norm_0: tensor(9.0748, device='cuda:0') (tensor(6.9947, device='cuda:0')) + | > loss_gen: 2.239954710006714 (2.3768403279153927) + | > loss_kl: 1.9568560123443604 (1.8405684813072807) + | > loss_feat: 6.551093578338623 (7.270942110764353) + | > loss_mel: 16.107431411743164 (17.727782337289103) + | > loss_duration: 1.4812703132629395 (1.4390661041987567) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.336605072021484 (30.6551994524504) + | > grad_norm_1: tensor(259.4493, device='cuda:0') (tensor(126.9815, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.3989 (0.362236876236765) + | > loader_time: 0.0068 (0.005637306916086297) + + + --> TIME: 2025-04-19 05:10:22 -- STEP: 101/142 -- GLOBAL_STEP: 432775 + | > loss_disc: 2.418283224105835 (2.4449099266883167) + | > loss_disc_real_0: 0.10734844207763672 (0.1234651809871787) + | > loss_disc_real_1: 0.2496597319841385 (0.20588622987270347) + | > loss_disc_real_2: 0.22239507734775543 (0.22617424065523808) + | > loss_disc_real_3: 0.22245962917804718 (0.22800418970608474) + | > loss_disc_real_4: 0.2368076890707016 (0.22818430520520352) + | > loss_disc_real_5: 0.19999445974826813 (0.2268434366082201) + | > loss_0: 2.418283224105835 (2.4449099266883167) + | > grad_norm_0: tensor(7.5555, device='cuda:0') (tensor(8.8101, device='cuda:0')) + | > loss_gen: 2.400117874145508 (2.3672127157154668) + | > loss_kl: 1.950164794921875 (1.8511799597504115) + | > loss_feat: 7.626544952392578 (7.205292092691554) + | > loss_mel: 18.70519256591797 (17.674797209182582) + | > loss_duration: 1.491568684577942 (1.4493135615150525) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.173587799072266 (30.54779556954261) + | > grad_norm_1: tensor(78.3738, device='cuda:0') (tensor(176.2774, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.4134 (0.37230308929292283) + | > loader_time: 0.0068 (0.0058537780648410895) + + + --> TIME: 2025-04-19 05:10:33 -- STEP: 126/142 -- GLOBAL_STEP: 432800 + | > loss_disc: 2.4960501194000244 (2.4514893728589255) + | > loss_disc_real_0: 0.18867884576320648 (0.1257560108862226) + | > loss_disc_real_1: 0.19980373978614807 (0.20719821053364915) + | > loss_disc_real_2: 0.1808188110589981 (0.2255877349821348) + | > loss_disc_real_3: 0.22064389288425446 (0.2277775684755946) + | > loss_disc_real_4: 0.2256390005350113 (0.22761835973887218) + | > loss_disc_real_5: 0.23018108308315277 (0.22877142330010733) + | > loss_0: 2.4960501194000244 (2.4514893728589255) + | > grad_norm_0: tensor(12.6766, device='cuda:0') (tensor(10.1339, device='cuda:0')) + | > loss_gen: 2.2359704971313477 (2.3632138116019124) + | > loss_kl: 1.8719120025634766 (1.8664714079054574) + | > loss_feat: 7.3393168449401855 (7.150647685641334) + | > loss_mel: 17.17343521118164 (17.628138587588353) + | > loss_duration: 1.4798154830932617 (1.4562155263764516) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.100452423095703 (30.46468701438298) + | > grad_norm_1: tensor(357.5327, device='cuda:0') (tensor(205.1112, device='cuda:0')) + | > current_lr_0: 0.00019081691894583383 + | > current_lr_1: 0.00019081691894583383 + | > step_time: 0.4378 (0.3832322321240864) + | > loader_time: 0.0079 (0.006126231617397731) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4881505966186523 (2.4881505966186523) + | > loss_disc_real_0: 0.21088328957557678 (0.21088328957557678) + | > loss_disc_real_1: 0.1882125735282898 (0.1882125735282898) + | > loss_disc_real_2: 0.23714958131313324 (0.23714958131313324) + | > loss_disc_real_3: 0.24332325160503387 (0.24332325160503387) + | > loss_disc_real_4: 0.23435112833976746 (0.23435112833976746) + | > loss_disc_real_5: 0.23731903731822968 (0.23731903731822968) + | > loss_0: 2.4881505966186523 (2.4881505966186523) + | > loss_gen: 2.469233751296997 (2.469233751296997) + | > loss_kl: 2.5916903018951416 (2.5916903018951416) + | > loss_feat: 5.999434947967529 (5.999434947967529) + | > loss_mel: 16.457544326782227 (16.457544326782227) + | > loss_duration: 1.7297905683517456 (1.7297905683517456) + | > loss_1: 29.24769401550293 (29.24769401550293) + + --> STEP: 1 + | > loss_disc: 2.4224700927734375 (2.4224700927734375) + | > loss_disc_real_0: 0.17482990026474 (0.17482990026474) + | > loss_disc_real_1: 0.22019906342029572 (0.22019906342029572) + | > loss_disc_real_2: 0.19466356933116913 (0.19466356933116913) + | > loss_disc_real_3: 0.18512439727783203 (0.18512439727783203) + | > loss_disc_real_4: 0.2289586216211319 (0.2289586216211319) + | > loss_disc_real_5: 0.19927208125591278 (0.19927208125591278) + | > loss_0: 2.4224700927734375 (2.4224700927734375) + | > loss_gen: 2.3361902236938477 (2.3361902236938477) + | > loss_kl: 2.843000650405884 (2.843000650405884) + | > loss_feat: 7.9969258308410645 (7.9969258308410645) + | > loss_mel: 19.773527145385742 (19.773527145385742) + | > loss_duration: 1.7995707988739014 (1.7995707988739014) + | > loss_1: 34.74921417236328 (34.74921417236328) + + --> STEP: 2 + | > loss_disc: 2.5985631942749023 (2.51051664352417) + | > loss_disc_real_0: 0.26739659905433655 (0.22111324965953827) + | > loss_disc_real_1: 0.20732532441616058 (0.21376219391822815) + | > loss_disc_real_2: 0.22819718718528748 (0.2114303782582283) + | > loss_disc_real_3: 0.20994782447814941 (0.19753611087799072) + | > loss_disc_real_4: 0.2675226032733917 (0.2482406124472618) + | > loss_disc_real_5: 0.22533917427062988 (0.21230562776327133) + | > loss_0: 2.5985631942749023 (2.51051664352417) + | > loss_gen: 2.342195749282837 (2.3391929864883423) + | > loss_kl: 2.229740858078003 (2.5363707542419434) + | > loss_feat: 7.284599781036377 (7.640762805938721) + | > loss_mel: 18.473224639892578 (19.12337589263916) + | > loss_duration: 1.7257177829742432 (1.7626442909240723) + | > loss_1: 32.055477142333984 (33.40234565734863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036163330078125 (+0.00011610984802246094) + | > avg_loss_disc: 2.51051664352417 (-0.14475059509277344) + | > avg_loss_disc_real_0: 0.22111324965953827 (+0.05764497071504593) + | > avg_loss_disc_real_1: 0.21376219391822815 (-0.046088457107543945) + | > avg_loss_disc_real_2: 0.2114303782582283 (-0.08253944665193558) + | > avg_loss_disc_real_3: 0.19753611087799072 (-0.07951875030994415) + | > avg_loss_disc_real_4: 0.2482406124472618 (-0.04450193792581558) + | > avg_loss_disc_real_5: 0.21230562776327133 (-0.06815455108880997) + | > avg_loss_0: 2.51051664352417 (-0.14475059509277344) + | > avg_loss_gen: 2.3391929864883423 (-0.050568580627441406) + | > avg_loss_kl: 2.5363707542419434 (-0.17805516719818115) + | > avg_loss_feat: 7.640762805938721 (+0.9967164993286133) + | > avg_loss_mel: 19.12337589263916 (+1.8850364685058594) + | > avg_loss_duration: 1.7626442909240723 (+0.020640194416046143) + | > avg_loss_1: 33.40234565734863 (+2.673768997192383) + + + > EPOCH: 377/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:10:41)  + + --> TIME: 2025-04-19 05:10:45 -- STEP: 9/142 -- GLOBAL_STEP: 432825 + | > loss_disc: 2.4097299575805664 (2.4008702172173395) + | > loss_disc_real_0: 0.07424847036600113 (0.106044492787785) + | > loss_disc_real_1: 0.19963501393795013 (0.21079354816012913) + | > loss_disc_real_2: 0.20189793407917023 (0.21797510815991294) + | > loss_disc_real_3: 0.22509264945983887 (0.2196667641401291) + | > loss_disc_real_4: 0.22964957356452942 (0.22261144551965925) + | > loss_disc_real_5: 0.18679800629615784 (0.22987952000564998) + | > loss_0: 2.4097299575805664 (2.4008702172173395) + | > grad_norm_0: tensor(8.0496, device='cuda:0') (tensor(8.2193, device='cuda:0')) + | > loss_gen: 2.5158660411834717 (2.393895732031928) + | > loss_kl: 1.8931368589401245 (1.7834415170881484) + | > loss_feat: 8.28774356842041 (7.365005281236437) + | > loss_mel: 19.285907745361328 (17.768361197577583) + | > loss_duration: 1.3933948278427124 (1.4107599788241916) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.37604904174805 (30.72146373324924) + | > grad_norm_1: tensor(68.2711, device='cuda:0') (tensor(212.3658, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.3257 (0.32707511054144967) + | > loader_time: 0.0042 (0.004548072814941406) + + + --> TIME: 2025-04-19 05:10:54 -- STEP: 34/142 -- GLOBAL_STEP: 432850 + | > loss_disc: 2.4356536865234375 (2.4156046895419854) + | > loss_disc_real_0: 0.13681049644947052 (0.1160163381958709) + | > loss_disc_real_1: 0.20648877322673798 (0.20633629358866634) + | > loss_disc_real_2: 0.21489229798316956 (0.22161084457355387) + | > loss_disc_real_3: 0.20945733785629272 (0.22373846260940328) + | > loss_disc_real_4: 0.22116118669509888 (0.22566280514001846) + | > loss_disc_real_5: 0.2464585155248642 (0.2328143851721988) + | > loss_0: 2.4356536865234375 (2.4156046895419854) + | > grad_norm_0: tensor(8.7903, device='cuda:0') (tensor(9.3557, device='cuda:0')) + | > loss_gen: 2.2562482357025146 (2.418933889445137) + | > loss_kl: 1.801506519317627 (1.791148609974805) + | > loss_feat: 6.463132858276367 (7.354520713581758) + | > loss_mel: 16.72197723388672 (17.82744749854593) + | > loss_duration: 1.458362340927124 (1.4206501736360437) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.70122718811035 (30.812701113083783) + | > grad_norm_1: tensor(247.5698, device='cuda:0') (tensor(234.7037, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.3649 (0.3391137193231022) + | > loader_time: 0.0059 (0.00516972121070413) + + + --> TIME: 2025-04-19 05:11:03 -- STEP: 59/142 -- GLOBAL_STEP: 432875 + | > loss_disc: 2.5789146423339844 (2.434663049245284) + | > loss_disc_real_0: 0.11902139335870743 (0.1199925413071099) + | > loss_disc_real_1: 0.2143249362707138 (0.20620180439140837) + | > loss_disc_real_2: 0.2666347920894623 (0.22531512404902507) + | > loss_disc_real_3: 0.267910361289978 (0.22589753063048346) + | > loss_disc_real_4: 0.25038060545921326 (0.2286901448742818) + | > loss_disc_real_5: 0.2843039929866791 (0.23244182613946623) + | > loss_0: 2.5789146423339844 (2.434663049245284) + | > grad_norm_0: tensor(28.2628, device='cuda:0') (tensor(11.2008, device='cuda:0')) + | > loss_gen: 2.3263344764709473 (2.3979407569109394) + | > loss_kl: 1.8052685260772705 (1.804953605441724) + | > loss_feat: 7.052499771118164 (7.212829654499636) + | > loss_mel: 16.612646102905273 (17.70841682563394) + | > loss_duration: 1.4491050243377686 (1.4315706996594446) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.245851516723633 (30.555711552248162) + | > grad_norm_1: tensor(596.1329, device='cuda:0') (tensor(288.6345, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.3778 (0.35037903058326847) + | > loader_time: 0.0059 (0.005480309664192847) + + + --> TIME: 2025-04-19 05:11:13 -- STEP: 84/142 -- GLOBAL_STEP: 432900 + | > loss_disc: 2.486156463623047 (2.4427533178102387) + | > loss_disc_real_0: 0.1484931856393814 (0.12276798797150452) + | > loss_disc_real_1: 0.19832558929920197 (0.20461463076727732) + | > loss_disc_real_2: 0.22660163044929504 (0.22528994260799318) + | > loss_disc_real_3: 0.2428702563047409 (0.22634806200152352) + | > loss_disc_real_4: 0.24232736229896545 (0.22834834350006922) + | > loss_disc_real_5: 0.22981445491313934 (0.23205088380546796) + | > loss_0: 2.486156463623047 (2.4427533178102387) + | > grad_norm_0: tensor(7.8756, device='cuda:0') (tensor(12.5449, device='cuda:0')) + | > loss_gen: 2.3316452503204346 (2.394573095298949) + | > loss_kl: 2.0054891109466553 (1.8285702821754275) + | > loss_feat: 6.149003982543945 (7.217458759035383) + | > loss_mel: 17.239959716796875 (17.78319086347307) + | > loss_duration: 1.5120244026184082 (1.4427285279546467) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.238122940063477 (30.666521435692196) + | > grad_norm_1: tensor(68.7599, device='cuda:0') (tensor(300.3704, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.3947 (0.36244564396994455) + | > loader_time: 0.0067 (0.0057625969250996905) + + + --> TIME: 2025-04-19 05:11:24 -- STEP: 109/142 -- GLOBAL_STEP: 432925 + | > loss_disc: 2.46934175491333 (2.4508475163661005) + | > loss_disc_real_0: 0.09269385784864426 (0.1239523673002873) + | > loss_disc_real_1: 0.19307315349578857 (0.20462765920599665) + | > loss_disc_real_2: 0.265534907579422 (0.2264857294909451) + | > loss_disc_real_3: 0.27339550852775574 (0.22707343046818304) + | > loss_disc_real_4: 0.23807992041110992 (0.22937647779600337) + | > loss_disc_real_5: 0.2967773973941803 (0.23293307177517392) + | > loss_0: 2.46934175491333 (2.4508475163661005) + | > grad_norm_0: tensor(6.7097, device='cuda:0') (tensor(11.7148, device='cuda:0')) + | > loss_gen: 2.4382832050323486 (2.3816147922375888) + | > loss_kl: 2.009011745452881 (1.8414973840800994) + | > loss_feat: 7.279356956481934 (7.176737417868518) + | > loss_mel: 18.91167640686035 (17.762020898521495) + | > loss_duration: 1.481968879699707 (1.4505718760534163) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.120296478271484 (30.61244227908073) + | > grad_norm_1: tensor(71.3154, device='cuda:0') (tensor(278.6575, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.4125 (0.37254429957188595) + | > loader_time: 0.0069 (0.005983162363734814) + + + --> TIME: 2025-04-19 05:11:35 -- STEP: 134/142 -- GLOBAL_STEP: 432950 + | > loss_disc: 2.500624895095825 (2.4592516867082503) + | > loss_disc_real_0: 0.1654902696609497 (0.1251898916688428) + | > loss_disc_real_1: 0.215315043926239 (0.20581557824095684) + | > loss_disc_real_2: 0.2105882614850998 (0.2263672749942808) + | > loss_disc_real_3: 0.23303525149822235 (0.22753148503712753) + | > loss_disc_real_4: 0.20402345061302185 (0.23061891122540432) + | > loss_disc_real_5: 0.23090308904647827 (0.2331197153943688) + | > loss_0: 2.500624895095825 (2.4592516867082503) + | > grad_norm_0: tensor(13.0456, device='cuda:0') (tensor(11.2104, device='cuda:0')) + | > loss_gen: 2.315194606781006 (2.3714018472984657) + | > loss_kl: 1.823136568069458 (1.8503993989816352) + | > loss_feat: 6.877490520477295 (7.156634401919237) + | > loss_mel: 17.653079986572266 (17.772084221911076) + | > loss_duration: 1.5048186779022217 (1.4571572266407866) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.17371940612793 (30.6076770184645) + | > grad_norm_1: tensor(515.7273, device='cuda:0') (tensor(265.7751, device='cuda:0')) + | > current_lr_0: 0.0001907930668309656 + | > current_lr_1: 0.0001907930668309656 + | > step_time: 0.4407 (0.3843600233988975) + | > loader_time: 0.0073 (0.006247064960536673) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3719565868377686 (2.3719565868377686) + | > loss_disc_real_0: 0.14090408384799957 (0.14090408384799957) + | > loss_disc_real_1: 0.21395868062973022 (0.21395868062973022) + | > loss_disc_real_2: 0.17055799067020416 (0.17055799067020416) + | > loss_disc_real_3: 0.19806700944900513 (0.19806700944900513) + | > loss_disc_real_4: 0.20646719634532928 (0.20646719634532928) + | > loss_disc_real_5: 0.19307982921600342 (0.19307982921600342) + | > loss_0: 2.3719565868377686 (2.3719565868377686) + | > loss_gen: 2.1871438026428223 (2.1871438026428223) + | > loss_kl: 2.614689350128174 (2.614689350128174) + | > loss_feat: 6.89426326751709 (6.89426326751709) + | > loss_mel: 18.80558204650879 (18.80558204650879) + | > loss_duration: 1.7211273908615112 (1.7211273908615112) + | > loss_1: 32.22280502319336 (32.22280502319336) + + --> STEP: 1 + | > loss_disc: 2.3841538429260254 (2.3841538429260254) + | > loss_disc_real_0: 0.12465810775756836 (0.12465810775756836) + | > loss_disc_real_1: 0.23759761452674866 (0.23759761452674866) + | > loss_disc_real_2: 0.20001305639743805 (0.20001305639743805) + | > loss_disc_real_3: 0.1967417448759079 (0.1967417448759079) + | > loss_disc_real_4: 0.1872618943452835 (0.1872618943452835) + | > loss_disc_real_5: 0.1936866194009781 (0.1936866194009781) + | > loss_0: 2.3841538429260254 (2.3841538429260254) + | > loss_gen: 2.1767489910125732 (2.1767489910125732) + | > loss_kl: 2.610687732696533 (2.610687732696533) + | > loss_feat: 7.475543022155762 (7.475543022155762) + | > loss_mel: 18.64764976501465 (18.64764976501465) + | > loss_duration: 1.7877757549285889 (1.7877757549285889) + | > loss_1: 32.69840621948242 (32.69840621948242) + + --> STEP: 2 + | > loss_disc: 2.4791903495788574 (2.4316720962524414) + | > loss_disc_real_0: 0.1255444586277008 (0.12510128319263458) + | > loss_disc_real_1: 0.2664945423603058 (0.2520460784435272) + | > loss_disc_real_2: 0.19805511832237244 (0.19903408735990524) + | > loss_disc_real_3: 0.21380503475666046 (0.20527338981628418) + | > loss_disc_real_4: 0.24714604020118713 (0.21720396727323532) + | > loss_disc_real_5: 0.26353517174720764 (0.22861089557409286) + | > loss_0: 2.4791903495788574 (2.4316720962524414) + | > loss_gen: 2.4016480445861816 (2.2891985177993774) + | > loss_kl: 2.1533467769622803 (2.3820172548294067) + | > loss_feat: 5.822945594787598 (6.64924430847168) + | > loss_mel: 16.131160736083984 (17.389405250549316) + | > loss_duration: 1.7065321207046509 (1.7471539378166199) + | > loss_1: 28.21563148498535 (30.457018852233887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00370025634765625 (+8.392333984375e-05) + | > avg_loss_disc: 2.4316720962524414 (-0.07884454727172852) + | > avg_loss_disc_real_0: 0.12510128319263458 (-0.09601196646690369) + | > avg_loss_disc_real_1: 0.2520460784435272 (+0.03828388452529907) + | > avg_loss_disc_real_2: 0.19903408735990524 (-0.012396290898323059) + | > avg_loss_disc_real_3: 0.20527338981628418 (+0.007737278938293457) + | > avg_loss_disc_real_4: 0.21720396727323532 (-0.03103664517402649) + | > avg_loss_disc_real_5: 0.22861089557409286 (+0.016305267810821533) + | > avg_loss_0: 2.4316720962524414 (-0.07884454727172852) + | > avg_loss_gen: 2.2891985177993774 (-0.049994468688964844) + | > avg_loss_kl: 2.3820172548294067 (-0.15435349941253662) + | > avg_loss_feat: 6.64924430847168 (-0.991518497467041) + | > avg_loss_mel: 17.389405250549316 (-1.7339706420898438) + | > avg_loss_duration: 1.7471539378166199 (-0.015490353107452393) + | > avg_loss_1: 30.457018852233887 (-2.945326805114746) + + + > EPOCH: 378/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:11:40)  + + --> TIME: 2025-04-19 05:11:46 -- STEP: 17/142 -- GLOBAL_STEP: 432975 + | > loss_disc: 2.494422674179077 (2.4146075809703156) + | > loss_disc_real_0: 0.17924106121063232 (0.12360593851874857) + | > loss_disc_real_1: 0.20245417952537537 (0.20439542041105382) + | > loss_disc_real_2: 0.2323135882616043 (0.22463269531726837) + | > loss_disc_real_3: 0.2321217805147171 (0.22407867627985337) + | > loss_disc_real_4: 0.23848684132099152 (0.2233713646145428) + | > loss_disc_real_5: 0.23264260590076447 (0.2220991972614737) + | > loss_0: 2.494422674179077 (2.4146075809703156) + | > grad_norm_0: tensor(16.4890, device='cuda:0') (tensor(10.2653, device='cuda:0')) + | > loss_gen: 2.2816522121429443 (2.4003391826854035) + | > loss_kl: 1.721083402633667 (1.7887386013479794) + | > loss_feat: 7.259219169616699 (7.25741299460916) + | > loss_mel: 17.53789520263672 (17.983264025519876) + | > loss_duration: 1.4031469821929932 (1.4070910916608923) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.2029972076416 (30.836845846737134) + | > grad_norm_1: tensor(210.4893, device='cuda:0') (tensor(253.3190, device='cuda:0')) + | > current_lr_0: 0.0001907692176976117 + | > current_lr_1: 0.0001907692176976117 + | > step_time: 0.3362 (0.32941265667186065) + | > loader_time: 0.0054 (0.004693087409524357) + + + --> TIME: 2025-04-19 05:11:55 -- STEP: 42/142 -- GLOBAL_STEP: 433000 + | > loss_disc: 2.4904415607452393 (2.4389536607833144) + | > loss_disc_real_0: 0.1310780793428421 (0.11943742455471129) + | > loss_disc_real_1: 0.2187267392873764 (0.2068793688501631) + | > loss_disc_real_2: 0.2042572796344757 (0.22840704094795955) + | > loss_disc_real_3: 0.21495173871517181 (0.22596872136706397) + | > loss_disc_real_4: 0.2222355455160141 (0.22553598348583495) + | > loss_disc_real_5: 0.23934687674045563 (0.2304871529340744) + | > loss_0: 2.4904415607452393 (2.4389536607833144) + | > grad_norm_0: tensor(15.3499, device='cuda:0') (tensor(10.8175, device='cuda:0')) + | > loss_gen: 2.248988628387451 (2.3743537834712436) + | > loss_kl: 1.8419090509414673 (1.8082373085476102) + | > loss_feat: 6.442563056945801 (7.148836828413463) + | > loss_mel: 16.722652435302734 (17.69861854825701) + | > loss_duration: 1.4690254926681519 (1.4228447987919761) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.725139617919922 (30.452891440618608) + | > grad_norm_1: tensor(332.6393, device='cuda:0') (tensor(256.4026, device='cuda:0')) + | > current_lr_0: 0.0001907692176976117 + | > current_lr_1: 0.0001907692176976117 + | > step_time: 0.3613 (0.34180772872198195) + | > loader_time: 0.0057 (0.005137029148283458) + + + --> TIME: 2025-04-19 05:12:05 -- STEP: 67/142 -- GLOBAL_STEP: 433025 + | > loss_disc: 2.510071039199829 (2.441405040114674) + | > loss_disc_real_0: 0.18110783398151398 (0.11752915927278462) + | > loss_disc_real_1: 0.23085540533065796 (0.20598066386891836) + | > loss_disc_real_2: 0.2235366404056549 (0.22620976126905698) + | > loss_disc_real_3: 0.21248359978199005 (0.2259121125758584) + | > loss_disc_real_4: 0.2241140604019165 (0.2262090008650253) + | > loss_disc_real_5: 0.21355511248111725 (0.23190002944042434) + | > loss_0: 2.510071039199829 (2.441405040114674) + | > grad_norm_0: tensor(13.3047, device='cuda:0') (tensor(11.6586, device='cuda:0')) + | > loss_gen: 2.3922250270843506 (2.3795564530500726) + | > loss_kl: 1.9337348937988281 (1.8226107412309789) + | > loss_feat: 7.604024410247803 (7.166688968886191) + | > loss_mel: 17.409982681274414 (17.690985110268663) + | > loss_duration: 1.406648874282837 (1.4347358995409154) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.74661636352539 (30.494577180093795) + | > grad_norm_1: tensor(448.2051, device='cuda:0') (tensor(298.7561, device='cuda:0')) + | > current_lr_0: 0.0001907692176976117 + | > current_lr_1: 0.0001907692176976117 + | > step_time: 0.3838 (0.3535830084957294) + | > loader_time: 0.0061 (0.005479157860599347) + + + --> TIME: 2025-04-19 05:12:15 -- STEP: 92/142 -- GLOBAL_STEP: 433050 + | > loss_disc: 2.534755229949951 (2.4571861676547844) + | > loss_disc_real_0: 0.09055104106664658 (0.11948366260722927) + | > loss_disc_real_1: 0.19342269003391266 (0.20610507820611415) + | > loss_disc_real_2: 0.23671625554561615 (0.22686719910606093) + | > loss_disc_real_3: 0.24435125291347504 (0.22757632366341093) + | > loss_disc_real_4: 0.23373782634735107 (0.228634056351755) + | > loss_disc_real_5: 0.2259199619293213 (0.23385250276845435) + | > loss_0: 2.534755229949951 (2.4571861676547844) + | > grad_norm_0: tensor(20.8441, device='cuda:0') (tensor(11.6331, device='cuda:0')) + | > loss_gen: 2.269139051437378 (2.3602719669756684) + | > loss_kl: 1.8972575664520264 (1.8459909376890764) + | > loss_feat: 7.196244716644287 (7.107669633367787) + | > loss_mel: 17.139436721801758 (17.705843438272893) + | > loss_duration: 1.441664695739746 (1.4467709997425908) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.943740844726562 (30.466546970865) + | > grad_norm_1: tensor(656.6262, device='cuda:0') (tensor(286.1283, device='cuda:0')) + | > current_lr_0: 0.0001907692176976117 + | > current_lr_1: 0.0001907692176976117 + | > step_time: 0.4132 (0.3658683792404506) + | > loader_time: 0.0077 (0.005752509054930314) + + + --> TIME: 2025-04-19 05:12:26 -- STEP: 117/142 -- GLOBAL_STEP: 433075 + | > loss_disc: 2.2110843658447266 (2.4606401349744225) + | > loss_disc_real_0: 0.12266578525304794 (0.12185037912976028) + | > loss_disc_real_1: 0.18655148148536682 (0.20663254676211593) + | > loss_disc_real_2: 0.19810406863689423 (0.2269477148850759) + | > loss_disc_real_3: 0.21112842857837677 (0.22744445400869745) + | > loss_disc_real_4: 0.2083957940340042 (0.22894863695160955) + | > loss_disc_real_5: 0.23937128484249115 (0.23410732343665555) + | > loss_0: 2.2110843658447266 (2.4606401349744225) + | > grad_norm_0: tensor(22.5523, device='cuda:0') (tensor(12.7279, device='cuda:0')) + | > loss_gen: 2.5080983638763428 (2.3659563064575195) + | > loss_kl: 1.9615747928619385 (1.8562416368060641) + | > loss_feat: 8.120488166809082 (7.092154837062216) + | > loss_mel: 19.557554244995117 (17.70582855257214) + | > loss_duration: 1.4769102334976196 (1.4540267561235998) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.62462615966797 (30.474208114493607) + | > grad_norm_1: tensor(440.7072, device='cuda:0') (tensor(292.6534, device='cuda:0')) + | > current_lr_0: 0.0001907692176976117 + | > current_lr_1: 0.0001907692176976117 + | > step_time: 0.4374 (0.3783048809084118) + | > loader_time: 0.0069 (0.006060683829152686) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5949575901031494 (2.5949575901031494) + | > loss_disc_real_0: 0.18503372371196747 (0.18503372371196747) + | > loss_disc_real_1: 0.27846160531044006 (0.27846160531044006) + | > loss_disc_real_2: 0.24045154452323914 (0.24045154452323914) + | > loss_disc_real_3: 0.2157822996377945 (0.2157822996377945) + | > loss_disc_real_4: 0.25837454199790955 (0.25837454199790955) + | > loss_disc_real_5: 0.20551274716854095 (0.20551274716854095) + | > loss_0: 2.5949575901031494 (2.5949575901031494) + | > loss_gen: 2.1848573684692383 (2.1848573684692383) + | > loss_kl: 2.662976026535034 (2.662976026535034) + | > loss_feat: 5.8205742835998535 (5.8205742835998535) + | > loss_mel: 15.815457344055176 (15.815457344055176) + | > loss_duration: 1.7640023231506348 (1.7640023231506348) + | > loss_1: 28.247865676879883 (28.247865676879883) + + --> STEP: 1 + | > loss_disc: 2.6531972885131836 (2.6531972885131836) + | > loss_disc_real_0: 0.19684170186519623 (0.19684170186519623) + | > loss_disc_real_1: 0.235838383436203 (0.235838383436203) + | > loss_disc_real_2: 0.24166251718997955 (0.24166251718997955) + | > loss_disc_real_3: 0.23253193497657776 (0.23253193497657776) + | > loss_disc_real_4: 0.21971210837364197 (0.21971210837364197) + | > loss_disc_real_5: 0.24237121641635895 (0.24237121641635895) + | > loss_0: 2.6531972885131836 (2.6531972885131836) + | > loss_gen: 2.171285629272461 (2.171285629272461) + | > loss_kl: 2.7798237800598145 (2.7798237800598145) + | > loss_feat: 6.430750846862793 (6.430750846862793) + | > loss_mel: 17.63446807861328 (17.63446807861328) + | > loss_duration: 1.7826732397079468 (1.7826732397079468) + | > loss_1: 30.799001693725586 (30.799001693725586) + + --> STEP: 2 + | > loss_disc: 2.585954189300537 (2.6195757389068604) + | > loss_disc_real_0: 0.2544068396091461 (0.22562427073717117) + | > loss_disc_real_1: 0.24503567814826965 (0.24043703079223633) + | > loss_disc_real_2: 0.23586109280586243 (0.238761804997921) + | > loss_disc_real_3: 0.22968991100788116 (0.23111092299222946) + | > loss_disc_real_4: 0.23491792380809784 (0.2273150160908699) + | > loss_disc_real_5: 0.23157843947410583 (0.2369748279452324) + | > loss_0: 2.585954189300537 (2.6195757389068604) + | > loss_gen: 2.364198923110962 (2.2677422761917114) + | > loss_kl: 2.391040563583374 (2.5854321718215942) + | > loss_feat: 6.4595184326171875 (6.44513463973999) + | > loss_mel: 16.18764877319336 (16.91105842590332) + | > loss_duration: 1.7105059623718262 (1.7465896010398865) + | > loss_1: 29.112913131713867 (29.955957412719727) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00337374210357666 (-0.00032651424407958984) + | > avg_loss_disc: 2.6195757389068604 (+0.18790364265441895) + | > avg_loss_disc_real_0: 0.22562427073717117 (+0.10052298754453659) + | > avg_loss_disc_real_1: 0.24043703079223633 (-0.011609047651290894) + | > avg_loss_disc_real_2: 0.238761804997921 (+0.03972771763801575) + | > avg_loss_disc_real_3: 0.23111092299222946 (+0.025837533175945282) + | > avg_loss_disc_real_4: 0.2273150160908699 (+0.010111048817634583) + | > avg_loss_disc_real_5: 0.2369748279452324 (+0.008363932371139526) + | > avg_loss_0: 2.6195757389068604 (+0.18790364265441895) + | > avg_loss_gen: 2.2677422761917114 (-0.021456241607666016) + | > avg_loss_kl: 2.5854321718215942 (+0.2034149169921875) + | > avg_loss_feat: 6.44513463973999 (-0.20410966873168945) + | > avg_loss_mel: 16.91105842590332 (-0.4783468246459961) + | > avg_loss_duration: 1.7465896010398865 (-0.0005643367767333984) + | > avg_loss_1: 29.955957412719727 (-0.5010614395141602) + + + > EPOCH: 379/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:12:39)  + + --> TIME: 2025-04-19 05:12:39 -- STEP: 0/142 -- GLOBAL_STEP: 433100 + | > loss_disc: 2.4186222553253174 (2.4186222553253174) + | > loss_disc_real_0: 0.1460391730070114 (0.1460391730070114) + | > loss_disc_real_1: 0.2514488399028778 (0.2514488399028778) + | > loss_disc_real_2: 0.23890703916549683 (0.23890703916549683) + | > loss_disc_real_3: 0.22671613097190857 (0.22671613097190857) + | > loss_disc_real_4: 0.24219079315662384 (0.24219079315662384) + | > loss_disc_real_5: 0.24100616574287415 (0.24100616574287415) + | > loss_0: 2.4186222553253174 (2.4186222553253174) + | > grad_norm_0: tensor(4.9773, device='cuda:0') (tensor(4.9773, device='cuda:0')) + | > loss_gen: 2.63942289352417 (2.63942289352417) + | > loss_kl: 1.7575474977493286 (1.7575474977493286) + | > loss_feat: 7.302092552185059 (7.302092552185059) + | > loss_mel: 18.088457107543945 (18.088457107543945) + | > loss_duration: 1.4218323230743408 (1.4218323230743408) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.209354400634766 (31.209354400634766) + | > grad_norm_1: tensor(260.2079, device='cuda:0') (tensor(260.2079, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.3547 (0.35468482971191406) + | > loader_time: 0.333 (0.3329639434814453) + + + --> TIME: 2025-04-19 05:12:48 -- STEP: 25/142 -- GLOBAL_STEP: 433125 + | > loss_disc: 2.3834264278411865 (2.4552853679656983) + | > loss_disc_real_0: 0.16091100871562958 (0.11894089251756668) + | > loss_disc_real_1: 0.20375549793243408 (0.2087617015838623) + | > loss_disc_real_2: 0.2095974236726761 (0.22692971765995026) + | > loss_disc_real_3: 0.2324068695306778 (0.23173626720905305) + | > loss_disc_real_4: 0.24186323583126068 (0.2284410071372986) + | > loss_disc_real_5: 0.22650732100009918 (0.2387286329269409) + | > loss_0: 2.3834264278411865 (2.4552853679656983) + | > grad_norm_0: tensor(8.2429, device='cuda:0') (tensor(7.4795, device='cuda:0')) + | > loss_gen: 2.553773880004883 (2.3703825378417975) + | > loss_kl: 1.9069291353225708 (1.8166341590881347) + | > loss_feat: 7.185794353485107 (7.313043537139893) + | > loss_mel: 16.955751419067383 (17.894930419921874) + | > loss_duration: 1.4535102844238281 (1.416684398651123) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.05575942993164 (30.811675415039062) + | > grad_norm_1: tensor(37.7212, device='cuda:0') (tensor(150.1586, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.3481 (0.33396374702453613) + | > loader_time: 0.006 (0.005003719329833984) + + + --> TIME: 2025-04-19 05:12:57 -- STEP: 50/142 -- GLOBAL_STEP: 433150 + | > loss_disc: 2.4355642795562744 (2.4451508331298824) + | > loss_disc_real_0: 0.0996459349989891 (0.11948306411504746) + | > loss_disc_real_1: 0.17557257413864136 (0.20497310101985933) + | > loss_disc_real_2: 0.2428974062204361 (0.22739391088485716) + | > loss_disc_real_3: 0.24112237989902496 (0.22862205803394317) + | > loss_disc_real_4: 0.22817914187908173 (0.22659846335649492) + | > loss_disc_real_5: 0.22705650329589844 (0.23640495359897615) + | > loss_0: 2.4355642795562744 (2.4451508331298824) + | > grad_norm_0: tensor(4.3952, device='cuda:0') (tensor(6.6824, device='cuda:0')) + | > loss_gen: 2.3403210639953613 (2.371041793823242) + | > loss_kl: 1.7956929206848145 (1.8452572584152223) + | > loss_feat: 7.093908309936523 (7.269579105377197) + | > loss_mel: 16.878894805908203 (17.725267677307126) + | > loss_duration: 1.4641799926757812 (1.4259569835662844) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.572996139526367 (30.63710292816162) + | > grad_norm_1: tensor(49.8953, device='cuda:0') (tensor(124.4781, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.37 (0.34594440937042237) + | > loader_time: 0.006 (0.005299067497253418) + + + --> TIME: 2025-04-19 05:13:07 -- STEP: 75/142 -- GLOBAL_STEP: 433175 + | > loss_disc: 2.2690012454986572 (2.4446106592814125) + | > loss_disc_real_0: 0.11508690565824509 (0.12253016879161199) + | > loss_disc_real_1: 0.18884140253067017 (0.20462417900562288) + | > loss_disc_real_2: 0.21108612418174744 (0.22620780567328136) + | > loss_disc_real_3: 0.19162122905254364 (0.22786361654599507) + | > loss_disc_real_4: 0.2287067472934723 (0.22764142096042633) + | > loss_disc_real_5: 0.22370514273643494 (0.2343104847272237) + | > loss_0: 2.2690012454986572 (2.4446106592814125) + | > grad_norm_0: tensor(10.0420, device='cuda:0') (tensor(7.0129, device='cuda:0')) + | > loss_gen: 2.5328421592712402 (2.3715416304270427) + | > loss_kl: 1.7335819005966187 (1.8664217472076416) + | > loss_feat: 8.22342586517334 (7.265204563140869) + | > loss_mel: 18.916906356811523 (17.773852767944337) + | > loss_duration: 1.425720453262329 (1.4380343405405682) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.83247756958008 (30.715055135091145) + | > grad_norm_1: tensor(305.9669, device='cuda:0') (tensor(146.6585, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.399 (0.35818846702575685) + | > loader_time: 0.0063 (0.005639518102010091) + + + --> TIME: 2025-04-19 05:13:17 -- STEP: 100/142 -- GLOBAL_STEP: 433200 + | > loss_disc: 2.4252541065216064 (2.446500141620635) + | > loss_disc_real_0: 0.11908441036939621 (0.12248213671147823) + | > loss_disc_real_1: 0.20755620300769806 (0.20441933661699296) + | > loss_disc_real_2: 0.21705138683319092 (0.22670859843492508) + | > loss_disc_real_3: 0.23174864053726196 (0.22788988828659057) + | > loss_disc_real_4: 0.21569719910621643 (0.22685789361596106) + | > loss_disc_real_5: 0.2409067004919052 (0.23372185483574867) + | > loss_0: 2.4252541065216064 (2.446500141620635) + | > grad_norm_0: tensor(4.3344, device='cuda:0') (tensor(7.4767, device='cuda:0')) + | > loss_gen: 2.2426323890686035 (2.369790115356446) + | > loss_kl: 1.9043946266174316 (1.8652071464061737) + | > loss_feat: 7.447946071624756 (7.244261355400085) + | > loss_mel: 17.66238021850586 (17.784120492935177) + | > loss_duration: 1.4951345920562744 (1.4484223115444184) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.752487182617188 (30.711801471710206) + | > grad_norm_1: tensor(139.1139, device='cuda:0') (tensor(174.7654, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.4139 (0.3706175112724304) + | > loader_time: 0.0072 (0.005899755954742431) + + + --> TIME: 2025-04-19 05:13:28 -- STEP: 125/142 -- GLOBAL_STEP: 433225 + | > loss_disc: 2.547621011734009 (2.453199251174926) + | > loss_disc_real_0: 0.11888542026281357 (0.12314434498548507) + | > loss_disc_real_1: 0.22251233458518982 (0.20503723013401032) + | > loss_disc_real_2: 0.23975904285907745 (0.22694334590435028) + | > loss_disc_real_3: 0.28844335675239563 (0.22927890253067015) + | > loss_disc_real_4: 0.26707467436790466 (0.22739433336257936) + | > loss_disc_real_5: 0.24761976301670074 (0.23436175632476808) + | > loss_0: 2.547621011734009 (2.453199251174926) + | > grad_norm_0: tensor(6.8788, device='cuda:0') (tensor(7.8197, device='cuda:0')) + | > loss_gen: 2.332364797592163 (2.3606934890747078) + | > loss_kl: 1.815653681755066 (1.8740604820251465) + | > loss_feat: 6.907196521759033 (7.1987510032653805) + | > loss_mel: 17.790891647338867 (17.75475214385987) + | > loss_duration: 1.5300016403198242 (1.455949076652527) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.37610626220703 (30.644206161499024) + | > grad_norm_1: tensor(212.9594, device='cuda:0') (tensor(183.1633, device='cuda:0')) + | > current_lr_0: 0.00019074537154539949 + | > current_lr_1: 0.00019074537154539949 + | > step_time: 0.4377 (0.3818912925720215) + | > loader_time: 0.0079 (0.006162483215332032) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5129363536834717 (2.5129363536834717) + | > loss_disc_real_0: 0.1453956514596939 (0.1453956514596939) + | > loss_disc_real_1: 0.20232149958610535 (0.20232149958610535) + | > loss_disc_real_2: 0.2469259351491928 (0.2469259351491928) + | > loss_disc_real_3: 0.24181731045246124 (0.24181731045246124) + | > loss_disc_real_4: 0.19757172465324402 (0.19757172465324402) + | > loss_disc_real_5: 0.234897181391716 (0.234897181391716) + | > loss_0: 2.5129363536834717 (2.5129363536834717) + | > loss_gen: 2.162675619125366 (2.162675619125366) + | > loss_kl: 2.5448269844055176 (2.5448269844055176) + | > loss_feat: 7.276857852935791 (7.276857852935791) + | > loss_mel: 19.90418815612793 (19.90418815612793) + | > loss_duration: 1.7325854301452637 (1.7325854301452637) + | > loss_1: 33.621131896972656 (33.621131896972656) + + --> STEP: 1 + | > loss_disc: 2.4650635719299316 (2.4650635719299316) + | > loss_disc_real_0: 0.1429663598537445 (0.1429663598537445) + | > loss_disc_real_1: 0.18768131732940674 (0.18768131732940674) + | > loss_disc_real_2: 0.2651804983615875 (0.2651804983615875) + | > loss_disc_real_3: 0.22843754291534424 (0.22843754291534424) + | > loss_disc_real_4: 0.20101124048233032 (0.20101124048233032) + | > loss_disc_real_5: 0.21363578736782074 (0.21363578736782074) + | > loss_0: 2.4650635719299316 (2.4650635719299316) + | > loss_gen: 2.124734878540039 (2.124734878540039) + | > loss_kl: 2.980862855911255 (2.980862855911255) + | > loss_feat: 7.385066986083984 (7.385066986083984) + | > loss_mel: 18.937984466552734 (18.937984466552734) + | > loss_duration: 1.7951717376708984 (1.7951717376708984) + | > loss_1: 33.223819732666016 (33.223819732666016) + + --> STEP: 2 + | > loss_disc: 2.391207218170166 (2.428135395050049) + | > loss_disc_real_0: 0.1027449369430542 (0.12285564839839935) + | > loss_disc_real_1: 0.22874051332473755 (0.20821091532707214) + | > loss_disc_real_2: 0.22535842657089233 (0.24526946246623993) + | > loss_disc_real_3: 0.2222166657447815 (0.22532710433006287) + | > loss_disc_real_4: 0.1922861933708191 (0.1966487169265747) + | > loss_disc_real_5: 0.1930290311574936 (0.20333240926265717) + | > loss_0: 2.391207218170166 (2.428135395050049) + | > loss_gen: 2.2002952098846436 (2.1625150442123413) + | > loss_kl: 2.1758551597595215 (2.578359007835388) + | > loss_feat: 7.992229461669922 (7.688648223876953) + | > loss_mel: 19.645999908447266 (19.2919921875) + | > loss_duration: 1.7355420589447021 (1.7653568983078003) + | > loss_1: 33.74992370605469 (33.48687171936035) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003317117691040039 (-5.6624412536621094e-05) + | > avg_loss_disc: 2.428135395050049 (-0.19144034385681152) + | > avg_loss_disc_real_0: 0.12285564839839935 (-0.10276862233877182) + | > avg_loss_disc_real_1: 0.20821091532707214 (-0.032226115465164185) + | > avg_loss_disc_real_2: 0.24526946246623993 (+0.006507657468318939) + | > avg_loss_disc_real_3: 0.22532710433006287 (-0.0057838186621665955) + | > avg_loss_disc_real_4: 0.1966487169265747 (-0.030666299164295197) + | > avg_loss_disc_real_5: 0.20333240926265717 (-0.033642418682575226) + | > avg_loss_0: 2.428135395050049 (-0.19144034385681152) + | > avg_loss_gen: 2.1625150442123413 (-0.10522723197937012) + | > avg_loss_kl: 2.578359007835388 (-0.007073163986206055) + | > avg_loss_feat: 7.688648223876953 (+1.243513584136963) + | > avg_loss_mel: 19.2919921875 (+2.3809337615966797) + | > avg_loss_duration: 1.7653568983078003 (+0.01876729726791382) + | > avg_loss_1: 33.48687171936035 (+3.530914306640625) + + + > EPOCH: 380/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:13:37)  + + --> TIME: 2025-04-19 05:13:41 -- STEP: 8/142 -- GLOBAL_STEP: 433250 + | > loss_disc: 2.442889928817749 (2.428552210330963) + | > loss_disc_real_0: 0.1259596198797226 (0.12045498751103878) + | > loss_disc_real_1: 0.20864351093769073 (0.21067333593964577) + | > loss_disc_real_2: 0.23951363563537598 (0.23144162446260452) + | > loss_disc_real_3: 0.2430528849363327 (0.22247759066522121) + | > loss_disc_real_4: 0.24330748617649078 (0.2260337807238102) + | > loss_disc_real_5: 0.21092191338539124 (0.22564656101167202) + | > loss_0: 2.442889928817749 (2.428552210330963) + | > grad_norm_0: tensor(4.8314, device='cuda:0') (tensor(7.4749, device='cuda:0')) + | > loss_gen: 2.255188465118408 (2.4563288390636444) + | > loss_kl: 1.7557040452957153 (1.8013916462659836) + | > loss_feat: 7.098679065704346 (7.404444336891174) + | > loss_mel: 17.309925079345703 (18.005786180496216) + | > loss_duration: 1.4195761680603027 (1.4127338826656342) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.839073181152344 (31.080684661865234) + | > grad_norm_1: tensor(72.7772, device='cuda:0') (tensor(221.1946, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.3254 (0.3245331346988678) + | > loader_time: 0.0049 (0.0045736730098724365) + + + --> TIME: 2025-04-19 05:13:49 -- STEP: 33/142 -- GLOBAL_STEP: 433275 + | > loss_disc: 2.4237289428710938 (2.4664664701981978) + | > loss_disc_real_0: 0.10497090965509415 (0.12077072928800728) + | > loss_disc_real_1: 0.1885818988084793 (0.21082653466499213) + | > loss_disc_real_2: 0.22368504106998444 (0.22904574509822961) + | > loss_disc_real_3: 0.241751566529274 (0.22863596858400287) + | > loss_disc_real_4: 0.24053123593330383 (0.2366058605186867) + | > loss_disc_real_5: 0.22578568756580353 (0.2326938094514789) + | > loss_0: 2.4237289428710938 (2.4664664701981978) + | > grad_norm_0: tensor(9.0868, device='cuda:0') (tensor(8.1653, device='cuda:0')) + | > loss_gen: 2.3013553619384766 (2.392562858986132) + | > loss_kl: 1.6083027124404907 (1.8216433199969204) + | > loss_feat: 6.760737895965576 (7.2677935975970644) + | > loss_mel: 17.35824203491211 (17.868475914001465) + | > loss_duration: 1.4384920597076416 (1.4207316998279456) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.467130661010742 (30.771207462657582) + | > grad_norm_1: tensor(313.7473, device='cuda:0') (tensor(179.0400, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.3563 (0.3370695908864339) + | > loader_time: 0.0059 (0.00514814347931833) + + + --> TIME: 2025-04-19 05:13:59 -- STEP: 58/142 -- GLOBAL_STEP: 433300 + | > loss_disc: 2.4368131160736084 (2.455630734049041) + | > loss_disc_real_0: 0.09073248505592346 (0.12030652390214903) + | > loss_disc_real_1: 0.19101734459400177 (0.20891685963704668) + | > loss_disc_real_2: 0.2448648065328598 (0.2268649398766715) + | > loss_disc_real_3: 0.2476464807987213 (0.22678751729685684) + | > loss_disc_real_4: 0.23052996397018433 (0.23459853745740036) + | > loss_disc_real_5: 0.2609749138355255 (0.23229738350572257) + | > loss_0: 2.4368131160736084 (2.455630734049041) + | > grad_norm_0: tensor(11.7338, device='cuda:0') (tensor(8.5622, device='cuda:0')) + | > loss_gen: 2.347417116165161 (2.380080749248636) + | > loss_kl: 2.0442349910736084 (1.8444323046454068) + | > loss_feat: 7.402743816375732 (7.303431716458551) + | > loss_mel: 19.03278160095215 (17.76485872268676) + | > loss_duration: 1.4783389568328857 (1.4330112481939379) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.30551528930664 (30.725814786450616) + | > grad_norm_1: tensor(227.1996, device='cuda:0') (tensor(205.3845, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.3765 (0.3485631038402689) + | > loader_time: 0.0061 (0.005511686719697096) + + + --> TIME: 2025-04-19 05:14:09 -- STEP: 83/142 -- GLOBAL_STEP: 433325 + | > loss_disc: 2.3504300117492676 (2.461046779012106) + | > loss_disc_real_0: 0.11118102073669434 (0.12221819265599711) + | > loss_disc_real_1: 0.1465480625629425 (0.20877205607402755) + | > loss_disc_real_2: 0.2371806800365448 (0.22602238407336087) + | > loss_disc_real_3: 0.2113361805677414 (0.22611721560179468) + | > loss_disc_real_4: 0.2016618847846985 (0.23255583697772889) + | > loss_disc_real_5: 0.23319150507450104 (0.2322463938988835) + | > loss_0: 2.3504300117492676 (2.461046779012106) + | > grad_norm_0: tensor(6.4710, device='cuda:0') (tensor(9.2741, device='cuda:0')) + | > loss_gen: 2.4757611751556396 (2.36697907907417) + | > loss_kl: 1.8659183979034424 (1.8473385342632431) + | > loss_feat: 7.411114692687988 (7.234971322209002) + | > loss_mel: 18.051000595092773 (17.686442513063724) + | > loss_duration: 1.528933048248291 (1.4432246397776773) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.332727432250977 (30.578955960560993) + | > grad_norm_1: tensor(193.4839, device='cuda:0') (tensor(216.4700, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.3958 (0.3608694047812956) + | > loader_time: 0.0067 (0.005796570375741246) + + + --> TIME: 2025-04-19 05:14:19 -- STEP: 108/142 -- GLOBAL_STEP: 433350 + | > loss_disc: 2.470695972442627 (2.4655083175058716) + | > loss_disc_real_0: 0.11332418769598007 (0.12380206001991476) + | > loss_disc_real_1: 0.25040462613105774 (0.20943845932682356) + | > loss_disc_real_2: 0.23275676369667053 (0.2264188565313816) + | > loss_disc_real_3: 0.2386629730463028 (0.2264730826848083) + | > loss_disc_real_4: 0.22423775494098663 (0.2326876292074168) + | > loss_disc_real_5: 0.22811931371688843 (0.23253576043579313) + | > loss_0: 2.470695972442627 (2.4655083175058716) + | > grad_norm_0: tensor(18.4762, device='cuda:0') (tensor(10.1318, device='cuda:0')) + | > loss_gen: 2.3871331214904785 (2.365082665725991) + | > loss_kl: 1.8490060567855835 (1.8628074769620542) + | > loss_feat: 7.1163225173950195 (7.212163130442302) + | > loss_mel: 18.270565032958984 (17.708756641105374) + | > loss_duration: 1.4901423454284668 (1.4523545503616333) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.113168716430664 (30.6011643409729) + | > grad_norm_1: tensor(499.2967, device='cuda:0') (tensor(233.7086, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.4202 (0.37135655570913245) + | > loader_time: 0.0066 (0.006016031459525779) + + + --> TIME: 2025-04-19 05:14:30 -- STEP: 133/142 -- GLOBAL_STEP: 433375 + | > loss_disc: 2.4644381999969482 (2.4631370046084986) + | > loss_disc_real_0: 0.07345211505889893 (0.1243046091071197) + | > loss_disc_real_1: 0.21012379229068756 (0.20941074360582165) + | > loss_disc_real_2: 0.18902276456356049 (0.2254390410686794) + | > loss_disc_real_3: 0.21440613269805908 (0.22690626340253012) + | > loss_disc_real_4: 0.21133755147457123 (0.23078604423462) + | > loss_disc_real_5: 0.19947341084480286 (0.23164550993675576) + | > loss_0: 2.4644381999969482 (2.4631370046084986) + | > grad_norm_0: tensor(8.8668, device='cuda:0') (tensor(11.0482, device='cuda:0')) + | > loss_gen: 2.331852674484253 (2.362671002409512) + | > loss_kl: 1.9096145629882812 (1.8669526765221043) + | > loss_feat: 7.658165454864502 (7.157647036072007) + | > loss_mel: 18.18566131591797 (17.650173122721515) + | > loss_duration: 1.5451980829238892 (1.4585242441722326) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.630491256713867 (30.495968022740872) + | > grad_norm_1: tensor(246.8553, device='cuda:0') (tensor(246.7523, device='cuda:0')) + | > current_lr_0: 0.0001907215283739563 + | > current_lr_1: 0.0001907215283739563 + | > step_time: 0.435 (0.3839610124889173) + | > loader_time: 0.0067 (0.006295023107887211) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5347695350646973 (2.5347695350646973) + | > loss_disc_real_0: 0.17095424234867096 (0.17095424234867096) + | > loss_disc_real_1: 0.22510559856891632 (0.22510559856891632) + | > loss_disc_real_2: 0.22715173661708832 (0.22715173661708832) + | > loss_disc_real_3: 0.2366027534008026 (0.2366027534008026) + | > loss_disc_real_4: 0.22496071457862854 (0.22496071457862854) + | > loss_disc_real_5: 0.27006450295448303 (0.27006450295448303) + | > loss_0: 2.5347695350646973 (2.5347695350646973) + | > loss_gen: 2.2800474166870117 (2.2800474166870117) + | > loss_kl: 2.657219409942627 (2.657219409942627) + | > loss_feat: 6.080470561981201 (6.080470561981201) + | > loss_mel: 17.437118530273438 (17.437118530273438) + | > loss_duration: 1.7106701135635376 (1.7106701135635376) + | > loss_1: 30.16552734375 (30.16552734375) + + --> STEP: 1 + | > loss_disc: 2.4874112606048584 (2.4874112606048584) + | > loss_disc_real_0: 0.1444491595029831 (0.1444491595029831) + | > loss_disc_real_1: 0.239690899848938 (0.239690899848938) + | > loss_disc_real_2: 0.2542528212070465 (0.2542528212070465) + | > loss_disc_real_3: 0.2291434407234192 (0.2291434407234192) + | > loss_disc_real_4: 0.24857264757156372 (0.24857264757156372) + | > loss_disc_real_5: 0.21938827633857727 (0.21938827633857727) + | > loss_0: 2.4874112606048584 (2.4874112606048584) + | > loss_gen: 2.2997922897338867 (2.2997922897338867) + | > loss_kl: 2.882178544998169 (2.882178544998169) + | > loss_feat: 6.224715709686279 (6.224715709686279) + | > loss_mel: 17.467233657836914 (17.467233657836914) + | > loss_duration: 1.7504209280014038 (1.7504209280014038) + | > loss_1: 30.62434196472168 (30.62434196472168) + + --> STEP: 2 + | > loss_disc: 2.376126289367676 (2.431768774986267) + | > loss_disc_real_0: 0.14912375807762146 (0.14678645879030228) + | > loss_disc_real_1: 0.18527568876743317 (0.21248329430818558) + | > loss_disc_real_2: 0.2449142336845398 (0.24958352744579315) + | > loss_disc_real_3: 0.23101507127285004 (0.2300792559981346) + | > loss_disc_real_4: 0.20652469992637634 (0.22754867374897003) + | > loss_disc_real_5: 0.20688092708587646 (0.21313460171222687) + | > loss_0: 2.376126289367676 (2.431768774986267) + | > loss_gen: 2.3003697395324707 (2.3000810146331787) + | > loss_kl: 2.1489925384521484 (2.5155855417251587) + | > loss_feat: 6.344421863555908 (6.284568786621094) + | > loss_mel: 16.231826782226562 (16.84953022003174) + | > loss_duration: 1.6914660930633545 (1.7209435105323792) + | > loss_1: 28.71707534790039 (29.670708656311035) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034720897674560547 (+0.00015497207641601562) + | > avg_loss_disc: 2.431768774986267 (+0.0036333799362182617) + | > avg_loss_disc_real_0: 0.14678645879030228 (+0.023930810391902924) + | > avg_loss_disc_real_1: 0.21248329430818558 (+0.004272378981113434) + | > avg_loss_disc_real_2: 0.24958352744579315 (+0.004314064979553223) + | > avg_loss_disc_real_3: 0.2300792559981346 (+0.004752151668071747) + | > avg_loss_disc_real_4: 0.22754867374897003 (+0.030899956822395325) + | > avg_loss_disc_real_5: 0.21313460171222687 (+0.009802192449569702) + | > avg_loss_0: 2.431768774986267 (+0.0036333799362182617) + | > avg_loss_gen: 2.3000810146331787 (+0.1375659704208374) + | > avg_loss_kl: 2.5155855417251587 (-0.06277346611022949) + | > avg_loss_feat: 6.284568786621094 (-1.4040794372558594) + | > avg_loss_mel: 16.84953022003174 (-2.4424619674682617) + | > avg_loss_duration: 1.7209435105323792 (-0.04441338777542114) + | > avg_loss_1: 29.670708656311035 (-3.8161630630493164) + + + > EPOCH: 381/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:14:36)  + + --> TIME: 2025-04-19 05:14:42 -- STEP: 16/142 -- GLOBAL_STEP: 433400 + | > loss_disc: 2.3579015731811523 (2.473248198628426) + | > loss_disc_real_0: 0.1194591298699379 (0.11585619626566768) + | > loss_disc_real_1: 0.22295048832893372 (0.20895493775606155) + | > loss_disc_real_2: 0.23481719195842743 (0.23264941480010748) + | > loss_disc_real_3: 0.21378526091575623 (0.23590134643018246) + | > loss_disc_real_4: 0.18113821744918823 (0.22842815332114697) + | > loss_disc_real_5: 0.2175675481557846 (0.23577389121055603) + | > loss_0: 2.3579015731811523 (2.473248198628426) + | > grad_norm_0: tensor(3.1191, device='cuda:0') (tensor(8.9812, device='cuda:0')) + | > loss_gen: 2.457058906555176 (2.3566072434186935) + | > loss_kl: 1.9075164794921875 (1.8562054112553596) + | > loss_feat: 7.916059494018555 (7.27643895149231) + | > loss_mel: 18.450767517089844 (17.67624068260193) + | > loss_duration: 1.3954135179519653 (1.4037750735878944) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.12681579589844 (30.56926703453064) + | > grad_norm_1: tensor(62.3939, device='cuda:0') (tensor(216.5481, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.34 (0.32975800335407257) + | > loader_time: 0.0055 (0.004710674285888672) + + + --> TIME: 2025-04-19 05:14:51 -- STEP: 41/142 -- GLOBAL_STEP: 433425 + | > loss_disc: 2.4338085651397705 (2.449598044883915) + | > loss_disc_real_0: 0.10640286654233932 (0.11758172911841695) + | > loss_disc_real_1: 0.21668943762779236 (0.2066165863013849) + | > loss_disc_real_2: 0.22533336281776428 (0.22831533885583644) + | > loss_disc_real_3: 0.24017855525016785 (0.2297106047229069) + | > loss_disc_real_4: 0.26224538683891296 (0.2297690180016727) + | > loss_disc_real_5: 0.2246616929769516 (0.2336548584990385) + | > loss_0: 2.4338085651397705 (2.449598044883915) + | > grad_norm_0: tensor(4.4479, device='cuda:0') (tensor(7.8874, device='cuda:0')) + | > loss_gen: 2.4081430435180664 (2.369657970056301) + | > loss_kl: 1.7583811283111572 (1.8418519148012487) + | > loss_feat: 7.324052810668945 (7.318643918851527) + | > loss_mel: 17.915794372558594 (17.84880852117771) + | > loss_duration: 1.42086660861969 (1.4209445366045323) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.827238082885742 (30.799906707391507) + | > grad_norm_1: tensor(49.1352, device='cuda:0') (tensor(172.9643, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.363 (0.3419826321485566) + | > loader_time: 0.0058 (0.005216400797774153) + + + --> TIME: 2025-04-19 05:15:01 -- STEP: 66/142 -- GLOBAL_STEP: 433450 + | > loss_disc: 2.381143808364868 (2.45279144157063) + | > loss_disc_real_0: 0.1279163956642151 (0.12197345348470139) + | > loss_disc_real_1: 0.18363867700099945 (0.20433740588751706) + | > loss_disc_real_2: 0.2210177183151245 (0.22574660543239478) + | > loss_disc_real_3: 0.21244245767593384 (0.22875441226995352) + | > loss_disc_real_4: 0.1967012733221054 (0.22924053172270456) + | > loss_disc_real_5: 0.22010348737239838 (0.23273004839817682) + | > loss_0: 2.381143808364868 (2.45279144157063) + | > grad_norm_0: tensor(6.8941, device='cuda:0') (tensor(7.6325, device='cuda:0')) + | > loss_gen: 2.450216293334961 (2.3609349691506587) + | > loss_kl: 1.9310418367385864 (1.8462287891994824) + | > loss_feat: 7.67055606842041 (7.21081576202855) + | > loss_mel: 18.6392765045166 (17.797639514460712) + | > loss_duration: 1.45493745803833 (1.4348819544821074) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.14603042602539 (30.65050087553082) + | > grad_norm_1: tensor(167.0999, device='cuda:0') (tensor(148.7208, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.3857 (0.35388764106866094) + | > loader_time: 0.0066 (0.00554601712660356) + + + --> TIME: 2025-04-19 05:15:11 -- STEP: 91/142 -- GLOBAL_STEP: 433475 + | > loss_disc: 2.529888868331909 (2.459637044550298) + | > loss_disc_real_0: 0.1463293731212616 (0.12294146295759704) + | > loss_disc_real_1: 0.22946664690971375 (0.2058832435817509) + | > loss_disc_real_2: 0.23985449969768524 (0.22700678630844578) + | > loss_disc_real_3: 0.26109346747398376 (0.23043670628097032) + | > loss_disc_real_4: 0.22731344401836395 (0.22953378892206883) + | > loss_disc_real_5: 0.26794788241386414 (0.23370925672761686) + | > loss_0: 2.529888868331909 (2.459637044550298) + | > grad_norm_0: tensor(12.3894, device='cuda:0') (tensor(8.8394, device='cuda:0')) + | > loss_gen: 2.2194817066192627 (2.3614561269571492) + | > loss_kl: 1.9532299041748047 (1.8572732262558989) + | > loss_feat: 6.92065954208374 (7.181923310835283) + | > loss_mel: 17.311874389648438 (17.864639523265136) + | > loss_duration: 1.4881043434143066 (1.4472251292113418) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.89335060119629 (30.71251732962472) + | > grad_norm_1: tensor(69.2881, device='cuda:0') (tensor(179.6356, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.4055 (0.36583230259654304) + | > loader_time: 0.007 (0.005826415596427499) + + + --> TIME: 2025-04-19 05:15:21 -- STEP: 116/142 -- GLOBAL_STEP: 433500 + | > loss_disc: 2.37882924079895 (2.4620436101124206) + | > loss_disc_real_0: 0.08795982599258423 (0.12298276879150292) + | > loss_disc_real_1: 0.18818524479866028 (0.20591644416081495) + | > loss_disc_real_2: 0.22439993917942047 (0.22675326453714534) + | > loss_disc_real_3: 0.2149503231048584 (0.23025825863768315) + | > loss_disc_real_4: 0.188053160905838 (0.22970817944612995) + | > loss_disc_real_5: 0.22529196739196777 (0.23395746111355978) + | > loss_0: 2.37882924079895 (2.4620436101124206) + | > grad_norm_0: tensor(5.2980, device='cuda:0') (tensor(9.7484, device='cuda:0')) + | > loss_gen: 2.3815317153930664 (2.3589485267113) + | > loss_kl: 1.8715258836746216 (1.862644008521376) + | > loss_feat: 7.340733528137207 (7.175335793659605) + | > loss_mel: 17.83283233642578 (17.82577434079401) + | > loss_duration: 1.4462075233459473 (1.4550930025248692) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.872831344604492 (30.67779568968148) + | > grad_norm_1: tensor(258.9121, device='cuda:0') (tensor(207.5121, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.4357 (0.3773669456613476) + | > loader_time: 0.0075 (0.00609809160232544) + + + --> TIME: 2025-04-19 05:15:33 -- STEP: 141/142 -- GLOBAL_STEP: 433525 + | > loss_disc: 2.4786205291748047 (2.4597876055020813) + | > loss_disc_real_0: 0.10805556178092957 (0.12315816796840505) + | > loss_disc_real_1: 0.24303825199604034 (0.20524862894775175) + | > loss_disc_real_2: 0.2289597988128662 (0.22644894564828127) + | > loss_disc_real_3: 0.2547161877155304 (0.22949603316208994) + | > loss_disc_real_4: 0.22913344204425812 (0.22858928989434074) + | > loss_disc_real_5: 0.26129910349845886 (0.23416631145680206) + | > loss_0: 2.4786205291748047 (2.4597876055020813) + | > grad_norm_0: tensor(18.5161, device='cuda:0') (tensor(10.3463, device='cuda:0')) + | > loss_gen: 2.530532121658325 (2.3579329598879983) + | > loss_kl: 1.5932846069335938 (1.8606568549541718) + | > loss_feat: 7.025365352630615 (7.157405528616398) + | > loss_mel: 16.542987823486328 (17.736999356154858) + | > loss_duration: 1.3182333707809448 (1.4607346497528941) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.010404586791992 (30.573729400093672) + | > grad_norm_1: tensor(353.8719, device='cuda:0') (tensor(222.6096, device='cuda:0')) + | > current_lr_0: 0.00019069768818290954 + | > current_lr_1: 0.00019069768818290954 + | > step_time: 0.2026 (0.38755227995257013) + | > loader_time: 0.003 (0.006320324349910656) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.574350357055664 (2.574350357055664) + | > loss_disc_real_0: 0.21303525567054749 (0.21303525567054749) + | > loss_disc_real_1: 0.19332246482372284 (0.19332246482372284) + | > loss_disc_real_2: 0.20853616297245026 (0.20853616297245026) + | > loss_disc_real_3: 0.215621680021286 (0.215621680021286) + | > loss_disc_real_4: 0.24717256426811218 (0.24717256426811218) + | > loss_disc_real_5: 0.28271743655204773 (0.28271743655204773) + | > loss_0: 2.574350357055664 (2.574350357055664) + | > loss_gen: 2.2506253719329834 (2.2506253719329834) + | > loss_kl: 2.408108711242676 (2.408108711242676) + | > loss_feat: 5.917016983032227 (5.917016983032227) + | > loss_mel: 17.080556869506836 (17.080556869506836) + | > loss_duration: 1.7399370670318604 (1.7399370670318604) + | > loss_1: 29.396244049072266 (29.396244049072266) + + --> STEP: 1 + | > loss_disc: 2.4296910762786865 (2.4296910762786865) + | > loss_disc_real_0: 0.19967374205589294 (0.19967374205589294) + | > loss_disc_real_1: 0.19120104610919952 (0.19120104610919952) + | > loss_disc_real_2: 0.22135312855243683 (0.22135312855243683) + | > loss_disc_real_3: 0.2248137891292572 (0.2248137891292572) + | > loss_disc_real_4: 0.22968848049640656 (0.22968848049640656) + | > loss_disc_real_5: 0.2625333070755005 (0.2625333070755005) + | > loss_0: 2.4296910762786865 (2.4296910762786865) + | > loss_gen: 2.5350186824798584 (2.5350186824798584) + | > loss_kl: 2.8694121837615967 (2.8694121837615967) + | > loss_feat: 6.97576379776001 (6.97576379776001) + | > loss_mel: 18.6301212310791 (18.6301212310791) + | > loss_duration: 1.7774689197540283 (1.7774689197540283) + | > loss_1: 32.787784576416016 (32.787784576416016) + + --> STEP: 2 + | > loss_disc: 2.4903359413146973 (2.460013508796692) + | > loss_disc_real_0: 0.2286345213651657 (0.21415413171052933) + | > loss_disc_real_1: 0.18856671452522278 (0.18988388031721115) + | > loss_disc_real_2: 0.2278560996055603 (0.22460461407899857) + | > loss_disc_real_3: 0.1965300440788269 (0.21067191660404205) + | > loss_disc_real_4: 0.2409820705652237 (0.23533527553081512) + | > loss_disc_real_5: 0.22093993425369263 (0.24173662066459656) + | > loss_0: 2.4903359413146973 (2.460013508796692) + | > loss_gen: 2.239537239074707 (2.3872779607772827) + | > loss_kl: 2.263327121734619 (2.566369652748108) + | > loss_feat: 5.373270511627197 (6.1745171546936035) + | > loss_mel: 16.43994903564453 (17.535035133361816) + | > loss_duration: 1.710263967514038 (1.7438664436340332) + | > loss_1: 28.02634620666504 (30.407065391540527) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034766197204589844 (+4.5299530029296875e-06) + | > avg_loss_disc: 2.460013508796692 (+0.028244733810424805) + | > avg_loss_disc_real_0: 0.21415413171052933 (+0.06736767292022705) + | > avg_loss_disc_real_1: 0.18988388031721115 (-0.022599413990974426) + | > avg_loss_disc_real_2: 0.22460461407899857 (-0.024978913366794586) + | > avg_loss_disc_real_3: 0.21067191660404205 (-0.01940733939409256) + | > avg_loss_disc_real_4: 0.23533527553081512 (+0.007786601781845093) + | > avg_loss_disc_real_5: 0.24173662066459656 (+0.02860201895236969) + | > avg_loss_0: 2.460013508796692 (+0.028244733810424805) + | > avg_loss_gen: 2.3872779607772827 (+0.087196946144104) + | > avg_loss_kl: 2.566369652748108 (+0.05078411102294922) + | > avg_loss_feat: 6.1745171546936035 (-0.11005163192749023) + | > avg_loss_mel: 17.535035133361816 (+0.6855049133300781) + | > avg_loss_duration: 1.7438664436340332 (+0.022922933101654053) + | > avg_loss_1: 30.407065391540527 (+0.7363567352294922) + + + > EPOCH: 382/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:15:35)  + + --> TIME: 2025-04-19 05:15:43 -- STEP: 24/142 -- GLOBAL_STEP: 433550 + | > loss_disc: 2.4667088985443115 (2.4452409545580545) + | > loss_disc_real_0: 0.1516784429550171 (0.11629901795337598) + | > loss_disc_real_1: 0.2113528549671173 (0.20700393368800482) + | > loss_disc_real_2: 0.2672215700149536 (0.23026511507729688) + | > loss_disc_real_3: 0.2493775486946106 (0.23072662328680357) + | > loss_disc_real_4: 0.2298140674829483 (0.22792601274947324) + | > loss_disc_real_5: 0.24561457335948944 (0.23036792750159898) + | > loss_0: 2.4667088985443115 (2.4452409545580545) + | > grad_norm_0: tensor(5.1867, device='cuda:0') (tensor(10.6671, device='cuda:0')) + | > loss_gen: 2.2773172855377197 (2.378390401601792) + | > loss_kl: 1.8507071733474731 (1.7868650555610657) + | > loss_feat: 6.497530937194824 (7.27941111723582) + | > loss_mel: 16.97248649597168 (17.687043070793155) + | > loss_duration: 1.456409215927124 (1.4189408471186955) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.05445098876953 (30.55065067609151) + | > grad_norm_1: tensor(221.7760, device='cuda:0') (tensor(263.7288, device='cuda:0')) + | > current_lr_0: 0.00019067385097188666 + | > current_lr_1: 0.00019067385097188666 + | > step_time: 0.342 (0.3325795233249664) + | > loader_time: 0.0057 (0.005122780799865723) + + + --> TIME: 2025-04-19 05:15:53 -- STEP: 49/142 -- GLOBAL_STEP: 433575 + | > loss_disc: 2.539263963699341 (2.429889401611016) + | > loss_disc_real_0: 0.12102712690830231 (0.11660898598481198) + | > loss_disc_real_1: 0.23126505315303802 (0.20197447495801107) + | > loss_disc_real_2: 0.2533579468727112 (0.22627286065597924) + | > loss_disc_real_3: 0.23916788399219513 (0.22686342013125516) + | > loss_disc_real_4: 0.22636756300926208 (0.22605407238006592) + | > loss_disc_real_5: 0.21294817328453064 (0.22900734720181445) + | > loss_0: 2.539263963699341 (2.429889401611016) + | > grad_norm_0: tensor(4.6262, device='cuda:0') (tensor(9.1471, device='cuda:0')) + | > loss_gen: 2.294372797012329 (2.384229407018545) + | > loss_kl: 1.8333345651626587 (1.8114618573869978) + | > loss_feat: 6.619813919067383 (7.329998259641687) + | > loss_mel: 17.415672302246094 (17.737598886295242) + | > loss_duration: 1.4348031282424927 (1.4260919507668937) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.59799575805664 (30.689380490050024) + | > grad_norm_1: tensor(59.8533, device='cuda:0') (tensor(228.6241, device='cuda:0')) + | > current_lr_0: 0.00019067385097188666 + | > current_lr_1: 0.00019067385097188666 + | > step_time: 0.3699 (0.3445430288509447) + | > loader_time: 0.0058 (0.005401684313404317) + + + --> TIME: 2025-04-19 05:16:02 -- STEP: 74/142 -- GLOBAL_STEP: 433600 + | > loss_disc: 2.525404214859009 (2.4528553292557995) + | > loss_disc_real_0: 0.13631057739257812 (0.12266554834472167) + | > loss_disc_real_1: 0.21261832118034363 (0.20557447885339328) + | > loss_disc_real_2: 0.2018500715494156 (0.22774027449053688) + | > loss_disc_real_3: 0.2272634655237198 (0.22797625270244237) + | > loss_disc_real_4: 0.24380356073379517 (0.22792607183391983) + | > loss_disc_real_5: 0.260623574256897 (0.2312988761711765) + | > loss_0: 2.525404214859009 (2.4528553292557995) + | > grad_norm_0: tensor(13.8609, device='cuda:0') (tensor(9.2210, device='cuda:0')) + | > loss_gen: 2.1663572788238525 (2.3752705857560446) + | > loss_kl: 1.7144579887390137 (1.8377953284495585) + | > loss_feat: 6.758764743804932 (7.272773710457054) + | > loss_mel: 17.684032440185547 (17.837944726686224) + | > loss_duration: 1.4623219966888428 (1.4386925069061482) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.785934448242188 (30.762476972631507) + | > grad_norm_1: tensor(309.1670, device='cuda:0') (tensor(224.5228, device='cuda:0')) + | > current_lr_0: 0.00019067385097188666 + | > current_lr_1: 0.00019067385097188666 + | > step_time: 0.3903 (0.3570131611179661) + | > loader_time: 0.0063 (0.005692230688559043) + + + --> TIME: 2025-04-19 05:16:13 -- STEP: 99/142 -- GLOBAL_STEP: 433625 + | > loss_disc: 2.5839505195617676 (2.459279826193144) + | > loss_disc_real_0: 0.12896043062210083 (0.12280339747667313) + | > loss_disc_real_1: 0.264699786901474 (0.20554350226214438) + | > loss_disc_real_2: 0.24907362461090088 (0.2278291116459201) + | > loss_disc_real_3: 0.2520730495452881 (0.22811548833293144) + | > loss_disc_real_4: 0.24487504363059998 (0.22807699502116502) + | > loss_disc_real_5: 0.2706541419029236 (0.2318588737586532) + | > loss_0: 2.5839505195617676 (2.459279826193144) + | > grad_norm_0: tensor(17.3019, device='cuda:0') (tensor(10.5030, device='cuda:0')) + | > loss_gen: 2.3416061401367188 (2.3708974929771034) + | > loss_kl: 1.8452411890029907 (1.8515367399562488) + | > loss_feat: 6.935688495635986 (7.215830152684992) + | > loss_mel: 18.166467666625977 (17.887103803230044) + | > loss_duration: 1.4860069751739502 (1.4495815122970428) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.77501106262207 (30.774949719207456) + | > grad_norm_1: tensor(532.9227, device='cuda:0') (tensor(246.2484, device='cuda:0')) + | > current_lr_0: 0.00019067385097188666 + | > current_lr_1: 0.00019067385097188666 + | > step_time: 0.4063 (0.36839290821191034) + | > loader_time: 0.0068 (0.005954308943314986) + + + --> TIME: 2025-04-19 05:16:24 -- STEP: 124/142 -- GLOBAL_STEP: 433650 + | > loss_disc: 2.4326112270355225 (2.4634133884983673) + | > loss_disc_real_0: 0.12334523350000381 (0.12316116546430896) + | > loss_disc_real_1: 0.18150228261947632 (0.20640417020167073) + | > loss_disc_real_2: 0.21550677716732025 (0.2273250341175064) + | > loss_disc_real_3: 0.24637195467948914 (0.228770766887934) + | > loss_disc_real_4: 0.22271612286567688 (0.22758861474933162) + | > loss_disc_real_5: 0.2059352993965149 (0.2330436299164449) + | > loss_0: 2.4326112270355225 (2.4634133884983673) + | > grad_norm_0: tensor(6.7748, device='cuda:0') (tensor(11.0554, device='cuda:0')) + | > loss_gen: 2.477473497390747 (2.3654587191920133) + | > loss_kl: 2.04292368888855 (nan) + | > loss_feat: 7.621753692626953 (7.17458285439399) + | > loss_mel: 17.603408813476562 (17.859677499340417) + | > loss_duration: 1.5059099197387695 (nan) + | > amp_scaler: 64.0 (126.96774193548387) + | > loss_1: 31.251468658447266 (nan) + | > grad_norm_1: tensor(221.9852, device='cuda:0') (tensor(256.7264, device='cuda:0')) + | > current_lr_0: 0.00019067385097188666 + | > current_lr_1: 0.00019067385097188666 + | > step_time: 0.437 (0.38029654756669073) + | > loader_time: 0.0075 (0.0062140783955973965) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.479513168334961 (2.479513168334961) + | > loss_disc_real_0: 0.18145206570625305 (0.18145206570625305) + | > loss_disc_real_1: 0.2737039029598236 (0.2737039029598236) + | > loss_disc_real_2: 0.2600419819355011 (0.2600419819355011) + | > loss_disc_real_3: 0.23511776328086853 (0.23511776328086853) + | > loss_disc_real_4: 0.23331087827682495 (0.23331087827682495) + | > loss_disc_real_5: 0.21196207404136658 (0.21196207404136658) + | > loss_0: 2.479513168334961 (2.479513168334961) + | > loss_gen: 2.577583074569702 (2.577583074569702) + | > loss_kl: 2.675002098083496 (2.675002098083496) + | > loss_feat: 6.923099040985107 (6.923099040985107) + | > loss_mel: 18.102787017822266 (18.102787017822266) + | > loss_duration: 1.7261724472045898 (1.7261724472045898) + | > loss_1: 32.004642486572266 (32.004642486572266) + + --> STEP: 1 + | > loss_disc: 2.624056339263916 (2.624056339263916) + | > loss_disc_real_0: 0.19165094196796417 (0.19165094196796417) + | > loss_disc_real_1: 0.28224170207977295 (0.28224170207977295) + | > loss_disc_real_2: 0.278124064207077 (0.278124064207077) + | > loss_disc_real_3: 0.23378364741802216 (0.23378364741802216) + | > loss_disc_real_4: 0.2783322334289551 (0.2783322334289551) + | > loss_disc_real_5: 0.24017897248268127 (0.24017897248268127) + | > loss_0: 2.624056339263916 (2.624056339263916) + | > loss_gen: 2.3693528175354004 (2.3693528175354004) + | > loss_kl: 2.779060125350952 (2.779060125350952) + | > loss_feat: 6.945791721343994 (6.945791721343994) + | > loss_mel: 18.055639266967773 (18.055639266967773) + | > loss_duration: 1.7833662033081055 (1.7833662033081055) + | > loss_1: 31.933208465576172 (31.933208465576172) + + --> STEP: 2 + | > loss_disc: 2.636648416519165 (2.6303523778915405) + | > loss_disc_real_0: 0.1297718733549118 (0.160711407661438) + | > loss_disc_real_1: 0.3527691662311554 (0.3175054341554642) + | > loss_disc_real_2: 0.2930121123790741 (0.28556808829307556) + | > loss_disc_real_3: 0.3095128536224365 (0.27164825052022934) + | > loss_disc_real_4: 0.2906741201877594 (0.28450317680835724) + | > loss_disc_real_5: 0.26027560234069824 (0.25022728741168976) + | > loss_0: 2.636648416519165 (2.6303523778915405) + | > loss_gen: 2.7215535640716553 (2.545453190803528) + | > loss_kl: 2.304201364517212 (2.541630744934082) + | > loss_feat: 7.376340866088867 (7.161066293716431) + | > loss_mel: 19.146526336669922 (18.601082801818848) + | > loss_duration: 1.714966893196106 (1.7491665482521057) + | > loss_1: 33.263587951660156 (32.598398208618164) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037626028060913086 (+0.0002859830856323242) + | > avg_loss_disc: 2.6303523778915405 (+0.17033886909484863) + | > avg_loss_disc_real_0: 0.160711407661438 (-0.05344272404909134) + | > avg_loss_disc_real_1: 0.3175054341554642 (+0.12762155383825302) + | > avg_loss_disc_real_2: 0.28556808829307556 (+0.060963474214076996) + | > avg_loss_disc_real_3: 0.27164825052022934 (+0.060976333916187286) + | > avg_loss_disc_real_4: 0.28450317680835724 (+0.049167901277542114) + | > avg_loss_disc_real_5: 0.25022728741168976 (+0.0084906667470932) + | > avg_loss_0: 2.6303523778915405 (+0.17033886909484863) + | > avg_loss_gen: 2.545453190803528 (+0.15817523002624512) + | > avg_loss_kl: 2.541630744934082 (-0.02473890781402588) + | > avg_loss_feat: 7.161066293716431 (+0.9865491390228271) + | > avg_loss_mel: 18.601082801818848 (+1.0660476684570312) + | > avg_loss_duration: 1.7491665482521057 (+0.00530010461807251) + | > avg_loss_1: 32.598398208618164 (+2.1913328170776367) + + + > EPOCH: 383/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:16:33)  + + --> TIME: 2025-04-19 05:16:37 -- STEP: 7/142 -- GLOBAL_STEP: 433675 + | > loss_disc: 2.545153856277466 (2.505987746374948) + | > loss_disc_real_0: 0.14533258974552155 (0.10793519552264895) + | > loss_disc_real_1: 0.2360667586326599 (0.21175075428826468) + | > loss_disc_real_2: 0.24446041882038116 (0.2371385714837483) + | > loss_disc_real_3: 0.22287745773792267 (0.23553521718297685) + | > loss_disc_real_4: 0.20375759899616241 (0.25140113702842165) + | > loss_disc_real_5: 0.2393554151058197 (0.26203547205243793) + | > loss_0: 2.545153856277466 (2.505987746374948) + | > grad_norm_0: tensor(6.3603, device='cuda:0') (tensor(11.5939, device='cuda:0')) + | > loss_gen: 2.197645664215088 (2.383401700428554) + | > loss_kl: 1.7009029388427734 (1.8214793716158186) + | > loss_feat: 6.176619052886963 (7.402974946158273) + | > loss_mel: 15.614688873291016 (17.793659482683456) + | > loss_duration: 1.4080966711044312 (1.4093171869005476) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.097951889038086 (30.81083243233817) + | > grad_norm_1: tensor(171.7447, device='cuda:0') (tensor(309.3554, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.324 (0.3354519435337612) + | > loader_time: 0.0043 (0.004414830889020648) + + + --> TIME: 2025-04-19 05:16:45 -- STEP: 32/142 -- GLOBAL_STEP: 433700 + | > loss_disc: 2.3578546047210693 (2.4640555307269096) + | > loss_disc_real_0: 0.10913623869419098 (0.1139270099811256) + | > loss_disc_real_1: 0.1739543080329895 (0.20956975780427456) + | > loss_disc_real_2: 0.20426253974437714 (0.23091929825022817) + | > loss_disc_real_3: 0.2295944094657898 (0.22772909747436643) + | > loss_disc_real_4: 0.19817443192005157 (0.2315368545241654) + | > loss_disc_real_5: 0.1903754025697708 (0.23567921575158834) + | > loss_0: 2.3578546047210693 (2.4640555307269096) + | > grad_norm_0: tensor(8.1990, device='cuda:0') (tensor(10.8027, device='cuda:0')) + | > loss_gen: 2.475888252258301 (2.3613889738917346) + | > loss_kl: 1.828626036643982 (1.8412370160222054) + | > loss_feat: 7.407896041870117 (7.223772794008255) + | > loss_mel: 17.251535415649414 (17.72834920883179) + | > loss_duration: 1.427311897277832 (1.4160544201731682) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.391254425048828 (30.570802450180054) + | > grad_norm_1: tensor(324.5241, device='cuda:0') (tensor(298.0612, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.3637 (0.34020087867975235) + | > loader_time: 0.0056 (0.005150459706783295) + + + --> TIME: 2025-04-19 05:16:55 -- STEP: 57/142 -- GLOBAL_STEP: 433725 + | > loss_disc: 2.4632365703582764 (2.44827043382745) + | > loss_disc_real_0: 0.1213148832321167 (0.1162005209347658) + | > loss_disc_real_1: 0.1866515427827835 (0.20758830533738723) + | > loss_disc_real_2: 0.19713963568210602 (0.22597026354388186) + | > loss_disc_real_3: 0.20399145781993866 (0.22648051354968757) + | > loss_disc_real_4: 0.2618364095687866 (0.23141255922484816) + | > loss_disc_real_5: 0.23457248508930206 (0.2344666136461392) + | > loss_0: 2.4632365703582764 (2.44827043382745) + | > grad_norm_0: tensor(5.4741, device='cuda:0') (tensor(10.1454, device='cuda:0')) + | > loss_gen: 2.298611879348755 (2.359212783345005) + | > loss_kl: 2.050168991088867 (1.8678702471549051) + | > loss_feat: 6.535397052764893 (7.168032503964608) + | > loss_mel: 15.789244651794434 (17.607171711168796) + | > loss_duration: 1.4824542999267578 (1.4287225530858625) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.15587615966797 (30.43100979453639) + | > grad_norm_1: tensor(193.6505, device='cuda:0') (tensor(256.9379, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.3754 (0.35055440350582723) + | > loader_time: 0.0057 (0.005418175145199424) + + + --> TIME: 2025-04-19 05:17:05 -- STEP: 82/142 -- GLOBAL_STEP: 433750 + | > loss_disc: 2.411104202270508 (2.4542584564627665) + | > loss_disc_real_0: 0.1264735907316208 (0.11942702531814575) + | > loss_disc_real_1: 0.20824292302131653 (0.20851860068193287) + | > loss_disc_real_2: 0.2434430718421936 (0.22482129658867672) + | > loss_disc_real_3: 0.23149046301841736 (0.2269857745708489) + | > loss_disc_real_4: 0.23030325770378113 (0.23058738050664343) + | > loss_disc_real_5: 0.2537502348423004 (0.23461554235801463) + | > loss_0: 2.411104202270508 (2.4542584564627665) + | > grad_norm_0: tensor(10.9623, device='cuda:0') (tensor(9.1982, device='cuda:0')) + | > loss_gen: 2.429379940032959 (2.355888718512) + | > loss_kl: 2.0007641315460205 (1.8793407431462916) + | > loss_feat: 7.225630760192871 (7.136133490539178) + | > loss_mel: 17.714466094970703 (17.69941732360096) + | > loss_duration: 1.529579520225525 (1.4405272006988528) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.89982032775879 (30.51130743724544) + | > grad_norm_1: tensor(246.5973, device='cuda:0') (tensor(224.8962, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.3939 (0.3621166392070491) + | > loader_time: 0.0062 (0.005708409518730351) + + + --> TIME: 2025-04-19 05:17:15 -- STEP: 107/142 -- GLOBAL_STEP: 433775 + | > loss_disc: 2.396294593811035 (2.463359188810685) + | > loss_disc_real_0: 0.11895477771759033 (0.1218765168407253) + | > loss_disc_real_1: 0.22656254470348358 (0.20880159833163853) + | > loss_disc_real_2: 0.24432313442230225 (0.22592769577124408) + | > loss_disc_real_3: 0.22153660655021667 (0.2270280000205352) + | > loss_disc_real_4: 0.20952528715133667 (0.22988494826811495) + | > loss_disc_real_5: 0.22531479597091675 (0.23456798939504356) + | > loss_0: 2.396294593811035 (2.463359188810685) + | > grad_norm_0: tensor(9.0091, device='cuda:0') (tensor(9.2162, device='cuda:0')) + | > loss_gen: 2.449206590652466 (2.3497618358825947) + | > loss_kl: 1.7442612648010254 (1.8845572516182874) + | > loss_feat: 6.6410136222839355 (7.083402294978917) + | > loss_mel: 16.6627254486084 (17.627570945525836) + | > loss_duration: 1.4729281663894653 (1.4497487344474442) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.970136642456055 (30.395040975552853) + | > grad_norm_1: tensor(134.2969, device='cuda:0') (tensor(218.4594, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.4164 (0.3722300328940989) + | > loader_time: 0.0071 (0.005954011578426183) + + + --> TIME: 2025-04-19 05:17:26 -- STEP: 132/142 -- GLOBAL_STEP: 433800 + | > loss_disc: 2.3864316940307617 (2.469161868095396) + | > loss_disc_real_0: 0.1422269195318222 (0.12303657722518299) + | > loss_disc_real_1: 0.17276988923549652 (0.2084192333347869) + | > loss_disc_real_2: 0.25883546471595764 (0.22609241092295357) + | > loss_disc_real_3: 0.26431146264076233 (0.22835119012178798) + | > loss_disc_real_4: 0.27082738280296326 (0.2303441847589883) + | > loss_disc_real_5: 0.2405046820640564 (0.23525295293692386) + | > loss_0: 2.3864316940307617 (2.469161868095396) + | > grad_norm_0: tensor(11.9773, device='cuda:0') (tensor(9.4465, device='cuda:0')) + | > loss_gen: 2.627329111099243 (2.3459139881712) + | > loss_kl: 1.8760433197021484 (1.8911180992921193) + | > loss_feat: 7.573611259460449 (7.064657016233965) + | > loss_mel: 18.778074264526367 (17.64354847416733) + | > loss_duration: 1.4645717144012451 (1.4563445069573147) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.31962966918945 (30.40158195206613) + | > grad_norm_1: tensor(378.0992, device='cuda:0') (tensor(219.8852, device='cuda:0')) + | > current_lr_0: 0.00019065001674051517 + | > current_lr_1: 0.00019065001674051517 + | > step_time: 0.4466 (0.3840015801516446) + | > loader_time: 0.0078 (0.006216294837720467) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.62333083152771 (2.62333083152771) + | > loss_disc_real_0: 0.27530068159103394 (0.27530068159103394) + | > loss_disc_real_1: 0.25852635502815247 (0.25852635502815247) + | > loss_disc_real_2: 0.2948116660118103 (0.2948116660118103) + | > loss_disc_real_3: 0.29807668924331665 (0.29807668924331665) + | > loss_disc_real_4: 0.264018714427948 (0.264018714427948) + | > loss_disc_real_5: 0.2726164758205414 (0.2726164758205414) + | > loss_0: 2.62333083152771 (2.62333083152771) + | > loss_gen: 2.853346824645996 (2.853346824645996) + | > loss_kl: 2.7459635734558105 (2.7459635734558105) + | > loss_feat: 6.631353378295898 (6.631353378295898) + | > loss_mel: 18.34832763671875 (18.34832763671875) + | > loss_duration: 1.7384295463562012 (1.7384295463562012) + | > loss_1: 32.317420959472656 (32.317420959472656) + + --> STEP: 1 + | > loss_disc: 2.5483791828155518 (2.5483791828155518) + | > loss_disc_real_0: 0.19796213507652283 (0.19796213507652283) + | > loss_disc_real_1: 0.26726338267326355 (0.26726338267326355) + | > loss_disc_real_2: 0.2685786783695221 (0.2685786783695221) + | > loss_disc_real_3: 0.2708278000354767 (0.2708278000354767) + | > loss_disc_real_4: 0.30137816071510315 (0.30137816071510315) + | > loss_disc_real_5: 0.24005542695522308 (0.24005542695522308) + | > loss_0: 2.5483791828155518 (2.5483791828155518) + | > loss_gen: 2.6447255611419678 (2.6447255611419678) + | > loss_kl: 2.959867477416992 (2.959867477416992) + | > loss_feat: 6.981097221374512 (6.981097221374512) + | > loss_mel: 18.08372688293457 (18.08372688293457) + | > loss_duration: 1.7976548671722412 (1.7976548671722412) + | > loss_1: 32.467071533203125 (32.467071533203125) + + --> STEP: 2 + | > loss_disc: 2.551811456680298 (2.550095319747925) + | > loss_disc_real_0: 0.2277252972126007 (0.21284371614456177) + | > loss_disc_real_1: 0.24349269270896912 (0.25537803769111633) + | > loss_disc_real_2: 0.254705011844635 (0.26164184510707855) + | > loss_disc_real_3: 0.23870648443698883 (0.25476714223623276) + | > loss_disc_real_4: 0.2512727379798889 (0.27632544934749603) + | > loss_disc_real_5: 0.22952362895011902 (0.23478952795267105) + | > loss_0: 2.551811456680298 (2.550095319747925) + | > loss_gen: 2.4198737144470215 (2.5322996377944946) + | > loss_kl: 2.440713882446289 (2.7002906799316406) + | > loss_feat: 6.794425010681152 (6.887761116027832) + | > loss_mel: 16.196189880371094 (17.139958381652832) + | > loss_duration: 1.689904808998108 (1.7437798380851746) + | > loss_1: 29.541107177734375 (31.00408935546875) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003428816795349121 (-0.0003337860107421875) + | > avg_loss_disc: 2.550095319747925 (-0.08025705814361572) + | > avg_loss_disc_real_0: 0.21284371614456177 (+0.05213230848312378) + | > avg_loss_disc_real_1: 0.25537803769111633 (-0.06212739646434784) + | > avg_loss_disc_real_2: 0.26164184510707855 (-0.02392624318599701) + | > avg_loss_disc_real_3: 0.25476714223623276 (-0.016881108283996582) + | > avg_loss_disc_real_4: 0.27632544934749603 (-0.008177727460861206) + | > avg_loss_disc_real_5: 0.23478952795267105 (-0.015437759459018707) + | > avg_loss_0: 2.550095319747925 (-0.08025705814361572) + | > avg_loss_gen: 2.5322996377944946 (-0.013153553009033203) + | > avg_loss_kl: 2.7002906799316406 (+0.1586599349975586) + | > avg_loss_feat: 6.887761116027832 (-0.27330517768859863) + | > avg_loss_mel: 17.139958381652832 (-1.4611244201660156) + | > avg_loss_duration: 1.7437798380851746 (-0.005386710166931152) + | > avg_loss_1: 31.00408935546875 (-1.594308853149414) + + + > EPOCH: 384/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:17:32)  + + --> TIME: 2025-04-19 05:17:38 -- STEP: 15/142 -- GLOBAL_STEP: 433825 + | > loss_disc: 2.3452577590942383 (2.388743273417155) + | > loss_disc_real_0: 0.08737190812826157 (0.10822996894518534) + | > loss_disc_real_1: 0.21757538616657257 (0.20212042729059856) + | > loss_disc_real_2: 0.2139156609773636 (0.22276494204998015) + | > loss_disc_real_3: 0.19944745302200317 (0.22053607702255248) + | > loss_disc_real_4: 0.19663481414318085 (0.22064457337061563) + | > loss_disc_real_5: 0.1935262233018875 (0.22655141254266103) + | > loss_0: 2.3452577590942383 (2.388743273417155) + | > grad_norm_0: tensor(4.7291, device='cuda:0') (tensor(8.3538, device='cuda:0')) + | > loss_gen: 2.548297643661499 (2.4151268800099692) + | > loss_kl: 1.775972843170166 (1.8493435621261596) + | > loss_feat: 8.276627540588379 (7.375015163421631) + | > loss_mel: 18.151996612548828 (17.67467664082845) + | > loss_duration: 1.404086709022522 (1.4090668280919394) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.156982421875 (30.723229090372723) + | > grad_norm_1: tensor(43.8562, device='cuda:0') (tensor(195.5007, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.3339 (0.32692224184672036) + | > loader_time: 0.0056 (0.004753780364990234) + + + --> TIME: 2025-04-19 05:17:47 -- STEP: 40/142 -- GLOBAL_STEP: 433850 + | > loss_disc: 2.392874002456665 (2.448753720521927) + | > loss_disc_real_0: 0.11352397501468658 (0.11836705319583415) + | > loss_disc_real_1: 0.19582915306091309 (0.2038937486708164) + | > loss_disc_real_2: 0.2334149330854416 (0.2264031022787094) + | > loss_disc_real_3: 0.23545552790164948 (0.23077561371028424) + | > loss_disc_real_4: 0.21285615861415863 (0.23180467039346694) + | > loss_disc_real_5: 0.2408672571182251 (0.2330851588398218) + | > loss_0: 2.392874002456665 (2.448753720521927) + | > grad_norm_0: tensor(5.7099, device='cuda:0') (tensor(7.4493, device='cuda:0')) + | > loss_gen: 2.3756916522979736 (2.3896563887596134) + | > loss_kl: 1.885118842124939 (1.8823931634426116) + | > loss_feat: 7.214232921600342 (7.291475892066956) + | > loss_mel: 18.232757568359375 (17.860783815383908) + | > loss_duration: 1.4040148258209229 (1.4234043151140212) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.11181640625 (30.847713565826417) + | > grad_norm_1: tensor(367.0584, device='cuda:0') (tensor(166.4513, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.359 (0.33983603119850164) + | > loader_time: 0.0061 (0.005207687616348265) + + + --> TIME: 2025-04-19 05:17:56 -- STEP: 65/142 -- GLOBAL_STEP: 433875 + | > loss_disc: 2.513861656188965 (2.4570701085604147) + | > loss_disc_real_0: 0.10862970352172852 (0.12050277705375965) + | > loss_disc_real_1: 0.1722583919763565 (0.20837622032715722) + | > loss_disc_real_2: 0.22050917148590088 (0.22454088421968313) + | > loss_disc_real_3: 0.2360544502735138 (0.2281418399168895) + | > loss_disc_real_4: 0.18891996145248413 (0.23012612553743217) + | > loss_disc_real_5: 0.21390947699546814 (0.2321153652209502) + | > loss_0: 2.513861656188965 (2.4570701085604147) + | > grad_norm_0: tensor(7.0257, device='cuda:0') (tensor(9.5118, device='cuda:0')) + | > loss_gen: 2.3112828731536865 (2.3804201089418853) + | > loss_kl: 1.987007975578308 (1.8924836232112003) + | > loss_feat: 7.159341812133789 (7.213861912947435) + | > loss_mel: 17.630653381347656 (17.811558312636155) + | > loss_duration: 1.462931513786316 (1.4358288123057439) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.551218032836914 (30.734152896587666) + | > grad_norm_1: tensor(263.7701, device='cuda:0') (tensor(213.8404, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.3891 (0.35190229782691373) + | > loader_time: 0.0068 (0.005535224767831654) + + + --> TIME: 2025-04-19 05:18:06 -- STEP: 90/142 -- GLOBAL_STEP: 433900 + | > loss_disc: 2.6010186672210693 (2.470336919360691) + | > loss_disc_real_0: 0.2455453723669052 (0.12497687314947446) + | > loss_disc_real_1: 0.21920208632946014 (0.20981529835197651) + | > loss_disc_real_2: 0.2189570963382721 (0.22502232293287913) + | > loss_disc_real_3: 0.21095217764377594 (0.22874181734191046) + | > loss_disc_real_4: 0.22079229354858398 (0.23042572687069576) + | > loss_disc_real_5: 0.22934193909168243 (0.23353851785262425) + | > loss_0: 2.6010186672210693 (2.470336919360691) + | > grad_norm_0: tensor(13.4880, device='cuda:0') (tensor(10.1406, device='cuda:0')) + | > loss_gen: 2.0246880054473877 (2.365749994913737) + | > loss_kl: 1.8950402736663818 (1.8895162436697217) + | > loss_feat: 4.6860833168029785 (7.106546343697442) + | > loss_mel: 13.574579238891602 (17.726652971903484) + | > loss_duration: 1.4883856773376465 (1.4472788161701629) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 23.668777465820312 (30.535744391547308) + | > grad_norm_1: tensor(179.2555, device='cuda:0') (tensor(234.1918, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.4001 (0.36330470244089763) + | > loader_time: 0.0066 (0.005798318650987411) + + + --> TIME: 2025-04-19 05:18:17 -- STEP: 115/142 -- GLOBAL_STEP: 433925 + | > loss_disc: 2.477853775024414 (2.47591089165729) + | > loss_disc_real_0: 0.09259247779846191 (0.1261039123586987) + | > loss_disc_real_1: 0.20413801074028015 (0.209494857425275) + | > loss_disc_real_2: 0.25302866101264954 (0.22617461564748184) + | > loss_disc_real_3: 0.2152169644832611 (0.22808262330034504) + | > loss_disc_real_4: 0.19796696305274963 (0.22942532987698264) + | > loss_disc_real_5: 0.222537562251091 (0.23297641678996708) + | > loss_0: 2.477853775024414 (2.47591089165729) + | > grad_norm_0: tensor(16.2867, device='cuda:0') (tensor(11.3411, device='cuda:0')) + | > loss_gen: 2.444225311279297 (2.35534150082132) + | > loss_kl: 1.9039970636367798 (1.890936711560125) + | > loss_feat: 7.5664849281311035 (7.064411909683891) + | > loss_mel: 17.88639259338379 (17.688464314004655) + | > loss_duration: 1.5019409656524658 (1.4545444851336273) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.303041458129883 (30.453698946082074) + | > grad_norm_1: tensor(353.2601, device='cuda:0') (tensor(255.8373, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.4234 (0.374079358059427) + | > loader_time: 0.0077 (0.00606101492176885) + + + --> TIME: 2025-04-19 05:18:28 -- STEP: 140/142 -- GLOBAL_STEP: 433950 + | > loss_disc: 2.557356834411621 (2.480299976893834) + | > loss_disc_real_0: 0.08384601026773453 (0.12804672393415661) + | > loss_disc_real_1: 0.20461134612560272 (0.20873244915689737) + | > loss_disc_real_2: 0.2127019762992859 (0.22567035928368567) + | > loss_disc_real_3: 0.23124167323112488 (0.22819176497203963) + | > loss_disc_real_4: 0.21082720160484314 (0.23024962044187955) + | > loss_disc_real_5: 0.21531666815280914 (0.23350023446338516) + | > loss_0: 2.557356834411621 (2.480299976893834) + | > grad_norm_0: tensor(12.2708, device='cuda:0') (tensor(12.4626, device='cuda:0')) + | > loss_gen: 2.2644805908203125 (2.358902016707828) + | > loss_kl: 1.988640308380127 (1.8952035427093505) + | > loss_feat: 6.4497761726379395 (7.037686841828482) + | > loss_mel: 15.432928085327148 (17.632508237021316) + | > loss_duration: 1.5177280902862549 (1.4615256667137146) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.653554916381836 (30.385826356070382) + | > grad_norm_1: tensor(317.0594, device='cuda:0') (tensor(275.0034, device='cuda:0')) + | > current_lr_0: 0.0001906261854884226 + | > current_lr_1: 0.0001906261854884226 + | > step_time: 0.4588 (0.38642750297273903) + | > loader_time: 0.0074 (0.006311219079153877) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.318066120147705 (2.318066120147705) + | > loss_disc_real_0: 0.2361903190612793 (0.2361903190612793) + | > loss_disc_real_1: 0.16722774505615234 (0.16722774505615234) + | > loss_disc_real_2: 0.23274348676204681 (0.23274348676204681) + | > loss_disc_real_3: 0.22606518864631653 (0.22606518864631653) + | > loss_disc_real_4: 0.2271193414926529 (0.2271193414926529) + | > loss_disc_real_5: 0.26586654782295227 (0.26586654782295227) + | > loss_0: 2.318066120147705 (2.318066120147705) + | > loss_gen: 2.8810057640075684 (2.8810057640075684) + | > loss_kl: 2.485229015350342 (2.485229015350342) + | > loss_feat: 7.556320667266846 (7.556320667266846) + | > loss_mel: 19.957353591918945 (19.957353591918945) + | > loss_duration: 1.7534000873565674 (1.7534000873565674) + | > loss_1: 34.63330841064453 (34.63330841064453) + + --> STEP: 1 + | > loss_disc: 2.493792772293091 (2.493792772293091) + | > loss_disc_real_0: 0.2739735245704651 (0.2739735245704651) + | > loss_disc_real_1: 0.23457103967666626 (0.23457103967666626) + | > loss_disc_real_2: 0.25688883662223816 (0.25688883662223816) + | > loss_disc_real_3: 0.22582632303237915 (0.22582632303237915) + | > loss_disc_real_4: 0.25440406799316406 (0.25440406799316406) + | > loss_disc_real_5: 0.19324685633182526 (0.19324685633182526) + | > loss_0: 2.493792772293091 (2.493792772293091) + | > loss_gen: 2.7132651805877686 (2.7132651805877686) + | > loss_kl: 2.9455971717834473 (2.9455971717834473) + | > loss_feat: 6.30027961730957 (6.30027961730957) + | > loss_mel: 18.07037353515625 (18.07037353515625) + | > loss_duration: 1.7749193906784058 (1.7749193906784058) + | > loss_1: 31.804433822631836 (31.804433822631836) + + --> STEP: 2 + | > loss_disc: 2.5192766189575195 (2.506534695625305) + | > loss_disc_real_0: 0.24778997898101807 (0.2608817517757416) + | > loss_disc_real_1: 0.25529006123542786 (0.24493055045604706) + | > loss_disc_real_2: 0.26292383670806885 (0.2599063366651535) + | > loss_disc_real_3: 0.2238299697637558 (0.22482814639806747) + | > loss_disc_real_4: 0.2618084251880646 (0.2581062465906143) + | > loss_disc_real_5: 0.28293246030807495 (0.2380896583199501) + | > loss_0: 2.5192766189575195 (2.506534695625305) + | > loss_gen: 2.7010934352874756 (2.707179307937622) + | > loss_kl: 2.2704312801361084 (2.608014225959778) + | > loss_feat: 7.274520397186279 (6.787400007247925) + | > loss_mel: 18.195064544677734 (18.132719039916992) + | > loss_duration: 1.7308104038238525 (1.7528648972511292) + | > loss_1: 32.17192077636719 (31.98817729949951) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003530263900756836 (+0.00010144710540771484) + | > avg_loss_disc: 2.506534695625305 (-0.04356062412261963) + | > avg_loss_disc_real_0: 0.2608817517757416 (+0.04803803563117981) + | > avg_loss_disc_real_1: 0.24493055045604706 (-0.010447487235069275) + | > avg_loss_disc_real_2: 0.2599063366651535 (-0.0017355084419250488) + | > avg_loss_disc_real_3: 0.22482814639806747 (-0.029938995838165283) + | > avg_loss_disc_real_4: 0.2581062465906143 (-0.018219202756881714) + | > avg_loss_disc_real_5: 0.2380896583199501 (+0.0033001303672790527) + | > avg_loss_0: 2.506534695625305 (-0.04356062412261963) + | > avg_loss_gen: 2.707179307937622 (+0.17487967014312744) + | > avg_loss_kl: 2.608014225959778 (-0.09227645397186279) + | > avg_loss_feat: 6.787400007247925 (-0.10036110877990723) + | > avg_loss_mel: 18.132719039916992 (+0.9927606582641602) + | > avg_loss_duration: 1.7528648972511292 (+0.00908505916595459) + | > avg_loss_1: 31.98817729949951 (+0.9840879440307617) + + + > EPOCH: 385/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:18:30)  + + --> TIME: 2025-04-19 05:18:39 -- STEP: 23/142 -- GLOBAL_STEP: 433975 + | > loss_disc: 2.320039749145508 (2.4092664407647173) + | > loss_disc_real_0: 0.08716428279876709 (0.10620027391806892) + | > loss_disc_real_1: 0.2314138263463974 (0.2033464824375899) + | > loss_disc_real_2: 0.1978643387556076 (0.22478089708348978) + | > loss_disc_real_3: 0.2371954321861267 (0.22802759901337002) + | > loss_disc_real_4: 0.2045341283082962 (0.2264980453511943) + | > loss_disc_real_5: 0.20528018474578857 (0.22896894488645636) + | > loss_0: 2.320039749145508 (2.4092664407647173) + | > grad_norm_0: tensor(10.1602, device='cuda:0') (tensor(10.6073, device='cuda:0')) + | > loss_gen: 2.5322787761688232 (2.3937194036400835) + | > loss_kl: 1.7958606481552124 (1.8387161545131518) + | > loss_feat: 7.047722816467285 (7.3334294402081035) + | > loss_mel: 18.645381927490234 (17.716367638629414) + | > loss_duration: 1.4259922504425049 (1.411554709724758) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.447235107421875 (30.693787616232168) + | > grad_norm_1: tensor(276.1312, device='cuda:0') (tensor(266.5075, device='cuda:0')) + | > current_lr_0: 0.00019060235721523656 + | > current_lr_1: 0.00019060235721523656 + | > step_time: 0.3543 (0.3395485256029212) + | > loader_time: 0.0052 (0.005079901736715565) + + + --> TIME: 2025-04-19 05:18:48 -- STEP: 48/142 -- GLOBAL_STEP: 434000 + | > loss_disc: 2.257194995880127 (2.4277782489856086) + | > loss_disc_real_0: 0.09572883695363998 (0.11372376093640924) + | > loss_disc_real_1: 0.17343153059482574 (0.20358995907008648) + | > loss_disc_real_2: 0.22244036197662354 (0.22668017664303383) + | > loss_disc_real_3: 0.231325164437294 (0.22671872936189175) + | > loss_disc_real_4: 0.2564540207386017 (0.22653469908982515) + | > loss_disc_real_5: 0.211400106549263 (0.23218487855046988) + | > loss_0: 2.257194995880127 (2.4277782489856086) + | > grad_norm_0: tensor(4.2769, device='cuda:0') (tensor(8.5045, device='cuda:0')) + | > loss_gen: 2.574306011199951 (2.366944342851639) + | > loss_kl: 1.880874514579773 (1.844621365269025) + | > loss_feat: 7.771328926086426 (7.194576879342397) + | > loss_mel: 18.98883056640625 (17.609255075454712) + | > loss_duration: 1.4375637769699097 (1.4241316393017769) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.65290451049805 (30.43952937920888) + | > grad_norm_1: tensor(144.1645, device='cuda:0') (tensor(206.7998, device='cuda:0')) + | > current_lr_0: 0.00019060235721523656 + | > current_lr_1: 0.00019060235721523656 + | > step_time: 0.3641 (0.3485686232646306) + | > loader_time: 0.0059 (0.005380421876907348) + + + --> TIME: 2025-04-19 05:18:58 -- STEP: 73/142 -- GLOBAL_STEP: 434025 + | > loss_disc: 2.560248374938965 (2.4512268974356455) + | > loss_disc_real_0: 0.15382710099220276 (0.12053076524848808) + | > loss_disc_real_1: 0.22695642709732056 (0.20519845371376977) + | > loss_disc_real_2: 0.2283010333776474 (0.2269423787724482) + | > loss_disc_real_3: 0.2270457148551941 (0.22814825769156627) + | > loss_disc_real_4: 0.2137426733970642 (0.22911711077984065) + | > loss_disc_real_5: 0.2376541942358017 (0.23366489863558992) + | > loss_0: 2.560248374938965 (2.4512268974356455) + | > grad_norm_0: tensor(7.6674, device='cuda:0') (tensor(8.0383, device='cuda:0')) + | > loss_gen: 2.2518110275268555 (2.3511399798197283) + | > loss_kl: 1.9216034412384033 (1.8527440649189362) + | > loss_feat: 6.972241401672363 (7.13511353976106) + | > loss_mel: 16.301712036132812 (17.577653127173853) + | > loss_duration: 1.490363359451294 (1.4365601931532768) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.937732696533203 (30.353211024036145) + | > grad_norm_1: tensor(240.3828, device='cuda:0') (tensor(176.6827, device='cuda:0')) + | > current_lr_0: 0.00019060235721523656 + | > current_lr_1: 0.00019060235721523656 + | > step_time: 0.3978 (0.35968405253266633) + | > loader_time: 0.0068 (0.005740077528235031) + + + --> TIME: 2025-04-19 05:19:08 -- STEP: 98/142 -- GLOBAL_STEP: 434050 + | > loss_disc: 2.3782057762145996 (2.4665131058011744) + | > loss_disc_real_0: 0.11418312042951584 (0.12341072326716111) + | > loss_disc_real_1: 0.19739539921283722 (0.2069198259589623) + | > loss_disc_real_2: 0.22619448602199554 (0.22754792488959372) + | > loss_disc_real_3: 0.19600100815296173 (0.22707558742591313) + | > loss_disc_real_4: 0.21589471399784088 (0.23039120831051652) + | > loss_disc_real_5: 0.21336477994918823 (0.234176049579163) + | > loss_0: 2.3782057762145996 (2.4665131058011744) + | > grad_norm_0: tensor(8.4712, device='cuda:0') (tensor(7.7714, device='cuda:0')) + | > loss_gen: 2.282688617706299 (2.333912309335203) + | > loss_kl: 1.9163860082626343 (1.8659848023434074) + | > loss_feat: 7.486048221588135 (7.053273916244507) + | > loss_mel: 18.60484504699707 (17.618462562561028) + | > loss_duration: 1.4324259757995605 (1.446793443086196) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.722393035888672 (30.318426969100017) + | > grad_norm_1: tensor(172.7374, device='cuda:0') (tensor(166.4280, device='cuda:0')) + | > current_lr_0: 0.00019060235721523656 + | > current_lr_1: 0.00019060235721523656 + | > step_time: 0.4131 (0.3706511234750553) + | > loader_time: 0.0067 (0.005997944851310885) + + + --> TIME: 2025-04-19 05:19:19 -- STEP: 123/142 -- GLOBAL_STEP: 434075 + | > loss_disc: 2.430161714553833 (2.468676917921238) + | > loss_disc_real_0: 0.08641829341650009 (0.12525074910826797) + | > loss_disc_real_1: 0.14342686533927917 (0.20586250140899562) + | > loss_disc_real_2: 0.18658775091171265 (0.22613605554026317) + | > loss_disc_real_3: 0.2160683125257492 (0.22792479565473106) + | > loss_disc_real_4: 0.20617587864398956 (0.23097891148512925) + | > loss_disc_real_5: 0.2328680455684662 (0.23426093585122892) + | > loss_0: 2.430161714553833 (2.468676917921238) + | > grad_norm_0: tensor(4.6786, device='cuda:0') (tensor(7.6677, device='cuda:0')) + | > loss_gen: 2.2405662536621094 (2.3256571195959084) + | > loss_kl: 1.836738109588623 (1.8701669782157835) + | > loss_feat: 7.9630937576293945 (7.0042757716605335) + | > loss_mel: 18.968149185180664 (17.531663468213576) + | > loss_duration: 1.4892109632492065 (1.4548687353366758) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.49775695800781 (30.186632001303074) + | > grad_norm_1: tensor(38.4126, device='cuda:0') (tensor(158.8115, device='cuda:0')) + | > current_lr_0: 0.00019060235721523656 + | > current_lr_1: 0.00019060235721523656 + | > step_time: 0.4462 (0.3828288171349502) + | > loader_time: 0.0079 (0.006263294840246681) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3773865699768066 (2.3773865699768066) + | > loss_disc_real_0: 0.15830962359905243 (0.15830962359905243) + | > loss_disc_real_1: 0.17458496987819672 (0.17458496987819672) + | > loss_disc_real_2: 0.18329477310180664 (0.18329477310180664) + | > loss_disc_real_3: 0.17116335034370422 (0.17116335034370422) + | > loss_disc_real_4: 0.27100709080696106 (0.27100709080696106) + | > loss_disc_real_5: 0.24218955636024475 (0.24218955636024475) + | > loss_0: 2.3773865699768066 (2.3773865699768066) + | > loss_gen: 2.386371612548828 (2.386371612548828) + | > loss_kl: 2.498244285583496 (2.498244285583496) + | > loss_feat: 7.5006022453308105 (7.5006022453308105) + | > loss_mel: 19.474475860595703 (19.474475860595703) + | > loss_duration: 1.743690848350525 (1.743690848350525) + | > loss_1: 33.60338592529297 (33.60338592529297) + + --> STEP: 1 + | > loss_disc: 2.5308172702789307 (2.5308172702789307) + | > loss_disc_real_0: 0.14941297471523285 (0.14941297471523285) + | > loss_disc_real_1: 0.1970193088054657 (0.1970193088054657) + | > loss_disc_real_2: 0.22474627196788788 (0.22474627196788788) + | > loss_disc_real_3: 0.21391020715236664 (0.21391020715236664) + | > loss_disc_real_4: 0.2879820466041565 (0.2879820466041565) + | > loss_disc_real_5: 0.2575031816959381 (0.2575031816959381) + | > loss_0: 2.5308172702789307 (2.5308172702789307) + | > loss_gen: 2.2322282791137695 (2.2322282791137695) + | > loss_kl: 2.664236545562744 (2.664236545562744) + | > loss_feat: 7.21553373336792 (7.21553373336792) + | > loss_mel: 19.600831985473633 (19.600831985473633) + | > loss_duration: 1.801682949066162 (1.801682949066162) + | > loss_1: 33.51451110839844 (33.51451110839844) + + --> STEP: 2 + | > loss_disc: 2.6015360355377197 (2.566176652908325) + | > loss_disc_real_0: 0.1917298436164856 (0.17057140916585922) + | > loss_disc_real_1: 0.22378703951835632 (0.210403174161911) + | > loss_disc_real_2: 0.2417314052581787 (0.2332388386130333) + | > loss_disc_real_3: 0.24281272292137146 (0.22836146503686905) + | > loss_disc_real_4: 0.2815627157688141 (0.2847723811864853) + | > loss_disc_real_5: 0.27965518832206726 (0.2685791850090027) + | > loss_0: 2.6015360355377197 (2.566176652908325) + | > loss_gen: 2.292921304702759 (2.262574791908264) + | > loss_kl: 2.6102607250213623 (2.6372486352920532) + | > loss_feat: 5.525367736816406 (6.370450735092163) + | > loss_mel: 14.211372375488281 (16.906102180480957) + | > loss_duration: 1.7335401773452759 (1.767611563205719) + | > loss_1: 26.37346076965332 (29.94398593902588) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033353567123413086 (-0.00019490718841552734) + | > avg_loss_disc: 2.566176652908325 (+0.05964195728302002) + | > avg_loss_disc_real_0: 0.17057140916585922 (-0.09031034260988235) + | > avg_loss_disc_real_1: 0.210403174161911 (-0.03452737629413605) + | > avg_loss_disc_real_2: 0.2332388386130333 (-0.02666749805212021) + | > avg_loss_disc_real_3: 0.22836146503686905 (+0.0035333186388015747) + | > avg_loss_disc_real_4: 0.2847723811864853 (+0.02666613459587097) + | > avg_loss_disc_real_5: 0.2685791850090027 (+0.030489526689052582) + | > avg_loss_0: 2.566176652908325 (+0.05964195728302002) + | > avg_loss_gen: 2.262574791908264 (-0.4446045160293579) + | > avg_loss_kl: 2.6372486352920532 (+0.02923440933227539) + | > avg_loss_feat: 6.370450735092163 (-0.4169492721557617) + | > avg_loss_mel: 16.906102180480957 (-1.2266168594360352) + | > avg_loss_duration: 1.767611563205719 (+0.014746665954589844) + | > avg_loss_1: 29.94398593902588 (-2.044191360473633) + + + > EPOCH: 386/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:19:30)  + + --> TIME: 2025-04-19 05:19:32 -- STEP: 6/142 -- GLOBAL_STEP: 434100 + | > loss_disc: 2.4922890663146973 (2.412357807159424) + | > loss_disc_real_0: 0.17364230751991272 (0.13215330118934313) + | > loss_disc_real_1: 0.1806231141090393 (0.20292933533589044) + | > loss_disc_real_2: 0.2319188117980957 (0.22745015968879065) + | > loss_disc_real_3: 0.208595871925354 (0.2194815253218015) + | > loss_disc_real_4: 0.2097889631986618 (0.21933210144440332) + | > loss_disc_real_5: 0.18266801536083221 (0.22080727169911066) + | > loss_0: 2.4922890663146973 (2.412357807159424) + | > grad_norm_0: tensor(6.5966, device='cuda:0') (tensor(6.8735, device='cuda:0')) + | > loss_gen: 2.151376485824585 (2.3730090061823526) + | > loss_kl: 1.8443325757980347 (1.8184764583905537) + | > loss_feat: 7.179548740386963 (7.063616196314494) + | > loss_mel: 17.452733993530273 (17.790369987487793) + | > loss_duration: 1.4108864068984985 (1.4133792122205098) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.03887939453125 (30.45885117848714) + | > grad_norm_1: tensor(94.5302, device='cuda:0') (tensor(89.4079, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.33 (0.3313607374827067) + | > loader_time: 0.0038 (0.004281401634216309) + + + --> TIME: 2025-04-19 05:19:41 -- STEP: 31/142 -- GLOBAL_STEP: 434125 + | > loss_disc: 2.491441011428833 (2.456888529562181) + | > loss_disc_real_0: 0.10081889480352402 (0.1206650210003699) + | > loss_disc_real_1: 0.2076958417892456 (0.20386171052532812) + | > loss_disc_real_2: 0.22804775834083557 (0.2273816995082363) + | > loss_disc_real_3: 0.22299891710281372 (0.22841347946274665) + | > loss_disc_real_4: 0.2203504741191864 (0.2301384311529898) + | > loss_disc_real_5: 0.2237299680709839 (0.23100651800632477) + | > loss_0: 2.491441011428833 (2.456888529562181) + | > grad_norm_0: tensor(10.0697, device='cuda:0') (tensor(7.0935, device='cuda:0')) + | > loss_gen: 2.3198559284210205 (2.359385198162448) + | > loss_kl: 1.8128629922866821 (1.814786918701664) + | > loss_feat: 6.954440116882324 (7.113677117132371) + | > loss_mel: 17.03839111328125 (17.70794136293473) + | > loss_duration: 1.4183580875396729 (1.4223078181666713) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.543909072875977 (30.41809857276178) + | > grad_norm_1: tensor(241.0703, device='cuda:0') (tensor(118.3562, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.3505 (0.3384100391018775) + | > loader_time: 0.0052 (0.004851910375779675) + + + --> TIME: 2025-04-19 05:19:50 -- STEP: 56/142 -- GLOBAL_STEP: 434150 + | > loss_disc: 2.4973928928375244 (2.4405816154820585) + | > loss_disc_real_0: 0.13600242137908936 (0.11875288427940436) + | > loss_disc_real_1: 0.2107580304145813 (0.20527009000735624) + | > loss_disc_real_2: 0.23605075478553772 (0.22745758747415884) + | > loss_disc_real_3: 0.2457253783941269 (0.2271890179919345) + | > loss_disc_real_4: 0.24514086544513702 (0.22514757407563074) + | > loss_disc_real_5: 0.2673441469669342 (0.23162431323102542) + | > loss_0: 2.4973928928375244 (2.4405816154820585) + | > grad_norm_0: tensor(9.9343, device='cuda:0') (tensor(8.0940, device='cuda:0')) + | > loss_gen: 2.341188430786133 (2.3725695737770636) + | > loss_kl: 1.9017679691314697 (1.8130100241729192) + | > loss_feat: 6.247953414916992 (7.199846753052303) + | > loss_mel: 16.28278350830078 (17.748759167534963) + | > loss_duration: 1.4646856784820557 (1.432687840291432) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.238378524780273 (30.56687351635524) + | > grad_norm_1: tensor(230.9647, device='cuda:0') (tensor(173.1835, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.375 (0.3490431862218039) + | > loader_time: 0.006 (0.005199027912957328) + + + --> TIME: 2025-04-19 05:20:00 -- STEP: 81/142 -- GLOBAL_STEP: 434175 + | > loss_disc: 2.4393861293792725 (2.4474412453027425) + | > loss_disc_real_0: 0.11661837249994278 (0.12123505219265267) + | > loss_disc_real_1: 0.2076636254787445 (0.20576267385924304) + | > loss_disc_real_2: 0.19106900691986084 (0.22775862522331286) + | > loss_disc_real_3: 0.20884759724140167 (0.22815853541280018) + | > loss_disc_real_4: 0.22715450823307037 (0.22530786196390787) + | > loss_disc_real_5: 0.23749323189258575 (0.2323150520707354) + | > loss_0: 2.4393861293792725 (2.4474412453027425) + | > grad_norm_0: tensor(9.4517, device='cuda:0') (tensor(8.1325, device='cuda:0')) + | > loss_gen: 2.223850965499878 (2.358221789936962) + | > loss_kl: 1.8235995769500732 (1.8435101994761713) + | > loss_feat: 7.270787715911865 (7.11590587945632) + | > loss_mel: 17.458816528320312 (17.68524932861328) + | > loss_duration: 1.4618793725967407 (1.4431741325943561) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.238933563232422 (30.446061499324845) + | > grad_norm_1: tensor(87.5010, device='cuda:0') (tensor(173.2235, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.3951 (0.3610878196763404) + | > loader_time: 0.0069 (0.005480913468349127) + + + --> TIME: 2025-04-19 05:20:11 -- STEP: 106/142 -- GLOBAL_STEP: 434200 + | > loss_disc: 2.374238967895508 (2.447808542341557) + | > loss_disc_real_0: 0.08853497356176376 (0.12215393395075258) + | > loss_disc_real_1: 0.19614270329475403 (0.2043526826320954) + | > loss_disc_real_2: 0.21775288879871368 (0.22640432557969722) + | > loss_disc_real_3: 0.21667352318763733 (0.22813468623273778) + | > loss_disc_real_4: 0.17649497091770172 (0.22568173847108516) + | > loss_disc_real_5: 0.20079614222049713 (0.2316145858955833) + | > loss_0: 2.374238967895508 (2.447808542341557) + | > grad_norm_0: tensor(9.7718, device='cuda:0') (tensor(8.4456, device='cuda:0')) + | > loss_gen: 2.494515895843506 (2.3540361264966583) + | > loss_kl: 1.9261690378189087 (1.8593958571272076) + | > loss_feat: 7.3805084228515625 (7.127341414397618) + | > loss_mel: 16.95135498046875 (17.694952568917902) + | > loss_duration: 1.4833195209503174 (1.453118498595256) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.235870361328125 (30.48884456562546) + | > grad_norm_1: tensor(291.6865, device='cuda:0') (tensor(182.7273, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.4123 (0.37222145188529543) + | > loader_time: 0.007 (0.005748573339210366) + + + --> TIME: 2025-04-19 05:20:22 -- STEP: 131/142 -- GLOBAL_STEP: 434225 + | > loss_disc: 2.37839674949646 (2.4519718002727022) + | > loss_disc_real_0: 0.1107860580086708 (0.12475097443880016) + | > loss_disc_real_1: 0.2298177182674408 (0.20552495292579856) + | > loss_disc_real_2: 0.20617011189460754 (0.22565326003627922) + | > loss_disc_real_3: 0.2184518277645111 (0.22807352927349905) + | > loss_disc_real_4: 0.2809586524963379 (0.22607431634691835) + | > loss_disc_real_5: 0.2258274406194687 (0.23137908309470606) + | > loss_0: 2.37839674949646 (2.4519718002727022) + | > grad_norm_0: tensor(12.9155, device='cuda:0') (tensor(9.3190, device='cuda:0')) + | > loss_gen: 2.5822277069091797 (2.3553207248221835) + | > loss_kl: 1.7632255554199219 (1.8682696145909432) + | > loss_feat: 6.8877034187316895 (7.108347212085287) + | > loss_mel: 18.129243850708008 (17.637746643474085) + | > loss_duration: 1.4795950651168823 (1.4589828998987913) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.841995239257812 (30.428667199520664) + | > grad_norm_1: tensor(181.6146, device='cuda:0') (tensor(198.2001, device='cuda:0')) + | > current_lr_0: 0.00019057853192058465 + | > current_lr_1: 0.00019057853192058465 + | > step_time: 0.4549 (0.3852835738931904) + | > loader_time: 0.0076 (0.00602172531244409) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3823394775390625 (2.3823394775390625) + | > loss_disc_real_0: 0.12334781140089035 (0.12334781140089035) + | > loss_disc_real_1: 0.198878213763237 (0.198878213763237) + | > loss_disc_real_2: 0.24548888206481934 (0.24548888206481934) + | > loss_disc_real_3: 0.2577543556690216 (0.2577543556690216) + | > loss_disc_real_4: 0.22591102123260498 (0.22591102123260498) + | > loss_disc_real_5: 0.22845332324504852 (0.22845332324504852) + | > loss_0: 2.3823394775390625 (2.3823394775390625) + | > loss_gen: 2.4340322017669678 (2.4340322017669678) + | > loss_kl: 2.674515962600708 (2.674515962600708) + | > loss_feat: 6.604177951812744 (6.604177951812744) + | > loss_mel: 18.622312545776367 (18.622312545776367) + | > loss_duration: 1.751871109008789 (1.751871109008789) + | > loss_1: 32.086910247802734 (32.086910247802734) + + --> STEP: 1 + | > loss_disc: 2.5768566131591797 (2.5768566131591797) + | > loss_disc_real_0: 0.17774753272533417 (0.17774753272533417) + | > loss_disc_real_1: 0.20537759363651276 (0.20537759363651276) + | > loss_disc_real_2: 0.275527685880661 (0.275527685880661) + | > loss_disc_real_3: 0.26961392164230347 (0.26961392164230347) + | > loss_disc_real_4: 0.2730405926704407 (0.2730405926704407) + | > loss_disc_real_5: 0.21043317019939423 (0.21043317019939423) + | > loss_0: 2.5768566131591797 (2.5768566131591797) + | > loss_gen: 2.242108106613159 (2.242108106613159) + | > loss_kl: 2.6353273391723633 (2.6353273391723633) + | > loss_feat: 7.199901580810547 (7.199901580810547) + | > loss_mel: 17.904142379760742 (17.904142379760742) + | > loss_duration: 1.8056570291519165 (1.8056570291519165) + | > loss_1: 31.78713607788086 (31.78713607788086) + + --> STEP: 2 + | > loss_disc: 2.519404172897339 (2.5481303930282593) + | > loss_disc_real_0: 0.18719825148582458 (0.18247289210557938) + | > loss_disc_real_1: 0.2111576795578003 (0.20826763659715652) + | > loss_disc_real_2: 0.23393788933753967 (0.25473278760910034) + | > loss_disc_real_3: 0.23962628841400146 (0.25462010502815247) + | > loss_disc_real_4: 0.26374807953834534 (0.268394336104393) + | > loss_disc_real_5: 0.2606104612350464 (0.2355218157172203) + | > loss_0: 2.519404172897339 (2.5481303930282593) + | > loss_gen: 2.39792799949646 (2.3200180530548096) + | > loss_kl: 2.499004364013672 (2.5671658515930176) + | > loss_feat: 7.088658332824707 (7.144279956817627) + | > loss_mel: 18.133922576904297 (18.01903247833252) + | > loss_duration: 1.7187120914459229 (1.7621845602989197) + | > loss_1: 31.838224411010742 (31.8126802444458) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037059783935546875 (+0.0003706216812133789) + | > avg_loss_disc: 2.5481303930282593 (-0.018046259880065918) + | > avg_loss_disc_real_0: 0.18247289210557938 (+0.011901482939720154) + | > avg_loss_disc_real_1: 0.20826763659715652 (-0.002135537564754486) + | > avg_loss_disc_real_2: 0.25473278760910034 (+0.021493948996067047) + | > avg_loss_disc_real_3: 0.25462010502815247 (+0.026258639991283417) + | > avg_loss_disc_real_4: 0.268394336104393 (-0.016378045082092285) + | > avg_loss_disc_real_5: 0.2355218157172203 (-0.03305736929178238) + | > avg_loss_0: 2.5481303930282593 (-0.018046259880065918) + | > avg_loss_gen: 2.3200180530548096 (+0.05744326114654541) + | > avg_loss_kl: 2.5671658515930176 (-0.07008278369903564) + | > avg_loss_feat: 7.144279956817627 (+0.7738292217254639) + | > avg_loss_mel: 18.01903247833252 (+1.1129302978515625) + | > avg_loss_duration: 1.7621845602989197 (-0.005427002906799316) + | > avg_loss_1: 31.8126802444458 (+1.8686943054199219) + + + > EPOCH: 387/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:20:28)  + + --> TIME: 2025-04-19 05:20:34 -- STEP: 14/142 -- GLOBAL_STEP: 434250 + | > loss_disc: 2.4204306602478027 (2.3953182867595126) + | > loss_disc_real_0: 0.11777465790510178 (0.11205324849912099) + | > loss_disc_real_1: 0.2281770408153534 (0.19750760921410151) + | > loss_disc_real_2: 0.21613571047782898 (0.22592955721276148) + | > loss_disc_real_3: 0.2106246054172516 (0.23013841147933686) + | > loss_disc_real_4: 0.2539915442466736 (0.2341483842049326) + | > loss_disc_real_5: 0.24904951453208923 (0.23267559174980437) + | > loss_0: 2.4204306602478027 (2.3953182867595126) + | > grad_norm_0: tensor(14.1399, device='cuda:0') (tensor(8.5570, device='cuda:0')) + | > loss_gen: 2.4591012001037598 (2.465145911489214) + | > loss_kl: 1.731946587562561 (1.8090356758662633) + | > loss_feat: 7.843281269073486 (7.640491996492658) + | > loss_mel: 17.730985641479492 (17.756762436458043) + | > loss_duration: 1.407227873802185 (1.4079534326280867) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.172542572021484 (31.079389572143555) + | > grad_norm_1: tensor(418.7187, device='cuda:0') (tensor(292.2820, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.3317 (0.32807724816458567) + | > loader_time: 0.0049 (0.004531059946332659) + + + --> TIME: 2025-04-19 05:20:43 -- STEP: 39/142 -- GLOBAL_STEP: 434275 + | > loss_disc: 2.434469699859619 (2.4184575447669396) + | > loss_disc_real_0: 0.1462951898574829 (0.11305455366770427) + | > loss_disc_real_1: 0.21673734486103058 (0.20440803200770646) + | > loss_disc_real_2: 0.2286098748445511 (0.22597611791048294) + | > loss_disc_real_3: 0.25954583287239075 (0.229227398450558) + | > loss_disc_real_4: 0.2406770884990692 (0.22778449112024063) + | > loss_disc_real_5: 0.2480555921792984 (0.2322793484498293) + | > loss_0: 2.434469699859619 (2.4184575447669396) + | > grad_norm_0: tensor(6.4688, device='cuda:0') (tensor(9.9935, device='cuda:0')) + | > loss_gen: 2.3890414237976074 (2.4013150533040366) + | > loss_kl: 1.8868281841278076 (1.832528001222855) + | > loss_feat: 6.517908573150635 (7.367983341217041) + | > loss_mel: 17.432294845581055 (17.665544387621765) + | > loss_duration: 1.4409151077270508 (1.4218474290309808) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.666988372802734 (30.68921812986716) + | > grad_norm_1: tensor(125.7716, device='cuda:0') (tensor(286.3145, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.3576 (0.34494797388712567) + | > loader_time: 0.0051 (0.004984030356773963) + + + --> TIME: 2025-04-19 05:20:52 -- STEP: 64/142 -- GLOBAL_STEP: 434300 + | > loss_disc: 2.424471378326416 (2.435788705945014) + | > loss_disc_real_0: 0.11462893337011337 (0.11645933100953698) + | > loss_disc_real_1: 0.1683025360107422 (0.20405304618179798) + | > loss_disc_real_2: 0.2396336942911148 (0.2261442302260548) + | > loss_disc_real_3: 0.22873014211654663 (0.22804962610825896) + | > loss_disc_real_4: 0.2546165883541107 (0.22991087660193443) + | > loss_disc_real_5: 0.21972282230854034 (0.23379182675853372) + | > loss_0: 2.424471378326416 (2.435788705945014) + | > grad_norm_0: tensor(4.5085, device='cuda:0') (tensor(9.0718, device='cuda:0')) + | > loss_gen: 2.3666391372680664 (2.3750400654971604) + | > loss_kl: 1.867517352104187 (1.8253756724298) + | > loss_feat: 7.055396556854248 (7.199255935847759) + | > loss_mel: 16.963123321533203 (17.477559193968766) + | > loss_duration: 1.46909499168396 (1.4338794518262148) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.721769332885742 (30.311110228300095) + | > grad_norm_1: tensor(154.2975, device='cuda:0') (tensor(248.9529, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.3826 (0.3550821729004383) + | > loader_time: 0.006 (0.0053397901356220245) + + + --> TIME: 2025-04-19 05:21:03 -- STEP: 89/142 -- GLOBAL_STEP: 434325 + | > loss_disc: 2.371511936187744 (2.4377455818519156) + | > loss_disc_real_0: 0.10323163121938705 (0.11818064020925693) + | > loss_disc_real_1: 0.2603387236595154 (0.2051610645283474) + | > loss_disc_real_2: 0.20844392478466034 (0.2245503252141931) + | > loss_disc_real_3: 0.2079695612192154 (0.22835754293404267) + | > loss_disc_real_4: 0.21070508658885956 (0.22857871567935087) + | > loss_disc_real_5: 0.20434148609638214 (0.23229912040608652) + | > loss_0: 2.371511936187744 (2.4377455818519156) + | > grad_norm_0: tensor(6.1665, device='cuda:0') (tensor(8.5685, device='cuda:0')) + | > loss_gen: 2.3586480617523193 (2.3662342918053114) + | > loss_kl: 1.856690526008606 (1.8481898200645874) + | > loss_feat: 7.858952045440674 (7.20636083838645) + | > loss_mel: 18.116622924804688 (17.490745426563723) + | > loss_duration: 1.4464991092681885 (1.4449933767318726) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.637413024902344 (30.356523749533665) + | > grad_norm_1: tensor(176.3685, device='cuda:0') (tensor(237.0776, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.4045 (0.3669825457455067) + | > loader_time: 0.0061 (0.0056556664156110084) + + + --> TIME: 2025-04-19 05:21:13 -- STEP: 114/142 -- GLOBAL_STEP: 434350 + | > loss_disc: 2.3555612564086914 (2.4423141730459115) + | > loss_disc_real_0: 0.15425793826580048 (0.1196603234530541) + | > loss_disc_real_1: 0.21797148883342743 (0.20523614658598313) + | > loss_disc_real_2: 0.21950040757656097 (0.22567483416774817) + | > loss_disc_real_3: 0.23520293831825256 (0.22847750560756316) + | > loss_disc_real_4: 0.22454655170440674 (0.2298927971145563) + | > loss_disc_real_5: 0.239067941904068 (0.23139828090604983) + | > loss_0: 2.3555612564086914 (2.4423141730459115) + | > grad_norm_0: tensor(4.8689, device='cuda:0') (tensor(8.8438, device='cuda:0')) + | > loss_gen: 2.4955437183380127 (2.3578274019977514) + | > loss_kl: 1.9925079345703125 (1.8561067863514549) + | > loss_feat: 8.112273216247559 (7.162427688899793) + | > loss_mel: 19.109989166259766 (17.444663725401224) + | > loss_duration: 1.4832441806793213 (1.4527546020976283) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 33.19356155395508 (30.273780320820055) + | > grad_norm_1: tensor(52.7904, device='cuda:0') (tensor(225.9348, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.4257 (0.37839316485220925) + | > loader_time: 0.007 (0.005909438718829238) + + + --> TIME: 2025-04-19 05:21:25 -- STEP: 139/142 -- GLOBAL_STEP: 434375 + | > loss_disc: 2.4502298831939697 (2.4457255559001894) + | > loss_disc_real_0: 0.08315343409776688 (0.12061098764697425) + | > loss_disc_real_1: 0.2327769547700882 (0.20549451790267614) + | > loss_disc_real_2: 0.1964980512857437 (0.22550530785279305) + | > loss_disc_real_3: 0.22833360731601715 (0.22871575863670102) + | > loss_disc_real_4: 0.20564976334571838 (0.22966206298886443) + | > loss_disc_real_5: 0.24441029131412506 (0.2317524921122215) + | > loss_0: 2.4502298831939697 (2.4457255559001894) + | > grad_norm_0: tensor(11.1143, device='cuda:0') (tensor(8.9659, device='cuda:0')) + | > loss_gen: 2.3697049617767334 (2.357277026279368) + | > loss_kl: 1.9376076459884644 (1.865250123490532) + | > loss_feat: 7.944399356842041 (7.142830694322106) + | > loss_mel: 19.099952697753906 (17.43668927391656) + | > loss_duration: 1.4846413135528564 (1.459067872102312) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.836307525634766 (30.261114998687084) + | > grad_norm_1: tensor(279.1844, device='cuda:0') (tensor(228.2612, device='cuda:0')) + | > current_lr_0: 0.00019055470960409457 + | > current_lr_1: 0.00019055470960409457 + | > step_time: 0.4517 (0.3901335472683255) + | > loader_time: 0.0075 (0.00616100023118712) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.462923765182495 (2.462923765182495) + | > loss_disc_real_0: 0.1799256056547165 (0.1799256056547165) + | > loss_disc_real_1: 0.21894948184490204 (0.21894948184490204) + | > loss_disc_real_2: 0.2530171871185303 (0.2530171871185303) + | > loss_disc_real_3: 0.23674029111862183 (0.23674029111862183) + | > loss_disc_real_4: 0.2248431146144867 (0.2248431146144867) + | > loss_disc_real_5: 0.252023309469223 (0.252023309469223) + | > loss_0: 2.462923765182495 (2.462923765182495) + | > loss_gen: 2.4114792346954346 (2.4114792346954346) + | > loss_kl: 2.559260129928589 (2.559260129928589) + | > loss_feat: 6.717347621917725 (6.717347621917725) + | > loss_mel: 17.956966400146484 (17.956966400146484) + | > loss_duration: 1.7349644899368286 (1.7349644899368286) + | > loss_1: 31.38001823425293 (31.38001823425293) + + --> STEP: 1 + | > loss_disc: 2.5448856353759766 (2.5448856353759766) + | > loss_disc_real_0: 0.1739630550146103 (0.1739630550146103) + | > loss_disc_real_1: 0.19037452340126038 (0.19037452340126038) + | > loss_disc_real_2: 0.23104485869407654 (0.23104485869407654) + | > loss_disc_real_3: 0.220880389213562 (0.220880389213562) + | > loss_disc_real_4: 0.19866955280303955 (0.19866955280303955) + | > loss_disc_real_5: 0.18920080363750458 (0.18920080363750458) + | > loss_0: 2.5448856353759766 (2.5448856353759766) + | > loss_gen: 2.171826124191284 (2.171826124191284) + | > loss_kl: 2.754887104034424 (2.754887104034424) + | > loss_feat: 6.386143684387207 (6.386143684387207) + | > loss_mel: 16.743398666381836 (16.743398666381836) + | > loss_duration: 1.7674071788787842 (1.7674071788787842) + | > loss_1: 29.82366180419922 (29.82366180419922) + + --> STEP: 2 + | > loss_disc: 2.452435255050659 (2.498660445213318) + | > loss_disc_real_0: 0.1831950694322586 (0.17857906222343445) + | > loss_disc_real_1: 0.19607745110988617 (0.19322598725557327) + | > loss_disc_real_2: 0.24134622514247894 (0.23619554191827774) + | > loss_disc_real_3: 0.2742389142513275 (0.24755965173244476) + | > loss_disc_real_4: 0.23023271560668945 (0.2144511342048645) + | > loss_disc_real_5: 0.22370269894599915 (0.20645175129175186) + | > loss_0: 2.452435255050659 (2.498660445213318) + | > loss_gen: 2.531019687652588 (2.351422905921936) + | > loss_kl: 2.2508981227874756 (2.5028926134109497) + | > loss_feat: 7.050498008728027 (6.718320846557617) + | > loss_mel: 17.536293029785156 (17.139845848083496) + | > loss_duration: 1.706615924835205 (1.7370115518569946) + | > loss_1: 31.0753231048584 (30.44949245452881) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003549814224243164 (-0.00015616416931152344) + | > avg_loss_disc: 2.498660445213318 (-0.049469947814941406) + | > avg_loss_disc_real_0: 0.17857906222343445 (-0.003893829882144928) + | > avg_loss_disc_real_1: 0.19322598725557327 (-0.015041649341583252) + | > avg_loss_disc_real_2: 0.23619554191827774 (-0.0185372456908226) + | > avg_loss_disc_real_3: 0.24755965173244476 (-0.007060453295707703) + | > avg_loss_disc_real_4: 0.2144511342048645 (-0.0539432018995285) + | > avg_loss_disc_real_5: 0.20645175129175186 (-0.029070064425468445) + | > avg_loss_0: 2.498660445213318 (-0.049469947814941406) + | > avg_loss_gen: 2.351422905921936 (+0.031404852867126465) + | > avg_loss_kl: 2.5028926134109497 (-0.06427323818206787) + | > avg_loss_feat: 6.718320846557617 (-0.42595911026000977) + | > avg_loss_mel: 17.139845848083496 (-0.8791866302490234) + | > avg_loss_duration: 1.7370115518569946 (-0.02517300844192505) + | > avg_loss_1: 30.44949245452881 (-1.3631877899169922) + + + > EPOCH: 388/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:21:27)  + + --> TIME: 2025-04-19 05:21:35 -- STEP: 22/142 -- GLOBAL_STEP: 434400 + | > loss_disc: 2.4229576587677 (2.4004881056872285) + | > loss_disc_real_0: 0.13972246646881104 (0.11694038337604566) + | > loss_disc_real_1: 0.21883031725883484 (0.20641515945846384) + | > loss_disc_real_2: 0.21365390717983246 (0.22301267900250174) + | > loss_disc_real_3: 0.2328917384147644 (0.2213154916058887) + | > loss_disc_real_4: 0.2669917643070221 (0.22243792902339588) + | > loss_disc_real_5: 0.24616460502147675 (0.22779313745823773) + | > loss_0: 2.4229576587677 (2.4004881056872285) + | > grad_norm_0: tensor(7.2119, device='cuda:0') (tensor(7.7219, device='cuda:0')) + | > loss_gen: 2.3915560245513916 (2.408983154730363) + | > loss_kl: 1.822999119758606 (1.8642921231009744) + | > loss_feat: 7.6007080078125 (7.580735791813243) + | > loss_mel: 18.545801162719727 (17.877120971679688) + | > loss_duration: 1.3893342018127441 (1.4095716855742715) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.750398635864258 (31.14070398157293) + | > grad_norm_1: tensor(300.0927, device='cuda:0') (tensor(230.3342, device='cuda:0')) + | > current_lr_0: 0.00019053089026539405 + | > current_lr_1: 0.00019053089026539405 + | > step_time: 0.3426 (0.3314426053654064) + | > loader_time: 0.0053 (0.004867076873779296) + + + --> TIME: 2025-04-19 05:21:45 -- STEP: 47/142 -- GLOBAL_STEP: 434425 + | > loss_disc: 2.439826011657715 (2.4281529112065097) + | > loss_disc_real_0: 0.08180566132068634 (0.11489370695136963) + | > loss_disc_real_1: 0.2024129033088684 (0.20694194829210322) + | > loss_disc_real_2: 0.21798019111156464 (0.2261176768769609) + | > loss_disc_real_3: 0.23032209277153015 (0.22554147402022748) + | > loss_disc_real_4: 0.20569337904453278 (0.22562018956275695) + | > loss_disc_real_5: 0.2426138073205948 (0.2290578766706142) + | > loss_0: 2.439826011657715 (2.4281529112065097) + | > grad_norm_0: tensor(10.5066, device='cuda:0') (tensor(8.2124, device='cuda:0')) + | > loss_gen: 2.432161569595337 (2.4007709482882884) + | > loss_kl: 1.871024250984192 (1.8464198290033544) + | > loss_feat: 7.486235618591309 (7.475674152374268) + | > loss_mel: 18.185789108276367 (17.835229670747797) + | > loss_duration: 1.419729232788086 (1.4237253386923607) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.394939422607422 (30.981820005051635) + | > grad_norm_1: tensor(157.2623, device='cuda:0') (tensor(253.4847, device='cuda:0')) + | > current_lr_0: 0.00019053089026539405 + | > current_lr_1: 0.00019053089026539405 + | > step_time: 0.3629 (0.3437198070769614) + | > loader_time: 0.0061 (0.005248221945255361) + + + --> TIME: 2025-04-19 05:21:54 -- STEP: 72/142 -- GLOBAL_STEP: 434450 + | > loss_disc: 2.4761126041412354 (2.443266034126282) + | > loss_disc_real_0: 0.13738715648651123 (0.11874707255305515) + | > loss_disc_real_1: 0.21318209171295166 (0.2074748126582967) + | > loss_disc_real_2: 0.208280548453331 (0.22579227863914436) + | > loss_disc_real_3: 0.2283809334039688 (0.2276064401699437) + | > loss_disc_real_4: 0.2489502727985382 (0.2281761606120401) + | > loss_disc_real_5: 0.2611571252346039 (0.23085478610462612) + | > loss_0: 2.4761126041412354 (2.443266034126282) + | > grad_norm_0: tensor(21.1045, device='cuda:0') (tensor(9.1996, device='cuda:0')) + | > loss_gen: 2.2724666595458984 (2.3895551562309265) + | > loss_kl: 1.7680797576904297 (1.8470156424575381) + | > loss_feat: 6.990891933441162 (7.362545417414771) + | > loss_mel: 17.184005737304688 (17.763081471125282) + | > loss_duration: 1.4474053382873535 (1.436089825299051) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.66284942626953 (30.798287524117363) + | > grad_norm_1: tensor(381.2728, device='cuda:0') (tensor(249.3634, device='cuda:0')) + | > current_lr_0: 0.00019053089026539405 + | > current_lr_1: 0.00019053089026539405 + | > step_time: 0.3943 (0.35570887393421596) + | > loader_time: 0.0064 (0.005531628926595052) + + + --> TIME: 2025-04-19 05:22:04 -- STEP: 97/142 -- GLOBAL_STEP: 434475 + | > loss_disc: 2.600656747817993 (2.4519489696345382) + | > loss_disc_real_0: 0.12168645858764648 (0.11879514130888526) + | > loss_disc_real_1: 0.18516282737255096 (0.20818419232196414) + | > loss_disc_real_2: 0.1873513013124466 (0.22496867563921152) + | > loss_disc_real_3: 0.1885364055633545 (0.22750481471572956) + | > loss_disc_real_4: 0.2367122918367386 (0.22836587278498816) + | > loss_disc_real_5: 0.2308269739151001 (0.23138220546786317) + | > loss_0: 2.600656747817993 (2.4519489696345382) + | > grad_norm_0: tensor(19.7212, device='cuda:0') (tensor(10.5488, device='cuda:0')) + | > loss_gen: 2.2285733222961426 (2.3816479063525655) + | > loss_kl: 1.753988265991211 (1.853443316577636) + | > loss_feat: 6.3337812423706055 (7.2741456474225545) + | > loss_mel: 16.066984176635742 (17.73141632866613) + | > loss_duration: 1.4704995155334473 (1.4477856036314034) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.85382652282715 (30.68843892677543) + | > grad_norm_1: tensor(196.6163, device='cuda:0') (tensor(272.2581, device='cuda:0')) + | > current_lr_0: 0.00019053089026539405 + | > current_lr_1: 0.00019053089026539405 + | > step_time: 0.4131 (0.3673109762447396) + | > loader_time: 0.0069 (0.005798762606591291) + + + --> TIME: 2025-04-19 05:22:15 -- STEP: 122/142 -- GLOBAL_STEP: 434500 + | > loss_disc: 2.5078186988830566 (2.4527469599833256) + | > loss_disc_real_0: 0.14052748680114746 (0.12106020031038855) + | > loss_disc_real_1: 0.21274146437644958 (0.20733711746383887) + | > loss_disc_real_2: 0.20449186861515045 (0.2256764666955979) + | > loss_disc_real_3: 0.1996895670890808 (0.22800439645032414) + | > loss_disc_real_4: 0.21846657991409302 (0.22860619902122217) + | > loss_disc_real_5: 0.2235412895679474 (0.23134692587324832) + | > loss_0: 2.5078186988830566 (2.4527469599833256) + | > grad_norm_0: tensor(14.0448, device='cuda:0') (tensor(11.2163, device='cuda:0')) + | > loss_gen: 2.1683738231658936 (2.3712841213726605) + | > loss_kl: 1.8072013854980469 (1.8637006243721383) + | > loss_feat: 6.583235263824463 (7.185504452126925) + | > loss_mel: 16.8275146484375 (17.600482299679616) + | > loss_duration: 1.4708151817321777 (1.4546058177948) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.857139587402344 (30.4755774951372) + | > grad_norm_1: tensor(390.0151, device='cuda:0') (tensor(279.4606, device='cuda:0')) + | > current_lr_0: 0.00019053089026539405 + | > current_lr_1: 0.00019053089026539405 + | > step_time: 0.4502 (0.37951705104014904) + | > loader_time: 0.0073 (0.006077447875601345) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4773073196411133 (2.4773073196411133) + | > loss_disc_real_0: 0.08700018376111984 (0.08700018376111984) + | > loss_disc_real_1: 0.21220460534095764 (0.21220460534095764) + | > loss_disc_real_2: 0.2269517332315445 (0.2269517332315445) + | > loss_disc_real_3: 0.19353248178958893 (0.19353248178958893) + | > loss_disc_real_4: 0.20138920843601227 (0.20138920843601227) + | > loss_disc_real_5: 0.24100452661514282 (0.24100452661514282) + | > loss_0: 2.4773073196411133 (2.4773073196411133) + | > loss_gen: 2.1348180770874023 (2.1348180770874023) + | > loss_kl: 2.710129737854004 (2.710129737854004) + | > loss_feat: 6.9207963943481445 (6.9207963943481445) + | > loss_mel: 17.773483276367188 (17.773483276367188) + | > loss_duration: 1.7495392560958862 (1.7495392560958862) + | > loss_1: 31.28876495361328 (31.28876495361328) + + --> STEP: 1 + | > loss_disc: 2.4357810020446777 (2.4357810020446777) + | > loss_disc_real_0: 0.09990181028842926 (0.09990181028842926) + | > loss_disc_real_1: 0.23003734648227692 (0.23003734648227692) + | > loss_disc_real_2: 0.24227125942707062 (0.24227125942707062) + | > loss_disc_real_3: 0.2070431262254715 (0.2070431262254715) + | > loss_disc_real_4: 0.2242428958415985 (0.2242428958415985) + | > loss_disc_real_5: 0.22076672315597534 (0.22076672315597534) + | > loss_0: 2.4357810020446777 (2.4357810020446777) + | > loss_gen: 2.246957778930664 (2.246957778930664) + | > loss_kl: 2.734316825866699 (2.734316825866699) + | > loss_feat: 7.095711708068848 (7.095711708068848) + | > loss_mel: 19.406171798706055 (19.406171798706055) + | > loss_duration: 1.765974521636963 (1.765974521636963) + | > loss_1: 33.2491340637207 (33.2491340637207) + + --> STEP: 2 + | > loss_disc: 2.541654109954834 (2.488717555999756) + | > loss_disc_real_0: 0.12344472110271454 (0.1116732656955719) + | > loss_disc_real_1: 0.2532528340816498 (0.24164509028196335) + | > loss_disc_real_2: 0.24434243142604828 (0.24330684542655945) + | > loss_disc_real_3: 0.23631098866462708 (0.22167705744504929) + | > loss_disc_real_4: 0.2505547106266022 (0.23739880323410034) + | > loss_disc_real_5: 0.20250676572322845 (0.2116367444396019) + | > loss_0: 2.541654109954834 (2.488717555999756) + | > loss_gen: 2.160602331161499 (2.2037800550460815) + | > loss_kl: 2.5979576110839844 (2.666137218475342) + | > loss_feat: 6.000198841094971 (6.547955274581909) + | > loss_mel: 18.151748657226562 (18.77896022796631) + | > loss_duration: 1.7255017757415771 (1.74573814868927) + | > loss_1: 30.636009216308594 (31.94257164001465) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037653446197509766 (+0.0002155303955078125) + | > avg_loss_disc: 2.488717555999756 (-0.009942889213562012) + | > avg_loss_disc_real_0: 0.1116732656955719 (-0.06690579652786255) + | > avg_loss_disc_real_1: 0.24164509028196335 (+0.048419103026390076) + | > avg_loss_disc_real_2: 0.24330684542655945 (+0.007111303508281708) + | > avg_loss_disc_real_3: 0.22167705744504929 (-0.025882594287395477) + | > avg_loss_disc_real_4: 0.23739880323410034 (+0.02294766902923584) + | > avg_loss_disc_real_5: 0.2116367444396019 (+0.005184993147850037) + | > avg_loss_0: 2.488717555999756 (-0.009942889213562012) + | > avg_loss_gen: 2.2037800550460815 (-0.1476428508758545) + | > avg_loss_kl: 2.666137218475342 (+0.1632446050643921) + | > avg_loss_feat: 6.547955274581909 (-0.170365571975708) + | > avg_loss_mel: 18.77896022796631 (+1.6391143798828125) + | > avg_loss_duration: 1.74573814868927 (+0.00872659683227539) + | > avg_loss_1: 31.94257164001465 (+1.4930791854858398) + + + > EPOCH: 389/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:22:26)  + + --> TIME: 2025-04-19 05:22:28 -- STEP: 5/142 -- GLOBAL_STEP: 434525 + | > loss_disc: 2.484891414642334 (2.456273889541626) + | > loss_disc_real_0: 0.09260234981775284 (0.12582636326551438) + | > loss_disc_real_1: 0.21815793216228485 (0.20747143924236297) + | > loss_disc_real_2: 0.2459876388311386 (0.2288203626871109) + | > loss_disc_real_3: 0.2746697664260864 (0.231404909491539) + | > loss_disc_real_4: 0.25244152545928955 (0.2309138685464859) + | > loss_disc_real_5: 0.27294644713401794 (0.24854653477668762) + | > loss_0: 2.484891414642334 (2.456273889541626) + | > grad_norm_0: tensor(11.8679, device='cuda:0') (tensor(12.6190, device='cuda:0')) + | > loss_gen: 2.4414334297180176 (2.4603538036346437) + | > loss_kl: 1.614995002746582 (1.7476467132568358) + | > loss_feat: 7.224696636199951 (7.509592342376709) + | > loss_mel: 18.051212310791016 (17.990576553344727) + | > loss_duration: 1.4114782810211182 (1.4074525594711305) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.74381446838379 (31.11562271118164) + | > grad_norm_1: tensor(320.6435, device='cuda:0') (tensor(383.9248, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.3245 (0.3237448692321777) + | > loader_time: 0.0042 (0.004416131973266601) + + + --> TIME: 2025-04-19 05:22:37 -- STEP: 30/142 -- GLOBAL_STEP: 434550 + | > loss_disc: 2.4176812171936035 (2.4461856683095298) + | > loss_disc_real_0: 0.10441503673791885 (0.11870581731200218) + | > loss_disc_real_1: 0.18271632492542267 (0.20572031835714977) + | > loss_disc_real_2: 0.22122134268283844 (0.22859409004449843) + | > loss_disc_real_3: 0.1995072066783905 (0.22914171119530996) + | > loss_disc_real_4: 0.24289916455745697 (0.2281018763780594) + | > loss_disc_real_5: 0.1777532398700714 (0.23137274384498596) + | > loss_0: 2.4176812171936035 (2.4461856683095298) + | > grad_norm_0: tensor(21.9479, device='cuda:0') (tensor(13.2435, device='cuda:0')) + | > loss_gen: 2.3173625469207764 (2.4098604202270506) + | > loss_kl: 1.7017924785614014 (1.7884396274884542) + | > loss_feat: 7.4541473388671875 (7.314861679077149) + | > loss_mel: 17.663562774658203 (17.622046534220377) + | > loss_duration: 1.445885181427002 (1.414336387316386) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.58275032043457 (30.54954465230306) + | > grad_norm_1: tensor(413.9998, device='cuda:0') (tensor(321.5338, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.3529 (0.336233393351237) + | > loader_time: 0.0055 (0.0049955288569132495) + + + --> TIME: 2025-04-19 05:22:46 -- STEP: 55/142 -- GLOBAL_STEP: 434575 + | > loss_disc: 2.3401646614074707 (2.4283695827830925) + | > loss_disc_real_0: 0.11707687377929688 (0.1172062908383933) + | > loss_disc_real_1: 0.2461409717798233 (0.20374361926859075) + | > loss_disc_real_2: 0.21827679872512817 (0.22562649087472395) + | > loss_disc_real_3: 0.20658434927463531 (0.22827555347572673) + | > loss_disc_real_4: 0.20533603429794312 (0.22812910838560624) + | > loss_disc_real_5: 0.2536376416683197 (0.2306235898624767) + | > loss_0: 2.3401646614074707 (2.4283695827830925) + | > grad_norm_0: tensor(6.6019, device='cuda:0') (tensor(11.5127, device='cuda:0')) + | > loss_gen: 2.4155659675598145 (2.401603755083951) + | > loss_kl: 1.9644734859466553 (1.808405752615495) + | > loss_feat: 6.537435531616211 (7.306464143232866) + | > loss_mel: 16.134206771850586 (17.5777772730047) + | > loss_duration: 1.4890215396881104 (1.4261559204621748) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.54070281982422 (30.520406861738724) + | > grad_norm_1: tensor(104.3696, device='cuda:0') (tensor(302.8017, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.3864 (0.3489156853068959) + | > loader_time: 0.0063 (0.005340931632302025) + + + --> TIME: 2025-04-19 05:22:56 -- STEP: 80/142 -- GLOBAL_STEP: 434600 + | > loss_disc: 2.4072954654693604 (2.4450854688882826) + | > loss_disc_real_0: 0.10861291736364365 (0.11776315118186176) + | > loss_disc_real_1: 0.18671520054340363 (0.2053712045773864) + | > loss_disc_real_2: 0.2394278645515442 (0.22662520967423916) + | > loss_disc_real_3: 0.23622745275497437 (0.22939436007291078) + | > loss_disc_real_4: 0.2196461260318756 (0.22976960707455873) + | > loss_disc_real_5: 0.2082725167274475 (0.23130171243101358) + | > loss_0: 2.4072954654693604 (2.4450854688882826) + | > grad_norm_0: tensor(8.0414, device='cuda:0') (tensor(10.5495, device='cuda:0')) + | > loss_gen: 2.4304919242858887 (2.38148612678051) + | > loss_kl: 1.9901728630065918 (1.8259871482849122) + | > loss_feat: 7.36471700668335 (7.247870397567749) + | > loss_mel: 18.177202224731445 (17.61426041126251) + | > loss_duration: 1.4585187435150146 (1.437815499305725) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.421104431152344 (30.507419538497924) + | > grad_norm_1: tensor(249.8154, device='cuda:0') (tensor(269.5775, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.4076 (0.36145237386226653) + | > loader_time: 0.0069 (0.005663555860519408) + + + --> TIME: 2025-04-19 05:23:07 -- STEP: 105/142 -- GLOBAL_STEP: 434625 + | > loss_disc: 2.4982762336730957 (2.449444187255132) + | > loss_disc_real_0: 0.09305601567029953 (0.12081009841391019) + | > loss_disc_real_1: 0.22165997326374054 (0.20563116385823205) + | > loss_disc_real_2: 0.225969597697258 (0.22563805026667458) + | > loss_disc_real_3: 0.26913121342658997 (0.22900580749625252) + | > loss_disc_real_4: 0.2276764214038849 (0.22966798430397398) + | > loss_disc_real_5: 0.25115787982940674 (0.23190400444325945) + | > loss_0: 2.4982762336730957 (2.449444187255132) + | > grad_norm_0: tensor(9.5412, device='cuda:0') (tensor(9.7884, device='cuda:0')) + | > loss_gen: 2.303617477416992 (2.371494581585839) + | > loss_kl: 1.9626425504684448 (1.8379656496502104) + | > loss_feat: 7.2825751304626465 (7.195681163242885) + | > loss_mel: 18.238834381103516 (17.733415385654993) + | > loss_duration: 1.4969463348388672 (1.4488044999894643) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.28461456298828 (30.587361381167458) + | > grad_norm_1: tensor(256.8670, device='cuda:0') (tensor(245.3811, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.4184 (0.37162471952892484) + | > loader_time: 0.0069 (0.005935339700608024) + + + --> TIME: 2025-04-19 05:23:18 -- STEP: 130/142 -- GLOBAL_STEP: 434650 + | > loss_disc: 2.5711779594421387 (2.4588210197595446) + | > loss_disc_real_0: 0.18673819303512573 (0.12525663934648038) + | > loss_disc_real_1: 0.21102191507816315 (0.20631569085212853) + | > loss_disc_real_2: 0.2107688933610916 (0.22587126436141822) + | > loss_disc_real_3: 0.23116503655910492 (0.22834084240289834) + | > loss_disc_real_4: 0.23663026094436646 (0.23019177053983395) + | > loss_disc_real_5: 0.2716982364654541 (0.23278768154291007) + | > loss_0: 2.5711779594421387 (2.4588210197595446) + | > grad_norm_0: tensor(6.6370, device='cuda:0') (tensor(9.6831, device='cuda:0')) + | > loss_gen: 2.186351776123047 (2.3671366453170775) + | > loss_kl: 1.9568098783493042 (1.8499130129814148) + | > loss_feat: 7.201243877410889 (7.182679444092971) + | > loss_mel: 17.928939819335938 (17.701446811969465) + | > loss_duration: 1.4685148000717163 (1.4558400851029616) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.74186134338379 (30.55701604989859) + | > grad_norm_1: tensor(65.8298, device='cuda:0') (tensor(228.9028, device='cuda:0')) + | > current_lr_0: 0.00019050707390411086 + | > current_lr_1: 0.00019050707390411086 + | > step_time: 0.451 (0.38313689048473654) + | > loader_time: 0.0076 (0.0061849685815664425) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4544684886932373 (2.4544684886932373) + | > loss_disc_real_0: 0.1529310792684555 (0.1529310792684555) + | > loss_disc_real_1: 0.17041611671447754 (0.17041611671447754) + | > loss_disc_real_2: 0.24810154736042023 (0.24810154736042023) + | > loss_disc_real_3: 0.24110263586044312 (0.24110263586044312) + | > loss_disc_real_4: 0.18544863164424896 (0.18544863164424896) + | > loss_disc_real_5: 0.3022787570953369 (0.3022787570953369) + | > loss_0: 2.4544684886932373 (2.4544684886932373) + | > loss_gen: 2.4267172813415527 (2.4267172813415527) + | > loss_kl: 2.306562662124634 (2.306562662124634) + | > loss_feat: 7.757755756378174 (7.757755756378174) + | > loss_mel: 19.391508102416992 (19.391508102416992) + | > loss_duration: 1.7262641191482544 (1.7262641191482544) + | > loss_1: 33.60880661010742 (33.60880661010742) + + --> STEP: 1 + | > loss_disc: 2.453308582305908 (2.453308582305908) + | > loss_disc_real_0: 0.12309186905622482 (0.12309186905622482) + | > loss_disc_real_1: 0.19212651252746582 (0.19212651252746582) + | > loss_disc_real_2: 0.2615962028503418 (0.2615962028503418) + | > loss_disc_real_3: 0.2736109793186188 (0.2736109793186188) + | > loss_disc_real_4: 0.18252359330654144 (0.18252359330654144) + | > loss_disc_real_5: 0.21960222721099854 (0.21960222721099854) + | > loss_0: 2.453308582305908 (2.453308582305908) + | > loss_gen: 2.277996063232422 (2.277996063232422) + | > loss_kl: 2.870990753173828 (2.870990753173828) + | > loss_feat: 7.058260440826416 (7.058260440826416) + | > loss_mel: 19.0279598236084 (19.0279598236084) + | > loss_duration: 1.75929594039917 (1.75929594039917) + | > loss_1: 32.994503021240234 (32.994503021240234) + + --> STEP: 2 + | > loss_disc: 2.4339118003845215 (2.443610191345215) + | > loss_disc_real_0: 0.1390894055366516 (0.13109063729643822) + | > loss_disc_real_1: 0.17401845753192902 (0.18307248502969742) + | > loss_disc_real_2: 0.26442766189575195 (0.2630119323730469) + | > loss_disc_real_3: 0.27936071157455444 (0.2764858454465866) + | > loss_disc_real_4: 0.22349007427692413 (0.2030068337917328) + | > loss_disc_real_5: 0.2073216736316681 (0.2134619504213333) + | > loss_0: 2.4339118003845215 (2.443610191345215) + | > loss_gen: 2.3477699756622314 (2.3128830194473267) + | > loss_kl: 2.4312450885772705 (2.6511179208755493) + | > loss_feat: 7.747959613800049 (7.403110027313232) + | > loss_mel: 18.780086517333984 (18.90402317047119) + | > loss_duration: 1.730539321899414 (1.744917631149292) + | > loss_1: 33.037601470947266 (33.01605224609375) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033349990844726562 (-0.0004303455352783203) + | > avg_loss_disc: 2.443610191345215 (-0.045107364654541016) + | > avg_loss_disc_real_0: 0.13109063729643822 (+0.019417371600866318) + | > avg_loss_disc_real_1: 0.18307248502969742 (-0.05857260525226593) + | > avg_loss_disc_real_2: 0.2630119323730469 (+0.019705086946487427) + | > avg_loss_disc_real_3: 0.2764858454465866 (+0.05480878800153732) + | > avg_loss_disc_real_4: 0.2030068337917328 (-0.034391969442367554) + | > avg_loss_disc_real_5: 0.2134619504213333 (+0.0018252059817314148) + | > avg_loss_0: 2.443610191345215 (-0.045107364654541016) + | > avg_loss_gen: 2.3128830194473267 (+0.10910296440124512) + | > avg_loss_kl: 2.6511179208755493 (-0.01501929759979248) + | > avg_loss_feat: 7.403110027313232 (+0.8551547527313232) + | > avg_loss_mel: 18.90402317047119 (+0.1250629425048828) + | > avg_loss_duration: 1.744917631149292 (-0.0008205175399780273) + | > avg_loss_1: 33.01605224609375 (+1.0734806060791016) + + + > EPOCH: 390/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:23:24)  + + --> TIME: 2025-04-19 05:23:30 -- STEP: 13/142 -- GLOBAL_STEP: 434675 + | > loss_disc: 2.4525463581085205 (2.41510244516226) + | > loss_disc_real_0: 0.12085767090320587 (0.11618841038300441) + | > loss_disc_real_1: 0.2062625139951706 (0.20054187109837165) + | > loss_disc_real_2: 0.2318153977394104 (0.23654418954482445) + | > loss_disc_real_3: 0.23488667607307434 (0.2293972774193837) + | > loss_disc_real_4: 0.2643561363220215 (0.22778460039542273) + | > loss_disc_real_5: 0.2214542180299759 (0.22227549094420213) + | > loss_0: 2.4525463581085205 (2.41510244516226) + | > grad_norm_0: tensor(7.9393, device='cuda:0') (tensor(6.2685, device='cuda:0')) + | > loss_gen: 2.2914958000183105 (2.4063548491551328) + | > loss_kl: 1.885993480682373 (1.826570749282837) + | > loss_feat: 7.305805206298828 (7.475829968085656) + | > loss_mel: 18.255287170410156 (18.221910770122822) + | > loss_duration: 1.4094048738479614 (1.4101509497715876) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.147987365722656 (31.340817084679237) + | > grad_norm_1: tensor(38.8540, device='cuda:0') (tensor(107.2753, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.3316 (0.3323689790872427) + | > loader_time: 0.0046 (0.00432883776151217) + + + --> TIME: 2025-04-19 05:23:39 -- STEP: 38/142 -- GLOBAL_STEP: 434700 + | > loss_disc: 2.4764301776885986 (2.425685468472933) + | > loss_disc_real_0: 0.10949607938528061 (0.11739783890937504) + | > loss_disc_real_1: 0.17398053407669067 (0.20549883771883815) + | > loss_disc_real_2: 0.2501704692840576 (0.22875075355956428) + | > loss_disc_real_3: 0.25459057092666626 (0.22768622127018476) + | > loss_disc_real_4: 0.22606462240219116 (0.2275861568356815) + | > loss_disc_real_5: 0.20098325610160828 (0.22443289662662305) + | > loss_0: 2.4764301776885986 (2.425685468472933) + | > grad_norm_0: tensor(7.8968, device='cuda:0') (tensor(7.2028, device='cuda:0')) + | > loss_gen: 2.3013405799865723 (2.3905511655305567) + | > loss_kl: 1.9037344455718994 (1.8265941017552425) + | > loss_feat: 7.370317459106445 (7.3631798091687655) + | > loss_mel: 17.419330596923828 (17.808461691203878) + | > loss_duration: 1.4566066265106201 (1.4247626285803945) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.45132827758789 (30.81354924252159) + | > grad_norm_1: tensor(208.9791, device='cuda:0') (tensor(154.1936, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.3644 (0.3464872774324919) + | > loader_time: 0.0054 (0.004877165744179172) + + + --> TIME: 2025-04-19 05:23:48 -- STEP: 63/142 -- GLOBAL_STEP: 434725 + | > loss_disc: 2.490983009338379 (2.421083737933446) + | > loss_disc_real_0: 0.13377712666988373 (0.11738369365533192) + | > loss_disc_real_1: 0.19180552661418915 (0.206306159259781) + | > loss_disc_real_2: 0.24596628546714783 (0.2267179075214598) + | > loss_disc_real_3: 0.24555861949920654 (0.2258342064562298) + | > loss_disc_real_4: 0.2522631287574768 (0.22630700965722403) + | > loss_disc_real_5: 0.23101215064525604 (0.22460886339346567) + | > loss_0: 2.490983009338379 (2.421083737933446) + | > grad_norm_0: tensor(6.3006, device='cuda:0') (tensor(7.9624, device='cuda:0')) + | > loss_gen: 2.342402458190918 (2.3894448847997762) + | > loss_kl: 1.9112226963043213 (1.8489917857306344) + | > loss_feat: 5.8242082595825195 (7.279569073328896) + | > loss_mel: 16.391151428222656 (17.639628607129296) + | > loss_duration: 1.4832820892333984 (1.4366330911242773) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.952268600463867 (30.59426734561012) + | > grad_norm_1: tensor(125.1360, device='cuda:0') (tensor(189.3552, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.3994 (0.36036795661562965) + | > loader_time: 0.0064 (0.0052858572157602444) + + + --> TIME: 2025-04-19 05:23:59 -- STEP: 88/142 -- GLOBAL_STEP: 434750 + | > loss_disc: 2.386467933654785 (2.4364127056165175) + | > loss_disc_real_0: 0.09591130167245865 (0.11924672109836881) + | > loss_disc_real_1: 0.16501836478710175 (0.20649270696396177) + | > loss_disc_real_2: 0.22581946849822998 (0.22821171835742213) + | > loss_disc_real_3: 0.2545018196105957 (0.22771025482903828) + | > loss_disc_real_4: 0.25450775027275085 (0.228966901417483) + | > loss_disc_real_5: 0.2290167510509491 (0.22771456092596054) + | > loss_0: 2.386467933654785 (2.4364127056165175) + | > grad_norm_0: tensor(7.8772, device='cuda:0') (tensor(8.1571, device='cuda:0')) + | > loss_gen: 2.406367301940918 (2.3796901161020454) + | > loss_kl: 1.9761686325073242 (1.8706267448988827) + | > loss_feat: 7.090237140655518 (7.219538331031799) + | > loss_mel: 18.672138214111328 (17.702175573869184) + | > loss_duration: 1.504014015197754 (1.4467442157593637) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.64892578125 (30.618774934248492) + | > grad_norm_1: tensor(212.8604, device='cuda:0') (tensor(188.1110, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.4063 (0.37348655408078973) + | > loader_time: 0.0069 (0.005604917352849786) + + + --> TIME: 2025-04-19 05:24:10 -- STEP: 113/142 -- GLOBAL_STEP: 434775 + | > loss_disc: 2.5052103996276855 (2.4394666684412316) + | > loss_disc_real_0: 0.17029346525669098 (0.11995255419638305) + | > loss_disc_real_1: 0.2570236623287201 (0.20680704660120264) + | > loss_disc_real_2: 0.22378604114055634 (0.22721234174956262) + | > loss_disc_real_3: 0.21501082181930542 (0.2274272079225135) + | > loss_disc_real_4: 0.22359654307365417 (0.22813666420700277) + | > loss_disc_real_5: 0.22356367111206055 (0.22850036977139193) + | > loss_0: 2.5052103996276855 (2.4394666684412316) + | > grad_norm_0: tensor(11.8745, device='cuda:0') (tensor(8.9084, device='cuda:0')) + | > loss_gen: 2.229508876800537 (2.375533646186895) + | > loss_kl: 1.8431103229522705 (1.8792482561769739) + | > loss_feat: 5.846241474151611 (7.197994481145808) + | > loss_mel: 15.622700691223145 (17.68542089715468) + | > loss_duration: 1.4778310060501099 (1.45264486295987) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.019393920898438 (30.590842120415342) + | > grad_norm_1: tensor(235.1534, device='cuda:0') (tensor(212.9394, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.4406 (0.3850961174585123) + | > loader_time: 0.0068 (0.005909871211094139) + + + --> TIME: 2025-04-19 05:24:21 -- STEP: 138/142 -- GLOBAL_STEP: 434800 + | > loss_disc: 2.5228137969970703 (2.452811928762905) + | > loss_disc_real_0: 0.08613591641187668 (0.12101384506061458) + | > loss_disc_real_1: 0.21108798682689667 (0.20834823665411575) + | > loss_disc_real_2: 0.24025744199752808 (0.22806566975254944) + | > loss_disc_real_3: 0.22905293107032776 (0.2284354078380958) + | > loss_disc_real_4: 0.2266797125339508 (0.22864646702141) + | > loss_disc_real_5: 0.24357949197292328 (0.230123371641705) + | > loss_0: 2.5228137969970703 (2.452811928762905) + | > grad_norm_0: tensor(13.9116, device='cuda:0') (tensor(9.8579, device='cuda:0')) + | > loss_gen: 2.3141024112701416 (2.3710238346155124) + | > loss_kl: 1.9399724006652832 (1.8881098342978435) + | > loss_feat: 6.830876350402832 (7.148820631745933) + | > loss_mel: 18.466339111328125 (17.66166709125906) + | > loss_duration: 1.5172975063323975 (1.4592343128245808) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.068588256835938 (30.528855724611144) + | > grad_norm_1: tensor(385.4114, device='cuda:0') (tensor(236.1046, device='cuda:0')) + | > current_lr_0: 0.00019048326051987284 + | > current_lr_1: 0.00019048326051987284 + | > step_time: 0.4637 (0.3971293663633043) + | > loader_time: 0.0075 (0.006207868672799374) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3968088626861572 (2.3968088626861572) + | > loss_disc_real_0: 0.12084634602069855 (0.12084634602069855) + | > loss_disc_real_1: 0.2489655762910843 (0.2489655762910843) + | > loss_disc_real_2: 0.22838115692138672 (0.22838115692138672) + | > loss_disc_real_3: 0.23959434032440186 (0.23959434032440186) + | > loss_disc_real_4: 0.2931845784187317 (0.2931845784187317) + | > loss_disc_real_5: 0.23572298884391785 (0.23572298884391785) + | > loss_0: 2.3968088626861572 (2.3968088626861572) + | > loss_gen: 2.441059112548828 (2.441059112548828) + | > loss_kl: 2.4768612384796143 (2.4768612384796143) + | > loss_feat: 6.10222053527832 (6.10222053527832) + | > loss_mel: 18.456369400024414 (18.456369400024414) + | > loss_duration: 1.7386597394943237 (1.7386597394943237) + | > loss_1: 31.215167999267578 (31.215167999267578) + + --> STEP: 1 + | > loss_disc: 2.773181438446045 (2.773181438446045) + | > loss_disc_real_0: 0.16565831005573273 (0.16565831005573273) + | > loss_disc_real_1: 0.30013307929039 (0.30013307929039) + | > loss_disc_real_2: 0.25813961029052734 (0.25813961029052734) + | > loss_disc_real_3: 0.28579065203666687 (0.28579065203666687) + | > loss_disc_real_4: 0.2980790436267853 (0.2980790436267853) + | > loss_disc_real_5: 0.2717849612236023 (0.2717849612236023) + | > loss_0: 2.773181438446045 (2.773181438446045) + | > loss_gen: 2.160724401473999 (2.160724401473999) + | > loss_kl: 2.943964719772339 (2.943964719772339) + | > loss_feat: 4.228754043579102 (4.228754043579102) + | > loss_mel: 15.20292854309082 (15.20292854309082) + | > loss_duration: 1.7602026462554932 (1.7602026462554932) + | > loss_1: 26.29657554626465 (26.29657554626465) + + --> STEP: 2 + | > loss_disc: 2.4501700401306152 (2.61167573928833) + | > loss_disc_real_0: 0.09804195165634155 (0.13185013085603714) + | > loss_disc_real_1: 0.277841717004776 (0.288987398147583) + | > loss_disc_real_2: 0.23328834772109985 (0.2457139790058136) + | > loss_disc_real_3: 0.1839926540851593 (0.23489165306091309) + | > loss_disc_real_4: 0.22339533269405365 (0.26073718816041946) + | > loss_disc_real_5: 0.239576056599617 (0.25568050891160965) + | > loss_0: 2.4501700401306152 (2.61167573928833) + | > loss_gen: 2.344559907913208 (2.2526421546936035) + | > loss_kl: 2.429037570953369 (2.686501145362854) + | > loss_feat: 7.63427209854126 (5.931513071060181) + | > loss_mel: 18.762365341186523 (16.982646942138672) + | > loss_duration: 1.737265706062317 (1.748734176158905) + | > loss_1: 32.907501220703125 (29.602038383483887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032236576080322266 (-0.00011134147644042969) + | > avg_loss_disc: 2.61167573928833 (+0.16806554794311523) + | > avg_loss_disc_real_0: 0.13185013085603714 (+0.0007594935595989227) + | > avg_loss_disc_real_1: 0.288987398147583 (+0.10591491311788559) + | > avg_loss_disc_real_2: 0.2457139790058136 (-0.017297953367233276) + | > avg_loss_disc_real_3: 0.23489165306091309 (-0.04159419238567352) + | > avg_loss_disc_real_4: 0.26073718816041946 (+0.057730354368686676) + | > avg_loss_disc_real_5: 0.25568050891160965 (+0.04221855849027634) + | > avg_loss_0: 2.61167573928833 (+0.16806554794311523) + | > avg_loss_gen: 2.2526421546936035 (-0.060240864753723145) + | > avg_loss_kl: 2.686501145362854 (+0.03538322448730469) + | > avg_loss_feat: 5.931513071060181 (-1.4715969562530518) + | > avg_loss_mel: 16.982646942138672 (-1.9213762283325195) + | > avg_loss_duration: 1.748734176158905 (+0.003816545009613037) + | > avg_loss_1: 29.602038383483887 (-3.4140138626098633) + + + > EPOCH: 391/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:24:24)  + + --> TIME: 2025-04-19 05:24:32 -- STEP: 21/142 -- GLOBAL_STEP: 434825 + | > loss_disc: 2.476250171661377 (2.445028997602917) + | > loss_disc_real_0: 0.06876042485237122 (0.11746399530342647) + | > loss_disc_real_1: 0.19446024298667908 (0.2024776495638348) + | > loss_disc_real_2: 0.1917523443698883 (0.22278296095984323) + | > loss_disc_real_3: 0.22940365970134735 (0.23046144488311948) + | > loss_disc_real_4: 0.22514761984348297 (0.22896658380826315) + | > loss_disc_real_5: 0.23803456127643585 (0.23846071887583958) + | > loss_0: 2.476250171661377 (2.445028997602917) + | > grad_norm_0: tensor(14.9484, device='cuda:0') (tensor(14.0733, device='cuda:0')) + | > loss_gen: 2.1861512660980225 (2.3958234559921996) + | > loss_kl: 1.7433099746704102 (1.8115693046933128) + | > loss_feat: 7.458728313446045 (7.462504545847575) + | > loss_mel: 18.427560806274414 (18.012644813174294) + | > loss_duration: 1.4405755996704102 (1.414401264417739) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.256324768066406 (31.096942901611328) + | > grad_norm_1: tensor(241.8043, device='cuda:0') (tensor(295.3162, device='cuda:0')) + | > current_lr_0: 0.00019045945011230786 + | > current_lr_1: 0.00019045945011230786 + | > step_time: 0.3425 (0.3307762145996094) + | > loader_time: 0.0052 (0.0047601064046223955) + + + --> TIME: 2025-04-19 05:24:41 -- STEP: 46/142 -- GLOBAL_STEP: 434850 + | > loss_disc: 2.363770008087158 (2.442078859909721) + | > loss_disc_real_0: 0.11030352115631104 (0.11550667055923006) + | > loss_disc_real_1: 0.19791242480278015 (0.20611923965423004) + | > loss_disc_real_2: 0.1874130368232727 (0.22429232623266138) + | > loss_disc_real_3: 0.19279123842716217 (0.22813494341528934) + | > loss_disc_real_4: 0.2066369354724884 (0.22700135118287543) + | > loss_disc_real_5: 0.21254320442676544 (0.23497672670561334) + | > loss_0: 2.363770008087158 (2.442078859909721) + | > grad_norm_0: tensor(14.0718, device='cuda:0') (tensor(12.2871, device='cuda:0')) + | > loss_gen: 2.3825061321258545 (2.3743595299513442) + | > loss_kl: 1.9864587783813477 (1.8263081726820574) + | > loss_feat: 8.032142639160156 (7.3506101836328925) + | > loss_mel: 18.565185546875 (17.8145723964857) + | > loss_duration: 1.421922206878662 (1.4268630950347236) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.388214111328125 (30.792713206747305) + | > grad_norm_1: tensor(130.9121, device='cuda:0') (tensor(306.5646, device='cuda:0')) + | > current_lr_0: 0.00019045945011230786 + | > current_lr_1: 0.00019045945011230786 + | > step_time: 0.3676 (0.3439853709676991) + | > loader_time: 0.0058 (0.00509241352910581) + + + --> TIME: 2025-04-19 05:24:51 -- STEP: 71/142 -- GLOBAL_STEP: 434875 + | > loss_disc: 2.5101635456085205 (2.445821503518333) + | > loss_disc_real_0: 0.11448846012353897 (0.11794195974796591) + | > loss_disc_real_1: 0.2260560393333435 (0.2055323493312782) + | > loss_disc_real_2: 0.25382229685783386 (0.22470505241776856) + | > loss_disc_real_3: 0.2287866622209549 (0.22967652451824133) + | > loss_disc_real_4: 0.21182675659656525 (0.22694025954730074) + | > loss_disc_real_5: 0.2253231406211853 (0.23694452286606105) + | > loss_0: 2.5101635456085205 (2.445821503518333) + | > grad_norm_0: tensor(8.8362, device='cuda:0') (tensor(12.0226, device='cuda:0')) + | > loss_gen: 2.2250211238861084 (2.376440555277005) + | > loss_kl: 1.8601319789886475 (1.8470459958197365) + | > loss_feat: 6.785284996032715 (7.313203556436888) + | > loss_mel: 16.742332458496094 (17.872150085341758) + | > loss_duration: 1.4707698822021484 (1.4371406612261919) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.083539962768555 (30.8459807382503) + | > grad_norm_1: tensor(386.3484, device='cuda:0') (tensor(305.5519, device='cuda:0')) + | > current_lr_0: 0.00019045945011230786 + | > current_lr_1: 0.00019045945011230786 + | > step_time: 0.4085 (0.35667506070204175) + | > loader_time: 0.0067 (0.005470715777974734) + + + --> TIME: 2025-04-19 05:25:01 -- STEP: 96/142 -- GLOBAL_STEP: 434900 + | > loss_disc: 2.43914794921875 (2.4523252372940374) + | > loss_disc_real_0: 0.15195365250110626 (0.11887899250723422) + | > loss_disc_real_1: 0.227619469165802 (0.2075664938117067) + | > loss_disc_real_2: 0.23517084121704102 (0.22523526567965746) + | > loss_disc_real_3: 0.2400790899991989 (0.22875376775239906) + | > loss_disc_real_4: 0.22551539540290833 (0.22714604840924343) + | > loss_disc_real_5: 0.20966017246246338 (0.23564876072729626) + | > loss_0: 2.43914794921875 (2.4523252372940374) + | > grad_norm_0: tensor(6.2327, device='cuda:0') (tensor(11.6353, device='cuda:0')) + | > loss_gen: 2.4424400329589844 (2.369603971640269) + | > loss_kl: 1.8940075635910034 (1.8648592482010524) + | > loss_feat: 7.098107814788818 (7.249884232878685) + | > loss_mel: 17.75839614868164 (17.88626618186633) + | > loss_duration: 1.470613956451416 (1.4481686043242614) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.663564682006836 (30.818782210350037) + | > grad_norm_1: tensor(51.7514, device='cuda:0') (tensor(287.4581, device='cuda:0')) + | > current_lr_0: 0.00019045945011230786 + | > current_lr_1: 0.00019045945011230786 + | > step_time: 0.4065 (0.3684662754336993) + | > loader_time: 0.0071 (0.0057842880487442025) + + + --> TIME: 2025-04-19 05:25:12 -- STEP: 121/142 -- GLOBAL_STEP: 434925 + | > loss_disc: 2.329540252685547 (2.451550889606318) + | > loss_disc_real_0: 0.1598152220249176 (0.1212714017053281) + | > loss_disc_real_1: 0.17742957174777985 (0.20682242069362608) + | > loss_disc_real_2: 0.20806290209293365 (0.22403133974587622) + | > loss_disc_real_3: 0.19192181527614594 (0.2290247762498777) + | > loss_disc_real_4: 0.21845415234565735 (0.22698243627370882) + | > loss_disc_real_5: 0.22672760486602783 (0.23496895277303112) + | > loss_0: 2.329540252685547 (2.451550889606318) + | > grad_norm_0: tensor(8.7790, device='cuda:0') (tensor(10.7320, device='cuda:0')) + | > loss_gen: 2.3871068954467773 (2.3679431292636335) + | > loss_kl: 2.061856746673584 (1.8728663684908025) + | > loss_feat: 7.654901504516602 (7.2363040309307) + | > loss_mel: 18.195232391357422 (17.847454449362004) + | > loss_duration: 1.5121328830718994 (1.455518077227695) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.81123161315918 (30.780086091727263) + | > grad_norm_1: tensor(240.6525, device='cuda:0') (tensor(261.5487, device='cuda:0')) + | > current_lr_0: 0.00019045945011230786 + | > current_lr_1: 0.00019045945011230786 + | > step_time: 0.4399 (0.37923922420533235) + | > loader_time: 0.007 (0.006083474671545109) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5340590476989746 (2.5340590476989746) + | > loss_disc_real_0: 0.20668485760688782 (0.20668485760688782) + | > loss_disc_real_1: 0.23218391835689545 (0.23218391835689545) + | > loss_disc_real_2: 0.27230286598205566 (0.27230286598205566) + | > loss_disc_real_3: 0.2698650658130646 (0.2698650658130646) + | > loss_disc_real_4: 0.26455336809158325 (0.26455336809158325) + | > loss_disc_real_5: 0.21886758506298065 (0.21886758506298065) + | > loss_0: 2.5340590476989746 (2.5340590476989746) + | > loss_gen: 2.55328631401062 (2.55328631401062) + | > loss_kl: 2.5515060424804688 (2.5515060424804688) + | > loss_feat: 7.45663595199585 (7.45663595199585) + | > loss_mel: 17.99435806274414 (17.99435806274414) + | > loss_duration: 1.747336983680725 (1.747336983680725) + | > loss_1: 32.303123474121094 (32.303123474121094) + + --> STEP: 1 + | > loss_disc: 2.5059542655944824 (2.5059542655944824) + | > loss_disc_real_0: 0.1657293736934662 (0.1657293736934662) + | > loss_disc_real_1: 0.2643905580043793 (0.2643905580043793) + | > loss_disc_real_2: 0.26743167638778687 (0.26743167638778687) + | > loss_disc_real_3: 0.24519935250282288 (0.24519935250282288) + | > loss_disc_real_4: 0.23336979746818542 (0.23336979746818542) + | > loss_disc_real_5: 0.237124964594841 (0.237124964594841) + | > loss_0: 2.5059542655944824 (2.5059542655944824) + | > loss_gen: 2.4084179401397705 (2.4084179401397705) + | > loss_kl: 2.8303661346435547 (2.8303661346435547) + | > loss_feat: 5.919297218322754 (5.919297218322754) + | > loss_mel: 17.73895263671875 (17.73895263671875) + | > loss_duration: 1.798095703125 (1.798095703125) + | > loss_1: 30.695131301879883 (30.695131301879883) + + --> STEP: 2 + | > loss_disc: 2.5466856956481934 (2.526319980621338) + | > loss_disc_real_0: 0.11349737644195557 (0.13961337506771088) + | > loss_disc_real_1: 0.2592209279537201 (0.2618057429790497) + | > loss_disc_real_2: 0.2802271246910095 (0.2738294005393982) + | > loss_disc_real_3: 0.24997609853744507 (0.24758772552013397) + | > loss_disc_real_4: 0.2205696552991867 (0.22696972638368607) + | > loss_disc_real_5: 0.23806120455265045 (0.23759308457374573) + | > loss_0: 2.5466856956481934 (2.526319980621338) + | > loss_gen: 2.4032249450683594 (2.405821442604065) + | > loss_kl: 2.4789865016937256 (2.65467631816864) + | > loss_feat: 7.640291213989258 (6.779794216156006) + | > loss_mel: 18.778017044067383 (18.258484840393066) + | > loss_duration: 1.7294777631759644 (1.7637867331504822) + | > loss_1: 33.029998779296875 (31.86256504058838) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032672882080078125 (+4.363059997558594e-05) + | > avg_loss_disc: 2.526319980621338 (-0.08535575866699219) + | > avg_loss_disc_real_0: 0.13961337506771088 (+0.007763244211673737) + | > avg_loss_disc_real_1: 0.2618057429790497 (-0.027181655168533325) + | > avg_loss_disc_real_2: 0.2738294005393982 (+0.028115421533584595) + | > avg_loss_disc_real_3: 0.24758772552013397 (+0.012696072459220886) + | > avg_loss_disc_real_4: 0.22696972638368607 (-0.0337674617767334) + | > avg_loss_disc_real_5: 0.23759308457374573 (-0.018087424337863922) + | > avg_loss_0: 2.526319980621338 (-0.08535575866699219) + | > avg_loss_gen: 2.405821442604065 (+0.15317928791046143) + | > avg_loss_kl: 2.65467631816864 (-0.03182482719421387) + | > avg_loss_feat: 6.779794216156006 (+0.8482811450958252) + | > avg_loss_mel: 18.258484840393066 (+1.2758378982543945) + | > avg_loss_duration: 1.7637867331504822 (+0.015052556991577148) + | > avg_loss_1: 31.86256504058838 (+2.260526657104492) + + + > EPOCH: 392/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:25:23)  + + --> TIME: 2025-04-19 05:25:25 -- STEP: 4/142 -- GLOBAL_STEP: 434950 + | > loss_disc: 2.4844486713409424 (2.4410828351974487) + | > loss_disc_real_0: 0.0808069184422493 (0.10863292403519154) + | > loss_disc_real_1: 0.1704716831445694 (0.19346707686781883) + | > loss_disc_real_2: 0.22286389768123627 (0.2410588599741459) + | > loss_disc_real_3: 0.24205228686332703 (0.2493879683315754) + | > loss_disc_real_4: 0.2598775327205658 (0.24491384625434875) + | > loss_disc_real_5: 0.23840287327766418 (0.22804145887494087) + | > loss_0: 2.4844486713409424 (2.4410828351974487) + | > grad_norm_0: tensor(10.1827, device='cuda:0') (tensor(10.8662, device='cuda:0')) + | > loss_gen: 2.4900729656219482 (2.3931921124458313) + | > loss_kl: 1.880213737487793 (1.8775674402713776) + | > loss_feat: 6.11476469039917 (6.996235489845276) + | > loss_mel: 16.53276824951172 (17.600786685943604) + | > loss_duration: 1.410632610321045 (1.4095012843608856) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.428451538085938 (30.27728271484375) + | > grad_norm_1: tensor(87.6091, device='cuda:0') (tensor(301.4125, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.3199 (0.3225827217102051) + | > loader_time: 0.0051 (0.004371345043182373) + + + --> TIME: 2025-04-19 05:25:34 -- STEP: 29/142 -- GLOBAL_STEP: 434975 + | > loss_disc: 2.45883846282959 (2.437956398931043) + | > loss_disc_real_0: 0.17023800313472748 (0.11850537642322738) + | > loss_disc_real_1: 0.2050493359565735 (0.20251159832395357) + | > loss_disc_real_2: 0.25313305854797363 (0.22699917544578685) + | > loss_disc_real_3: 0.2185734659433365 (0.23013170731478724) + | > loss_disc_real_4: 0.263105571269989 (0.2316460943427579) + | > loss_disc_real_5: 0.2200024574995041 (0.23303479572822308) + | > loss_0: 2.45883846282959 (2.437956398931043) + | > grad_norm_0: tensor(8.1658, device='cuda:0') (tensor(11.2726, device='cuda:0')) + | > loss_gen: 2.526916265487671 (2.410425827420991) + | > loss_kl: 1.8198773860931396 (1.842618177677023) + | > loss_feat: 7.502802848815918 (7.258229831169391) + | > loss_mel: 17.189645767211914 (17.75013364594558) + | > loss_duration: 1.4150075912475586 (1.4177054249007126) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.45425033569336 (30.679113124978954) + | > grad_norm_1: tensor(408.0825, device='cuda:0') (tensor(298.9123, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.3431 (0.33501812507366313) + | > loader_time: 0.0051 (0.0047433458525558995) + + + --> TIME: 2025-04-19 05:25:43 -- STEP: 54/142 -- GLOBAL_STEP: 435000 + | > loss_disc: 2.585447311401367 (2.426063188800106) + | > loss_disc_real_0: 0.16214619576931 (0.11515828608362763) + | > loss_disc_real_1: 0.18233969807624817 (0.20190423671846036) + | > loss_disc_real_2: 0.23559539020061493 (0.22497640356973367) + | > loss_disc_real_3: 0.23217788338661194 (0.22808043437975425) + | > loss_disc_real_4: 0.20706672966480255 (0.22719316018952262) + | > loss_disc_real_5: 0.24398864805698395 (0.2323520420878022) + | > loss_0: 2.585447311401367 (2.426063188800106) + | > grad_norm_0: tensor(6.4369, device='cuda:0') (tensor(12.1398, device='cuda:0')) + | > loss_gen: 2.1440303325653076 (2.4026153661586616) + | > loss_kl: 2.028766393661499 (1.8545917029734011) + | > loss_feat: 6.580653190612793 (7.240031321843465) + | > loss_mel: 16.645206451416016 (17.76929548051622) + | > loss_duration: 1.447580099105835 (1.4271738153916818) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.846237182617188 (30.693707819338197) + | > grad_norm_1: tensor(109.8037, device='cuda:0') (tensor(324.4964, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.3807 (0.3515539919888532) + | > loader_time: 0.0063 (0.0052681587360523365) + + + --> TIME: 2025-04-19 05:25:53 -- STEP: 79/142 -- GLOBAL_STEP: 435025 + | > loss_disc: 2.408344030380249 (2.442681366884256) + | > loss_disc_real_0: 0.1101880595088005 (0.1165872551972353) + | > loss_disc_real_1: 0.19890853762626648 (0.20361749941035162) + | > loss_disc_real_2: 0.24283213913440704 (0.22588813003105454) + | > loss_disc_real_3: 0.23170045018196106 (0.2286263347803792) + | > loss_disc_real_4: 0.22490182518959045 (0.22853805728351015) + | > loss_disc_real_5: 0.2618808150291443 (0.2346009289539313) + | > loss_0: 2.408344030380249 (2.442681366884256) + | > grad_norm_0: tensor(9.9322, device='cuda:0') (tensor(11.6068, device='cuda:0')) + | > loss_gen: 2.3993866443634033 (2.390095710754394) + | > loss_kl: 1.79649019241333 (1.854844794997686) + | > loss_feat: 6.74101448059082 (7.160736482354659) + | > loss_mel: 16.64437484741211 (17.653511264656153) + | > loss_duration: 1.4977238178253174 (1.438652035556262) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.078990936279297 (30.49784042261824) + | > grad_norm_1: tensor(239.2424, device='cuda:0') (tensor(310.0696, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.3857 (0.3669025234029263) + | > loader_time: 0.0061 (0.005605733847316307) + + + --> TIME: 2025-04-19 05:26:04 -- STEP: 104/142 -- GLOBAL_STEP: 435050 + | > loss_disc: 2.3767850399017334 (2.4532957420899324) + | > loss_disc_real_0: 0.15707342326641083 (0.11969244616249433) + | > loss_disc_real_1: 0.19001947343349457 (0.20393483727597272) + | > loss_disc_real_2: 0.2253803312778473 (0.22568305452855733) + | > loss_disc_real_3: 0.22526687383651733 (0.22872101314938986) + | > loss_disc_real_4: 0.22448696196079254 (0.23087741062045097) + | > loss_disc_real_5: 0.23500996828079224 (0.23373857584710306) + | > loss_0: 2.3767850399017334 (2.4532957420899324) + | > grad_norm_0: tensor(5.5847, device='cuda:0') (tensor(11.4911, device='cuda:0')) + | > loss_gen: 2.3099563121795654 (2.374870884876985) + | > loss_kl: 1.9904003143310547 (1.862986330802624) + | > loss_feat: 7.021313667297363 (7.109294941792121) + | > loss_mel: 17.333478927612305 (17.65089568724998) + | > loss_duration: 1.4925719499588013 (1.4485835467393584) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.147720336914062 (30.446631504939152) + | > grad_norm_1: tensor(208.9188, device='cuda:0') (tensor(313.2600, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.4083 (0.3759859571090111) + | > loader_time: 0.0064 (0.005879177496983455) + + + --> TIME: 2025-04-19 05:26:15 -- STEP: 129/142 -- GLOBAL_STEP: 435075 + | > loss_disc: 2.4665298461914062 (2.456330994302913) + | > loss_disc_real_0: 0.07107482105493546 (0.12141102483106214) + | > loss_disc_real_1: 0.20934289693832397 (0.20447300721046535) + | > loss_disc_real_2: 0.20208995044231415 (0.2261127484168193) + | > loss_disc_real_3: 0.21184614300727844 (0.2291437088288078) + | > loss_disc_real_4: 0.23715782165527344 (0.23015854848447695) + | > loss_disc_real_5: 0.22486768662929535 (0.23283487569916156) + | > loss_0: 2.4665298461914062 (2.456330994302913) + | > grad_norm_0: tensor(21.4211, device='cuda:0') (tensor(12.4311, device='cuda:0')) + | > loss_gen: 2.2879931926727295 (2.3679098491520847) + | > loss_kl: 1.8779220581054688 (1.8670406905255577) + | > loss_feat: 7.091300964355469 (7.081857773684716) + | > loss_mel: 16.7533016204834 (17.663642010947523) + | > loss_duration: 1.4702904224395752 (1.4554310186888824) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.48080825805664 (30.435881503792697) + | > grad_norm_1: tensor(398.7287, device='cuda:0') (tensor(319.7646, device='cuda:0')) + | > current_lr_0: 0.0001904356426810438 + | > current_lr_1: 0.0001904356426810438 + | > step_time: 0.4435 (0.3866244951883952) + | > loader_time: 0.0072 (0.006143427634424018) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.588991403579712 (2.588991403579712) + | > loss_disc_real_0: 0.16954512894153595 (0.16954512894153595) + | > loss_disc_real_1: 0.19373641908168793 (0.19373641908168793) + | > loss_disc_real_2: 0.28087538480758667 (0.28087538480758667) + | > loss_disc_real_3: 0.2949419319629669 (0.2949419319629669) + | > loss_disc_real_4: 0.23822027444839478 (0.23822027444839478) + | > loss_disc_real_5: 0.2677309811115265 (0.2677309811115265) + | > loss_0: 2.588991403579712 (2.588991403579712) + | > loss_gen: 2.254379987716675 (2.254379987716675) + | > loss_kl: 2.678187131881714 (2.678187131881714) + | > loss_feat: 6.654506206512451 (6.654506206512451) + | > loss_mel: 19.067447662353516 (19.067447662353516) + | > loss_duration: 1.7477343082427979 (1.7477343082427979) + | > loss_1: 32.40225601196289 (32.40225601196289) + + --> STEP: 1 + | > loss_disc: 2.6509511470794678 (2.6509511470794678) + | > loss_disc_real_0: 0.22670121490955353 (0.22670121490955353) + | > loss_disc_real_1: 0.2236706167459488 (0.2236706167459488) + | > loss_disc_real_2: 0.2660583257675171 (0.2660583257675171) + | > loss_disc_real_3: 0.270929753780365 (0.270929753780365) + | > loss_disc_real_4: 0.22024887800216675 (0.22024887800216675) + | > loss_disc_real_5: 0.257968932390213 (0.257968932390213) + | > loss_0: 2.6509511470794678 (2.6509511470794678) + | > loss_gen: 2.2652382850646973 (2.2652382850646973) + | > loss_kl: 2.798072576522827 (2.798072576522827) + | > loss_feat: 6.063684463500977 (6.063684463500977) + | > loss_mel: 17.09518051147461 (17.09518051147461) + | > loss_duration: 1.802890658378601 (1.802890658378601) + | > loss_1: 30.025066375732422 (30.025066375732422) + + --> STEP: 2 + | > loss_disc: 2.6022276878356934 (2.6265894174575806) + | > loss_disc_real_0: 0.2342243790626526 (0.23046279698610306) + | > loss_disc_real_1: 0.22806712985038757 (0.22586887329816818) + | > loss_disc_real_2: 0.24260957539081573 (0.2543339505791664) + | > loss_disc_real_3: 0.2929460108280182 (0.2819378823041916) + | > loss_disc_real_4: 0.22360150516033173 (0.22192519158124924) + | > loss_disc_real_5: 0.21235451102256775 (0.23516172170639038) + | > loss_0: 2.6022276878356934 (2.6265894174575806) + | > loss_gen: 2.3099637031555176 (2.2876009941101074) + | > loss_kl: 2.3868274688720703 (2.5924500226974487) + | > loss_feat: 7.09234094619751 (6.578012704849243) + | > loss_mel: 19.511920928955078 (18.303550720214844) + | > loss_duration: 1.7296028137207031 (1.766246736049652) + | > loss_1: 33.03065490722656 (31.527860641479492) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036145448684692383 (+0.0003472566604614258) + | > avg_loss_disc: 2.6265894174575806 (+0.10026943683624268) + | > avg_loss_disc_real_0: 0.23046279698610306 (+0.09084942191839218) + | > avg_loss_disc_real_1: 0.22586887329816818 (-0.0359368696808815) + | > avg_loss_disc_real_2: 0.2543339505791664 (-0.01949544996023178) + | > avg_loss_disc_real_3: 0.2819378823041916 (+0.03435015678405762) + | > avg_loss_disc_real_4: 0.22192519158124924 (-0.005044534802436829) + | > avg_loss_disc_real_5: 0.23516172170639038 (-0.0024313628673553467) + | > avg_loss_0: 2.6265894174575806 (+0.10026943683624268) + | > avg_loss_gen: 2.2876009941101074 (-0.11822044849395752) + | > avg_loss_kl: 2.5924500226974487 (-0.062226295471191406) + | > avg_loss_feat: 6.578012704849243 (-0.2017815113067627) + | > avg_loss_mel: 18.303550720214844 (+0.045065879821777344) + | > avg_loss_duration: 1.766246736049652 (+0.002460002899169922) + | > avg_loss_1: 31.527860641479492 (-0.3347043991088867) + + + > EPOCH: 393/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:26:22)  + + --> TIME: 2025-04-19 05:26:27 -- STEP: 12/142 -- GLOBAL_STEP: 435100 + | > loss_disc: 2.406345844268799 (2.4332855542500815) + | > loss_disc_real_0: 0.07016557455062866 (0.14103112897525233) + | > loss_disc_real_1: 0.1746765822172165 (0.19862625127037367) + | > loss_disc_real_2: 0.2194911539554596 (0.22350169097383818) + | > loss_disc_real_3: 0.2277466356754303 (0.2278220442434152) + | > loss_disc_real_4: 0.20849861204624176 (0.22136009857058525) + | > loss_disc_real_5: 0.21490487456321716 (0.231332466006279) + | > loss_0: 2.406345844268799 (2.4332855542500815) + | > grad_norm_0: tensor(9.4680, device='cuda:0') (tensor(21.2389, device='cuda:0')) + | > loss_gen: 2.4434022903442383 (2.427174707253774) + | > loss_kl: 1.7905986309051514 (1.8036190768082936) + | > loss_feat: 7.179998874664307 (7.536497751871745) + | > loss_mel: 17.15765380859375 (18.001005172729492) + | > loss_duration: 1.4246898889541626 (1.4112486739953358) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.996341705322266 (31.179545084635418) + | > grad_norm_1: tensor(251.1228, device='cuda:0') (tensor(322.1237, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.331 (0.32550412416458124) + | > loader_time: 0.0048 (0.004557649294535319) + + + --> TIME: 2025-04-19 05:26:36 -- STEP: 37/142 -- GLOBAL_STEP: 435125 + | > loss_disc: 2.5356762409210205 (2.430532713194151) + | > loss_disc_real_0: 0.1064111515879631 (0.12396545758521235) + | > loss_disc_real_1: 0.22380413115024567 (0.20523036049829946) + | > loss_disc_real_2: 0.2917957007884979 (0.2240364209220216) + | > loss_disc_real_3: 0.21776683628559113 (0.22519469824997154) + | > loss_disc_real_4: 0.23661082983016968 (0.22447511432944117) + | > loss_disc_real_5: 0.20816294848918915 (0.2343109411162299) + | > loss_0: 2.5356762409210205 (2.430532713194151) + | > grad_norm_0: tensor(11.1819, device='cuda:0') (tensor(13.6239, device='cuda:0')) + | > loss_gen: 2.179640531539917 (2.400527225958334) + | > loss_kl: 1.8607993125915527 (1.8350047356373556) + | > loss_feat: 6.6228508949279785 (7.276232667871423) + | > loss_mel: 17.034854888916016 (17.601374187984984) + | > loss_duration: 1.451197862625122 (1.4229368325826284) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.149343490600586 (30.53607554049105) + | > grad_norm_1: tensor(344.2624, device='cuda:0') (tensor(286.8123, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.3536 (0.33902228845132376) + | > loader_time: 0.0052 (0.004967921488993877) + + + --> TIME: 2025-04-19 05:26:45 -- STEP: 62/142 -- GLOBAL_STEP: 435150 + | > loss_disc: 2.488809585571289 (2.4237482047850087) + | > loss_disc_real_0: 0.12283611297607422 (0.11998210144379447) + | > loss_disc_real_1: 0.20539632439613342 (0.20217447535645577) + | > loss_disc_real_2: 0.2464110255241394 (0.2245056494589775) + | > loss_disc_real_3: 0.22374962270259857 (0.2261991445575991) + | > loss_disc_real_4: 0.2315400242805481 (0.22485853467256792) + | > loss_disc_real_5: 0.22189222276210785 (0.23118214213079022) + | > loss_0: 2.488809585571289 (2.4237482047850087) + | > grad_norm_0: tensor(7.0808, device='cuda:0') (tensor(12.6552, device='cuda:0')) + | > loss_gen: 2.3061296939849854 (2.3933450483506733) + | > loss_kl: 1.8392913341522217 (1.8338455923141972) + | > loss_feat: 7.5279741287231445 (7.274976707273914) + | > loss_mel: 18.109111785888672 (17.73866344267323) + | > loss_duration: 1.4865505695343018 (1.4337317943572998) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.269058227539062 (30.674562669569447) + | > grad_norm_1: tensor(185.9800, device='cuda:0') (tensor(304.0966, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.3829 (0.350544979495387) + | > loader_time: 0.0061 (0.005308205081570533) + + + --> TIME: 2025-04-19 05:26:55 -- STEP: 87/142 -- GLOBAL_STEP: 435175 + | > loss_disc: 2.3695194721221924 (2.4362496809027667) + | > loss_disc_real_0: 0.12173284590244293 (0.12009317166675096) + | > loss_disc_real_1: 0.19948701560497284 (0.20352444360996114) + | > loss_disc_real_2: 0.2044859081506729 (0.2246472864657983) + | > loss_disc_real_3: 0.21964503824710846 (0.22808161600567828) + | > loss_disc_real_4: 0.2382938265800476 (0.22686136916451072) + | > loss_disc_real_5: 0.20273976027965546 (0.23313732702156592) + | > loss_0: 2.3695194721221924 (2.4362496809027667) + | > grad_norm_0: tensor(6.6217, device='cuda:0') (tensor(11.3181, device='cuda:0')) + | > loss_gen: 2.394124984741211 (2.377337738015187) + | > loss_kl: 1.8875247240066528 (1.8423743042452583) + | > loss_feat: 7.761028289794922 (7.190310116471915) + | > loss_mel: 19.180580139160156 (17.711344280462164) + | > loss_duration: 1.4813451766967773 (1.4446517050951373) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.7046012878418 (30.56601824705628) + | > grad_norm_1: tensor(274.7831, device='cuda:0') (tensor(272.8194, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.4026 (0.36245307155039125) + | > loader_time: 0.0072 (0.005612924181181809) + + + --> TIME: 2025-04-19 05:27:06 -- STEP: 112/142 -- GLOBAL_STEP: 435200 + | > loss_disc: 2.486199378967285 (2.4476334665502826) + | > loss_disc_real_0: 0.18511484563350677 (0.12381922284008137) + | > loss_disc_real_1: 0.22557760775089264 (0.20508509994085347) + | > loss_disc_real_2: 0.2170155644416809 (0.22543083942894424) + | > loss_disc_real_3: 0.2431580275297165 (0.22754095973713057) + | > loss_disc_real_4: 0.23062308132648468 (0.22700329337801253) + | > loss_disc_real_5: 0.21731875836849213 (0.23367967483188426) + | > loss_0: 2.486199378967285 (2.4476334665502826) + | > grad_norm_0: tensor(5.9716, device='cuda:0') (tensor(10.9557, device='cuda:0')) + | > loss_gen: 2.269871711730957 (2.370352796145848) + | > loss_kl: 1.923073649406433 (1.8563544345753533) + | > loss_feat: 6.69331169128418 (7.164247342518398) + | > loss_mel: 17.082637786865234 (17.71226141282492) + | > loss_duration: 1.4542365074157715 (1.4526837532009396) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.423131942749023 (30.555899841444834) + | > grad_norm_1: tensor(143.0506, device='cuda:0') (tensor(266.4814, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.4254 (0.3742303039346423) + | > loader_time: 0.0081 (0.005919765148844037) + + + --> TIME: 2025-04-19 05:27:17 -- STEP: 137/142 -- GLOBAL_STEP: 435225 + | > loss_disc: 2.470658779144287 (2.457642610925828) + | > loss_disc_real_0: 0.20526599884033203 (0.1261291383446133) + | > loss_disc_real_1: 0.23293696343898773 (0.20605217583858185) + | > loss_disc_real_2: 0.2229575663805008 (0.22544396271670822) + | > loss_disc_real_3: 0.22720617055892944 (0.22808199946897745) + | > loss_disc_real_4: 0.2553216814994812 (0.22659430527774088) + | > loss_disc_real_5: 0.2450876086950302 (0.23544005853851346) + | > loss_0: 2.470658779144287 (2.457642610925828) + | > grad_norm_0: tensor(16.6877, device='cuda:0') (tensor(10.6766, device='cuda:0')) + | > loss_gen: 2.5113766193389893 (2.3660143622516716) + | > loss_kl: 1.8710410594940186 (1.8663542688327983) + | > loss_feat: 6.870011329650879 (7.150514132785101) + | > loss_mel: 18.124305725097656 (17.68259622754843) + | > loss_duration: 1.4741926193237305 (1.4588530176747452) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.850929260253906 (30.52433215788681) + | > grad_norm_1: tensor(362.8549, device='cuda:0') (tensor(256.5400, device='cuda:0')) + | > current_lr_0: 0.00019041183822570866 + | > current_lr_1: 0.00019041183822570866 + | > step_time: 0.4524 (0.3860446623642079) + | > loader_time: 0.0076 (0.00619908840986934) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.659606456756592 (2.659606456756592) + | > loss_disc_real_0: 0.318920373916626 (0.318920373916626) + | > loss_disc_real_1: 0.24680542945861816 (0.24680542945861816) + | > loss_disc_real_2: 0.2279558777809143 (0.2279558777809143) + | > loss_disc_real_3: 0.2783471941947937 (0.2783471941947937) + | > loss_disc_real_4: 0.20100420713424683 (0.20100420713424683) + | > loss_disc_real_5: 0.26005178689956665 (0.26005178689956665) + | > loss_0: 2.659606456756592 (2.659606456756592) + | > loss_gen: 2.611243486404419 (2.611243486404419) + | > loss_kl: 2.616501808166504 (2.616501808166504) + | > loss_feat: 6.3599724769592285 (6.3599724769592285) + | > loss_mel: 19.018098831176758 (19.018098831176758) + | > loss_duration: 1.748105764389038 (1.748105764389038) + | > loss_1: 32.35392379760742 (32.35392379760742) + + --> STEP: 1 + | > loss_disc: 2.733208656311035 (2.733208656311035) + | > loss_disc_real_0: 0.3321220576763153 (0.3321220576763153) + | > loss_disc_real_1: 0.31483951210975647 (0.31483951210975647) + | > loss_disc_real_2: 0.269140362739563 (0.269140362739563) + | > loss_disc_real_3: 0.2818301320075989 (0.2818301320075989) + | > loss_disc_real_4: 0.20780180394649506 (0.20780180394649506) + | > loss_disc_real_5: 0.2721240818500519 (0.2721240818500519) + | > loss_0: 2.733208656311035 (2.733208656311035) + | > loss_gen: 2.8026933670043945 (2.8026933670043945) + | > loss_kl: 2.9028098583221436 (2.9028098583221436) + | > loss_feat: 5.8790283203125 (5.8790283203125) + | > loss_mel: 17.472158432006836 (17.472158432006836) + | > loss_duration: 1.7762000560760498 (1.7762000560760498) + | > loss_1: 30.832889556884766 (30.832889556884766) + + --> STEP: 2 + | > loss_disc: 2.54246187210083 (2.6378352642059326) + | > loss_disc_real_0: 0.33945441246032715 (0.33578823506832123) + | > loss_disc_real_1: 0.23254716396331787 (0.27369333803653717) + | > loss_disc_real_2: 0.22027398645877838 (0.24470717459917068) + | > loss_disc_real_3: 0.2142390012741089 (0.24803456664085388) + | > loss_disc_real_4: 0.22275297343730927 (0.21527738869190216) + | > loss_disc_real_5: 0.2536373436450958 (0.26288071274757385) + | > loss_0: 2.54246187210083 (2.6378352642059326) + | > loss_gen: 2.833879232406616 (2.8182862997055054) + | > loss_kl: 2.4868974685668945 (2.694853663444519) + | > loss_feat: 8.432469367980957 (7.1557488441467285) + | > loss_mel: 19.341259002685547 (18.40670871734619) + | > loss_duration: 1.7377073764801025 (1.7569537162780762) + | > loss_1: 34.83221435546875 (32.83255195617676) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003229856491088867 (-0.0003846883773803711) + | > avg_loss_disc: 2.6378352642059326 (+0.01124584674835205) + | > avg_loss_disc_real_0: 0.33578823506832123 (+0.10532543808221817) + | > avg_loss_disc_real_1: 0.27369333803653717 (+0.04782446473836899) + | > avg_loss_disc_real_2: 0.24470717459917068 (-0.009626775979995728) + | > avg_loss_disc_real_3: 0.24803456664085388 (-0.03390331566333771) + | > avg_loss_disc_real_4: 0.21527738869190216 (-0.006647802889347076) + | > avg_loss_disc_real_5: 0.26288071274757385 (+0.02771899104118347) + | > avg_loss_0: 2.6378352642059326 (+0.01124584674835205) + | > avg_loss_gen: 2.8182862997055054 (+0.530685305595398) + | > avg_loss_kl: 2.694853663444519 (+0.10240364074707031) + | > avg_loss_feat: 7.1557488441467285 (+0.5777361392974854) + | > avg_loss_mel: 18.40670871734619 (+0.10315799713134766) + | > avg_loss_duration: 1.7569537162780762 (-0.009293019771575928) + | > avg_loss_1: 32.83255195617676 (+1.3046913146972656) + + + > EPOCH: 394/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:27:21)  + + --> TIME: 2025-04-19 05:27:28 -- STEP: 20/142 -- GLOBAL_STEP: 435250 + | > loss_disc: 2.4241788387298584 (2.4935574412345884) + | > loss_disc_real_0: 0.06419079750776291 (0.130656586587429) + | > loss_disc_real_1: 0.2219003289937973 (0.21395471394062043) + | > loss_disc_real_2: 0.18772339820861816 (0.2372317872941494) + | > loss_disc_real_3: 0.24211540818214417 (0.226782239228487) + | > loss_disc_real_4: 0.22679927945137024 (0.2297696866095066) + | > loss_disc_real_5: 0.25158143043518066 (0.22978197485208512) + | > loss_0: 2.4241788387298584 (2.4935574412345884) + | > grad_norm_0: tensor(4.9720, device='cuda:0') (tensor(8.4544, device='cuda:0')) + | > loss_gen: 2.3374264240264893 (2.3242125153541564) + | > loss_kl: 1.8489121198654175 (1.8364356815814973) + | > loss_feat: 7.757301330566406 (7.21834237575531) + | > loss_mel: 19.24850845336914 (17.980594635009766) + | > loss_duration: 1.4532335996627808 (1.411862415075302) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.645381927490234 (30.77144775390625) + | > grad_norm_1: tensor(66.1684, device='cuda:0') (tensor(167.0159, device='cuda:0')) + | > current_lr_0: 0.00019038803674593043 + | > current_lr_1: 0.00019038803674593043 + | > step_time: 0.3402 (0.3298255443572997) + | > loader_time: 0.0055 (0.004828941822052002) + + + --> TIME: 2025-04-19 05:27:37 -- STEP: 45/142 -- GLOBAL_STEP: 435275 + | > loss_disc: 2.486703872680664 (2.4781023449367945) + | > loss_disc_real_0: 0.10402901470661163 (0.12524953832228977) + | > loss_disc_real_1: 0.17763197422027588 (0.20670684178670248) + | > loss_disc_real_2: 0.21655447781085968 (0.23021866579850514) + | > loss_disc_real_3: 0.22423209249973297 (0.2283876445558336) + | > loss_disc_real_4: 0.22289550304412842 (0.23159425225522784) + | > loss_disc_real_5: 0.2505643963813782 (0.23336426715056102) + | > loss_0: 2.486703872680664 (2.4781023449367945) + | > grad_norm_0: tensor(3.7403, device='cuda:0') (tensor(8.6113, device='cuda:0')) + | > loss_gen: 2.2137184143066406 (2.333648411432902) + | > loss_kl: 1.9329112768173218 (1.8460920227898492) + | > loss_feat: 6.879986763000488 (7.120833333333334) + | > loss_mel: 18.040143966674805 (17.787841924031575) + | > loss_duration: 1.4521862268447876 (1.425781708293491) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.51894760131836 (30.514197497897676) + | > grad_norm_1: tensor(100.0053, device='cuda:0') (tensor(192.2666, device='cuda:0')) + | > current_lr_0: 0.00019038803674593043 + | > current_lr_1: 0.00019038803674593043 + | > step_time: 0.367 (0.34338765144348143) + | > loader_time: 0.006 (0.005183929867214625) + + + --> TIME: 2025-04-19 05:27:47 -- STEP: 70/142 -- GLOBAL_STEP: 435300 + | > loss_disc: 2.4820609092712402 (2.470771939413887) + | > loss_disc_real_0: 0.13615764677524567 (0.12345988952687809) + | > loss_disc_real_1: 0.21804821491241455 (0.20643985526902334) + | > loss_disc_real_2: 0.2164468914270401 (0.22970376312732696) + | > loss_disc_real_3: 0.23394347727298737 (0.230319302209786) + | > loss_disc_real_4: 0.2593173384666443 (0.2289628927196775) + | > loss_disc_real_5: 0.2556394338607788 (0.23410625138453076) + | > loss_0: 2.4820609092712402 (2.470771939413887) + | > grad_norm_0: tensor(3.9946, device='cuda:0') (tensor(8.3953, device='cuda:0')) + | > loss_gen: 2.3663835525512695 (2.3406682423182903) + | > loss_kl: 1.7846472263336182 (1.8591538378170558) + | > loss_feat: 6.83501672744751 (7.084144892011368) + | > loss_mel: 18.195295333862305 (17.656386252811977) + | > loss_duration: 1.4688233137130737 (1.4368478349276952) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.650165557861328 (30.37720105307443) + | > grad_norm_1: tensor(52.7280, device='cuda:0') (tensor(185.1981, device='cuda:0')) + | > current_lr_0: 0.00019038803674593043 + | > current_lr_1: 0.00019038803674593043 + | > step_time: 0.3881 (0.3553563015801566) + | > loader_time: 0.007 (0.0055215324674333845) + + + --> TIME: 2025-04-19 05:27:57 -- STEP: 95/142 -- GLOBAL_STEP: 435325 + | > loss_disc: 2.56815505027771 (2.4713986321499473) + | > loss_disc_real_0: 0.10622318089008331 (0.1222410355743609) + | > loss_disc_real_1: 0.18280261754989624 (0.20648013055324554) + | > loss_disc_real_2: 0.2235376238822937 (0.2292535181108274) + | > loss_disc_real_3: 0.24836120009422302 (0.2302524298429489) + | > loss_disc_real_4: 0.2023620307445526 (0.231158741053782) + | > loss_disc_real_5: 0.23684293031692505 (0.23437535307909313) + | > loss_0: 2.56815505027771 (2.4713986321499473) + | > grad_norm_0: tensor(11.5927, device='cuda:0') (tensor(7.9919, device='cuda:0')) + | > loss_gen: 2.1971349716186523 (2.3370619472704433) + | > loss_kl: 1.951655626296997 (1.864885503367374) + | > loss_feat: 6.4785380363464355 (7.068091613367983) + | > loss_mel: 17.78938102722168 (17.608231434069175) + | > loss_duration: 1.4923820495605469 (1.447219843613474) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.90909194946289 (30.325490349217464) + | > grad_norm_1: tensor(155.3575, device='cuda:0') (tensor(177.2362, device='cuda:0')) + | > current_lr_0: 0.00019038803674593043 + | > current_lr_1: 0.00019038803674593043 + | > step_time: 0.4027 (0.36625878685399105) + | > loader_time: 0.0066 (0.005805625413593493) + + + --> TIME: 2025-04-19 05:28:08 -- STEP: 120/142 -- GLOBAL_STEP: 435350 + | > loss_disc: 2.5540478229522705 (2.4744280099868776) + | > loss_disc_real_0: 0.12747006118297577 (0.1230034756163756) + | > loss_disc_real_1: 0.17709980905056 (0.20602641043563683) + | > loss_disc_real_2: 0.2127556949853897 (0.22906840058664482) + | > loss_disc_real_3: 0.25018060207366943 (0.22972520602246124) + | > loss_disc_real_4: 0.2387097179889679 (0.23099557235836982) + | > loss_disc_real_5: 0.26351961493492126 (0.23386217455069225) + | > loss_0: 2.5540478229522705 (2.4744280099868776) + | > grad_norm_0: tensor(12.3782, device='cuda:0') (tensor(7.6760, device='cuda:0')) + | > loss_gen: 2.2989635467529297 (2.3300367474555967) + | > loss_kl: 1.839590072631836 (1.868751405676206) + | > loss_feat: 6.816989421844482 (7.039283573627471) + | > loss_mel: 18.801898956298828 (17.63761352698008) + | > loss_duration: 1.4960410594940186 (1.4537381956974664) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.253482818603516 (30.329423459370933) + | > grad_norm_1: tensor(259.1870, device='cuda:0') (tensor(169.0167, device='cuda:0')) + | > current_lr_0: 0.00019038803674593043 + | > current_lr_1: 0.00019038803674593043 + | > step_time: 0.4337 (0.37713499665260314) + | > loader_time: 0.0071 (0.006061051289240519) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4833834171295166 (2.4833834171295166) + | > loss_disc_real_0: 0.21755175292491913 (0.21755175292491913) + | > loss_disc_real_1: 0.21797214448451996 (0.21797214448451996) + | > loss_disc_real_2: 0.22639921307563782 (0.22639921307563782) + | > loss_disc_real_3: 0.2799226939678192 (0.2799226939678192) + | > loss_disc_real_4: 0.2267325222492218 (0.2267325222492218) + | > loss_disc_real_5: 0.26832979917526245 (0.26832979917526245) + | > loss_0: 2.4833834171295166 (2.4833834171295166) + | > loss_gen: 2.4805922508239746 (2.4805922508239746) + | > loss_kl: 2.327460527420044 (2.327460527420044) + | > loss_feat: 5.849737167358398 (5.849737167358398) + | > loss_mel: 18.01988410949707 (18.01988410949707) + | > loss_duration: 1.753487467765808 (1.753487467765808) + | > loss_1: 30.431161880493164 (30.431161880493164) + + --> STEP: 1 + | > loss_disc: 2.677762269973755 (2.677762269973755) + | > loss_disc_real_0: 0.2770838737487793 (0.2770838737487793) + | > loss_disc_real_1: 0.22342383861541748 (0.22342383861541748) + | > loss_disc_real_2: 0.24413561820983887 (0.24413561820983887) + | > loss_disc_real_3: 0.28626394271850586 (0.28626394271850586) + | > loss_disc_real_4: 0.28534406423568726 (0.28534406423568726) + | > loss_disc_real_5: 0.26879632472991943 (0.26879632472991943) + | > loss_0: 2.677762269973755 (2.677762269973755) + | > loss_gen: 2.5170464515686035 (2.5170464515686035) + | > loss_kl: 2.648008108139038 (2.648008108139038) + | > loss_feat: 6.6989521980285645 (6.6989521980285645) + | > loss_mel: 16.819536209106445 (16.819536209106445) + | > loss_duration: 1.7804957628250122 (1.7804957628250122) + | > loss_1: 30.464038848876953 (30.464038848876953) + + --> STEP: 2 + | > loss_disc: 2.7042980194091797 (2.6910301446914673) + | > loss_disc_real_0: 0.24684961140155792 (0.2619667425751686) + | > loss_disc_real_1: 0.224602609872818 (0.22401322424411774) + | > loss_disc_real_2: 0.2586236894130707 (0.2513796538114548) + | > loss_disc_real_3: 0.28674814105033875 (0.2865060418844223) + | > loss_disc_real_4: 0.2580358386039734 (0.2716899514198303) + | > loss_disc_real_5: 0.26056069135665894 (0.2646785080432892) + | > loss_0: 2.7042980194091797 (2.6910301446914673) + | > loss_gen: 2.308133840560913 (2.4125901460647583) + | > loss_kl: 2.3389909267425537 (2.493499517440796) + | > loss_feat: 6.067672252655029 (6.383312225341797) + | > loss_mel: 16.92347526550293 (16.871505737304688) + | > loss_duration: 1.731381893157959 (1.7559388279914856) + | > loss_1: 29.36965560913086 (29.916847229003906) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003394484519958496 (+0.0001646280288696289) + | > avg_loss_disc: 2.6910301446914673 (+0.05319488048553467) + | > avg_loss_disc_real_0: 0.2619667425751686 (-0.07382149249315262) + | > avg_loss_disc_real_1: 0.22401322424411774 (-0.049680113792419434) + | > avg_loss_disc_real_2: 0.2513796538114548 (+0.006672479212284088) + | > avg_loss_disc_real_3: 0.2865060418844223 (+0.03847147524356842) + | > avg_loss_disc_real_4: 0.2716899514198303 (+0.05641256272792816) + | > avg_loss_disc_real_5: 0.2646785080432892 (+0.001797795295715332) + | > avg_loss_0: 2.6910301446914673 (+0.05319488048553467) + | > avg_loss_gen: 2.4125901460647583 (-0.40569615364074707) + | > avg_loss_kl: 2.493499517440796 (-0.20135414600372314) + | > avg_loss_feat: 6.383312225341797 (-0.7724366188049316) + | > avg_loss_mel: 16.871505737304688 (-1.535202980041504) + | > avg_loss_duration: 1.7559388279914856 (-0.0010148882865905762) + | > avg_loss_1: 29.916847229003906 (-2.9157047271728516) + + + > EPOCH: 395/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:28:19)  + + --> TIME: 2025-04-19 05:28:21 -- STEP: 3/142 -- GLOBAL_STEP: 435375 + | > loss_disc: 2.3817615509033203 (2.427185535430908) + | > loss_disc_real_0: 0.07248115539550781 (0.10232289383808772) + | > loss_disc_real_1: 0.1649463176727295 (0.18807320296764374) + | > loss_disc_real_2: 0.2340065836906433 (0.23416382571061453) + | > loss_disc_real_3: 0.21135438978672028 (0.24844482044378915) + | > loss_disc_real_4: 0.2597937285900116 (0.2634848753611247) + | > loss_disc_real_5: 0.2894895076751709 (0.29227108756701153) + | > loss_0: 2.3817615509033203 (2.427185535430908) + | > grad_norm_0: tensor(8.4230, device='cuda:0') (tensor(8.2213, device='cuda:0')) + | > loss_gen: 2.4364752769470215 (2.380070686340332) + | > loss_kl: 1.7652628421783447 (1.8070083061854045) + | > loss_feat: 6.764815807342529 (7.045853455861409) + | > loss_mel: 18.140512466430664 (17.250322977701824) + | > loss_duration: 1.42769455909729 (1.4146169026692708) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.534761428833008 (29.897871653238933) + | > grad_norm_1: tensor(212.3534, device='cuda:0') (tensor(263.5114, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.3236 (0.3239290714263916) + | > loader_time: 0.004 (0.004638195037841797) + + + --> TIME: 2025-04-19 05:28:29 -- STEP: 28/142 -- GLOBAL_STEP: 435400 + | > loss_disc: 2.4501543045043945 (2.4756301982062205) + | > loss_disc_real_0: 0.11129715293645859 (0.1207081994840077) + | > loss_disc_real_1: 0.18301273882389069 (0.21045285037585668) + | > loss_disc_real_2: 0.19895245134830475 (0.22845209283488138) + | > loss_disc_real_3: 0.2122938185930252 (0.2271049294088568) + | > loss_disc_real_4: 0.2131669670343399 (0.2322018247629915) + | > loss_disc_real_5: 0.20883192121982574 (0.239662126238857) + | > loss_0: 2.4501543045043945 (2.4756301982062205) + | > grad_norm_0: tensor(6.3817, device='cuda:0') (tensor(10.6228, device='cuda:0')) + | > loss_gen: 2.2627625465393066 (2.353293716907501) + | > loss_kl: 1.7726585865020752 (1.781137432370867) + | > loss_feat: 6.745031833648682 (7.100457957812718) + | > loss_mel: 17.7487850189209 (17.585926260266984) + | > loss_duration: 1.457092523574829 (1.418047696352005) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.986330032348633 (30.238862923213414) + | > grad_norm_1: tensor(174.6810, device='cuda:0') (tensor(238.2828, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.3512 (0.33706216301236835) + | > loader_time: 0.0056 (0.005049271242959159) + + + --> TIME: 2025-04-19 05:28:39 -- STEP: 53/142 -- GLOBAL_STEP: 435425 + | > loss_disc: 2.409092664718628 (2.4642092911702282) + | > loss_disc_real_0: 0.10686077922582626 (0.12199907454679597) + | > loss_disc_real_1: 0.21390622854232788 (0.20799070681041143) + | > loss_disc_real_2: 0.2238837033510208 (0.22662052259130297) + | > loss_disc_real_3: 0.22171175479888916 (0.22681464170509913) + | > loss_disc_real_4: 0.18852436542510986 (0.2293599233874735) + | > loss_disc_real_5: 0.22572176158428192 (0.23958968975633946) + | > loss_0: 2.409092664718628 (2.4642092911702282) + | > grad_norm_0: tensor(7.1469, device='cuda:0') (tensor(9.4284, device='cuda:0')) + | > loss_gen: 2.2564916610717773 (2.3562190082837953) + | > loss_kl: 1.7687729597091675 (1.799257699048744) + | > loss_feat: 6.885179042816162 (7.089134315274796) + | > loss_mel: 17.295578002929688 (17.583511532477615) + | > loss_duration: 1.4421393871307373 (1.4261207670535685) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.64815902709961 (30.254243058978386) + | > grad_norm_1: tensor(83.3888, device='cuda:0') (tensor(232.9412, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.376 (0.352441243405612) + | > loader_time: 0.0061 (0.005450752546202461) + + + --> TIME: 2025-04-19 05:28:49 -- STEP: 78/142 -- GLOBAL_STEP: 435450 + | > loss_disc: 2.495851993560791 (2.468665725145585) + | > loss_disc_real_0: 0.15681461989879608 (0.12188905171858959) + | > loss_disc_real_1: 0.2167627215385437 (0.2082837808590669) + | > loss_disc_real_2: 0.24111901223659515 (0.22726750889649758) + | > loss_disc_real_3: 0.2483593076467514 (0.2281887181676351) + | > loss_disc_real_4: 0.23792125284671783 (0.22988667816687852) + | > loss_disc_real_5: 0.2632283568382263 (0.2378291122806378) + | > loss_0: 2.495851993560791 (2.468665725145585) + | > grad_norm_0: tensor(10.3830, device='cuda:0') (tensor(8.9058, device='cuda:0')) + | > loss_gen: 2.396254539489746 (2.3558781819465833) + | > loss_kl: 1.8489662408828735 (1.8176756501197815) + | > loss_feat: 7.752803802490234 (7.108507871627809) + | > loss_mel: 19.01603126525879 (17.592327606983673) + | > loss_duration: 1.4922538995742798 (1.4382578042837293) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.506309509277344 (30.31264693920429) + | > grad_norm_1: tensor(224.4207, device='cuda:0') (tensor(220.7337, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.4045 (0.3666334029955742) + | > loader_time: 0.0063 (0.005769313910068611) + + + --> TIME: 2025-04-19 05:29:00 -- STEP: 103/142 -- GLOBAL_STEP: 435475 + | > loss_disc: 2.4120078086853027 (2.471829759264456) + | > loss_disc_real_0: 0.11455511301755905 (0.12317666715209924) + | > loss_disc_real_1: 0.22182303667068481 (0.2083731499111768) + | > loss_disc_real_2: 0.21231332421302795 (0.22672955865420183) + | > loss_disc_real_3: 0.18452784419059753 (0.22751641490505736) + | > loss_disc_real_4: 0.1925322562456131 (0.22942836568193528) + | > loss_disc_real_5: 0.2271237075328827 (0.23738097305436737) + | > loss_0: 2.4120078086853027 (2.471829759264456) + | > grad_norm_0: tensor(4.1124, device='cuda:0') (tensor(8.7882, device='cuda:0')) + | > loss_gen: 2.33695125579834 (2.341859900835649) + | > loss_kl: 1.8651484251022339 (1.8249561150097153) + | > loss_feat: 7.044510364532471 (7.052177859741509) + | > loss_mel: 16.558298110961914 (17.564198021749853) + | > loss_duration: 1.5076454877853394 (1.4484640892269547) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.31255340576172 (30.231655880085473) + | > grad_norm_1: tensor(76.7521, device='cuda:0') (tensor(222.7238, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.4256 (0.3787070686377368) + | > loader_time: 0.0073 (0.0060295132757390586) + + + --> TIME: 2025-04-19 05:29:11 -- STEP: 128/142 -- GLOBAL_STEP: 435500 + | > loss_disc: 2.474392890930176 (2.466781107708812) + | > loss_disc_real_0: 0.14675843715667725 (0.12407138530397788) + | > loss_disc_real_1: 0.1783788949251175 (0.20741813292261213) + | > loss_disc_real_2: 0.21767118573188782 (0.2265263090375811) + | > loss_disc_real_3: 0.2059229016304016 (0.2274825470522046) + | > loss_disc_real_4: 0.19695228338241577 (0.22933266533073038) + | > loss_disc_real_5: 0.24359963834285736 (0.23665937478654087) + | > loss_0: 2.474392890930176 (2.466781107708812) + | > grad_norm_0: tensor(6.2665, device='cuda:0') (tensor(9.0228, device='cuda:0')) + | > loss_gen: 2.2644295692443848 (2.346201609820128) + | > loss_kl: 1.9422024488449097 (1.8353763036429882) + | > loss_feat: 6.958188056945801 (7.037924960255624) + | > loss_mel: 18.21170425415039 (17.506325274705887) + | > loss_duration: 1.4778002500534058 (1.455254074186087) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.854324340820312 (30.181082159280777) + | > grad_norm_1: tensor(236.0378, device='cuda:0') (tensor(226.7019, device='cuda:0')) + | > current_lr_0: 0.0001903642382413372 + | > current_lr_1: 0.0001903642382413372 + | > step_time: 0.4462 (0.3923535291105509) + | > loader_time: 0.0072 (0.006341736763715744) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6410973072052 (2.6410973072052) + | > loss_disc_real_0: 0.23158274590969086 (0.23158274590969086) + | > loss_disc_real_1: 0.20919199287891388 (0.20919199287891388) + | > loss_disc_real_2: 0.22547805309295654 (0.22547805309295654) + | > loss_disc_real_3: 0.27953892946243286 (0.27953892946243286) + | > loss_disc_real_4: 0.2619019150733948 (0.2619019150733948) + | > loss_disc_real_5: 0.2281220555305481 (0.2281220555305481) + | > loss_0: 2.6410973072052 (2.6410973072052) + | > loss_gen: 2.3129427433013916 (2.3129427433013916) + | > loss_kl: 2.5396268367767334 (2.5396268367767334) + | > loss_feat: 6.060770511627197 (6.060770511627197) + | > loss_mel: 16.965621948242188 (16.965621948242188) + | > loss_duration: 1.7328813076019287 (1.7328813076019287) + | > loss_1: 29.61184310913086 (29.61184310913086) + + --> STEP: 1 + | > loss_disc: 2.6023218631744385 (2.6023218631744385) + | > loss_disc_real_0: 0.211854487657547 (0.211854487657547) + | > loss_disc_real_1: 0.21118296682834625 (0.21118296682834625) + | > loss_disc_real_2: 0.24153880774974823 (0.24153880774974823) + | > loss_disc_real_3: 0.23848366737365723 (0.23848366737365723) + | > loss_disc_real_4: 0.25918906927108765 (0.25918906927108765) + | > loss_disc_real_5: 0.20665891468524933 (0.20665891468524933) + | > loss_0: 2.6023218631744385 (2.6023218631744385) + | > loss_gen: 2.376737356185913 (2.376737356185913) + | > loss_kl: 2.883887767791748 (2.883887767791748) + | > loss_feat: 7.579119682312012 (7.579119682312012) + | > loss_mel: 19.21881866455078 (19.21881866455078) + | > loss_duration: 1.759921908378601 (1.759921908378601) + | > loss_1: 33.818485260009766 (33.818485260009766) + + --> STEP: 2 + | > loss_disc: 2.5396065711975098 (2.570964217185974) + | > loss_disc_real_0: 0.18228931725025177 (0.19707190245389938) + | > loss_disc_real_1: 0.21180464327335358 (0.21149380505084991) + | > loss_disc_real_2: 0.22150366008281708 (0.23152123391628265) + | > loss_disc_real_3: 0.21784141659736633 (0.22816254198551178) + | > loss_disc_real_4: 0.2549702525138855 (0.2570796608924866) + | > loss_disc_real_5: 0.24472576379776 (0.22569233924150467) + | > loss_0: 2.5396065711975098 (2.570964217185974) + | > loss_gen: 2.32124662399292 (2.3489919900894165) + | > loss_kl: 2.4861433506011963 (2.685015559196472) + | > loss_feat: 6.689455986022949 (7.1342878341674805) + | > loss_mel: 17.234045028686523 (18.226431846618652) + | > loss_duration: 1.7210676670074463 (1.7404947876930237) + | > loss_1: 30.45195960998535 (32.13522243499756) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037140846252441406 (+0.00031960010528564453) + | > avg_loss_disc: 2.570964217185974 (-0.12006592750549316) + | > avg_loss_disc_real_0: 0.19707190245389938 (-0.06489484012126923) + | > avg_loss_disc_real_1: 0.21149380505084991 (-0.012519419193267822) + | > avg_loss_disc_real_2: 0.23152123391628265 (-0.01985841989517212) + | > avg_loss_disc_real_3: 0.22816254198551178 (-0.05834349989891052) + | > avg_loss_disc_real_4: 0.2570796608924866 (-0.01461029052734375) + | > avg_loss_disc_real_5: 0.22569233924150467 (-0.038986168801784515) + | > avg_loss_0: 2.570964217185974 (-0.12006592750549316) + | > avg_loss_gen: 2.3489919900894165 (-0.0635981559753418) + | > avg_loss_kl: 2.685015559196472 (+0.19151604175567627) + | > avg_loss_feat: 7.1342878341674805 (+0.7509756088256836) + | > avg_loss_mel: 18.226431846618652 (+1.3549261093139648) + | > avg_loss_duration: 1.7404947876930237 (-0.015444040298461914) + | > avg_loss_1: 32.13522243499756 (+2.2183752059936523) + + + > EPOCH: 396/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:29:19)  + + --> TIME: 2025-04-19 05:29:23 -- STEP: 11/142 -- GLOBAL_STEP: 435525 + | > loss_disc: 2.4706811904907227 (2.4590237140655518) + | > loss_disc_real_0: 0.0761026069521904 (0.10692669451236725) + | > loss_disc_real_1: 0.1813875436782837 (0.21512130851095373) + | > loss_disc_real_2: 0.248818501830101 (0.23373240503397855) + | > loss_disc_real_3: 0.22960196435451508 (0.22994300045750357) + | > loss_disc_real_4: 0.22439491748809814 (0.23011884906075217) + | > loss_disc_real_5: 0.28880465030670166 (0.23521204428239303) + | > loss_0: 2.4706811904907227 (2.4590237140655518) + | > grad_norm_0: tensor(10.4700, device='cuda:0') (tensor(8.2087, device='cuda:0')) + | > loss_gen: 2.313490390777588 (2.365251974626021) + | > loss_kl: 1.8949589729309082 (1.8018293272365222) + | > loss_feat: 7.164340019226074 (7.240883870558306) + | > loss_mel: 18.556808471679688 (17.914689670909535) + | > loss_duration: 1.3909683227539062 (1.4058641520413486) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.320566177368164 (30.728519092906605) + | > grad_norm_1: tensor(189.0347, device='cuda:0') (tensor(210.8221, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.3291 (0.32464820688421075) + | > loader_time: 0.005 (0.004406994039362127) + + + --> TIME: 2025-04-19 05:29:32 -- STEP: 36/142 -- GLOBAL_STEP: 435550 + | > loss_disc: 2.4121029376983643 (2.4523098468780518) + | > loss_disc_real_0: 0.09902802854776382 (0.11580334044992924) + | > loss_disc_real_1: 0.1995105892419815 (0.20715698972344398) + | > loss_disc_real_2: 0.2207910418510437 (0.22761561514602768) + | > loss_disc_real_3: 0.21204577386379242 (0.2273205498026477) + | > loss_disc_real_4: 0.1668866127729416 (0.23079990057481659) + | > loss_disc_real_5: 0.21634899079799652 (0.23325799529751143) + | > loss_0: 2.4121029376983643 (2.4523098468780518) + | > grad_norm_0: tensor(6.3366, device='cuda:0') (tensor(9.3259, device='cuda:0')) + | > loss_gen: 2.281494617462158 (2.3775846891933012) + | > loss_kl: 1.8124701976776123 (1.8307632042302027) + | > loss_feat: 7.485271453857422 (7.267597675323486) + | > loss_mel: 18.044504165649414 (17.794401354259914) + | > loss_duration: 1.4198405742645264 (1.4186639388402302) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.043581008911133 (30.689011361863876) + | > grad_norm_1: tensor(73.8467, device='cuda:0') (tensor(240.4936, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.3551 (0.3380637831158108) + | > loader_time: 0.0053 (0.00494951672024197) + + + --> TIME: 2025-04-19 05:29:42 -- STEP: 61/142 -- GLOBAL_STEP: 435575 + | > loss_disc: 2.4061927795410156 (2.436410665512085) + | > loss_disc_real_0: 0.10806252062320709 (0.11318205480204253) + | > loss_disc_real_1: 0.19514983892440796 (0.20592767024626496) + | > loss_disc_real_2: 0.23546288907527924 (0.22543745329145526) + | > loss_disc_real_3: 0.23194420337677002 (0.2259404554230268) + | > loss_disc_real_4: 0.22039824724197388 (0.22879095009115877) + | > loss_disc_real_5: 0.22303080558776855 (0.2334821678575922) + | > loss_0: 2.4061927795410156 (2.436410665512085) + | > grad_norm_0: tensor(10.1057, device='cuda:0') (tensor(9.1401, device='cuda:0')) + | > loss_gen: 2.4988181591033936 (2.390434925673438) + | > loss_kl: 1.822868824005127 (1.8401768578857671) + | > loss_feat: 7.291682720184326 (7.287526458990379) + | > loss_mel: 18.401594161987305 (17.887364231172164) + | > loss_duration: 1.4563184976577759 (1.4326996881453717) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.471281051635742 (30.83820246086746) + | > grad_norm_1: tensor(94.3656, device='cuda:0') (tensor(252.1225, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.3816 (0.349841317192453) + | > loader_time: 0.0065 (0.005335608466726835) + + + --> TIME: 2025-04-19 05:29:52 -- STEP: 86/142 -- GLOBAL_STEP: 435600 + | > loss_disc: 2.491973876953125 (2.434086344962896) + | > loss_disc_real_0: 0.08005341142416 (0.11523327309378358) + | > loss_disc_real_1: 0.22881105542182922 (0.2055987701859585) + | > loss_disc_real_2: 0.24615901708602905 (0.2250924167591472) + | > loss_disc_real_3: 0.2426244616508484 (0.22664777154839316) + | > loss_disc_real_4: 0.24342438578605652 (0.2293192156872084) + | > loss_disc_real_5: 0.22660821676254272 (0.23266303521949191) + | > loss_0: 2.491973876953125 (2.434086344962896) + | > grad_norm_0: tensor(13.4285, device='cuda:0') (tensor(9.8050, device='cuda:0')) + | > loss_gen: 2.3419620990753174 (2.3870769378750825) + | > loss_kl: 1.9219890832901 (1.858869118745937) + | > loss_feat: 6.830631732940674 (7.249033495437267) + | > loss_mel: 18.45343589782715 (17.87558851685635) + | > loss_duration: 1.4831222295761108 (1.4438954730366547) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.03114128112793 (30.814463681952898) + | > grad_norm_1: tensor(277.5407, device='cuda:0') (tensor(264.3647, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.3979 (0.36185636631278123) + | > loader_time: 0.0067 (0.005647983661917753) + + + --> TIME: 2025-04-19 05:30:02 -- STEP: 111/142 -- GLOBAL_STEP: 435625 + | > loss_disc: 2.4975814819335938 (2.438778660318872) + | > loss_disc_real_0: 0.16375592350959778 (0.11845669633633382) + | > loss_disc_real_1: 0.22960934042930603 (0.20563458094188758) + | > loss_disc_real_2: 0.24205094575881958 (0.22472866819248544) + | > loss_disc_real_3: 0.2436840534210205 (0.22683414239604194) + | > loss_disc_real_4: 0.24933509528636932 (0.22808185327160466) + | > loss_disc_real_5: 0.25776082277297974 (0.23199601149236834) + | > loss_0: 2.4975814819335938 (2.438778660318872) + | > grad_norm_0: tensor(14.6536, device='cuda:0') (tensor(11.2601, device='cuda:0')) + | > loss_gen: 2.463533878326416 (2.3838743991679974) + | > loss_kl: 1.9336439371109009 (1.870358894537161) + | > loss_feat: 6.870155334472656 (7.2118192750054435) + | > loss_mel: 18.961772918701172 (17.806945921064507) + | > loss_duration: 1.496018409729004 (1.4515216125024326) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.72512435913086 (30.724520176380604) + | > grad_norm_1: tensor(189.7129, device='cuda:0') (tensor(273.0325, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.4249 (0.3725931923668665) + | > loader_time: 0.0072 (0.005933458740646775) + + + --> TIME: 2025-04-19 05:30:13 -- STEP: 136/142 -- GLOBAL_STEP: 435650 + | > loss_disc: 2.393279790878296 (2.4488210923531466) + | > loss_disc_real_0: 0.12162170559167862 (0.12018963732921026) + | > loss_disc_real_1: 0.19443348050117493 (0.2067894569652922) + | > loss_disc_real_2: 0.22286421060562134 (0.22600284394095926) + | > loss_disc_real_3: 0.20479227602481842 (0.2267593178240692) + | > loss_disc_real_4: 0.22226588428020477 (0.229136260555071) + | > loss_disc_real_5: 0.20992562174797058 (0.2325934932074126) + | > loss_0: 2.393279790878296 (2.4488210923531466) + | > grad_norm_0: tensor(33.0044, device='cuda:0') (tensor(12.0198, device='cuda:0')) + | > loss_gen: 2.4615659713745117 (2.3821161529597115) + | > loss_kl: 1.967942714691162 (1.8799155897953932) + | > loss_feat: 7.550363063812256 (7.168020662139444) + | > loss_mel: 18.862930297851562 (17.743923888486982) + | > loss_duration: 1.5103168487548828 (1.457812368869781) + | > amp_scaler: 128.0 (64.94117647058823) + | > loss_1: 32.353118896484375 (30.631788772695206) + | > grad_norm_1: tensor(493.5945, device='cuda:0') (tensor(281.5859, device='cuda:0')) + | > current_lr_0: 0.00019034044271155703 + | > current_lr_1: 0.00019034044271155703 + | > step_time: 0.4473 (0.384666120304781) + | > loader_time: 0.0073 (0.006212052176980411) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.486206531524658 (2.486206531524658) + | > loss_disc_real_0: 0.21264027059078217 (0.21264027059078217) + | > loss_disc_real_1: 0.21171553432941437 (0.21171553432941437) + | > loss_disc_real_2: 0.22588561475276947 (0.22588561475276947) + | > loss_disc_real_3: 0.25708919763565063 (0.25708919763565063) + | > loss_disc_real_4: 0.2004980742931366 (0.2004980742931366) + | > loss_disc_real_5: 0.22909118235111237 (0.22909118235111237) + | > loss_0: 2.486206531524658 (2.486206531524658) + | > loss_gen: 2.466341257095337 (2.466341257095337) + | > loss_kl: 2.538600444793701 (2.538600444793701) + | > loss_feat: 7.445999622344971 (7.445999622344971) + | > loss_mel: 18.554691314697266 (18.554691314697266) + | > loss_duration: 1.7594330310821533 (1.7594330310821533) + | > loss_1: 32.76506423950195 (32.76506423950195) + + --> STEP: 1 + | > loss_disc: 2.632075786590576 (2.632075786590576) + | > loss_disc_real_0: 0.30126675963401794 (0.30126675963401794) + | > loss_disc_real_1: 0.2614080309867859 (0.2614080309867859) + | > loss_disc_real_2: 0.2643886208534241 (0.2643886208534241) + | > loss_disc_real_3: 0.23276959359645844 (0.23276959359645844) + | > loss_disc_real_4: 0.23114268481731415 (0.23114268481731415) + | > loss_disc_real_5: 0.25872763991355896 (0.25872763991355896) + | > loss_0: 2.632075786590576 (2.632075786590576) + | > loss_gen: 2.6593539714813232 (2.6593539714813232) + | > loss_kl: 2.910003423690796 (2.910003423690796) + | > loss_feat: 7.019974708557129 (7.019974708557129) + | > loss_mel: 19.144378662109375 (19.144378662109375) + | > loss_duration: 1.8130066394805908 (1.8130066394805908) + | > loss_1: 33.546714782714844 (33.546714782714844) + + --> STEP: 2 + | > loss_disc: 2.576538562774658 (2.604307174682617) + | > loss_disc_real_0: 0.2564551830291748 (0.2788609713315964) + | > loss_disc_real_1: 0.2135370373725891 (0.2374725341796875) + | > loss_disc_real_2: 0.2373265027999878 (0.25085756182670593) + | > loss_disc_real_3: 0.24547995626926422 (0.23912477493286133) + | > loss_disc_real_4: 0.2485383301973343 (0.23984050750732422) + | > loss_disc_real_5: 0.20845703780651093 (0.23359233886003494) + | > loss_0: 2.576538562774658 (2.604307174682617) + | > loss_gen: 2.5124850273132324 (2.585919499397278) + | > loss_kl: 2.416485548019409 (2.6632444858551025) + | > loss_feat: 7.428849697113037 (7.224412202835083) + | > loss_mel: 18.24018096923828 (18.692279815673828) + | > loss_duration: 1.7418622970581055 (1.7774344682693481) + | > loss_1: 32.33986282348633 (32.943288803100586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036737918853759766 (-4.029273986816406e-05) + | > avg_loss_disc: 2.604307174682617 (+0.033342957496643066) + | > avg_loss_disc_real_0: 0.2788609713315964 (+0.08178906887769699) + | > avg_loss_disc_real_1: 0.2374725341796875 (+0.025978729128837585) + | > avg_loss_disc_real_2: 0.25085756182670593 (+0.01933632791042328) + | > avg_loss_disc_real_3: 0.23912477493286133 (+0.010962232947349548) + | > avg_loss_disc_real_4: 0.23984050750732422 (-0.017239153385162354) + | > avg_loss_disc_real_5: 0.23359233886003494 (+0.007899999618530273) + | > avg_loss_0: 2.604307174682617 (+0.033342957496643066) + | > avg_loss_gen: 2.585919499397278 (+0.23692750930786133) + | > avg_loss_kl: 2.6632444858551025 (-0.02177107334136963) + | > avg_loss_feat: 7.224412202835083 (+0.09012436866760254) + | > avg_loss_mel: 18.692279815673828 (+0.4658479690551758) + | > avg_loss_duration: 1.7774344682693481 (+0.03693968057632446) + | > avg_loss_1: 32.943288803100586 (+0.8080663681030273) + + + > EPOCH: 397/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:30:17)  + + --> TIME: 2025-04-19 05:30:25 -- STEP: 19/142 -- GLOBAL_STEP: 435675 + | > loss_disc: 2.3404579162597656 (2.414867551703202) + | > loss_disc_real_0: 0.09900718927383423 (0.10882557928562164) + | > loss_disc_real_1: 0.21437425911426544 (0.2062086384547384) + | > loss_disc_real_2: 0.23952174186706543 (0.2228790803959495) + | > loss_disc_real_3: 0.19549939036369324 (0.2257351992945922) + | > loss_disc_real_4: 0.22614607214927673 (0.23445986603435717) + | > loss_disc_real_5: 0.19931629300117493 (0.22910585293644353) + | > loss_0: 2.3404579162597656 (2.414867551703202) + | > grad_norm_0: tensor(5.9852, device='cuda:0') (tensor(7.5368, device='cuda:0')) + | > loss_gen: 2.4229423999786377 (2.3828757060201546) + | > loss_kl: 2.0099077224731445 (1.8135093136837608) + | > loss_feat: 7.704402446746826 (7.375580034757915) + | > loss_mel: 17.649568557739258 (17.519386090730364) + | > loss_duration: 1.4205864667892456 (1.414260745048523) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.207406997680664 (30.505612222771894) + | > grad_norm_1: tensor(195.6522, device='cuda:0') (tensor(207.6273, device='cuda:0')) + | > current_lr_0: 0.0001903166501562181 + | > current_lr_1: 0.0001903166501562181 + | > step_time: 0.3379 (0.3311722153111508) + | > loader_time: 0.0052 (0.004957562998721474) + + + --> TIME: 2025-04-19 05:30:34 -- STEP: 44/142 -- GLOBAL_STEP: 435700 + | > loss_disc: 2.5082831382751465 (2.435873947360299) + | > loss_disc_real_0: 0.16739173233509064 (0.11257035319100726) + | > loss_disc_real_1: 0.1984354704618454 (0.20678028125654568) + | > loss_disc_real_2: 0.21303211152553558 (0.2227748524058949) + | > loss_disc_real_3: 0.19863265752792358 (0.22834338349374858) + | > loss_disc_real_4: 0.19996511936187744 (0.22816448658704758) + | > loss_disc_real_5: 0.20947220921516418 (0.2328628965399482) + | > loss_0: 2.5082831382751465 (2.435873947360299) + | > grad_norm_0: tensor(7.3965, device='cuda:0') (tensor(7.6072, device='cuda:0')) + | > loss_gen: 2.205165386199951 (2.359862544319846) + | > loss_kl: 1.832842469215393 (1.8151134658943524) + | > loss_feat: 7.250372409820557 (7.2436757846312085) + | > loss_mel: 17.48455810546875 (17.590830802917484) + | > loss_duration: 1.4494335651397705 (1.427467622540214) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.22237205505371 (30.436950466849588) + | > grad_norm_1: tensor(168.0290, device='cuda:0') (tensor(182.3297, device='cuda:0')) + | > current_lr_0: 0.0001903166501562181 + | > current_lr_1: 0.0001903166501562181 + | > step_time: 0.3702 (0.346904462034052) + | > loader_time: 0.0055 (0.005336100404912776) + + + --> TIME: 2025-04-19 05:30:44 -- STEP: 69/142 -- GLOBAL_STEP: 435725 + | > loss_disc: 2.428248167037964 (2.4483040001081378) + | > loss_disc_real_0: 0.14323562383651733 (0.11583200097084044) + | > loss_disc_real_1: 0.16964435577392578 (0.20725971244383548) + | > loss_disc_real_2: 0.24531716108322144 (0.22413589643395465) + | > loss_disc_real_3: 0.21976152062416077 (0.2288483592911043) + | > loss_disc_real_4: 0.24490268528461456 (0.23074881386929663) + | > loss_disc_real_5: 0.24663491547107697 (0.23512972804947177) + | > loss_0: 2.428248167037964 (2.4483040001081378) + | > grad_norm_0: tensor(11.2229, device='cuda:0') (tensor(7.2458, device='cuda:0')) + | > loss_gen: 2.3580687046051025 (2.35864534239838) + | > loss_kl: 1.8730237483978271 (1.8432079968245134) + | > loss_feat: 7.037476062774658 (7.256998794666235) + | > loss_mel: 18.398054122924805 (17.610195740409523) + | > loss_duration: 1.5112404823303223 (1.436679848726245) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.1778621673584 (30.505727961443473) + | > grad_norm_1: tensor(240.2985, device='cuda:0') (tensor(172.5840, device='cuda:0')) + | > current_lr_0: 0.0001903166501562181 + | > current_lr_1: 0.0001903166501562181 + | > step_time: 0.4128 (0.36178388111833204) + | > loader_time: 0.0063 (0.005693055581355442) + + + --> TIME: 2025-04-19 05:30:54 -- STEP: 94/142 -- GLOBAL_STEP: 435750 + | > loss_disc: 2.4544081687927246 (2.4428540189215457) + | > loss_disc_real_0: 0.12076248973608017 (0.11681602705032268) + | > loss_disc_real_1: 0.20722903311252594 (0.20642586480429836) + | > loss_disc_real_2: 0.23223701119422913 (0.22436489164829254) + | > loss_disc_real_3: 0.22627466917037964 (0.22903386392491928) + | > loss_disc_real_4: 0.2156323492527008 (0.22880600345261554) + | > loss_disc_real_5: 0.2305476814508438 (0.23289784836642285) + | > loss_0: 2.4544081687927246 (2.4428540189215457) + | > grad_norm_0: tensor(14.1667, device='cuda:0') (tensor(7.9218, device='cuda:0')) + | > loss_gen: 2.220092535018921 (2.353433464435821) + | > loss_kl: 1.8091371059417725 (1.8606780551849527) + | > loss_feat: 6.847514629364014 (7.201664833312339) + | > loss_mel: 17.961776733398438 (17.643678989816216) + | > loss_duration: 1.4897236824035645 (1.4471655247059274) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.328243255615234 (30.50662109699655) + | > grad_norm_1: tensor(165.5451, device='cuda:0') (tensor(177.5531, device='cuda:0')) + | > current_lr_0: 0.0001903166501562181 + | > current_lr_1: 0.0001903166501562181 + | > step_time: 0.4108 (0.37412305588417855) + | > loader_time: 0.0069 (0.005989389216646236) + + + --> TIME: 2025-04-19 05:31:05 -- STEP: 119/142 -- GLOBAL_STEP: 435775 + | > loss_disc: 2.4385108947753906 (2.452840129868321) + | > loss_disc_real_0: 0.10425245016813278 (0.11835968970250682) + | > loss_disc_real_1: 0.1908525973558426 (0.20684232832003044) + | > loss_disc_real_2: 0.21321792900562286 (0.22505875382603718) + | > loss_disc_real_3: 0.24513660371303558 (0.22915964617448695) + | > loss_disc_real_4: 0.26911109685897827 (0.23013420177607977) + | > loss_disc_real_5: 0.2477129101753235 (0.23487574049905569) + | > loss_0: 2.4385108947753906 (2.452840129868321) + | > grad_norm_0: tensor(11.1829, device='cuda:0') (tensor(8.5949, device='cuda:0')) + | > loss_gen: 2.3057425022125244 (2.3533905474077756) + | > loss_kl: 1.9228289127349854 (1.8660211052213396) + | > loss_feat: 6.900138854980469 (7.190087907454547) + | > loss_mel: 18.027027130126953 (17.662451479615296) + | > loss_duration: 1.519163966178894 (1.4536807206498474) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.67490005493164 (30.5256319446724) + | > grad_norm_1: tensor(291.6254, device='cuda:0') (tensor(204.3903, device='cuda:0')) + | > current_lr_0: 0.0001903166501562181 + | > current_lr_1: 0.0001903166501562181 + | > step_time: 0.4376 (0.3865109591924843) + | > loader_time: 0.0072 (0.00626481480959083) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6403985023498535 (2.6403985023498535) + | > loss_disc_real_0: 0.1434883177280426 (0.1434883177280426) + | > loss_disc_real_1: 0.24221159517765045 (0.24221159517765045) + | > loss_disc_real_2: 0.26145145297050476 (0.26145145297050476) + | > loss_disc_real_3: 0.2809748649597168 (0.2809748649597168) + | > loss_disc_real_4: 0.2179907262325287 (0.2179907262325287) + | > loss_disc_real_5: 0.2593517601490021 (0.2593517601490021) + | > loss_0: 2.6403985023498535 (2.6403985023498535) + | > loss_gen: 2.1768641471862793 (2.1768641471862793) + | > loss_kl: 2.670485734939575 (2.670485734939575) + | > loss_feat: 5.516856670379639 (5.516856670379639) + | > loss_mel: 16.816455841064453 (16.816455841064453) + | > loss_duration: 1.7568098306655884 (1.7568098306655884) + | > loss_1: 28.93747329711914 (28.93747329711914) + + --> STEP: 1 + | > loss_disc: 2.5758213996887207 (2.5758213996887207) + | > loss_disc_real_0: 0.15560953319072723 (0.15560953319072723) + | > loss_disc_real_1: 0.26073160767555237 (0.26073160767555237) + | > loss_disc_real_2: 0.24765384197235107 (0.24765384197235107) + | > loss_disc_real_3: 0.2343684881925583 (0.2343684881925583) + | > loss_disc_real_4: 0.23268014192581177 (0.23268014192581177) + | > loss_disc_real_5: 0.21307547390460968 (0.21307547390460968) + | > loss_0: 2.5758213996887207 (2.5758213996887207) + | > loss_gen: 2.276371479034424 (2.276371479034424) + | > loss_kl: 2.8168370723724365 (2.8168370723724365) + | > loss_feat: 7.169724464416504 (7.169724464416504) + | > loss_mel: 18.165077209472656 (18.165077209472656) + | > loss_duration: 1.7832947969436646 (1.7832947969436646) + | > loss_1: 32.2113037109375 (32.2113037109375) + + --> STEP: 2 + | > loss_disc: 2.5291802883148193 (2.55250084400177) + | > loss_disc_real_0: 0.15246543288230896 (0.1540374830365181) + | > loss_disc_real_1: 0.27107182145118713 (0.26590171456336975) + | > loss_disc_real_2: 0.2349437028169632 (0.24129877239465714) + | > loss_disc_real_3: 0.22553777694702148 (0.2299531325697899) + | > loss_disc_real_4: 0.24079634249210358 (0.23673824220895767) + | > loss_disc_real_5: 0.2195950150489807 (0.2163352444767952) + | > loss_0: 2.5291802883148193 (2.55250084400177) + | > loss_gen: 2.2880859375 (2.282228708267212) + | > loss_kl: 2.2439639568328857 (2.530400514602661) + | > loss_feat: 7.0754714012146 (7.122597932815552) + | > loss_mel: 16.796539306640625 (17.48080825805664) + | > loss_duration: 1.7119303941726685 (1.7476125955581665) + | > loss_1: 30.115991592407227 (31.163647651672363) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003596186637878418 (-7.76052474975586e-05) + | > avg_loss_disc: 2.55250084400177 (-0.05180633068084717) + | > avg_loss_disc_real_0: 0.1540374830365181 (-0.12482348829507828) + | > avg_loss_disc_real_1: 0.26590171456336975 (+0.02842918038368225) + | > avg_loss_disc_real_2: 0.24129877239465714 (-0.009558789432048798) + | > avg_loss_disc_real_3: 0.2299531325697899 (-0.009171642363071442) + | > avg_loss_disc_real_4: 0.23673824220895767 (-0.0031022652983665466) + | > avg_loss_disc_real_5: 0.2163352444767952 (-0.017257094383239746) + | > avg_loss_0: 2.55250084400177 (-0.05180633068084717) + | > avg_loss_gen: 2.282228708267212 (-0.3036907911300659) + | > avg_loss_kl: 2.530400514602661 (-0.1328439712524414) + | > avg_loss_feat: 7.122597932815552 (-0.10181427001953125) + | > avg_loss_mel: 17.48080825805664 (-1.2114715576171875) + | > avg_loss_duration: 1.7476125955581665 (-0.02982187271118164) + | > avg_loss_1: 31.163647651672363 (-1.7796411514282227) + + + > EPOCH: 398/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:31:18)  + + --> TIME: 2025-04-19 05:31:19 -- STEP: 2/142 -- GLOBAL_STEP: 435800 + | > loss_disc: 2.4373912811279297 (2.4859848022460938) + | > loss_disc_real_0: 0.1415206640958786 (0.14136271178722382) + | > loss_disc_real_1: 0.23248997330665588 (0.22008268535137177) + | > loss_disc_real_2: 0.2017095983028412 (0.21599430590867996) + | > loss_disc_real_3: 0.24481892585754395 (0.24036063253879547) + | > loss_disc_real_4: 0.22055178880691528 (0.24108058214187622) + | > loss_disc_real_5: 0.2148081511259079 (0.2261463776230812) + | > loss_0: 2.4373912811279297 (2.4859848022460938) + | > grad_norm_0: tensor(5.8912, device='cuda:0') (tensor(5.4470, device='cuda:0')) + | > loss_gen: 2.3152682781219482 (2.31731915473938) + | > loss_kl: 1.819519281387329 (1.7846149802207947) + | > loss_feat: 7.0503668785095215 (6.920463800430298) + | > loss_mel: 17.301292419433594 (17.388572692871094) + | > loss_duration: 1.4114331007003784 (1.3991909623146057) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.897878646850586 (29.810161590576172) + | > grad_norm_1: tensor(154.5540, device='cuda:0') (tensor(136.6765, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.3404 (0.3442424535751343) + | > loader_time: 0.0036 (0.003808259963989258) + + + --> TIME: 2025-04-19 05:31:28 -- STEP: 27/142 -- GLOBAL_STEP: 435825 + | > loss_disc: 2.508795738220215 (2.471726338068644) + | > loss_disc_real_0: 0.0887274518609047 (0.11918272823095322) + | > loss_disc_real_1: 0.19999076426029205 (0.20748849765018182) + | > loss_disc_real_2: 0.2235664278268814 (0.22268654571639168) + | > loss_disc_real_3: 0.21108092367649078 (0.23020755196059192) + | > loss_disc_real_4: 0.22334380447864532 (0.229514354357013) + | > loss_disc_real_5: 0.20573806762695312 (0.23853064411216313) + | > loss_0: 2.508795738220215 (2.471726338068644) + | > grad_norm_0: tensor(7.2002, device='cuda:0') (tensor(6.7387, device='cuda:0')) + | > loss_gen: 2.2409989833831787 (2.340467762064051) + | > loss_kl: 1.8907173871994019 (1.8146034302534881) + | > loss_feat: 6.879429340362549 (7.262088281136972) + | > loss_mel: 17.166704177856445 (17.604356165285463) + | > loss_duration: 1.4240541458129883 (1.4159290260738797) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.601905822753906 (30.43744475753219) + | > grad_norm_1: tensor(166.8987, device='cuda:0') (tensor(176.4095, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.3474 (0.33836879553618254) + | > loader_time: 0.0049 (0.0048254066043429905) + + + --> TIME: 2025-04-19 05:31:37 -- STEP: 52/142 -- GLOBAL_STEP: 435850 + | > loss_disc: 2.550413131713867 (2.4529580657298755) + | > loss_disc_real_0: 0.16381192207336426 (0.11988154927698466) + | > loss_disc_real_1: 0.21527718007564545 (0.20568131999327585) + | > loss_disc_real_2: 0.2595621645450592 (0.22404796888048834) + | > loss_disc_real_3: 0.2252785861492157 (0.23053095489740372) + | > loss_disc_real_4: 0.2755233645439148 (0.22915916803937691) + | > loss_disc_real_5: 0.2322578877210617 (0.23457792429969862) + | > loss_0: 2.550413131713867 (2.4529580657298755) + | > grad_norm_0: tensor(5.6154, device='cuda:0') (tensor(7.5744, device='cuda:0')) + | > loss_gen: 2.310231924057007 (2.360854433133052) + | > loss_kl: 1.7722193002700806 (1.8250582470343664) + | > loss_feat: 6.633471488952637 (7.213125274731563) + | > loss_mel: 18.637611389160156 (17.65299197343679) + | > loss_duration: 1.4798686504364014 (1.426490902900696) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.833402633666992 (30.47852087020874) + | > grad_norm_1: tensor(240.1420, device='cuda:0') (tensor(183.3773, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.372 (0.3489384697033809) + | > loader_time: 0.0055 (0.005216470131507287) + + + --> TIME: 2025-04-19 05:31:47 -- STEP: 77/142 -- GLOBAL_STEP: 435875 + | > loss_disc: 2.4459292888641357 (2.461288157995646) + | > loss_disc_real_0: 0.09983613342046738 (0.12032074945700633) + | > loss_disc_real_1: 0.16819940507411957 (0.20560990699699946) + | > loss_disc_real_2: 0.2351793348789215 (0.2242341126714434) + | > loss_disc_real_3: 0.23201435804367065 (0.2314192306298714) + | > loss_disc_real_4: 0.2304045408964157 (0.23104881621026374) + | > loss_disc_real_5: 0.25193536281585693 (0.23442205186788137) + | > loss_0: 2.4459292888641357 (2.461288157995646) + | > grad_norm_0: tensor(8.0569, device='cuda:0') (tensor(7.6149, device='cuda:0')) + | > loss_gen: 2.396862030029297 (2.3541141268494843) + | > loss_kl: 1.8452092409133911 (1.8298718944772498) + | > loss_feat: 6.998012065887451 (7.218433293429288) + | > loss_mel: 18.72812271118164 (17.67988888629071) + | > loss_duration: 1.47747004032135 (1.4383338829139611) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.445676803588867 (30.52064219388095) + | > grad_norm_1: tensor(82.5077, device='cuda:0') (tensor(177.6222, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.3906 (0.36125843865530827) + | > loader_time: 0.0067 (0.00557861699686422) + + + --> TIME: 2025-04-19 05:31:57 -- STEP: 102/142 -- GLOBAL_STEP: 435900 + | > loss_disc: 2.627495527267456 (2.463038771760231) + | > loss_disc_real_0: 0.24812665581703186 (0.1226056855829323) + | > loss_disc_real_1: 0.22148093581199646 (0.20776683778739444) + | > loss_disc_real_2: 0.2300592064857483 (0.22460053013820275) + | > loss_disc_real_3: 0.23996862769126892 (0.23090852928512237) + | > loss_disc_real_4: 0.2638859450817108 (0.22953285116191005) + | > loss_disc_real_5: 0.22751925885677338 (0.23387157566407146) + | > loss_0: 2.627495527267456 (2.463038771760231) + | > grad_norm_0: tensor(13.4765, device='cuda:0') (tensor(7.8799, device='cuda:0')) + | > loss_gen: 2.172396421432495 (2.3521138289395505) + | > loss_kl: 1.8543522357940674 (1.838918181026683) + | > loss_feat: 5.961528778076172 (7.1716034365635295) + | > loss_mel: 15.39820384979248 (17.67724547666662) + | > loss_duration: 1.4641246795654297 (1.449537560051563) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 26.850605010986328 (30.48941862816904) + | > grad_norm_1: tensor(96.9235, device='cuda:0') (tensor(171.5991, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.4225 (0.371989649884841) + | > loader_time: 0.0072 (0.005865756203146543) + + + --> TIME: 2025-04-19 05:32:08 -- STEP: 127/142 -- GLOBAL_STEP: 435925 + | > loss_disc: 2.563107967376709 (2.4639405828761314) + | > loss_disc_real_0: 0.14047406613826752 (0.12396458407321315) + | > loss_disc_real_1: 0.2012191265821457 (0.20700498500208217) + | > loss_disc_real_2: 0.21500840783119202 (0.2250567498638874) + | > loss_disc_real_3: 0.21896414458751678 (0.23066262161637854) + | > loss_disc_real_4: 0.24676856398582458 (0.22927373917553368) + | > loss_disc_real_5: 0.22351570427417755 (0.23344828528682077) + | > loss_0: 2.563107967376709 (2.4639405828761314) + | > grad_norm_0: tensor(17.8312, device='cuda:0') (tensor(8.7251, device='cuda:0')) + | > loss_gen: 2.1834683418273926 (2.348855502962128) + | > loss_kl: 1.786476492881775 (1.8461996007153367) + | > loss_feat: 6.560535430908203 (7.143954810195082) + | > loss_mel: 16.31253433227539 (17.6468734591026) + | > loss_duration: 1.4733963012695312 (1.4562867495018668) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.316410064697266 (30.442170225729154) + | > grad_norm_1: tensor(335.4395, device='cuda:0') (tensor(187.4281, device='cuda:0')) + | > current_lr_0: 0.00019029286057494854 + | > current_lr_1: 0.00019029286057494854 + | > step_time: 0.4472 (0.3844886032615121) + | > loader_time: 0.0076 (0.006130742275808741) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4605610370635986 (2.4605610370635986) + | > loss_disc_real_0: 0.16370157897472382 (0.16370157897472382) + | > loss_disc_real_1: 0.199336439371109 (0.199336439371109) + | > loss_disc_real_2: 0.22762443125247955 (0.22762443125247955) + | > loss_disc_real_3: 0.2607642114162445 (0.2607642114162445) + | > loss_disc_real_4: 0.2253626585006714 (0.2253626585006714) + | > loss_disc_real_5: 0.2732628881931305 (0.2732628881931305) + | > loss_0: 2.4605610370635986 (2.4605610370635986) + | > loss_gen: 2.4841620922088623 (2.4841620922088623) + | > loss_kl: 2.310981035232544 (2.310981035232544) + | > loss_feat: 6.516911029815674 (6.516911029815674) + | > loss_mel: 17.78685760498047 (17.78685760498047) + | > loss_duration: 1.723923683166504 (1.723923683166504) + | > loss_1: 30.822834014892578 (30.822834014892578) + + --> STEP: 1 + | > loss_disc: 2.573052406311035 (2.573052406311035) + | > loss_disc_real_0: 0.1762106865644455 (0.1762106865644455) + | > loss_disc_real_1: 0.21271274983882904 (0.21271274983882904) + | > loss_disc_real_2: 0.23513783514499664 (0.23513783514499664) + | > loss_disc_real_3: 0.2466524839401245 (0.2466524839401245) + | > loss_disc_real_4: 0.23981842398643494 (0.23981842398643494) + | > loss_disc_real_5: 0.25120648741722107 (0.25120648741722107) + | > loss_0: 2.573052406311035 (2.573052406311035) + | > loss_gen: 2.2147417068481445 (2.2147417068481445) + | > loss_kl: 2.579193592071533 (2.579193592071533) + | > loss_feat: 5.131350040435791 (5.131350040435791) + | > loss_mel: 14.108842849731445 (14.108842849731445) + | > loss_duration: 1.7659598588943481 (1.7659598588943481) + | > loss_1: 25.800086975097656 (25.800086975097656) + + --> STEP: 2 + | > loss_disc: 2.5554862022399902 (2.5642693042755127) + | > loss_disc_real_0: 0.14086312055587769 (0.1585369035601616) + | > loss_disc_real_1: 0.21023541688919067 (0.21147408336400986) + | > loss_disc_real_2: 0.23171140253543854 (0.2334246188402176) + | > loss_disc_real_3: 0.24311329424381256 (0.24488288909196854) + | > loss_disc_real_4: 0.24909108877182007 (0.2444547563791275) + | > loss_disc_real_5: 0.1937854290008545 (0.22249595820903778) + | > loss_0: 2.5554862022399902 (2.5642693042755127) + | > loss_gen: 2.072838544845581 (2.143790125846863) + | > loss_kl: 2.300816535949707 (2.44000506401062) + | > loss_feat: 5.269586086273193 (5.200468063354492) + | > loss_mel: 15.143622398376465 (14.626232624053955) + | > loss_duration: 1.7366487979888916 (1.7513043284416199) + | > loss_1: 26.52351188659668 (26.161799430847168) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038077831268310547 (+0.00021159648895263672) + | > avg_loss_disc: 2.5642693042755127 (+0.011768460273742676) + | > avg_loss_disc_real_0: 0.1585369035601616 (+0.004499420523643494) + | > avg_loss_disc_real_1: 0.21147408336400986 (-0.054427631199359894) + | > avg_loss_disc_real_2: 0.2334246188402176 (-0.007874153554439545) + | > avg_loss_disc_real_3: 0.24488288909196854 (+0.01492975652217865) + | > avg_loss_disc_real_4: 0.2444547563791275 (+0.00771651417016983) + | > avg_loss_disc_real_5: 0.22249595820903778 (+0.006160713732242584) + | > avg_loss_0: 2.5642693042755127 (+0.011768460273742676) + | > avg_loss_gen: 2.143790125846863 (-0.13843858242034912) + | > avg_loss_kl: 2.44000506401062 (-0.09039545059204102) + | > avg_loss_feat: 5.200468063354492 (-1.9221298694610596) + | > avg_loss_mel: 14.626232624053955 (-2.8545756340026855) + | > avg_loss_duration: 1.7513043284416199 (+0.003691732883453369) + | > avg_loss_1: 26.161799430847168 (-5.001848220825195) + + > BEST MODEL : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/best_model_435940.pth + + > EPOCH: 399/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:32:18)  + + --> TIME: 2025-04-19 05:32:22 -- STEP: 10/142 -- GLOBAL_STEP: 435950 + | > loss_disc: 2.3851723670959473 (2.37983980178833) + | > loss_disc_real_0: 0.10930247604846954 (0.10431078970432281) + | > loss_disc_real_1: 0.19093644618988037 (0.2007538989186287) + | > loss_disc_real_2: 0.20808489620685577 (0.22506791353225708) + | > loss_disc_real_3: 0.25548529624938965 (0.21798461824655532) + | > loss_disc_real_4: 0.2554760277271271 (0.22874603122472764) + | > loss_disc_real_5: 0.24837203323841095 (0.215793377161026) + | > loss_0: 2.3851723670959473 (2.37983980178833) + | > grad_norm_0: tensor(18.4372, device='cuda:0') (tensor(14.6117, device='cuda:0')) + | > loss_gen: 2.5900821685791016 (2.4671926259994508) + | > loss_kl: 1.882246971130371 (1.815765130519867) + | > loss_feat: 7.9053425788879395 (7.606401109695435) + | > loss_mel: 18.563640594482422 (18.050661277770995) + | > loss_duration: 1.3724699020385742 (1.4031251668930054) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.31378173828125 (31.343145179748536) + | > grad_norm_1: tensor(459.8127, device='cuda:0') (tensor(332.4659, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.3352 (0.32904925346374514) + | > loader_time: 0.0048 (0.004479122161865234) + + + --> TIME: 2025-04-19 05:32:31 -- STEP: 35/142 -- GLOBAL_STEP: 435975 + | > loss_disc: 2.350710868835449 (2.3982226371765143) + | > loss_disc_real_0: 0.10306619852781296 (0.10745932289532252) + | > loss_disc_real_1: 0.21502457559108734 (0.20011274346283503) + | > loss_disc_real_2: 0.2277529090642929 (0.22102223890168327) + | > loss_disc_real_3: 0.19377785921096802 (0.225676315171378) + | > loss_disc_real_4: 0.20189757645130157 (0.22873335140092033) + | > loss_disc_real_5: 0.20996038615703583 (0.22769679980618612) + | > loss_0: 2.350710868835449 (2.3982226371765143) + | > grad_norm_0: tensor(7.9920, device='cuda:0') (tensor(11.9077, device='cuda:0')) + | > loss_gen: 2.3968045711517334 (2.4424024036952425) + | > loss_kl: 1.9195858240127563 (1.820165773800441) + | > loss_feat: 7.9685845375061035 (7.472947079794747) + | > loss_mel: 17.616004943847656 (17.824239785330636) + | > loss_duration: 1.4450898170471191 (1.4202888556889126) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.3460693359375 (30.980043738228932) + | > grad_norm_1: tensor(222.3108, device='cuda:0') (tensor(317.7265, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.3573 (0.33969326700483043) + | > loader_time: 0.0052 (0.004927185603550502) + + + --> TIME: 2025-04-19 05:32:40 -- STEP: 60/142 -- GLOBAL_STEP: 436000 + | > loss_disc: 2.5046746730804443 (2.4246628284454346) + | > loss_disc_real_0: 0.11818287521600723 (0.11241025738418102) + | > loss_disc_real_1: 0.18676181137561798 (0.2019849496583144) + | > loss_disc_real_2: 0.26390889286994934 (0.22638298446933428) + | > loss_disc_real_3: 0.24764125049114227 (0.22761944731076558) + | > loss_disc_real_4: 0.2434876263141632 (0.23042628690600395) + | > loss_disc_real_5: 0.22819668054580688 (0.2309105485677719) + | > loss_0: 2.5046746730804443 (2.4246628284454346) + | > grad_norm_0: tensor(16.6799, device='cuda:0') (tensor(10.4970, device='cuda:0')) + | > loss_gen: 2.354194164276123 (2.406433343887329) + | > loss_kl: 1.8432928323745728 (1.835377585887909) + | > loss_feat: 7.177532196044922 (7.366952808698018) + | > loss_mel: 17.85126304626465 (17.68721017837525) + | > loss_duration: 1.4672820568084717 (1.4330030004183452) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.693565368652344 (30.728976917266845) + | > grad_norm_1: tensor(274.1912, device='cuda:0') (tensor(269.2814, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.3767 (0.35104186137517285) + | > loader_time: 0.0059 (0.005204455057779948) + + + --> TIME: 2025-04-19 05:32:50 -- STEP: 85/142 -- GLOBAL_STEP: 436025 + | > loss_disc: 2.5424880981445312 (2.4390808974995335) + | > loss_disc_real_0: 0.13723988831043243 (0.11660625846946941) + | > loss_disc_real_1: 0.2588079273700714 (0.2030693027903052) + | > loss_disc_real_2: 0.23510761559009552 (0.22684467943275677) + | > loss_disc_real_3: 0.22198174893856049 (0.2277713649413165) + | > loss_disc_real_4: 0.21570242941379547 (0.2293149667627671) + | > loss_disc_real_5: 0.23730385303497314 (0.23199147915138918) + | > loss_0: 2.5424880981445312 (2.4390808974995335) + | > grad_norm_0: tensor(6.0549, device='cuda:0') (tensor(9.9047, device='cuda:0')) + | > loss_gen: 2.3495798110961914 (2.392308768104105) + | > loss_kl: 1.9192816019058228 (1.8449661703670726) + | > loss_feat: 6.914753437042236 (7.30289285884184) + | > loss_mel: 17.59786605834961 (17.69990292717429) + | > loss_duration: 1.4829673767089844 (1.4434222445768468) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.264446258544922 (30.683492974674) + | > grad_norm_1: tensor(69.9053, device='cuda:0') (tensor(238.4276, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.3977 (0.3632357288809383) + | > loader_time: 0.0066 (0.005474197163301355) + + + --> TIME: 2025-04-19 05:33:00 -- STEP: 110/142 -- GLOBAL_STEP: 436050 + | > loss_disc: 2.409928321838379 (2.4503199382261798) + | > loss_disc_real_0: 0.09647037088871002 (0.11959443180398507) + | > loss_disc_real_1: 0.2163996696472168 (0.20376940857280385) + | > loss_disc_real_2: 0.24178946018218994 (0.22671571522951126) + | > loss_disc_real_3: 0.2347489446401596 (0.2278889913450588) + | > loss_disc_real_4: 0.2442379891872406 (0.22986218441616404) + | > loss_disc_real_5: 0.23486639559268951 (0.23271048204465347) + | > loss_0: 2.409928321838379 (2.4503199382261798) + | > grad_norm_0: tensor(6.1666, device='cuda:0') (tensor(10.3987, device='cuda:0')) + | > loss_gen: 2.465944290161133 (2.3791912230578345) + | > loss_kl: 1.9720447063446045 (1.8588869094848632) + | > loss_feat: 7.116298675537109 (7.24209338534962) + | > loss_mel: 18.130863189697266 (17.70007390108976) + | > loss_duration: 1.473578691482544 (1.4521796627478165) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.15873146057129 (30.63242515217174) + | > grad_norm_1: tensor(213.1521, device='cuda:0') (tensor(251.4369, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.4332 (0.37360439517281274) + | > loader_time: 0.0069 (0.0057283618233420625) + + + --> TIME: 2025-04-19 05:33:12 -- STEP: 135/142 -- GLOBAL_STEP: 436075 + | > loss_disc: 2.547308921813965 (2.4508190914436625) + | > loss_disc_real_0: 0.15966705977916718 (0.12023145021111876) + | > loss_disc_real_1: 0.22346676886081696 (0.20406295337058877) + | > loss_disc_real_2: 0.24225017428398132 (0.22634985733915258) + | > loss_disc_real_3: 0.25296661257743835 (0.22767275053041952) + | > loss_disc_real_4: 0.26543909311294556 (0.2291094875997967) + | > loss_disc_real_5: 0.23641732335090637 (0.23302891861509395) + | > loss_0: 2.547308921813965 (2.4508190914436625) + | > grad_norm_0: tensor(11.8587, device='cuda:0') (tensor(10.5376, device='cuda:0')) + | > loss_gen: 2.418551206588745 (2.377093924416437) + | > loss_kl: 1.8111761808395386 (1.8679575699347037) + | > loss_feat: 7.077332973480225 (7.219165953883419) + | > loss_mel: 17.11647605895996 (17.646193610297317) + | > loss_duration: 1.4707685708999634 (1.4588012174323752) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.894304275512695 (30.569212355437102) + | > grad_norm_1: tensor(162.4544, device='cuda:0') (tensor(261.9301, device='cuda:0')) + | > current_lr_0: 0.00019026907396737667 + | > current_lr_1: 0.00019026907396737667 + | > step_time: 0.4761 (0.38582501234831634) + | > loader_time: 0.0073 (0.005998625578703703) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.489400863647461 (2.489400863647461) + | > loss_disc_real_0: 0.13458314538002014 (0.13458314538002014) + | > loss_disc_real_1: 0.2138998657464981 (0.2138998657464981) + | > loss_disc_real_2: 0.2476944625377655 (0.2476944625377655) + | > loss_disc_real_3: 0.2753394544124603 (0.2753394544124603) + | > loss_disc_real_4: 0.24119864404201508 (0.24119864404201508) + | > loss_disc_real_5: 0.28173890709877014 (0.28173890709877014) + | > loss_0: 2.489400863647461 (2.489400863647461) + | > loss_gen: 2.4387056827545166 (2.4387056827545166) + | > loss_kl: 2.480422258377075 (2.480422258377075) + | > loss_feat: 6.602665901184082 (6.602665901184082) + | > loss_mel: 18.79323959350586 (18.79323959350586) + | > loss_duration: 1.723825454711914 (1.723825454711914) + | > loss_1: 32.038856506347656 (32.038856506347656) + + --> STEP: 1 + | > loss_disc: 2.555330276489258 (2.555330276489258) + | > loss_disc_real_0: 0.11709149181842804 (0.11709149181842804) + | > loss_disc_real_1: 0.2194146066904068 (0.2194146066904068) + | > loss_disc_real_2: 0.2935091257095337 (0.2935091257095337) + | > loss_disc_real_3: 0.2877887785434723 (0.2877887785434723) + | > loss_disc_real_4: 0.2617529630661011 (0.2617529630661011) + | > loss_disc_real_5: 0.27963346242904663 (0.27963346242904663) + | > loss_0: 2.555330276489258 (2.555330276489258) + | > loss_gen: 2.3365747928619385 (2.3365747928619385) + | > loss_kl: 2.5916876792907715 (2.5916876792907715) + | > loss_feat: 7.4179511070251465 (7.4179511070251465) + | > loss_mel: 18.7424259185791 (18.7424259185791) + | > loss_duration: 1.7913678884506226 (1.7913678884506226) + | > loss_1: 32.8800048828125 (32.8800048828125) + + --> STEP: 2 + | > loss_disc: 2.4947378635406494 (2.5250340700149536) + | > loss_disc_real_0: 0.17683374881744385 (0.14696262031793594) + | > loss_disc_real_1: 0.22899924218654633 (0.22420692443847656) + | > loss_disc_real_2: 0.23586241900920868 (0.2646857723593712) + | > loss_disc_real_3: 0.2837839126586914 (0.28578634560108185) + | > loss_disc_real_4: 0.19847890734672546 (0.23011593520641327) + | > loss_disc_real_5: 0.255027711391449 (0.2673305869102478) + | > loss_0: 2.4947378635406494 (2.5250340700149536) + | > loss_gen: 2.391573905944824 (2.3640743494033813) + | > loss_kl: 2.4819252490997314 (2.5368064641952515) + | > loss_feat: 6.568681716918945 (6.993316411972046) + | > loss_mel: 17.718259811401367 (18.230342864990234) + | > loss_duration: 1.7160894870758057 (1.7537286877632141) + | > loss_1: 30.87653160095215 (31.878268241882324) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038808584213256836 (+7.30752944946289e-05) + | > avg_loss_disc: 2.5250340700149536 (-0.03923523426055908) + | > avg_loss_disc_real_0: 0.14696262031793594 (-0.011574283242225647) + | > avg_loss_disc_real_1: 0.22420692443847656 (+0.012732841074466705) + | > avg_loss_disc_real_2: 0.2646857723593712 (+0.031261153519153595) + | > avg_loss_disc_real_3: 0.28578634560108185 (+0.04090345650911331) + | > avg_loss_disc_real_4: 0.23011593520641327 (-0.014338821172714233) + | > avg_loss_disc_real_5: 0.2673305869102478 (+0.04483462870121002) + | > avg_loss_0: 2.5250340700149536 (-0.03923523426055908) + | > avg_loss_gen: 2.3640743494033813 (+0.22028422355651855) + | > avg_loss_kl: 2.5368064641952515 (+0.09680140018463135) + | > avg_loss_feat: 6.993316411972046 (+1.7928483486175537) + | > avg_loss_mel: 18.230342864990234 (+3.6041102409362793) + | > avg_loss_duration: 1.7537286877632141 (+0.0024243593215942383) + | > avg_loss_1: 31.878268241882324 (+5.716468811035156) + + + > EPOCH: 400/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:33:16)  + + --> TIME: 2025-04-19 05:33:23 -- STEP: 18/142 -- GLOBAL_STEP: 436100 + | > loss_disc: 2.388611316680908 (2.425840457280477) + | > loss_disc_real_0: 0.10057848691940308 (0.11120564035243458) + | > loss_disc_real_1: 0.18780076503753662 (0.20619316895802817) + | > loss_disc_real_2: 0.23187290132045746 (0.22458343538973066) + | > loss_disc_real_3: 0.2388467639684677 (0.2269100397825241) + | > loss_disc_real_4: 0.2518591582775116 (0.22418252544270623) + | > loss_disc_real_5: 0.24802295863628387 (0.23555707352028954) + | > loss_0: 2.388611316680908 (2.425840457280477) + | > grad_norm_0: tensor(3.8443, device='cuda:0') (tensor(9.4551, device='cuda:0')) + | > loss_gen: 2.311516761779785 (2.3721699582205877) + | > loss_kl: 1.8822484016418457 (1.7988317608833313) + | > loss_feat: 7.184865951538086 (7.315764983495076) + | > loss_mel: 17.396400451660156 (17.511729346381294) + | > loss_duration: 1.4068944454193115 (1.4117012752426996) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.181926727294922 (30.41019736395942) + | > grad_norm_1: tensor(188.5304, device='cuda:0') (tensor(264.6609, device='cuda:0')) + | > current_lr_0: 0.00019024529033313074 + | > current_lr_1: 0.00019024529033313074 + | > step_time: 0.3385 (0.330752862824334) + | > loader_time: 0.005 (0.0047111378775702584) + + + --> TIME: 2025-04-19 05:33:32 -- STEP: 43/142 -- GLOBAL_STEP: 436125 + | > loss_disc: 2.4058196544647217 (2.415857170903406) + | > loss_disc_real_0: 0.1217573806643486 (0.11428836403891098) + | > loss_disc_real_1: 0.20361004769802094 (0.20233059553212898) + | > loss_disc_real_2: 0.2087794989347458 (0.2223976791598076) + | > loss_disc_real_3: 0.2197713702917099 (0.22810493063095005) + | > loss_disc_real_4: 0.1951216757297516 (0.22463919430278068) + | > loss_disc_real_5: 0.21702176332473755 (0.23353413267191067) + | > loss_0: 2.4058196544647217 (2.415857170903406) + | > grad_norm_0: tensor(10.2168, device='cuda:0') (tensor(8.6482, device='cuda:0')) + | > loss_gen: 2.318878412246704 (2.3990216532418893) + | > loss_kl: 1.8051866292953491 (1.8241880106371502) + | > loss_feat: 6.911484241485596 (7.329752766808798) + | > loss_mel: 17.508222579956055 (17.627919086190158) + | > loss_duration: 1.4586142301559448 (1.4258822258128674) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.00238609313965 (30.6067636622939) + | > grad_norm_1: tensor(203.3082, device='cuda:0') (tensor(220.6579, device='cuda:0')) + | > current_lr_0: 0.00019024529033313074 + | > current_lr_1: 0.00019024529033313074 + | > step_time: 0.3628 (0.34316190453462836) + | > loader_time: 0.0056 (0.005138630090757859) + + + --> TIME: 2025-04-19 05:33:42 -- STEP: 68/142 -- GLOBAL_STEP: 436150 + | > loss_disc: 2.481423854827881 (2.4352464746026437) + | > loss_disc_real_0: 0.11908083409070969 (0.11825970713706578) + | > loss_disc_real_1: 0.2542755901813507 (0.2065566530560746) + | > loss_disc_real_2: 0.2517121732234955 (0.22387104569112554) + | > loss_disc_real_3: 0.21197500824928284 (0.22691360707668698) + | > loss_disc_real_4: 0.23071669042110443 (0.22497812307932796) + | > loss_disc_real_5: 0.19782891869544983 (0.23231921980486198) + | > loss_0: 2.481423854827881 (2.4352464746026437) + | > grad_norm_0: tensor(11.0996, device='cuda:0') (tensor(8.0920, device='cuda:0')) + | > loss_gen: 2.5368189811706543 (2.3993868126588715) + | > loss_kl: 1.8633265495300293 (1.8417016530738157) + | > loss_feat: 8.236451148986816 (7.292761150528403) + | > loss_mel: 18.648710250854492 (17.66296149702633) + | > loss_duration: 1.4458613395690918 (1.4352142688106089) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.73116683959961 (30.632025157704074) + | > grad_norm_1: tensor(119.6699, device='cuda:0') (tensor(186.5601, device='cuda:0')) + | > current_lr_0: 0.00019024529033313074 + | > current_lr_1: 0.00019024529033313074 + | > step_time: 0.3873 (0.3546941736165216) + | > loader_time: 0.0059 (0.005479451488046086) + + + --> TIME: 2025-04-19 05:33:52 -- STEP: 93/142 -- GLOBAL_STEP: 436175 + | > loss_disc: 2.426271915435791 (2.454160116052115) + | > loss_disc_real_0: 0.13772478699684143 (0.11981731220599144) + | > loss_disc_real_1: 0.20262984931468964 (0.20962174316888216) + | > loss_disc_real_2: 0.2583802342414856 (0.22644199783443122) + | > loss_disc_real_3: 0.225712388753891 (0.2289351668729577) + | > loss_disc_real_4: 0.20603829622268677 (0.2272182042880725) + | > loss_disc_real_5: 0.21731941401958466 (0.23387898360529252) + | > loss_0: 2.426271915435791 (2.454160116052115) + | > grad_norm_0: tensor(6.3507, device='cuda:0') (tensor(8.2983, device='cuda:0')) + | > loss_gen: 2.3976712226867676 (2.373244944439141) + | > loss_kl: 1.8808224201202393 (1.8617366436989076) + | > loss_feat: 6.943048477172852 (7.214876518454603) + | > loss_mel: 17.64236068725586 (17.735551905888386) + | > loss_duration: 1.4708218574523926 (1.4472779958478863) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.33472442626953 (30.63268792757424) + | > grad_norm_1: tensor(110.3811, device='cuda:0') (tensor(192.2016, device='cuda:0')) + | > current_lr_0: 0.00019024529033313074 + | > current_lr_1: 0.00019024529033313074 + | > step_time: 0.3989 (0.36582984206497043) + | > loader_time: 0.0066 (0.005790290012154528) + + + --> TIME: 2025-04-19 05:34:03 -- STEP: 118/142 -- GLOBAL_STEP: 436200 + | > loss_disc: 2.4437146186828613 (2.459899809400914) + | > loss_disc_real_0: 0.14387845993041992 (0.12217294544739238) + | > loss_disc_real_1: 0.2026478350162506 (0.20925602417881206) + | > loss_disc_real_2: 0.2054637223482132 (0.22578472694603063) + | > loss_disc_real_3: 0.22095999121665955 (0.22927685284008414) + | > loss_disc_real_4: 0.23591431975364685 (0.2287493748432499) + | > loss_disc_real_5: 0.23585374653339386 (0.2341186538086099) + | > loss_0: 2.4437146186828613 (2.459899809400914) + | > grad_norm_0: tensor(9.1866, device='cuda:0') (tensor(9.0359, device='cuda:0')) + | > loss_gen: 2.4667325019836426 (2.366186000533024) + | > loss_kl: 1.98692786693573 (1.866836530677343) + | > loss_feat: 7.385987281799316 (7.161111560918517) + | > loss_mel: 18.008174896240234 (17.681495836225597) + | > loss_duration: 1.4504108428955078 (1.4538837236873172) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.298233032226562 (30.52951352070954) + | > grad_norm_1: tensor(366.8266, device='cuda:0') (tensor(207.1436, device='cuda:0')) + | > current_lr_0: 0.00019024529033313074 + | > current_lr_1: 0.00019024529033313074 + | > step_time: 0.4419 (0.37817035287113526) + | > loader_time: 0.0068 (0.006072561619645458) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5127294063568115 (2.5127294063568115) + | > loss_disc_real_0: 0.2211172878742218 (0.2211172878742218) + | > loss_disc_real_1: 0.24929110705852509 (0.24929110705852509) + | > loss_disc_real_2: 0.18718647956848145 (0.18718647956848145) + | > loss_disc_real_3: 0.2033538520336151 (0.2033538520336151) + | > loss_disc_real_4: 0.20203101634979248 (0.20203101634979248) + | > loss_disc_real_5: 0.26545387506484985 (0.26545387506484985) + | > loss_0: 2.5127294063568115 (2.5127294063568115) + | > loss_gen: 2.3568100929260254 (2.3568100929260254) + | > loss_kl: 2.522822856903076 (2.522822856903076) + | > loss_feat: 5.37093448638916 (5.37093448638916) + | > loss_mel: 15.091726303100586 (15.091726303100586) + | > loss_duration: 1.751206636428833 (1.751206636428833) + | > loss_1: 27.0935001373291 (27.0935001373291) + + --> STEP: 1 + | > loss_disc: 2.5997042655944824 (2.5997042655944824) + | > loss_disc_real_0: 0.1836494505405426 (0.1836494505405426) + | > loss_disc_real_1: 0.24461805820465088 (0.24461805820465088) + | > loss_disc_real_2: 0.24133619666099548 (0.24133619666099548) + | > loss_disc_real_3: 0.22478201985359192 (0.22478201985359192) + | > loss_disc_real_4: 0.2656734883785248 (0.2656734883785248) + | > loss_disc_real_5: 0.2112693041563034 (0.2112693041563034) + | > loss_0: 2.5997042655944824 (2.5997042655944824) + | > loss_gen: 2.3586010932922363 (2.3586010932922363) + | > loss_kl: 3.04052472114563 (3.04052472114563) + | > loss_feat: 7.006330966949463 (7.006330966949463) + | > loss_mel: 18.085603713989258 (18.085603713989258) + | > loss_duration: 1.8118038177490234 (1.8118038177490234) + | > loss_1: 32.30286407470703 (32.30286407470703) + + --> STEP: 2 + | > loss_disc: 2.499401807785034 (2.5495530366897583) + | > loss_disc_real_0: 0.21933695673942566 (0.20149320363998413) + | > loss_disc_real_1: 0.2503271996974945 (0.2474726289510727) + | > loss_disc_real_2: 0.2271338552236557 (0.2342350259423256) + | > loss_disc_real_3: 0.2137070894241333 (0.2192445546388626) + | > loss_disc_real_4: 0.22084471583366394 (0.24325910210609436) + | > loss_disc_real_5: 0.21472038328647614 (0.21299484372138977) + | > loss_0: 2.499401807785034 (2.5495530366897583) + | > loss_gen: 2.5164411067962646 (2.4375211000442505) + | > loss_kl: 2.4630041122436523 (2.751764416694641) + | > loss_feat: 7.698779106140137 (7.3525550365448) + | > loss_mel: 18.849367141723633 (18.467485427856445) + | > loss_duration: 1.7028534412384033 (1.7573286294937134) + | > loss_1: 33.230445861816406 (32.76665496826172) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036678314208984375 (-0.0002130270004272461) + | > avg_loss_disc: 2.5495530366897583 (+0.024518966674804688) + | > avg_loss_disc_real_0: 0.20149320363998413 (+0.05453058332204819) + | > avg_loss_disc_real_1: 0.2474726289510727 (+0.02326570451259613) + | > avg_loss_disc_real_2: 0.2342350259423256 (-0.030450746417045593) + | > avg_loss_disc_real_3: 0.2192445546388626 (-0.06654179096221924) + | > avg_loss_disc_real_4: 0.24325910210609436 (+0.013143166899681091) + | > avg_loss_disc_real_5: 0.21299484372138977 (-0.05433574318885803) + | > avg_loss_0: 2.5495530366897583 (+0.024518966674804688) + | > avg_loss_gen: 2.4375211000442505 (+0.07344675064086914) + | > avg_loss_kl: 2.751764416694641 (+0.21495795249938965) + | > avg_loss_feat: 7.3525550365448 (+0.3592386245727539) + | > avg_loss_mel: 18.467485427856445 (+0.23714256286621094) + | > avg_loss_duration: 1.7573286294937134 (+0.0035999417304992676) + | > avg_loss_1: 32.76665496826172 (+0.8883867263793945) + + + > EPOCH: 401/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:34:15)  + + --> TIME: 2025-04-19 05:34:16 -- STEP: 1/142 -- GLOBAL_STEP: 436225 + | > loss_disc: 2.3236138820648193 (2.3236138820648193) + | > loss_disc_real_0: 0.1000523567199707 (0.1000523567199707) + | > loss_disc_real_1: 0.23287956416606903 (0.23287956416606903) + | > loss_disc_real_2: 0.2129463404417038 (0.2129463404417038) + | > loss_disc_real_3: 0.20474405586719513 (0.20474405586719513) + | > loss_disc_real_4: 0.1915772408246994 (0.1915772408246994) + | > loss_disc_real_5: 0.20651376247406006 (0.20651376247406006) + | > loss_0: 2.3236138820648193 (2.3236138820648193) + | > grad_norm_0: tensor(10.2053, device='cuda:0') (tensor(10.2053, device='cuda:0')) + | > loss_gen: 2.3604676723480225 (2.3604676723480225) + | > loss_kl: 1.8469839096069336 (1.8469839096069336) + | > loss_feat: 8.186077117919922 (8.186077117919922) + | > loss_mel: 18.505691528320312 (18.505691528320312) + | > loss_duration: 1.373998999595642 (1.373998999595642) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.27322006225586 (32.27322006225586) + | > grad_norm_1: tensor(231.6139, device='cuda:0') (tensor(231.6139, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.346 (0.34603285789489746) + | > loader_time: 0.0051 (0.005106210708618164) + + + --> TIME: 2025-04-19 05:34:25 -- STEP: 26/142 -- GLOBAL_STEP: 436250 + | > loss_disc: 2.4096081256866455 (2.4334204288629384) + | > loss_disc_real_0: 0.08585711568593979 (0.10643896116660191) + | > loss_disc_real_1: 0.21560752391815186 (0.2065311549947812) + | > loss_disc_real_2: 0.19758246839046478 (0.22752100401199782) + | > loss_disc_real_3: 0.1695089340209961 (0.22604666421046624) + | > loss_disc_real_4: 0.20662987232208252 (0.23007611815746015) + | > loss_disc_real_5: 0.2319025993347168 (0.23645084064740401) + | > loss_0: 2.4096081256866455 (2.4334204288629384) + | > grad_norm_0: tensor(7.5457, device='cuda:0') (tensor(10.1726, device='cuda:0')) + | > loss_gen: 2.488239049911499 (2.4024970806561976) + | > loss_kl: 1.768463134765625 (1.7948394738710844) + | > loss_feat: 6.929861545562744 (7.445053118925828) + | > loss_mel: 17.41913604736328 (17.925032505622283) + | > loss_duration: 1.4251832962036133 (1.4172490881039546) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.0308837890625 (30.98467122591459) + | > grad_norm_1: tensor(220.1987, device='cuda:0') (tensor(252.4823, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.3483 (0.3344992215816791) + | > loader_time: 0.0059 (0.004999243296109713) + + + --> TIME: 2025-04-19 05:34:34 -- STEP: 51/142 -- GLOBAL_STEP: 436275 + | > loss_disc: 2.5324530601501465 (2.436924261205337) + | > loss_disc_real_0: 0.10215642303228378 (0.11179366108833574) + | > loss_disc_real_1: 0.17437508702278137 (0.20573669467486588) + | > loss_disc_real_2: 0.21714235842227936 (0.22687286021662692) + | > loss_disc_real_3: 0.24797223508358002 (0.22617782360198452) + | > loss_disc_real_4: 0.2377973198890686 (0.22792520826938106) + | > loss_disc_real_5: 0.24115672707557678 (0.2365080302252489) + | > loss_0: 2.5324530601501465 (2.436924261205337) + | > grad_norm_0: tensor(9.7064, device='cuda:0') (tensor(10.4126, device='cuda:0')) + | > loss_gen: 2.13666033744812 (2.3878462595098155) + | > loss_kl: 1.9276745319366455 (1.824134868734023) + | > loss_feat: 6.700799465179443 (7.3355081876118975) + | > loss_mel: 16.350988388061523 (17.658580162945924) + | > loss_duration: 1.4132192134857178 (1.4236138381210028) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.529340744018555 (30.629683326272403) + | > grad_norm_1: tensor(357.0532, device='cuda:0') (tensor(275.0707, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.3693 (0.3466585897931866) + | > loader_time: 0.0061 (0.005369784785251992) + + + --> TIME: 2025-04-19 05:34:44 -- STEP: 76/142 -- GLOBAL_STEP: 436300 + | > loss_disc: 2.535614013671875 (2.4463254276074866) + | > loss_disc_real_0: 0.11438068002462387 (0.11483598765181868) + | > loss_disc_real_1: 0.2316211313009262 (0.20566951777589948) + | > loss_disc_real_2: 0.24795225262641907 (0.2271225572024521) + | > loss_disc_real_3: 0.2659565508365631 (0.22609298264509753) + | > loss_disc_real_4: 0.2780877351760864 (0.22945880282082057) + | > loss_disc_real_5: 0.25188949704170227 (0.2354185477291283) + | > loss_0: 2.535614013671875 (2.4463254276074866) + | > grad_norm_0: tensor(10.4152, device='cuda:0') (tensor(10.4635, device='cuda:0')) + | > loss_gen: 2.4555039405822754 (2.3816582936989636) + | > loss_kl: 1.8607070446014404 (1.8442320776613135) + | > loss_feat: 7.275963306427002 (7.245938972422951) + | > loss_mel: 18.710739135742188 (17.66512284780804) + | > loss_duration: 1.4808846712112427 (1.4363259406466231) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.783796310424805 (30.57327822635048) + | > grad_norm_1: tensor(386.0253, device='cuda:0') (tensor(282.8749, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.3916 (0.35882004938627543) + | > loader_time: 0.0064 (0.005677333003596256) + + + --> TIME: 2025-04-19 05:34:54 -- STEP: 101/142 -- GLOBAL_STEP: 436325 + | > loss_disc: 2.5780835151672363 (2.4542554510702) + | > loss_disc_real_0: 0.07242933660745621 (0.11800411100139713) + | > loss_disc_real_1: 0.26548510789871216 (0.20685732305640042) + | > loss_disc_real_2: 0.23777057230472565 (0.22644745993732227) + | > loss_disc_real_3: 0.252499520778656 (0.2270863520922047) + | > loss_disc_real_4: 0.2390589714050293 (0.22923388369012587) + | > loss_disc_real_5: 0.23921164870262146 (0.23526531295611128) + | > loss_0: 2.5780835151672363 (2.4542554510702) + | > grad_norm_0: tensor(18.7150, device='cuda:0') (tensor(11.1834, device='cuda:0')) + | > loss_gen: 2.3078360557556152 (2.3757582418989425) + | > loss_kl: 1.9027589559555054 (1.8635469297371288) + | > loss_feat: 7.6631646156311035 (7.207195489713461) + | > loss_mel: 18.63370704650879 (17.71364709646395) + | > loss_duration: 1.4832181930541992 (1.4467265204627915) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.990684509277344 (30.606874314865262) + | > grad_norm_1: tensor(416.7610, device='cuda:0') (tensor(294.7320, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.4156 (0.36911237593924645) + | > loader_time: 0.0066 (0.005940501052554291) + + + --> TIME: 2025-04-19 05:35:05 -- STEP: 126/142 -- GLOBAL_STEP: 436350 + | > loss_disc: 2.500187635421753 (2.456529723273383) + | > loss_disc_real_0: 0.19878999888896942 (0.12068241751856273) + | > loss_disc_real_1: 0.18259747326374054 (0.20614734966130482) + | > loss_disc_real_2: 0.22276821732521057 (0.22631001176815185) + | > loss_disc_real_3: 0.19723764061927795 (0.2273907455660048) + | > loss_disc_real_4: 0.23862949013710022 (0.2291105617842977) + | > loss_disc_real_5: 0.24821887910366058 (0.23602257952803657) + | > loss_0: 2.500187635421753 (2.456529723273383) + | > grad_norm_0: tensor(29.2621, device='cuda:0') (tensor(12.5461, device='cuda:0')) + | > loss_gen: 2.2082743644714355 (2.378047008363027) + | > loss_kl: 1.9280920028686523 (1.8684415751033359) + | > loss_feat: 6.663580417633057 (7.1987507986643955) + | > loss_mel: 17.116596221923828 (17.72488446462722) + | > loss_duration: 1.4743810892105103 (1.4539074434174433) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.39092445373535 (30.62403146047441) + | > grad_norm_1: tensor(185.9645, device='cuda:0') (tensor(306.9163, device='cuda:0')) + | > current_lr_0: 0.0001902215096718391 + | > current_lr_1: 0.0001902215096718391 + | > step_time: 0.4379 (0.3806838648659842) + | > loader_time: 0.0071 (0.006203329752361964) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.499476432800293 (2.499476432800293) + | > loss_disc_real_0: 0.12054997682571411 (0.12054997682571411) + | > loss_disc_real_1: 0.24578413367271423 (0.24578413367271423) + | > loss_disc_real_2: 0.2210765928030014 (0.2210765928030014) + | > loss_disc_real_3: 0.23251104354858398 (0.23251104354858398) + | > loss_disc_real_4: 0.1972142606973648 (0.1972142606973648) + | > loss_disc_real_5: 0.2428237646818161 (0.2428237646818161) + | > loss_0: 2.499476432800293 (2.499476432800293) + | > loss_gen: 2.1626405715942383 (2.1626405715942383) + | > loss_kl: 2.5411462783813477 (2.5411462783813477) + | > loss_feat: 6.676079273223877 (6.676079273223877) + | > loss_mel: 16.635295867919922 (16.635295867919922) + | > loss_duration: 1.7645570039749146 (1.7645570039749146) + | > loss_1: 29.779720306396484 (29.779720306396484) + + --> STEP: 1 + | > loss_disc: 2.4703540802001953 (2.4703540802001953) + | > loss_disc_real_0: 0.12648053467273712 (0.12648053467273712) + | > loss_disc_real_1: 0.24760740995407104 (0.24760740995407104) + | > loss_disc_real_2: 0.25796905159950256 (0.25796905159950256) + | > loss_disc_real_3: 0.20205777883529663 (0.20205777883529663) + | > loss_disc_real_4: 0.2006310671567917 (0.2006310671567917) + | > loss_disc_real_5: 0.22912591695785522 (0.22912591695785522) + | > loss_0: 2.4703540802001953 (2.4703540802001953) + | > loss_gen: 2.114166498184204 (2.114166498184204) + | > loss_kl: 2.729041814804077 (2.729041814804077) + | > loss_feat: 6.999340057373047 (6.999340057373047) + | > loss_mel: 17.521514892578125 (17.521514892578125) + | > loss_duration: 1.7966004610061646 (1.7966004610061646) + | > loss_1: 31.160663604736328 (31.160663604736328) + + --> STEP: 2 + | > loss_disc: 2.464262008666992 (2.4673080444335938) + | > loss_disc_real_0: 0.07531826198101044 (0.10089939832687378) + | > loss_disc_real_1: 0.26372724771499634 (0.2556673288345337) + | > loss_disc_real_2: 0.2620973289012909 (0.26003319025039673) + | > loss_disc_real_3: 0.21242472529411316 (0.2072412520647049) + | > loss_disc_real_4: 0.20903274416923523 (0.20483190566301346) + | > loss_disc_real_5: 0.2126944363117218 (0.2209101766347885) + | > loss_0: 2.464262008666992 (2.4673080444335938) + | > loss_gen: 2.162384271621704 (2.138275384902954) + | > loss_kl: 2.378356695175171 (2.553699254989624) + | > loss_feat: 7.405759811401367 (7.202549934387207) + | > loss_mel: 18.675270080566406 (18.098392486572266) + | > loss_duration: 1.7053554058074951 (1.7509779334068298) + | > loss_1: 32.327125549316406 (31.743894577026367) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038450956344604492 (+0.00017726421356201172) + | > avg_loss_disc: 2.4673080444335938 (-0.08224499225616455) + | > avg_loss_disc_real_0: 0.10089939832687378 (-0.10059380531311035) + | > avg_loss_disc_real_1: 0.2556673288345337 (+0.008194699883460999) + | > avg_loss_disc_real_2: 0.26003319025039673 (+0.025798164308071136) + | > avg_loss_disc_real_3: 0.2072412520647049 (-0.012003302574157715) + | > avg_loss_disc_real_4: 0.20483190566301346 (-0.0384271964430809) + | > avg_loss_disc_real_5: 0.2209101766347885 (+0.007915332913398743) + | > avg_loss_0: 2.4673080444335938 (-0.08224499225616455) + | > avg_loss_gen: 2.138275384902954 (-0.2992457151412964) + | > avg_loss_kl: 2.553699254989624 (-0.1980651617050171) + | > avg_loss_feat: 7.202549934387207 (-0.15000510215759277) + | > avg_loss_mel: 18.098392486572266 (-0.3690929412841797) + | > avg_loss_duration: 1.7509779334068298 (-0.006350696086883545) + | > avg_loss_1: 31.743894577026367 (-1.0227603912353516) + + + > EPOCH: 402/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:35:14)  + + --> TIME: 2025-04-19 05:35:17 -- STEP: 9/142 -- GLOBAL_STEP: 436375 + | > loss_disc: 2.4448628425598145 (2.4373465643988714) + | > loss_disc_real_0: 0.2148789018392563 (0.12823933280176586) + | > loss_disc_real_1: 0.254850298166275 (0.2107194893889957) + | > loss_disc_real_2: 0.2552383542060852 (0.23413512276278603) + | > loss_disc_real_3: 0.19776983559131622 (0.2250605672597885) + | > loss_disc_real_4: 0.2357301265001297 (0.21600950592093998) + | > loss_disc_real_5: 0.2459009885787964 (0.23890082703696358) + | > loss_0: 2.4448628425598145 (2.4373465643988714) + | > grad_norm_0: tensor(17.8475, device='cuda:0') (tensor(12.5216, device='cuda:0')) + | > loss_gen: 2.5131289958953857 (2.4356890519460044) + | > loss_kl: 1.7886067628860474 (1.8183039294348822) + | > loss_feat: 7.4804582595825195 (7.2165488137139215) + | > loss_mel: 17.492185592651367 (16.939217779371475) + | > loss_duration: 1.3955559730529785 (1.4142663876215618) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.669933319091797 (29.824025683932835) + | > grad_norm_1: tensor(307.1612, device='cuda:0') (tensor(284.3542, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.3281 (0.3262695206536187) + | > loader_time: 0.005 (0.004507409201727973) + + + --> TIME: 2025-04-19 05:35:26 -- STEP: 34/142 -- GLOBAL_STEP: 436400 + | > loss_disc: 2.4816951751708984 (2.440780190860524) + | > loss_disc_real_0: 0.16423729062080383 (0.11712001833845587) + | > loss_disc_real_1: 0.24433325231075287 (0.20795504792648203) + | > loss_disc_real_2: 0.2422325760126114 (0.2286184285493458) + | > loss_disc_real_3: 0.24414753913879395 (0.23207331229658687) + | > loss_disc_real_4: 0.2396736890077591 (0.22444868000114665) + | > loss_disc_real_5: 0.24583379924297333 (0.23776858343797572) + | > loss_0: 2.4816951751708984 (2.440780190860524) + | > grad_norm_0: tensor(24.5328, device='cuda:0') (tensor(11.9144, device='cuda:0')) + | > loss_gen: 2.3320863246917725 (2.3892669958226818) + | > loss_kl: 1.7445870637893677 (1.8368592016837175) + | > loss_feat: 7.704150199890137 (7.229709527071784) + | > loss_mel: 18.34637451171875 (17.309396771823657) + | > loss_duration: 1.4480831623077393 (1.4216370828011458) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.57528305053711 (30.186869565178366) + | > grad_norm_1: tensor(476.7148, device='cuda:0') (tensor(291.7190, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.3578 (0.3508377005072201) + | > loader_time: 0.006 (0.005282984060399672) + + + --> TIME: 2025-04-19 05:35:36 -- STEP: 59/142 -- GLOBAL_STEP: 436425 + | > loss_disc: 2.343198776245117 (2.438368284096152) + | > loss_disc_real_0: 0.10201143473386765 (0.11513546551183118) + | > loss_disc_real_1: 0.16179484128952026 (0.2071654448064707) + | > loss_disc_real_2: 0.18981006741523743 (0.22724043666306187) + | > loss_disc_real_3: 0.23518861830234528 (0.2298925794282202) + | > loss_disc_real_4: 0.19643284380435944 (0.2231640795529899) + | > loss_disc_real_5: 0.1983761042356491 (0.23422043732667375) + | > loss_0: 2.343198776245117 (2.438368284096152) + | > grad_norm_0: tensor(14.9302, device='cuda:0') (tensor(10.9002, device='cuda:0')) + | > loss_gen: 2.517674446105957 (2.3862602549084158) + | > loss_kl: 1.885801911354065 (1.8297191999726377) + | > loss_feat: 8.095138549804688 (7.199295658176228) + | > loss_mel: 18.508256912231445 (17.442735752816926) + | > loss_duration: 1.4460376501083374 (1.4323602611735713) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.45290756225586 (30.290371102801824) + | > grad_norm_1: tensor(406.6082, device='cuda:0') (tensor(276.6583, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.3747 (0.3577922037092306) + | > loader_time: 0.0062 (0.005585027953325693) + + + --> TIME: 2025-04-19 05:35:46 -- STEP: 84/142 -- GLOBAL_STEP: 436450 + | > loss_disc: 2.5273725986480713 (2.4546350183941064) + | > loss_disc_real_0: 0.15315237641334534 (0.11974731859351907) + | > loss_disc_real_1: 0.22145704925060272 (0.2095855593326546) + | > loss_disc_real_2: 0.2303377389907837 (0.22605483180710248) + | > loss_disc_real_3: 0.2285715788602829 (0.22899204847358523) + | > loss_disc_real_4: 0.23831018805503845 (0.22714716267018092) + | > loss_disc_real_5: 0.24551597237586975 (0.23477316967078618) + | > loss_0: 2.5273725986480713 (2.4546350183941064) + | > grad_norm_0: tensor(9.3180, device='cuda:0') (tensor(11.6455, device='cuda:0')) + | > loss_gen: 2.1852753162384033 (2.3670468926429753) + | > loss_kl: 1.9451053142547607 (1.8327289464927854) + | > loss_feat: 5.928797721862793 (7.062455103510902) + | > loss_mel: 15.183088302612305 (17.377711545853387) + | > loss_duration: 1.4926717281341553 (1.4439984361330667) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 26.734939575195312 (30.083940891992476) + | > grad_norm_1: tensor(284.5324, device='cuda:0') (tensor(275.9739, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.3972 (0.36767928089414326) + | > loader_time: 0.0066 (0.005860961618877593) + + + --> TIME: 2025-04-19 05:35:56 -- STEP: 109/142 -- GLOBAL_STEP: 436475 + | > loss_disc: 2.4138267040252686 (2.4533655337237428) + | > loss_disc_real_0: 0.0800226628780365 (0.11998195652294597) + | > loss_disc_real_1: 0.18701449036598206 (0.2096036337657806) + | > loss_disc_real_2: 0.22332654893398285 (0.22526316303725635) + | > loss_disc_real_3: 0.254581481218338 (0.22828340516724718) + | > loss_disc_real_4: 0.22314013540744781 (0.22689139022739654) + | > loss_disc_real_5: 0.22633928060531616 (0.23362360853667652) + | > loss_0: 2.4138267040252686 (2.4533655337237428) + | > grad_norm_0: tensor(9.2977, device='cuda:0') (tensor(11.0807, device='cuda:0')) + | > loss_gen: 2.2739689350128174 (2.3591636364613104) + | > loss_kl: 1.8633791208267212 (1.8357877414160912) + | > loss_feat: 6.796262741088867 (7.070411117798692) + | > loss_mel: 17.717796325683594 (17.493316720385067) + | > loss_duration: 1.4877851009368896 (1.4529029874626647) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.139192581176758 (30.2115822013365) + | > grad_norm_1: tensor(63.9173, device='cuda:0') (tensor(261.7626, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.4112 (0.37710651345209245) + | > loader_time: 0.0072 (0.006086741018732754) + + + --> TIME: 2025-04-19 05:36:07 -- STEP: 134/142 -- GLOBAL_STEP: 436500 + | > loss_disc: 2.526986598968506 (2.4571782564049327) + | > loss_disc_real_0: 0.13856451213359833 (0.12198710619513668) + | > loss_disc_real_1: 0.19488325715065002 (0.20927229401335787) + | > loss_disc_real_2: 0.24209333956241608 (0.22485832603120093) + | > loss_disc_real_3: 0.23635706305503845 (0.22901709155360264) + | > loss_disc_real_4: 0.2127300351858139 (0.22714486506892673) + | > loss_disc_real_5: 0.22659939527511597 (0.23288028954125162) + | > loss_0: 2.526986598968506 (2.4571782564049327) + | > grad_norm_0: tensor(16.6083, device='cuda:0') (tensor(11.1635, device='cuda:0')) + | > loss_gen: 2.2735753059387207 (2.3522710177435804) + | > loss_kl: 1.8893028497695923 (1.842711649723907) + | > loss_feat: 6.949696063995361 (7.033059444000472) + | > loss_mel: 16.951156616210938 (17.4504315817534) + | > loss_duration: 1.519881248474121 (1.4596840899382062) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.58361053466797 (30.138157716437952) + | > grad_norm_1: tensor(365.1162, device='cuda:0') (tensor(257.8802, device='cuda:0')) + | > current_lr_0: 0.0001901977319831301 + | > current_lr_1: 0.0001901977319831301 + | > step_time: 0.4399 (0.38849446844698776) + | > loader_time: 0.0075 (0.006382403089039361) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.494426727294922 (2.494426727294922) + | > loss_disc_real_0: 0.17463959753513336 (0.17463959753513336) + | > loss_disc_real_1: 0.233015239238739 (0.233015239238739) + | > loss_disc_real_2: 0.23592829704284668 (0.23592829704284668) + | > loss_disc_real_3: 0.1917639970779419 (0.1917639970779419) + | > loss_disc_real_4: 0.2345789074897766 (0.2345789074897766) + | > loss_disc_real_5: 0.23147252202033997 (0.23147252202033997) + | > loss_0: 2.494426727294922 (2.494426727294922) + | > loss_gen: 2.3762881755828857 (2.3762881755828857) + | > loss_kl: 2.5235066413879395 (2.5235066413879395) + | > loss_feat: 6.58974027633667 (6.58974027633667) + | > loss_mel: 16.963733673095703 (16.963733673095703) + | > loss_duration: 1.719632863998413 (1.719632863998413) + | > loss_1: 30.172901153564453 (30.172901153564453) + + --> STEP: 1 + | > loss_disc: 2.582796335220337 (2.582796335220337) + | > loss_disc_real_0: 0.1518443077802658 (0.1518443077802658) + | > loss_disc_real_1: 0.18949104845523834 (0.18949104845523834) + | > loss_disc_real_2: 0.21948885917663574 (0.21948885917663574) + | > loss_disc_real_3: 0.22416408360004425 (0.22416408360004425) + | > loss_disc_real_4: 0.23658138513565063 (0.23658138513565063) + | > loss_disc_real_5: 0.23687206208705902 (0.23687206208705902) + | > loss_0: 2.582796335220337 (2.582796335220337) + | > loss_gen: 2.1482291221618652 (2.1482291221618652) + | > loss_kl: 2.722625255584717 (2.722625255584717) + | > loss_feat: 7.507307052612305 (7.507307052612305) + | > loss_mel: 18.351497650146484 (18.351497650146484) + | > loss_duration: 1.787996768951416 (1.787996768951416) + | > loss_1: 32.51765823364258 (32.51765823364258) + + --> STEP: 2 + | > loss_disc: 2.675832509994507 (2.629314422607422) + | > loss_disc_real_0: 0.2273450791835785 (0.18959469348192215) + | > loss_disc_real_1: 0.22962896525859833 (0.20956000685691833) + | > loss_disc_real_2: 0.23594516515731812 (0.22771701216697693) + | > loss_disc_real_3: 0.20896528661251068 (0.21656468510627747) + | > loss_disc_real_4: 0.22059695422649384 (0.22858916968107224) + | > loss_disc_real_5: 0.25094860792160034 (0.24391033500432968) + | > loss_0: 2.675832509994507 (2.629314422607422) + | > loss_gen: 2.1958634853363037 (2.1720463037490845) + | > loss_kl: 2.2722675800323486 (2.4974464178085327) + | > loss_feat: 6.803773880004883 (7.155540466308594) + | > loss_mel: 14.229985237121582 (16.290741443634033) + | > loss_duration: 1.6968700885772705 (1.7424334287643433) + | > loss_1: 27.198760986328125 (29.85820960998535) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004220128059387207 (+0.0003750324249267578) + | > avg_loss_disc: 2.629314422607422 (+0.16200637817382812) + | > avg_loss_disc_real_0: 0.18959469348192215 (+0.08869529515504837) + | > avg_loss_disc_real_1: 0.20956000685691833 (-0.046107321977615356) + | > avg_loss_disc_real_2: 0.22771701216697693 (-0.0323161780834198) + | > avg_loss_disc_real_3: 0.21656468510627747 (+0.00932343304157257) + | > avg_loss_disc_real_4: 0.22858916968107224 (+0.023757264018058777) + | > avg_loss_disc_real_5: 0.24391033500432968 (+0.023000158369541168) + | > avg_loss_0: 2.629314422607422 (+0.16200637817382812) + | > avg_loss_gen: 2.1720463037490845 (+0.03377091884613037) + | > avg_loss_kl: 2.4974464178085327 (-0.05625283718109131) + | > avg_loss_feat: 7.155540466308594 (-0.04700946807861328) + | > avg_loss_mel: 16.290741443634033 (-1.8076510429382324) + | > avg_loss_duration: 1.7424334287643433 (-0.008544504642486572) + | > avg_loss_1: 29.85820960998535 (-1.8856849670410156) + + + > EPOCH: 403/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:36:13)  + + --> TIME: 2025-04-19 05:36:19 -- STEP: 17/142 -- GLOBAL_STEP: 436525 + | > loss_disc: 2.3591694831848145 (2.4074274091159595) + | > loss_disc_real_0: 0.10566498339176178 (0.1163788963766659) + | > loss_disc_real_1: 0.2172188013792038 (0.20758632964947643) + | > loss_disc_real_2: 0.23603150248527527 (0.2248009890317917) + | > loss_disc_real_3: 0.21084722876548767 (0.217080154839684) + | > loss_disc_real_4: 0.21176937222480774 (0.22089155456599066) + | > loss_disc_real_5: 0.19908453524112701 (0.22554135760840247) + | > loss_0: 2.3591694831848145 (2.4074274091159595) + | > grad_norm_0: tensor(8.2562, device='cuda:0') (tensor(8.7835, device='cuda:0')) + | > loss_gen: 2.3554389476776123 (2.3918432628407196) + | > loss_kl: 1.6941879987716675 (1.8203559062060188) + | > loss_feat: 6.851958274841309 (7.4495201952317185) + | > loss_mel: 17.104093551635742 (17.895565257352942) + | > loss_duration: 1.4056546688079834 (1.4106175478766947) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.411333084106445 (30.967902127434225) + | > grad_norm_1: tensor(116.7576, device='cuda:0') (tensor(161.4961, device='cuda:0')) + | > current_lr_0: 0.00019017395726663222 + | > current_lr_1: 0.00019017395726663222 + | > step_time: 0.3363 (0.3295232548433192) + | > loader_time: 0.0053 (0.004839644712560317) + + + --> TIME: 2025-04-19 05:36:28 -- STEP: 42/142 -- GLOBAL_STEP: 436550 + | > loss_disc: 2.567593574523926 (2.420843726112729) + | > loss_disc_real_0: 0.11486964672803879 (0.12030775951487678) + | > loss_disc_real_1: 0.24079345166683197 (0.2053015058239301) + | > loss_disc_real_2: 0.2172548472881317 (0.22307012513989494) + | > loss_disc_real_3: 0.20892345905303955 (0.22384543362117948) + | > loss_disc_real_4: 0.24184012413024902 (0.22321596351407824) + | > loss_disc_real_5: 0.23712795972824097 (0.2266940563207581) + | > loss_0: 2.567593574523926 (2.420843726112729) + | > grad_norm_0: tensor(13.7296, device='cuda:0') (tensor(10.5095, device='cuda:0')) + | > loss_gen: 2.4470698833465576 (2.3922672214962195) + | > loss_kl: 1.8407634496688843 (1.819397523289635) + | > loss_feat: 6.952566146850586 (7.338765223821004) + | > loss_mel: 17.55245018005371 (17.642599151248024) + | > loss_duration: 1.4693975448608398 (1.4234506090482075) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.262245178222656 (30.616479692004976) + | > grad_norm_1: tensor(124.8565, device='cuda:0') (tensor(254.2755, device='cuda:0')) + | > current_lr_0: 0.00019017395726663222 + | > current_lr_1: 0.00019017395726663222 + | > step_time: 0.3635 (0.3424461171740577) + | > loader_time: 0.0058 (0.005229722885858446) + + + --> TIME: 2025-04-19 05:36:37 -- STEP: 67/142 -- GLOBAL_STEP: 436575 + | > loss_disc: 2.460287570953369 (2.4341990983308253) + | > loss_disc_real_0: 0.09896411746740341 (0.12151022021895025) + | > loss_disc_real_1: 0.19444455206394196 (0.2058966913774832) + | > loss_disc_real_2: 0.22779782116413116 (0.22366268198881575) + | > loss_disc_real_3: 0.2046073079109192 (0.22460060288657002) + | > loss_disc_real_4: 0.20466800034046173 (0.22504832584466508) + | > loss_disc_real_5: 0.23711231350898743 (0.2313150536658159) + | > loss_0: 2.460287570953369 (2.4341990983308253) + | > grad_norm_0: tensor(11.1191, device='cuda:0') (tensor(10.2228, device='cuda:0')) + | > loss_gen: 2.465182065963745 (2.386854079232287) + | > loss_kl: 1.8765356540679932 (1.8304813726624447) + | > loss_feat: 7.654738903045654 (7.286990727951277) + | > loss_mel: 17.358741760253906 (17.64094663022169) + | > loss_duration: 1.4012136459350586 (1.4339874146589593) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.756412506103516 (30.579260185583315) + | > grad_norm_1: tensor(91.0698, device='cuda:0') (tensor(245.6399, device='cuda:0')) + | > current_lr_0: 0.00019017395726663222 + | > current_lr_1: 0.00019017395726663222 + | > step_time: 0.3796 (0.3538199253936312) + | > loader_time: 0.0064 (0.005525094359668334) + + + --> TIME: 2025-04-19 05:36:48 -- STEP: 92/142 -- GLOBAL_STEP: 436600 + | > loss_disc: 2.5245888233184814 (2.4426321128140325) + | > loss_disc_real_0: 0.09118510037660599 (0.12319991608028827) + | > loss_disc_real_1: 0.23743654787540436 (0.20616356024275656) + | > loss_disc_real_2: 0.26659998297691345 (0.2253638842831487) + | > loss_disc_real_3: 0.23330765962600708 (0.22654406807344893) + | > loss_disc_real_4: 0.2851470112800598 (0.22680503470094307) + | > loss_disc_real_5: 0.2455851286649704 (0.23171798868671709) + | > loss_0: 2.5245888233184814 (2.4426321128140325) + | > grad_norm_0: tensor(7.7945, device='cuda:0') (tensor(10.3348, device='cuda:0')) + | > loss_gen: 2.404284954071045 (2.373745793881623) + | > loss_kl: 2.019944906234741 (1.8408639068188875) + | > loss_feat: 6.427525043487549 (7.222143111021622) + | > loss_mel: 16.482227325439453 (17.672338724136353) + | > loss_duration: 1.4267323017120361 (1.445680865774984) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.76071548461914 (30.55477243921031) + | > grad_norm_1: tensor(152.6878, device='cuda:0') (tensor(237.5316, device='cuda:0')) + | > current_lr_0: 0.00019017395726663222 + | > current_lr_1: 0.00019017395726663222 + | > step_time: 0.4117 (0.3652314310488494) + | > loader_time: 0.0064 (0.005819750868755841) + + + --> TIME: 2025-04-19 05:36:58 -- STEP: 117/142 -- GLOBAL_STEP: 436625 + | > loss_disc: 2.315863847732544 (2.444787186435145) + | > loss_disc_real_0: 0.10757514089345932 (0.12292012171103404) + | > loss_disc_real_1: 0.1852332502603531 (0.20537258124249613) + | > loss_disc_real_2: 0.24138116836547852 (0.2258846665549482) + | > loss_disc_real_3: 0.2400597482919693 (0.2269471566646527) + | > loss_disc_real_4: 0.21045547723770142 (0.22685852519467345) + | > loss_disc_real_5: 0.22477403283119202 (0.23227813203110653) + | > loss_0: 2.315863847732544 (2.444787186435145) + | > grad_norm_0: tensor(11.1664, device='cuda:0') (tensor(10.4972, device='cuda:0')) + | > loss_gen: 2.52224063873291 (2.3669687315948997) + | > loss_kl: 1.8235524892807007 (1.8446322414610121) + | > loss_feat: 7.910007953643799 (7.184697945912679) + | > loss_mel: 18.531999588012695 (17.64372106698843) + | > loss_duration: 1.4667530059814453 (1.4531952968010535) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.254554748535156 (30.49321528377696) + | > grad_norm_1: tensor(270.0028, device='cuda:0') (tensor(247.2536, device='cuda:0')) + | > current_lr_0: 0.00019017395726663222 + | > current_lr_1: 0.00019017395726663222 + | > step_time: 0.4321 (0.37585402553917) + | > loader_time: 0.007 (0.006092157119359728) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4837427139282227 (2.4837427139282227) + | > loss_disc_real_0: 0.23072944581508636 (0.23072944581508636) + | > loss_disc_real_1: 0.21409112215042114 (0.21409112215042114) + | > loss_disc_real_2: 0.26797398924827576 (0.26797398924827576) + | > loss_disc_real_3: 0.25825703144073486 (0.25825703144073486) + | > loss_disc_real_4: 0.2213168442249298 (0.2213168442249298) + | > loss_disc_real_5: 0.2271900475025177 (0.2271900475025177) + | > loss_0: 2.4837427139282227 (2.4837427139282227) + | > loss_gen: 2.5244405269622803 (2.5244405269622803) + | > loss_kl: 2.4785096645355225 (2.4785096645355225) + | > loss_feat: 6.8251872062683105 (6.8251872062683105) + | > loss_mel: 18.404125213623047 (18.404125213623047) + | > loss_duration: 1.7410839796066284 (1.7410839796066284) + | > loss_1: 31.973344802856445 (31.973344802856445) + + --> STEP: 1 + | > loss_disc: 2.7083003520965576 (2.7083003520965576) + | > loss_disc_real_0: 0.251365602016449 (0.251365602016449) + | > loss_disc_real_1: 0.23233218491077423 (0.23233218491077423) + | > loss_disc_real_2: 0.27014198899269104 (0.27014198899269104) + | > loss_disc_real_3: 0.2730651795864105 (0.2730651795864105) + | > loss_disc_real_4: 0.2643139064311981 (0.2643139064311981) + | > loss_disc_real_5: 0.20714856684207916 (0.20714856684207916) + | > loss_0: 2.7083003520965576 (2.7083003520965576) + | > loss_gen: 2.309230089187622 (2.309230089187622) + | > loss_kl: 2.680962324142456 (2.680962324142456) + | > loss_feat: 6.228874683380127 (6.228874683380127) + | > loss_mel: 16.142980575561523 (16.142980575561523) + | > loss_duration: 1.7950536012649536 (1.7950536012649536) + | > loss_1: 29.157102584838867 (29.157102584838867) + + --> STEP: 2 + | > loss_disc: 2.58099365234375 (2.644647002220154) + | > loss_disc_real_0: 0.20042669773101807 (0.22589614987373352) + | > loss_disc_real_1: 0.2746821939945221 (0.25350718945264816) + | > loss_disc_real_2: 0.2528470754623413 (0.2614945322275162) + | > loss_disc_real_3: 0.2693621814250946 (0.27121368050575256) + | > loss_disc_real_4: 0.25082701444625854 (0.25757046043872833) + | > loss_disc_real_5: 0.3053007423877716 (0.2562246546149254) + | > loss_0: 2.58099365234375 (2.644647002220154) + | > loss_gen: 2.5418860912323 (2.425558090209961) + | > loss_kl: 2.512584686279297 (2.5967735052108765) + | > loss_feat: 6.4471659660339355 (6.338020324707031) + | > loss_mel: 17.996158599853516 (17.06956958770752) + | > loss_duration: 1.7346227169036865 (1.76483815908432) + | > loss_1: 31.232418060302734 (30.1947603225708) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003402113914489746 (-0.0008180141448974609) + | > avg_loss_disc: 2.644647002220154 (+0.015332579612731934) + | > avg_loss_disc_real_0: 0.22589614987373352 (+0.03630145639181137) + | > avg_loss_disc_real_1: 0.25350718945264816 (+0.04394718259572983) + | > avg_loss_disc_real_2: 0.2614945322275162 (+0.033777520060539246) + | > avg_loss_disc_real_3: 0.27121368050575256 (+0.0546489953994751) + | > avg_loss_disc_real_4: 0.25757046043872833 (+0.028981290757656097) + | > avg_loss_disc_real_5: 0.2562246546149254 (+0.012314319610595703) + | > avg_loss_0: 2.644647002220154 (+0.015332579612731934) + | > avg_loss_gen: 2.425558090209961 (+0.25351178646087646) + | > avg_loss_kl: 2.5967735052108765 (+0.09932708740234375) + | > avg_loss_feat: 6.338020324707031 (-0.8175201416015625) + | > avg_loss_mel: 17.06956958770752 (+0.7788281440734863) + | > avg_loss_duration: 1.76483815908432 (+0.022404730319976807) + | > avg_loss_1: 30.1947603225708 (+0.3365507125854492) + + + > EPOCH: 404/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:37:11)  + + --> TIME: 2025-04-19 05:37:12 -- STEP: 0/142 -- GLOBAL_STEP: 436650 + | > loss_disc: 2.441448450088501 (2.441448450088501) + | > loss_disc_real_0: 0.15367387235164642 (0.15367387235164642) + | > loss_disc_real_1: 0.21566960215568542 (0.21566960215568542) + | > loss_disc_real_2: 0.2507692277431488 (0.2507692277431488) + | > loss_disc_real_3: 0.2389058768749237 (0.2389058768749237) + | > loss_disc_real_4: 0.2312334179878235 (0.2312334179878235) + | > loss_disc_real_5: 0.2239551842212677 (0.2239551842212677) + | > loss_0: 2.441448450088501 (2.441448450088501) + | > grad_norm_0: tensor(10.1673, device='cuda:0') (tensor(10.1673, device='cuda:0')) + | > loss_gen: 2.548205614089966 (2.548205614089966) + | > loss_kl: 1.7365339994430542 (1.7365339994430542) + | > loss_feat: 7.297649383544922 (7.297649383544922) + | > loss_mel: 18.070329666137695 (18.070329666137695) + | > loss_duration: 1.4309263229370117 (1.4309263229370117) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.08364486694336 (31.08364486694336) + | > grad_norm_1: tensor(240.7857, device='cuda:0') (tensor(240.7857, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.3425 (0.34250903129577637) + | > loader_time: 0.3199 (0.31990718841552734) + + + --> TIME: 2025-04-19 05:37:20 -- STEP: 25/142 -- GLOBAL_STEP: 436675 + | > loss_disc: 2.48721981048584 (2.417309465408325) + | > loss_disc_real_0: 0.14625626802444458 (0.11673978835344315) + | > loss_disc_real_1: 0.22983679175376892 (0.2021200531721115) + | > loss_disc_real_2: 0.23172271251678467 (0.22714393377304076) + | > loss_disc_real_3: 0.25235095620155334 (0.22659059703350068) + | > loss_disc_real_4: 0.20814941823482513 (0.22123991549015046) + | > loss_disc_real_5: 0.22627747058868408 (0.22880871891975402) + | > loss_0: 2.48721981048584 (2.417309465408325) + | > grad_norm_0: tensor(5.9438, device='cuda:0') (tensor(6.2294, device='cuda:0')) + | > loss_gen: 2.4454095363616943 (2.386876277923584) + | > loss_kl: 1.790954828262329 (1.7973838567733764) + | > loss_feat: 7.340582370758057 (7.427372150421142) + | > loss_mel: 16.562379837036133 (17.76211784362793) + | > loss_duration: 1.4414305686950684 (1.416957688331604) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.58075714111328 (30.79070770263672) + | > grad_norm_1: tensor(215.5522, device='cuda:0') (tensor(145.6931, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.3451 (0.3417816257476807) + | > loader_time: 0.0053 (0.004638042449951172) + + + --> TIME: 2025-04-19 05:37:30 -- STEP: 50/142 -- GLOBAL_STEP: 436700 + | > loss_disc: 2.502363681793213 (2.4441138601303094) + | > loss_disc_real_0: 0.16105999052524567 (0.12320723876357079) + | > loss_disc_real_1: 0.2012077271938324 (0.20597918540239335) + | > loss_disc_real_2: 0.19671018421649933 (0.22642448753118516) + | > loss_disc_real_3: 0.22057244181632996 (0.22712090522050857) + | > loss_disc_real_4: 0.2179577648639679 (0.22495910048484802) + | > loss_disc_real_5: 0.24877536296844482 (0.2333880665898323) + | > loss_0: 2.502363681793213 (2.4441138601303094) + | > grad_norm_0: tensor(14.3150, device='cuda:0') (tensor(7.9134, device='cuda:0')) + | > loss_gen: 2.2619004249572754 (2.374594130516053) + | > loss_kl: 1.7874385118484497 (1.8164747500419616) + | > loss_feat: 6.59753942489624 (7.312473230361938) + | > loss_mel: 17.134071350097656 (17.73997175216675) + | > loss_duration: 1.4647605419158936 (1.4256379270553592) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.245710372924805 (30.669151611328125) + | > grad_norm_1: tensor(99.6835, device='cuda:0') (tensor(175.1798, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.3699 (0.35445701599121093) + | > loader_time: 0.0059 (0.005063567161560058) + + + --> TIME: 2025-04-19 05:37:40 -- STEP: 75/142 -- GLOBAL_STEP: 436725 + | > loss_disc: 2.5478949546813965 (2.4519162209828687) + | > loss_disc_real_0: 0.09759782999753952 (0.12581565837065378) + | > loss_disc_real_1: 0.21295367181301117 (0.20516963462034865) + | > loss_disc_real_2: 0.2213277965784073 (0.22680106441179912) + | > loss_disc_real_3: 0.24154630303382874 (0.22803106069564819) + | > loss_disc_real_4: 0.2590538561344147 (0.22618668794631958) + | > loss_disc_real_5: 0.22517457604408264 (0.2322833812236786) + | > loss_0: 2.5478949546813965 (2.4519162209828687) + | > grad_norm_0: tensor(14.7898, device='cuda:0') (tensor(8.9424, device='cuda:0')) + | > loss_gen: 2.2914977073669434 (2.3696890576680505) + | > loss_kl: 1.7970784902572632 (1.8363023503621418) + | > loss_feat: 7.065995216369629 (7.3027525202433266) + | > loss_mel: 16.096092224121094 (17.73929389953613) + | > loss_duration: 1.423011064529419 (1.4376933002471923) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.673675537109375 (30.685731048583985) + | > grad_norm_1: tensor(353.5085, device='cuda:0') (tensor(201.6408, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.3957 (0.3643107795715332) + | > loader_time: 0.0068 (0.0054257774353027345) + + + --> TIME: 2025-04-19 05:37:50 -- STEP: 100/142 -- GLOBAL_STEP: 436750 + | > loss_disc: 2.3701252937316895 (2.4519554138183586) + | > loss_disc_real_0: 0.10734977573156357 (0.1257637124508619) + | > loss_disc_real_1: 0.18354502320289612 (0.20538179725408554) + | > loss_disc_real_2: 0.21487031877040863 (0.22615632817149162) + | > loss_disc_real_3: 0.2195615917444229 (0.2273554238677025) + | > loss_disc_real_4: 0.2172946035861969 (0.22522911086678504) + | > loss_disc_real_5: 0.22607654333114624 (0.23236060991883278) + | > loss_0: 2.3701252937316895 (2.4519554138183586) + | > grad_norm_0: tensor(7.6214, device='cuda:0') (tensor(9.8061, device='cuda:0')) + | > loss_gen: 2.5905494689941406 (2.371432194709779) + | > loss_kl: 1.9411678314208984 (1.8495129799842835) + | > loss_feat: 8.156919479370117 (7.2094238758087155) + | > loss_mel: 18.82888412475586 (17.694311075210567) + | > loss_duration: 1.4727783203125 (1.4481249415874482) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.990299224853516 (30.572805042266847) + | > grad_norm_1: tensor(396.3975, device='cuda:0') (tensor(223.8717, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.414 (0.37327193975448625) + | > loader_time: 0.0069 (0.005742316246032714) + + + --> TIME: 2025-04-19 05:38:01 -- STEP: 125/142 -- GLOBAL_STEP: 436775 + | > loss_disc: 2.451554536819458 (2.451814693450927) + | > loss_disc_real_0: 0.13384075462818146 (0.12602941858768457) + | > loss_disc_real_1: 0.19052088260650635 (0.2050247666835785) + | > loss_disc_real_2: 0.24202167987823486 (0.22598958098888397) + | > loss_disc_real_3: 0.26852354407310486 (0.22805577051639556) + | > loss_disc_real_4: 0.213180273771286 (0.22567324674129485) + | > loss_disc_real_5: 0.22879454493522644 (0.23278629410266877) + | > loss_0: 2.451554536819458 (2.451814693450927) + | > grad_norm_0: tensor(8.3534, device='cuda:0') (tensor(10.1815, device='cuda:0')) + | > loss_gen: 2.5029520988464355 (2.3683532199859627) + | > loss_kl: 1.8063677549362183 (1.8565001859664918) + | > loss_feat: 6.940989971160889 (7.177514812469482) + | > loss_mel: 18.187164306640625 (17.67062848663331) + | > loss_duration: 1.5293147563934326 (1.455673734664917) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.966787338256836 (30.528670349121093) + | > grad_norm_1: tensor(121.6394, device='cuda:0') (tensor(237.4891, device='cuda:0')) + | > current_lr_0: 0.00019015018552197388 + | > current_lr_1: 0.00019015018552197388 + | > step_time: 0.4376 (0.38413128471374525) + | > loader_time: 0.0075 (0.006018861770629882) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.500100612640381 (2.500100612640381) + | > loss_disc_real_0: 0.14818398654460907 (0.14818398654460907) + | > loss_disc_real_1: 0.24248342216014862 (0.24248342216014862) + | > loss_disc_real_2: 0.27793219685554504 (0.27793219685554504) + | > loss_disc_real_3: 0.28555604815483093 (0.28555604815483093) + | > loss_disc_real_4: 0.20599427819252014 (0.20599427819252014) + | > loss_disc_real_5: 0.30405285954475403 (0.30405285954475403) + | > loss_0: 2.500100612640381 (2.500100612640381) + | > loss_gen: 2.5399820804595947 (2.5399820804595947) + | > loss_kl: 2.5659921169281006 (2.5659921169281006) + | > loss_feat: 7.579315662384033 (7.579315662384033) + | > loss_mel: 19.431583404541016 (19.431583404541016) + | > loss_duration: 1.758798360824585 (1.758798360824585) + | > loss_1: 33.87567138671875 (33.87567138671875) + + --> STEP: 1 + | > loss_disc: 2.574763774871826 (2.574763774871826) + | > loss_disc_real_0: 0.1465567797422409 (0.1465567797422409) + | > loss_disc_real_1: 0.24346277117729187 (0.24346277117729187) + | > loss_disc_real_2: 0.2787080407142639 (0.2787080407142639) + | > loss_disc_real_3: 0.26917457580566406 (0.26917457580566406) + | > loss_disc_real_4: 0.2672834098339081 (0.2672834098339081) + | > loss_disc_real_5: 0.30514028668403625 (0.30514028668403625) + | > loss_0: 2.574763774871826 (2.574763774871826) + | > loss_gen: 2.5426058769226074 (2.5426058769226074) + | > loss_kl: 2.712437629699707 (2.712437629699707) + | > loss_feat: 8.042460441589355 (8.042460441589355) + | > loss_mel: 18.75445556640625 (18.75445556640625) + | > loss_duration: 1.8097411394119263 (1.8097411394119263) + | > loss_1: 33.86170196533203 (33.86170196533203) + + --> STEP: 2 + | > loss_disc: 2.497042417526245 (2.5359030961990356) + | > loss_disc_real_0: 0.17538177967071533 (0.16096927970647812) + | > loss_disc_real_1: 0.20516394078731537 (0.22431335598230362) + | > loss_disc_real_2: 0.2525997757911682 (0.26565390825271606) + | > loss_disc_real_3: 0.2514708340167999 (0.260322704911232) + | > loss_disc_real_4: 0.23712404072284698 (0.25220372527837753) + | > loss_disc_real_5: 0.23652316629886627 (0.27083172649145126) + | > loss_0: 2.497042417526245 (2.5359030961990356) + | > loss_gen: 2.3935599327087402 (2.468082904815674) + | > loss_kl: 2.334792137145996 (2.5236148834228516) + | > loss_feat: 6.8845367431640625 (7.463498592376709) + | > loss_mel: 17.070199966430664 (17.912327766418457) + | > loss_duration: 1.71976900100708 (1.7647550702095032) + | > loss_1: 30.402856826782227 (32.13227939605713) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003519415855407715 (+0.00011730194091796875) + | > avg_loss_disc: 2.5359030961990356 (-0.10874390602111816) + | > avg_loss_disc_real_0: 0.16096927970647812 (-0.0649268701672554) + | > avg_loss_disc_real_1: 0.22431335598230362 (-0.029193833470344543) + | > avg_loss_disc_real_2: 0.26565390825271606 (+0.00415937602519989) + | > avg_loss_disc_real_3: 0.260322704911232 (-0.010890975594520569) + | > avg_loss_disc_real_4: 0.25220372527837753 (-0.0053667351603507996) + | > avg_loss_disc_real_5: 0.27083172649145126 (+0.014607071876525879) + | > avg_loss_0: 2.5359030961990356 (-0.10874390602111816) + | > avg_loss_gen: 2.468082904815674 (+0.04252481460571289) + | > avg_loss_kl: 2.5236148834228516 (-0.0731586217880249) + | > avg_loss_feat: 7.463498592376709 (+1.1254782676696777) + | > avg_loss_mel: 17.912327766418457 (+0.8427581787109375) + | > avg_loss_duration: 1.7647550702095032 (-8.308887481689453e-05) + | > avg_loss_1: 32.13227939605713 (+1.9375190734863281) + + + > EPOCH: 405/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:38:10)  + + --> TIME: 2025-04-19 05:38:13 -- STEP: 8/142 -- GLOBAL_STEP: 436800 + | > loss_disc: 2.4781579971313477 (2.3745627403259277) + | > loss_disc_real_0: 0.10894312709569931 (0.11361388769000769) + | > loss_disc_real_1: 0.17406907677650452 (0.20037063024938107) + | > loss_disc_real_2: 0.24168983101844788 (0.22873255610466003) + | > loss_disc_real_3: 0.20219558477401733 (0.22345639765262604) + | > loss_disc_real_4: 0.2408865988254547 (0.21539640799164772) + | > loss_disc_real_5: 0.18088819086551666 (0.2214046809822321) + | > loss_0: 2.4781579971313477 (2.3745627403259277) + | > grad_norm_0: tensor(4.7174, device='cuda:0') (tensor(10.4305, device='cuda:0')) + | > loss_gen: 2.317115545272827 (2.420417219400406) + | > loss_kl: 1.7051948308944702 (1.7329613268375397) + | > loss_feat: 7.74607515335083 (7.588345110416412) + | > loss_mel: 17.466936111450195 (17.4639949798584) + | > loss_duration: 1.3975231647491455 (1.4067416191101074) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.632844924926758 (30.612459897994995) + | > grad_norm_1: tensor(94.1312, device='cuda:0') (tensor(251.2245, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.3234 (0.3237336277961731) + | > loader_time: 0.0051 (0.004401057958602905) + + + --> TIME: 2025-04-19 05:38:22 -- STEP: 33/142 -- GLOBAL_STEP: 436825 + | > loss_disc: 2.4527697563171387 (2.4187431913433644) + | > loss_disc_real_0: 0.11986663192510605 (0.11342722696788383) + | > loss_disc_real_1: 0.2081814557313919 (0.2062824382023378) + | > loss_disc_real_2: 0.2183326631784439 (0.22717803129644104) + | > loss_disc_real_3: 0.23617468774318695 (0.22644574985359656) + | > loss_disc_real_4: 0.25386136770248413 (0.2273553271185268) + | > loss_disc_real_5: 0.2162245661020279 (0.23078633348147073) + | > loss_0: 2.4527697563171387 (2.4187431913433644) + | > grad_norm_0: tensor(6.3747, device='cuda:0') (tensor(10.1046, device='cuda:0')) + | > loss_gen: 2.373779773712158 (2.4237040245171744) + | > loss_kl: 1.721798300743103 (1.7997101147969563) + | > loss_feat: 7.861845970153809 (7.44145578326601) + | > loss_mel: 19.44276237487793 (17.856564810781766) + | > loss_duration: 1.4200541973114014 (1.4156248280496309) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.82024002075195 (30.937059633659594) + | > grad_norm_1: tensor(274.9174, device='cuda:0') (tensor(267.7109, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.3563 (0.33696523579684173) + | > loader_time: 0.0057 (0.004981373295639501) + + + --> TIME: 2025-04-19 05:38:31 -- STEP: 58/142 -- GLOBAL_STEP: 436850 + | > loss_disc: 2.4407081604003906 (2.4351611589563302) + | > loss_disc_real_0: 0.17045025527477264 (0.11729268575536794) + | > loss_disc_real_1: 0.1721767783164978 (0.20671508923686785) + | > loss_disc_real_2: 0.20197859406471252 (0.2251944043513002) + | > loss_disc_real_3: 0.22409075498580933 (0.22689213629426627) + | > loss_disc_real_4: 0.21222855150699615 (0.2265927948314568) + | > loss_disc_real_5: 0.229783833026886 (0.23089938276800617) + | > loss_0: 2.4407081604003906 (2.4351611589563302) + | > grad_norm_0: tensor(19.2319, device='cuda:0') (tensor(11.4593, device='cuda:0')) + | > loss_gen: 2.3619871139526367 (2.3992890242872567) + | > loss_kl: 1.8090379238128662 (1.8141047769579395) + | > loss_feat: 7.109425067901611 (7.336886923888634) + | > loss_mel: 19.01810646057129 (17.816689014434814) + | > loss_duration: 1.4537787437438965 (1.4298762925739945) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.752336502075195 (30.796845995146654) + | > grad_norm_1: tensor(256.9057, device='cuda:0') (tensor(293.6383, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.374 (0.3484882774024174) + | > loader_time: 0.0061 (0.005307896383877458) + + + --> TIME: 2025-04-19 05:38:41 -- STEP: 83/142 -- GLOBAL_STEP: 436875 + | > loss_disc: 2.5282180309295654 (2.4425959989249) + | > loss_disc_real_0: 0.07456298172473907 (0.12065515514597835) + | > loss_disc_real_1: 0.20322462916374207 (0.20693324596048837) + | > loss_disc_real_2: 0.22550852596759796 (0.22493495406156563) + | > loss_disc_real_3: 0.23465748131275177 (0.22690339296697135) + | > loss_disc_real_4: 0.2120646983385086 (0.2269699817680451) + | > loss_disc_real_5: 0.21715085208415985 (0.23223597493516393) + | > loss_0: 2.5282180309295654 (2.4425959989249) + | > grad_norm_0: tensor(10.1735, device='cuda:0') (tensor(13.4670, device='cuda:0')) + | > loss_gen: 2.159062385559082 (2.399829034345697) + | > loss_kl: 2.002734422683716 (1.8355648776134812) + | > loss_feat: 6.306212902069092 (7.293367293943842) + | > loss_mel: 16.71854019165039 (17.783244604087734) + | > loss_duration: 1.525266170501709 (1.4420687118208553) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.711816787719727 (30.754074533301665) + | > grad_norm_1: tensor(125.7591, device='cuda:0') (tensor(295.9223, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.3961 (0.36067146852792026) + | > loader_time: 0.0063 (0.005649885499333762) + + + --> TIME: 2025-04-19 05:38:52 -- STEP: 108/142 -- GLOBAL_STEP: 436900 + | > loss_disc: 2.4106760025024414 (2.4438997749929086) + | > loss_disc_real_0: 0.10292992740869522 (0.12017453896502654) + | > loss_disc_real_1: 0.19161245226860046 (0.20655491285853916) + | > loss_disc_real_2: 0.22215639054775238 (0.225378329831141) + | > loss_disc_real_3: 0.2024037092924118 (0.22709779869075175) + | > loss_disc_real_4: 0.16982142627239227 (0.22687600001140876) + | > loss_disc_real_5: 0.2284328192472458 (0.23284025117754936) + | > loss_0: 2.4106760025024414 (2.4438997749929086) + | > grad_norm_0: tensor(10.7139, device='cuda:0') (tensor(13.3342, device='cuda:0')) + | > loss_gen: 2.3797266483306885 (2.3971946040789285) + | > loss_kl: 1.9537837505340576 (1.8491684595743816) + | > loss_feat: 7.390214920043945 (7.261374654593291) + | > loss_mel: 18.350603103637695 (17.68147846504493) + | > loss_duration: 1.4838104248046875 (1.4507683670079266) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.55813980102539 (30.639984713660347) + | > grad_norm_1: tensor(228.0421, device='cuda:0') (tensor(298.0265, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.4176 (0.37211059420197096) + | > loader_time: 0.007 (0.005944508093374747) + + + --> TIME: 2025-04-19 05:39:03 -- STEP: 133/142 -- GLOBAL_STEP: 436925 + | > loss_disc: 2.4304916858673096 (2.44902557358706) + | > loss_disc_real_0: 0.14406616985797882 (0.12145274003645531) + | > loss_disc_real_1: 0.2036948800086975 (0.20611655868982015) + | > loss_disc_real_2: 0.2066032886505127 (0.22567782265351233) + | > loss_disc_real_3: 0.18386413156986237 (0.2273544288219366) + | > loss_disc_real_4: 0.17345589399337769 (0.2276746286709506) + | > loss_disc_real_5: 0.20783422887325287 (0.23363518692497023) + | > loss_0: 2.4304916858673096 (2.44902557358706) + | > grad_norm_0: tensor(6.7049, device='cuda:0') (tensor(12.8091, device='cuda:0')) + | > loss_gen: 2.24617600440979 (2.388145047022884) + | > loss_kl: 2.0094351768493652 (1.8582243578774589) + | > loss_feat: 6.971685409545898 (7.227143409556913) + | > loss_mel: 17.55157470703125 (17.678894860403865) + | > loss_duration: 1.5262891054153442 (1.4571229891669482) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.305160522460938 (30.60953080026727) + | > grad_norm_1: tensor(108.7417, device='cuda:0') (tensor(286.5515, device='cuda:0')) + | > current_lr_0: 0.00019012641674878364 + | > current_lr_1: 0.00019012641674878364 + | > step_time: 0.4373 (0.3837421998045498) + | > loader_time: 0.007 (0.006202927209380875) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.599409580230713 (2.599409580230713) + | > loss_disc_real_0: 0.14194300770759583 (0.14194300770759583) + | > loss_disc_real_1: 0.3048924505710602 (0.3048924505710602) + | > loss_disc_real_2: 0.24238404631614685 (0.24238404631614685) + | > loss_disc_real_3: 0.26409175992012024 (0.26409175992012024) + | > loss_disc_real_4: 0.23792128264904022 (0.23792128264904022) + | > loss_disc_real_5: 0.2799750864505768 (0.2799750864505768) + | > loss_0: 2.599409580230713 (2.599409580230713) + | > loss_gen: 2.2931275367736816 (2.2931275367736816) + | > loss_kl: 2.3653817176818848 (2.3653817176818848) + | > loss_feat: 6.1799445152282715 (6.1799445152282715) + | > loss_mel: 18.17901611328125 (18.17901611328125) + | > loss_duration: 1.7460569143295288 (1.7460569143295288) + | > loss_1: 30.763526916503906 (30.763526916503906) + + --> STEP: 1 + | > loss_disc: 2.5889194011688232 (2.5889194011688232) + | > loss_disc_real_0: 0.09986835718154907 (0.09986835718154907) + | > loss_disc_real_1: 0.2984069883823395 (0.2984069883823395) + | > loss_disc_real_2: 0.223102867603302 (0.223102867603302) + | > loss_disc_real_3: 0.2646745443344116 (0.2646745443344116) + | > loss_disc_real_4: 0.21000942587852478 (0.21000942587852478) + | > loss_disc_real_5: 0.22327446937561035 (0.22327446937561035) + | > loss_0: 2.5889194011688232 (2.5889194011688232) + | > loss_gen: 2.171318769454956 (2.171318769454956) + | > loss_kl: 3.034036159515381 (3.034036159515381) + | > loss_feat: 7.136049747467041 (7.136049747467041) + | > loss_mel: 18.68519401550293 (18.68519401550293) + | > loss_duration: 1.7686340808868408 (1.7686340808868408) + | > loss_1: 32.795230865478516 (32.795230865478516) + + --> STEP: 2 + | > loss_disc: 2.289996862411499 (2.439458131790161) + | > loss_disc_real_0: 0.12153814733028412 (0.1107032522559166) + | > loss_disc_real_1: 0.2635558247566223 (0.2809814065694809) + | > loss_disc_real_2: 0.20423424243927002 (0.213668555021286) + | > loss_disc_real_3: 0.19714751839637756 (0.2309110313653946) + | > loss_disc_real_4: 0.19073240458965302 (0.2003709152340889) + | > loss_disc_real_5: 0.23869548738002777 (0.23098497837781906) + | > loss_0: 2.289996862411499 (2.439458131790161) + | > loss_gen: 2.46890926361084 (2.320114016532898) + | > loss_kl: 2.4104363918304443 (2.7222362756729126) + | > loss_feat: 7.28986930847168 (7.21295952796936) + | > loss_mel: 17.758014678955078 (18.221604347229004) + | > loss_duration: 1.7279856204986572 (1.748309850692749) + | > loss_1: 31.655216217041016 (32.225223541259766) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003411531448364258 (-0.00010788440704345703) + | > avg_loss_disc: 2.439458131790161 (-0.09644496440887451) + | > avg_loss_disc_real_0: 0.1107032522559166 (-0.05026602745056152) + | > avg_loss_disc_real_1: 0.2809814065694809 (+0.05666805058717728) + | > avg_loss_disc_real_2: 0.213668555021286 (-0.051985353231430054) + | > avg_loss_disc_real_3: 0.2309110313653946 (-0.029411673545837402) + | > avg_loss_disc_real_4: 0.2003709152340889 (-0.051832810044288635) + | > avg_loss_disc_real_5: 0.23098497837781906 (-0.0398467481136322) + | > avg_loss_0: 2.439458131790161 (-0.09644496440887451) + | > avg_loss_gen: 2.320114016532898 (-0.14796888828277588) + | > avg_loss_kl: 2.7222362756729126 (+0.19862139225006104) + | > avg_loss_feat: 7.21295952796936 (-0.25053906440734863) + | > avg_loss_mel: 18.221604347229004 (+0.3092765808105469) + | > avg_loss_duration: 1.748309850692749 (-0.01644521951675415) + | > avg_loss_1: 32.225223541259766 (+0.09294414520263672) + + + > EPOCH: 406/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:39:08)  + + --> TIME: 2025-04-19 05:39:14 -- STEP: 16/142 -- GLOBAL_STEP: 436950 + | > loss_disc: 2.5134952068328857 (2.4675315618515015) + | > loss_disc_real_0: 0.09539289772510529 (0.12185312388464808) + | > loss_disc_real_1: 0.21985772252082825 (0.20226579159498215) + | > loss_disc_real_2: 0.2583988308906555 (0.22902285307645798) + | > loss_disc_real_3: 0.21800488233566284 (0.2284856140613556) + | > loss_disc_real_4: 0.2105879783630371 (0.23561831004917622) + | > loss_disc_real_5: 0.2402220517396927 (0.23761871550232172) + | > loss_0: 2.5134952068328857 (2.4675315618515015) + | > grad_norm_0: tensor(11.6715, device='cuda:0') (tensor(9.2119, device='cuda:0')) + | > loss_gen: 2.511758804321289 (2.3709618747234344) + | > loss_kl: 1.8819185495376587 (1.8181740194559097) + | > loss_feat: 7.215949058532715 (7.206151843070984) + | > loss_mel: 17.305910110473633 (17.289770424365994) + | > loss_duration: 1.3948701620101929 (1.4070524126291275) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.310407638549805 (30.09211039543152) + | > grad_norm_1: tensor(256.8119, device='cuda:0') (tensor(177.0390, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.338 (0.32725299894809723) + | > loader_time: 0.0053 (0.004682287573814392) + + + --> TIME: 2025-04-19 05:39:23 -- STEP: 41/142 -- GLOBAL_STEP: 436975 + | > loss_disc: 2.4709107875823975 (2.4525926752788263) + | > loss_disc_real_0: 0.10962145775556564 (0.12233348536055262) + | > loss_disc_real_1: 0.24658408761024475 (0.20589924658217082) + | > loss_disc_real_2: 0.20709377527236938 (0.2278406536433755) + | > loss_disc_real_3: 0.2393072247505188 (0.2274473389474357) + | > loss_disc_real_4: 0.2088492512702942 (0.2286965905166254) + | > loss_disc_real_5: 0.2750348448753357 (0.23480683129008223) + | > loss_0: 2.4709107875823975 (2.4525926752788263) + | > grad_norm_0: tensor(4.8487, device='cuda:0') (tensor(7.5826, device='cuda:0')) + | > loss_gen: 2.338789463043213 (2.3694524706863778) + | > loss_kl: 1.9345000982284546 (1.8379840879905514) + | > loss_feat: 7.155760765075684 (7.247351902287181) + | > loss_mel: 17.63861656188965 (17.63355371428699) + | > loss_duration: 1.4216456413269043 (1.420859819505273) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.48931312561035 (30.509201980218656) + | > grad_norm_1: tensor(198.5697, device='cuda:0') (tensor(149.6937, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.3596 (0.34038851319289803) + | > loader_time: 0.0052 (0.005065598138948766) + + + --> TIME: 2025-04-19 05:39:33 -- STEP: 66/142 -- GLOBAL_STEP: 437000 + | > loss_disc: 2.33333683013916 (2.4617333628914575) + | > loss_disc_real_0: 0.10550089180469513 (0.12419372130975578) + | > loss_disc_real_1: 0.1897796094417572 (0.20692844305074576) + | > loss_disc_real_2: 0.196929469704628 (0.2276205892364184) + | > loss_disc_real_3: 0.18550556898117065 (0.2285451970317147) + | > loss_disc_real_4: 0.19522905349731445 (0.22877313196659088) + | > loss_disc_real_5: 0.21940137445926666 (0.2366548605037458) + | > loss_0: 2.33333683013916 (2.4617333628914575) + | > grad_norm_0: tensor(4.7157, device='cuda:0') (tensor(7.3573, device='cuda:0')) + | > loss_gen: 2.285564661026001 (2.3565569827050874) + | > loss_kl: 1.890041470527649 (1.8479086883140332) + | > loss_feat: 8.01939868927002 (7.184943574847597) + | > loss_mel: 19.447542190551758 (17.64222671046401) + | > loss_duration: 1.4375838041305542 (1.4328125065023252) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.08013153076172 (30.464448350848574) + | > grad_norm_1: tensor(91.4617, device='cuda:0') (tensor(141.9656, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.3887 (0.35358911210840405) + | > loader_time: 0.0064 (0.00543751861109878) + + + --> TIME: 2025-04-19 05:39:43 -- STEP: 91/142 -- GLOBAL_STEP: 437025 + | > loss_disc: 2.406553268432617 (2.458972936148172) + | > loss_disc_real_0: 0.1018209457397461 (0.12241450566184389) + | > loss_disc_real_1: 0.23583365976810455 (0.20668313431215815) + | > loss_disc_real_2: 0.2152508795261383 (0.22734414041042328) + | > loss_disc_real_3: 0.19654729962348938 (0.2278533180008878) + | > loss_disc_real_4: 0.2155161052942276 (0.2280870491004252) + | > loss_disc_real_5: 0.23617927730083466 (0.23532821107041704) + | > loss_0: 2.406553268432617 (2.458972936148172) + | > grad_norm_0: tensor(14.2383, device='cuda:0') (tensor(8.4560, device='cuda:0')) + | > loss_gen: 2.4236652851104736 (2.3612944555806594) + | > loss_kl: 1.910131573677063 (1.8592293380380986) + | > loss_feat: 7.44528341293335 (7.202408759148566) + | > loss_mel: 18.26287078857422 (17.77261077964699) + | > loss_duration: 1.4806122779846191 (1.4440233275130552) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.522563934326172 (30.63956658918779) + | > grad_norm_1: tensor(485.0789, device='cuda:0') (tensor(184.7949, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.3991 (0.3649828695988918) + | > loader_time: 0.0071 (0.005734375544956752) + + + --> TIME: 2025-04-19 05:39:54 -- STEP: 116/142 -- GLOBAL_STEP: 437050 + | > loss_disc: 2.3678417205810547 (2.461841727125234) + | > loss_disc_real_0: 0.09201764315366745 (0.12187816517363334) + | > loss_disc_real_1: 0.22067378461360931 (0.20655420409708192) + | > loss_disc_real_2: 0.20510990917682648 (0.22709801716023478) + | > loss_disc_real_3: 0.2424807846546173 (0.2285579582997437) + | > loss_disc_real_4: 0.22516770660877228 (0.2298916853193579) + | > loss_disc_real_5: 0.24454312026500702 (0.23457227672996192) + | > loss_0: 2.3678417205810547 (2.461841727125234) + | > grad_norm_0: tensor(15.1426, device='cuda:0') (tensor(8.8877, device='cuda:0')) + | > loss_gen: 2.5664784908294678 (2.3563535007937206) + | > loss_kl: 1.698651909828186 (1.8573531611212368) + | > loss_feat: 7.40720272064209 (7.18651400763413) + | > loss_mel: 17.393701553344727 (17.754215051387913) + | > loss_duration: 1.4659764766693115 (1.4516115866858381) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.532012939453125 (30.606047268571526) + | > grad_norm_1: tensor(564.1194, device='cuda:0') (tensor(203.8007, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.4267 (0.37546010264034935) + | > loader_time: 0.0072 (0.006026605079913962) + + + --> TIME: 2025-04-19 05:40:05 -- STEP: 141/142 -- GLOBAL_STEP: 437075 + | > loss_disc: 2.206068992614746 (2.4637350376616136) + | > loss_disc_real_0: 0.06169964000582695 (0.12208280915803943) + | > loss_disc_real_1: 0.12987037003040314 (0.20529225870227139) + | > loss_disc_real_2: 0.17065030336380005 (0.22654958365234076) + | > loss_disc_real_3: 0.19496947526931763 (0.22852776120317744) + | > loss_disc_real_4: 0.14717473089694977 (0.22929365193167478) + | > loss_disc_real_5: 0.1774943768978119 (0.23549646308236089) + | > loss_0: 2.206068992614746 (2.4637350376616136) + | > grad_norm_0: tensor(18.0415, device='cuda:0') (tensor(9.4108, device='cuda:0')) + | > loss_gen: 2.915036916732788 (2.3569847861080304) + | > loss_kl: 1.7694458961486816 (1.8658293916824016) + | > loss_feat: 8.745903015136719 (7.172728382949288) + | > loss_mel: 18.83403968811035 (17.69915799891694) + | > loss_duration: 1.3158857822418213 (1.4573703640741658) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.5803108215332 (30.55207082058521) + | > grad_norm_1: tensor(395.3489, device='cuda:0') (tensor(219.4123, device='cuda:0')) + | > current_lr_0: 0.00019010265094669004 + | > current_lr_1: 0.00019010265094669004 + | > step_time: 0.2027 (0.3858545283053784) + | > loader_time: 0.0028 (0.0062317949660280906) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4988632202148438 (2.4988632202148438) + | > loss_disc_real_0: 0.20972035825252533 (0.20972035825252533) + | > loss_disc_real_1: 0.30035850405693054 (0.30035850405693054) + | > loss_disc_real_2: 0.2531657814979553 (0.2531657814979553) + | > loss_disc_real_3: 0.2229384332895279 (0.2229384332895279) + | > loss_disc_real_4: 0.24784931540489197 (0.24784931540489197) + | > loss_disc_real_5: 0.2645556628704071 (0.2645556628704071) + | > loss_0: 2.4988632202148438 (2.4988632202148438) + | > loss_gen: 2.690997362136841 (2.690997362136841) + | > loss_kl: 2.749694585800171 (2.749694585800171) + | > loss_feat: 5.876916885375977 (5.876916885375977) + | > loss_mel: 16.103796005249023 (16.103796005249023) + | > loss_duration: 1.7739804983139038 (1.7739804983139038) + | > loss_1: 29.19538688659668 (29.19538688659668) + + --> STEP: 1 + | > loss_disc: 2.666470527648926 (2.666470527648926) + | > loss_disc_real_0: 0.24122591316699982 (0.24122591316699982) + | > loss_disc_real_1: 0.30804014205932617 (0.30804014205932617) + | > loss_disc_real_2: 0.235383078455925 (0.235383078455925) + | > loss_disc_real_3: 0.23807674646377563 (0.23807674646377563) + | > loss_disc_real_4: 0.23424668610095978 (0.23424668610095978) + | > loss_disc_real_5: 0.23766423761844635 (0.23766423761844635) + | > loss_0: 2.666470527648926 (2.666470527648926) + | > loss_gen: 2.6790072917938232 (2.6790072917938232) + | > loss_kl: 2.8609976768493652 (2.8609976768493652) + | > loss_feat: 7.24888277053833 (7.24888277053833) + | > loss_mel: 19.192855834960938 (19.192855834960938) + | > loss_duration: 1.761202335357666 (1.761202335357666) + | > loss_1: 33.74294662475586 (33.74294662475586) + + --> STEP: 2 + | > loss_disc: 2.6970129013061523 (2.681741714477539) + | > loss_disc_real_0: 0.38472938537597656 (0.3129776492714882) + | > loss_disc_real_1: 0.31676962971687317 (0.31240488588809967) + | > loss_disc_real_2: 0.2606010437011719 (0.24799206107854843) + | > loss_disc_real_3: 0.2518996596336365 (0.24498820304870605) + | > loss_disc_real_4: 0.23997104167938232 (0.23710886389017105) + | > loss_disc_real_5: 0.2507367432117462 (0.24420049041509628) + | > loss_0: 2.6970129013061523 (2.681741714477539) + | > loss_gen: 2.90181827545166 (2.7904127836227417) + | > loss_kl: 2.094468355178833 (2.477733016014099) + | > loss_feat: 6.666534423828125 (6.9577085971832275) + | > loss_mel: 18.045637130737305 (18.61924648284912) + | > loss_duration: 1.7356985807418823 (1.7484504580497742) + | > loss_1: 31.444156646728516 (32.59355163574219) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004082441329956055 (+0.0006709098815917969) + | > avg_loss_disc: 2.681741714477539 (+0.24228358268737793) + | > avg_loss_disc_real_0: 0.3129776492714882 (+0.2022743970155716) + | > avg_loss_disc_real_1: 0.31240488588809967 (+0.031423479318618774) + | > avg_loss_disc_real_2: 0.24799206107854843 (+0.03432350605726242) + | > avg_loss_disc_real_3: 0.24498820304870605 (+0.014077171683311462) + | > avg_loss_disc_real_4: 0.23710886389017105 (+0.03673794865608215) + | > avg_loss_disc_real_5: 0.24420049041509628 (+0.013215512037277222) + | > avg_loss_0: 2.681741714477539 (+0.24228358268737793) + | > avg_loss_gen: 2.7904127836227417 (+0.47029876708984375) + | > avg_loss_kl: 2.477733016014099 (-0.24450325965881348) + | > avg_loss_feat: 6.9577085971832275 (-0.2552509307861328) + | > avg_loss_mel: 18.61924648284912 (+0.3976421356201172) + | > avg_loss_duration: 1.7484504580497742 (+0.00014060735702514648) + | > avg_loss_1: 32.59355163574219 (+0.3683280944824219) + + + > EPOCH: 407/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:40:07)  + + --> TIME: 2025-04-19 05:40:16 -- STEP: 24/142 -- GLOBAL_STEP: 437100 + | > loss_disc: 2.5249059200286865 (2.427287499109904) + | > loss_disc_real_0: 0.09610838443040848 (0.11317905193815629) + | > loss_disc_real_1: 0.2083531767129898 (0.2074059204508861) + | > loss_disc_real_2: 0.25967150926589966 (0.22710609373946986) + | > loss_disc_real_3: 0.20481930673122406 (0.2263970449566841) + | > loss_disc_real_4: 0.24857555329799652 (0.22689032120009264) + | > loss_disc_real_5: 0.25624755024909973 (0.23095604466895261) + | > loss_0: 2.5249059200286865 (2.427287499109904) + | > grad_norm_0: tensor(7.7513, device='cuda:0') (tensor(10.6560, device='cuda:0')) + | > loss_gen: 2.4000332355499268 (2.4057683050632477) + | > loss_kl: 1.928843379020691 (1.8374617993831635) + | > loss_feat: 6.403237819671631 (7.240754226843516) + | > loss_mel: 17.047924041748047 (17.571517467498786) + | > loss_duration: 1.4507330656051636 (1.4120017538468044) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.230772018432617 (30.4675034681956) + | > grad_norm_1: tensor(121.3337, device='cuda:0') (tensor(292.6824, device='cuda:0')) + | > current_lr_0: 0.0001900788881153217 + | > current_lr_1: 0.0001900788881153217 + | > step_time: 0.3507 (0.33389467000961304) + | > loader_time: 0.0055 (0.004870394865671793) + + + --> TIME: 2025-04-19 05:40:25 -- STEP: 49/142 -- GLOBAL_STEP: 437125 + | > loss_disc: 2.489097833633423 (2.4495777694546446) + | > loss_disc_real_0: 0.12167984247207642 (0.11613587016353802) + | > loss_disc_real_1: 0.2472943663597107 (0.20888436357585752) + | > loss_disc_real_2: 0.22602367401123047 (0.22521733781512904) + | > loss_disc_real_3: 0.20710402727127075 (0.22780792871300054) + | > loss_disc_real_4: 0.186090350151062 (0.22595024412992049) + | > loss_disc_real_5: 0.19576597213745117 (0.2349640800028431) + | > loss_0: 2.489097833633423 (2.4495777694546446) + | > grad_norm_0: tensor(7.8442, device='cuda:0') (tensor(10.0371, device='cuda:0')) + | > loss_gen: 2.2962656021118164 (2.375944064587963) + | > loss_kl: 1.7960176467895508 (1.8317397467944088) + | > loss_feat: 6.926528453826904 (7.218291302116549) + | > loss_mel: 17.704055786132812 (17.62127068577981) + | > loss_duration: 1.4415937662124634 (1.425100681733112) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.164459228515625 (30.472346286384425) + | > grad_norm_1: tensor(215.2103, device='cuda:0') (tensor(282.3348, device='cuda:0')) + | > current_lr_0: 0.0001900788881153217 + | > current_lr_1: 0.0001900788881153217 + | > step_time: 0.3699 (0.35088666604489704) + | > loader_time: 0.0055 (0.005204356446558115) + + + --> TIME: 2025-04-19 05:40:35 -- STEP: 74/142 -- GLOBAL_STEP: 437150 + | > loss_disc: 2.37502384185791 (2.449501047263274) + | > loss_disc_real_0: 0.10078010708093643 (0.11775637136117832) + | > loss_disc_real_1: 0.20145195722579956 (0.2078677268849837) + | > loss_disc_real_2: 0.19551299512386322 (0.22522449936415698) + | > loss_disc_real_3: 0.21695420145988464 (0.22854985941100764) + | > loss_disc_real_4: 0.23632876574993134 (0.22545714233372663) + | > loss_disc_real_5: 0.20313091576099396 (0.23470237690049248) + | > loss_0: 2.37502384185791 (2.449501047263274) + | > grad_norm_0: tensor(11.7897, device='cuda:0') (tensor(9.3856, device='cuda:0')) + | > loss_gen: 2.289124011993408 (2.3688437487628007) + | > loss_kl: 1.8147940635681152 (1.8505169813697402) + | > loss_feat: 7.656500816345215 (7.217213997969756) + | > loss_mel: 18.243762969970703 (17.603724544112747) + | > loss_duration: 1.4626679420471191 (1.4376408377209224) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.466848373413086 (30.477940018112594) + | > grad_norm_1: tensor(297.2528, device='cuda:0') (tensor(251.3535, device='cuda:0')) + | > current_lr_0: 0.0001900788881153217 + | > current_lr_1: 0.0001900788881153217 + | > step_time: 0.3904 (0.36114229060508113) + | > loader_time: 0.006 (0.0053960858164606865) + + + --> TIME: 2025-04-19 05:40:45 -- STEP: 99/142 -- GLOBAL_STEP: 437175 + | > loss_disc: 2.501616954803467 (2.455235221169211) + | > loss_disc_real_0: 0.13141845166683197 (0.11901533912227612) + | > loss_disc_real_1: 0.25095227360725403 (0.20821141278502917) + | > loss_disc_real_2: 0.2425404191017151 (0.2243259654502676) + | > loss_disc_real_3: 0.2442765235900879 (0.22808684679594907) + | > loss_disc_real_4: 0.23908527195453644 (0.22733792798085647) + | > loss_disc_real_5: 0.23047269880771637 (0.23510545748050768) + | > loss_0: 2.501616954803467 (2.455235221169211) + | > grad_norm_0: tensor(25.2985, device='cuda:0') (tensor(9.4526, device='cuda:0')) + | > loss_gen: 2.30570912361145 (2.3620912354401877) + | > loss_kl: 1.8182345628738403 (1.8592766270493015) + | > loss_feat: 6.8333845138549805 (7.168222769342288) + | > loss_mel: 17.4016056060791 (17.626092766270503) + | > loss_duration: 1.4913280010223389 (1.4476131441617253) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.850261688232422 (30.463296485669684) + | > grad_norm_1: tensor(484.1892, device='cuda:0') (tensor(244.0036, device='cuda:0')) + | > current_lr_0: 0.0001900788881153217 + | > current_lr_1: 0.0001900788881153217 + | > step_time: 0.4051 (0.3712132868140634) + | > loader_time: 0.0076 (0.005694001612037117) + + + --> TIME: 2025-04-19 05:40:56 -- STEP: 124/142 -- GLOBAL_STEP: 437200 + | > loss_disc: 2.4584362506866455 (2.47297248340422) + | > loss_disc_real_0: 0.10335518419742584 (0.1236872254239936) + | > loss_disc_real_1: 0.19091260433197021 (0.20880345675733783) + | > loss_disc_real_2: 0.20944885909557343 (0.22621757044426857) + | > loss_disc_real_3: 0.2304711937904358 (0.22898828791033837) + | > loss_disc_real_4: 0.22143475711345673 (0.2285790993802009) + | > loss_disc_real_5: 0.24541884660720825 (0.23410982758768142) + | > loss_0: 2.4584362506866455 (2.47297248340422) + | > grad_norm_0: tensor(10.7784, device='cuda:0') (tensor(10.7406, device='cuda:0')) + | > loss_gen: 2.2222771644592285 (2.3506668671484916) + | > loss_kl: 2.038604736328125 (1.8678195447691026) + | > loss_feat: 7.1546783447265625 (7.103914760774182) + | > loss_mel: 18.10375213623047 (17.57725639497081) + | > loss_duration: 1.5150142908096313 (1.4543455829543452) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.034326553344727 (30.35400310639412) + | > grad_norm_1: tensor(316.8585, device='cuda:0') (tensor(250.6364, device='cuda:0')) + | > current_lr_0: 0.0001900788881153217 + | > current_lr_1: 0.0001900788881153217 + | > step_time: 0.4408 (0.38218043311949695) + | > loader_time: 0.0072 (0.005983879489283406) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.604491710662842 (2.604491710662842) + | > loss_disc_real_0: 0.25642645359039307 (0.25642645359039307) + | > loss_disc_real_1: 0.20218218863010406 (0.20218218863010406) + | > loss_disc_real_2: 0.24891044199466705 (0.24891044199466705) + | > loss_disc_real_3: 0.25810641050338745 (0.25810641050338745) + | > loss_disc_real_4: 0.22720475494861603 (0.22720475494861603) + | > loss_disc_real_5: 0.2352544069290161 (0.2352544069290161) + | > loss_0: 2.604491710662842 (2.604491710662842) + | > loss_gen: 2.350616931915283 (2.350616931915283) + | > loss_kl: 2.2665481567382812 (2.2665481567382812) + | > loss_feat: 6.86812686920166 (6.86812686920166) + | > loss_mel: 18.36653709411621 (18.36653709411621) + | > loss_duration: 1.755375623703003 (1.755375623703003) + | > loss_1: 31.60720443725586 (31.60720443725586) + + --> STEP: 1 + | > loss_disc: 2.620913505554199 (2.620913505554199) + | > loss_disc_real_0: 0.27030378580093384 (0.27030378580093384) + | > loss_disc_real_1: 0.20521041750907898 (0.20521041750907898) + | > loss_disc_real_2: 0.2687681317329407 (0.2687681317329407) + | > loss_disc_real_3: 0.25624749064445496 (0.25624749064445496) + | > loss_disc_real_4: 0.22329775989055634 (0.22329775989055634) + | > loss_disc_real_5: 0.2117316573858261 (0.2117316573858261) + | > loss_0: 2.620913505554199 (2.620913505554199) + | > loss_gen: 2.4425628185272217 (2.4425628185272217) + | > loss_kl: 2.779693126678467 (2.779693126678467) + | > loss_feat: 6.019596099853516 (6.019596099853516) + | > loss_mel: 17.112253189086914 (17.112253189086914) + | > loss_duration: 1.7980358600616455 (1.7980358600616455) + | > loss_1: 30.152141571044922 (30.152141571044922) + + --> STEP: 2 + | > loss_disc: 2.4262280464172363 (2.5235707759857178) + | > loss_disc_real_0: 0.1819455921649933 (0.22612468898296356) + | > loss_disc_real_1: 0.2141972780227661 (0.20970384776592255) + | > loss_disc_real_2: 0.22263866662979126 (0.24570339918136597) + | > loss_disc_real_3: 0.25402823090553284 (0.2551378607749939) + | > loss_disc_real_4: 0.20263350009918213 (0.21296562999486923) + | > loss_disc_real_5: 0.22285786271095276 (0.21729476004838943) + | > loss_0: 2.4262280464172363 (2.5235707759857178) + | > loss_gen: 2.616499900817871 (2.5295313596725464) + | > loss_kl: 2.390597343444824 (2.5851452350616455) + | > loss_feat: 8.327351570129395 (7.173473834991455) + | > loss_mel: 18.693286895751953 (17.902770042419434) + | > loss_duration: 1.7239844799041748 (1.7610101699829102) + | > loss_1: 33.7517204284668 (31.95193099975586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032672882080078125 (-0.0008151531219482422) + | > avg_loss_disc: 2.5235707759857178 (-0.1581709384918213) + | > avg_loss_disc_real_0: 0.22612468898296356 (-0.08685296028852463) + | > avg_loss_disc_real_1: 0.20970384776592255 (-0.10270103812217712) + | > avg_loss_disc_real_2: 0.24570339918136597 (-0.0022886618971824646) + | > avg_loss_disc_real_3: 0.2551378607749939 (+0.010149657726287842) + | > avg_loss_disc_real_4: 0.21296562999486923 (-0.02414323389530182) + | > avg_loss_disc_real_5: 0.21729476004838943 (-0.026905730366706848) + | > avg_loss_0: 2.5235707759857178 (-0.1581709384918213) + | > avg_loss_gen: 2.5295313596725464 (-0.2608814239501953) + | > avg_loss_kl: 2.5851452350616455 (+0.10741221904754639) + | > avg_loss_feat: 7.173473834991455 (+0.21576523780822754) + | > avg_loss_mel: 17.902770042419434 (-0.7164764404296875) + | > avg_loss_duration: 1.7610101699829102 (+0.012559711933135986) + | > avg_loss_1: 31.95193099975586 (-0.6416206359863281) + + + > EPOCH: 408/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:41:06)  + + --> TIME: 2025-04-19 05:41:09 -- STEP: 7/142 -- GLOBAL_STEP: 437225 + | > loss_disc: 2.399887800216675 (2.4348272936684743) + | > loss_disc_real_0: 0.12538449466228485 (0.12216164703880038) + | > loss_disc_real_1: 0.23701642453670502 (0.2098703235387802) + | > loss_disc_real_2: 0.23577898740768433 (0.2305105733019965) + | > loss_disc_real_3: 0.22229352593421936 (0.2366782341684614) + | > loss_disc_real_4: 0.2219095230102539 (0.23121883400848933) + | > loss_disc_real_5: 0.22369854152202606 (0.23764161552701676) + | > loss_0: 2.399887800216675 (2.4348272936684743) + | > grad_norm_0: tensor(8.5300, device='cuda:0') (tensor(10.7153, device='cuda:0')) + | > loss_gen: 2.348499059677124 (2.405167988368443) + | > loss_kl: 1.8653565645217896 (1.8191109555108207) + | > loss_feat: 7.215907573699951 (7.408987658364432) + | > loss_mel: 17.003747940063477 (17.783396857125418) + | > loss_duration: 1.4152084589004517 (1.4106745038713728) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.84872055053711 (30.827337537493026) + | > grad_norm_1: tensor(210.8473, device='cuda:0') (tensor(266.0666, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.3231 (0.3255073002406529) + | > loader_time: 0.004 (0.004888364246913365) + + + --> TIME: 2025-04-19 05:41:17 -- STEP: 32/142 -- GLOBAL_STEP: 437250 + | > loss_disc: 2.3781325817108154 (2.4084796980023384) + | > loss_disc_real_0: 0.08148416131734848 (0.11396518931724131) + | > loss_disc_real_1: 0.1843760907649994 (0.20695121586322784) + | > loss_disc_real_2: 0.20514817535877228 (0.2234290954656899) + | > loss_disc_real_3: 0.21159331500530243 (0.22755524097010493) + | > loss_disc_real_4: 0.21764980256557465 (0.22343251528218389) + | > loss_disc_real_5: 0.22009892761707306 (0.23120567807927728) + | > loss_0: 2.3781325817108154 (2.4084796980023384) + | > grad_norm_0: tensor(8.3626, device='cuda:0') (tensor(8.9920, device='cuda:0')) + | > loss_gen: 2.350090980529785 (2.3974010869860645) + | > loss_kl: 1.7788296937942505 (1.8102298229932785) + | > loss_feat: 7.645256519317627 (7.501311585307121) + | > loss_mel: 17.672847747802734 (17.962353110313412) + | > loss_duration: 1.4103480577468872 (1.4170249588787556) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.85737419128418 (31.088320672512054) + | > grad_norm_1: tensor(66.1859, device='cuda:0') (tensor(206.7391, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.3627 (0.33902011811733246) + | > loader_time: 0.0053 (0.005037814378738403) + + + --> TIME: 2025-04-19 05:41:27 -- STEP: 57/142 -- GLOBAL_STEP: 437275 + | > loss_disc: 2.4969987869262695 (2.436715824562206) + | > loss_disc_real_0: 0.13136254251003265 (0.118372720061687) + | > loss_disc_real_1: 0.1766916960477829 (0.21092087030410767) + | > loss_disc_real_2: 0.2309049367904663 (0.22395710542536618) + | > loss_disc_real_3: 0.24651221930980682 (0.22846425860597377) + | > loss_disc_real_4: 0.23305349051952362 (0.2255950870743969) + | > loss_disc_real_5: 0.23771922290325165 (0.231345061408846) + | > loss_0: 2.4969987869262695 (2.436715824562206) + | > grad_norm_0: tensor(3.7327, device='cuda:0') (tensor(7.5053, device='cuda:0')) + | > loss_gen: 2.2567427158355713 (2.366039414154856) + | > loss_kl: 1.7926326990127563 (1.8317179784440158) + | > loss_feat: 7.171411991119385 (7.330145501253898) + | > loss_mel: 17.792762756347656 (17.836113293965642) + | > loss_duration: 1.4775464534759521 (1.4269193850065534) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.49109649658203 (30.790935717130964) + | > grad_norm_1: tensor(104.4033, device='cuda:0') (tensor(154.1034, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.3734 (0.35004200851708117) + | > loader_time: 0.0058 (0.005302374823051586) + + + --> TIME: 2025-04-19 05:41:37 -- STEP: 82/142 -- GLOBAL_STEP: 437300 + | > loss_disc: 2.420025587081909 (2.4391913152322533) + | > loss_disc_real_0: 0.11184505373239517 (0.11906371238391573) + | > loss_disc_real_1: 0.2258748561143875 (0.20804228706330788) + | > loss_disc_real_2: 0.21516458690166473 (0.2236591455776517) + | > loss_disc_real_3: 0.2033388614654541 (0.2273632329775066) + | > loss_disc_real_4: 0.16156929731369019 (0.22656264301480317) + | > loss_disc_real_5: 0.2400856465101242 (0.23194229493780835) + | > loss_0: 2.420025587081909 (2.4391913152322533) + | > grad_norm_0: tensor(3.5207, device='cuda:0') (tensor(7.1088, device='cuda:0')) + | > loss_gen: 2.3147058486938477 (2.3672441651181484) + | > loss_kl: 2.020298480987549 (1.8340439621995135) + | > loss_feat: 7.856643199920654 (7.278028587015664) + | > loss_mel: 18.760665893554688 (17.857626542812437) + | > loss_duration: 1.5268555879592896 (1.4372626455818736) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.47916793823242 (30.774205882374833) + | > grad_norm_1: tensor(96.4359, device='cuda:0') (tensor(144.1973, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.3965 (0.36280017364315864) + | > loader_time: 0.0059 (0.005570472740545504) + + + --> TIME: 2025-04-19 05:41:47 -- STEP: 107/142 -- GLOBAL_STEP: 437325 + | > loss_disc: 2.4557011127471924 (2.4436116441387985) + | > loss_disc_real_0: 0.1656007617712021 (0.12000056996802304) + | > loss_disc_real_1: 0.17932571470737457 (0.2061944553785235) + | > loss_disc_real_2: 0.19488385319709778 (0.2241292354938026) + | > loss_disc_real_3: 0.20267094671726227 (0.22717830190591723) + | > loss_disc_real_4: 0.2143106311559677 (0.22721839494237275) + | > loss_disc_real_5: 0.2036486119031906 (0.23110667706649995) + | > loss_0: 2.4557011127471924 (2.4436116441387985) + | > grad_norm_0: tensor(11.7507, device='cuda:0') (tensor(8.1916, device='cuda:0')) + | > loss_gen: 2.421100616455078 (2.3605320587336465) + | > loss_kl: 1.8105154037475586 (1.848043514189319) + | > loss_feat: 6.592179775238037 (7.2084110429353805) + | > loss_mel: 15.681517601013184 (17.759495289526246) + | > loss_duration: 1.4675273895263672 (1.447622257972432) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.972841262817383 (30.624104134390286) + | > grad_norm_1: tensor(125.3719, device='cuda:0') (tensor(177.4993, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.4173 (0.3736800946921946) + | > loader_time: 0.0067 (0.005885099696221751) + + + --> TIME: 2025-04-19 05:41:58 -- STEP: 132/142 -- GLOBAL_STEP: 437350 + | > loss_disc: 2.4141478538513184 (2.449340601762135) + | > loss_disc_real_0: 0.10824008285999298 (0.12106443744039896) + | > loss_disc_real_1: 0.20802128314971924 (0.2068994396337957) + | > loss_disc_real_2: 0.23259337246418 (0.22518030226682173) + | > loss_disc_real_3: 0.22336708009243011 (0.22816249852379164) + | > loss_disc_real_4: 0.22489871084690094 (0.2283425129046946) + | > loss_disc_real_5: 0.2272491753101349 (0.23114977743137965) + | > loss_0: 2.4141478538513184 (2.449340601762135) + | > grad_norm_0: tensor(6.2488, device='cuda:0') (tensor(8.7735, device='cuda:0')) + | > loss_gen: 2.3975205421447754 (2.353947883302516) + | > loss_kl: 1.8929669857025146 (1.8520972475861057) + | > loss_feat: 6.819980621337891 (7.13409635514924) + | > loss_mel: 18.044092178344727 (17.684962431589756) + | > loss_duration: 1.477666974067688 (1.4543996037858906) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.632226943969727 (30.4795035015453) + | > grad_norm_1: tensor(130.9277, device='cuda:0') (tensor(195.8011, device='cuda:0')) + | > current_lr_0: 0.00019005512825430728 + | > current_lr_1: 0.00019005512825430728 + | > step_time: 0.4445 (0.38505879676703253) + | > loader_time: 0.008 (0.0061742410515293914) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6980881690979004 (2.6980881690979004) + | > loss_disc_real_0: 0.31860366463661194 (0.31860366463661194) + | > loss_disc_real_1: 0.2878802418708801 (0.2878802418708801) + | > loss_disc_real_2: 0.23571617901325226 (0.23571617901325226) + | > loss_disc_real_3: 0.24634608626365662 (0.24634608626365662) + | > loss_disc_real_4: 0.2491828203201294 (0.2491828203201294) + | > loss_disc_real_5: 0.2938380539417267 (0.2938380539417267) + | > loss_0: 2.6980881690979004 (2.6980881690979004) + | > loss_gen: 2.582526445388794 (2.582526445388794) + | > loss_kl: 2.6471445560455322 (2.6471445560455322) + | > loss_feat: 5.297209739685059 (5.297209739685059) + | > loss_mel: 15.149178504943848 (15.149178504943848) + | > loss_duration: 1.737100601196289 (1.737100601196289) + | > loss_1: 27.41316032409668 (27.41316032409668) + + --> STEP: 1 + | > loss_disc: 2.459853172302246 (2.459853172302246) + | > loss_disc_real_0: 0.20664463937282562 (0.20664463937282562) + | > loss_disc_real_1: 0.22743922472000122 (0.22743922472000122) + | > loss_disc_real_2: 0.23920509219169617 (0.23920509219169617) + | > loss_disc_real_3: 0.26222628355026245 (0.26222628355026245) + | > loss_disc_real_4: 0.21541264653205872 (0.21541264653205872) + | > loss_disc_real_5: 0.2760762870311737 (0.2760762870311737) + | > loss_0: 2.459853172302246 (2.459853172302246) + | > loss_gen: 2.6089866161346436 (2.6089866161346436) + | > loss_kl: 2.827888011932373 (2.827888011932373) + | > loss_feat: 7.483386516571045 (7.483386516571045) + | > loss_mel: 18.897136688232422 (18.897136688232422) + | > loss_duration: 1.7926985025405884 (1.7926985025405884) + | > loss_1: 33.6100959777832 (33.6100959777832) + + --> STEP: 2 + | > loss_disc: 2.6379621028900146 (2.5489076375961304) + | > loss_disc_real_0: 0.3017885982990265 (0.25421661883592606) + | > loss_disc_real_1: 0.30389735102653503 (0.2656682878732681) + | > loss_disc_real_2: 0.22009797394275665 (0.2296515330672264) + | > loss_disc_real_3: 0.22432637214660645 (0.24327632784843445) + | > loss_disc_real_4: 0.23833389580249786 (0.2268732711672783) + | > loss_disc_real_5: 0.2505405843257904 (0.26330843567848206) + | > loss_0: 2.6379621028900146 (2.5489076375961304) + | > loss_gen: 2.4897561073303223 (2.549371361732483) + | > loss_kl: 2.584080934524536 (2.7059844732284546) + | > loss_feat: 5.822869300842285 (6.653127908706665) + | > loss_mel: 15.301350593566895 (17.099243640899658) + | > loss_duration: 1.7066426277160645 (1.7496705651283264) + | > loss_1: 27.904699325561523 (30.757397651672363) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032728910446166992 (+5.602836608886719e-06) + | > avg_loss_disc: 2.5489076375961304 (+0.025336861610412598) + | > avg_loss_disc_real_0: 0.25421661883592606 (+0.028091929852962494) + | > avg_loss_disc_real_1: 0.2656682878732681 (+0.05596444010734558) + | > avg_loss_disc_real_2: 0.2296515330672264 (-0.016051866114139557) + | > avg_loss_disc_real_3: 0.24327632784843445 (-0.011861532926559448) + | > avg_loss_disc_real_4: 0.2268732711672783 (+0.013907641172409058) + | > avg_loss_disc_real_5: 0.26330843567848206 (+0.04601367563009262) + | > avg_loss_0: 2.5489076375961304 (+0.025336861610412598) + | > avg_loss_gen: 2.549371361732483 (+0.019840002059936523) + | > avg_loss_kl: 2.7059844732284546 (+0.12083923816680908) + | > avg_loss_feat: 6.653127908706665 (-0.52034592628479) + | > avg_loss_mel: 17.099243640899658 (-0.8035264015197754) + | > avg_loss_duration: 1.7496705651283264 (-0.01133960485458374) + | > avg_loss_1: 30.757397651672363 (-1.194533348083496) + + + > EPOCH: 409/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:42:04)  + + --> TIME: 2025-04-19 05:42:10 -- STEP: 15/142 -- GLOBAL_STEP: 437375 + | > loss_disc: 2.3871517181396484 (2.3836164474487305) + | > loss_disc_real_0: 0.12395524978637695 (0.11104596356550853) + | > loss_disc_real_1: 0.2300802618265152 (0.19906685451666514) + | > loss_disc_real_2: 0.23398885130882263 (0.2196936974922816) + | > loss_disc_real_3: 0.23177039623260498 (0.22435582677523294) + | > loss_disc_real_4: 0.20682814717292786 (0.21830830872058868) + | > loss_disc_real_5: 0.21253809332847595 (0.21801478465398152) + | > loss_0: 2.3871517181396484 (2.3836164474487305) + | > grad_norm_0: tensor(4.1859, device='cuda:0') (tensor(6.2249, device='cuda:0')) + | > loss_gen: 2.561405658721924 (2.4010586102803546) + | > loss_kl: 1.763181447982788 (1.8146913528442383) + | > loss_feat: 7.271778583526611 (7.579957930246989) + | > loss_mel: 16.363988876342773 (17.790338389078777) + | > loss_duration: 1.4236177206039429 (1.4068334182103475) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.38397216796875 (30.992879486083986) + | > grad_norm_1: tensor(131.0161, device='cuda:0') (tensor(134.5887, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.3357 (0.3434614181518555) + | > loader_time: 0.0051 (0.0045950889587402345) + + + --> TIME: 2025-04-19 05:42:19 -- STEP: 40/142 -- GLOBAL_STEP: 437400 + | > loss_disc: 2.4009151458740234 (2.414946347475052) + | > loss_disc_real_0: 0.11433438211679459 (0.1142037084326148) + | > loss_disc_real_1: 0.19060619175434113 (0.20213760435581207) + | > loss_disc_real_2: 0.1986168473958969 (0.2239748690277338) + | > loss_disc_real_3: 0.2257319539785385 (0.22859881781041622) + | > loss_disc_real_4: 0.19851461052894592 (0.2244403913617134) + | > loss_disc_real_5: 0.2295299470424652 (0.22989296726882458) + | > loss_0: 2.4009151458740234 (2.414946347475052) + | > grad_norm_0: tensor(7.0597, device='cuda:0') (tensor(6.9530, device='cuda:0')) + | > loss_gen: 2.309509038925171 (2.38730291724205) + | > loss_kl: 1.8109608888626099 (1.8417861759662628) + | > loss_feat: 7.536596298217773 (7.382773685455322) + | > loss_mel: 17.56109619140625 (17.661853981018066) + | > loss_duration: 1.3968478441238403 (1.4180940955877306) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.61501121520996 (30.691810750961302) + | > grad_norm_1: tensor(244.4734, device='cuda:0') (tensor(166.6078, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.3607 (0.34676927924156187) + | > loader_time: 0.0057 (0.005078142881393433) + + + --> TIME: 2025-04-19 05:42:28 -- STEP: 65/142 -- GLOBAL_STEP: 437425 + | > loss_disc: 2.4396023750305176 (2.4141623166891244) + | > loss_disc_real_0: 0.1743595004081726 (0.1144405528329886) + | > loss_disc_real_1: 0.2244815081357956 (0.20267129930166097) + | > loss_disc_real_2: 0.2302754521369934 (0.22384092440971962) + | > loss_disc_real_3: 0.22403401136398315 (0.22722228971811442) + | > loss_disc_real_4: 0.22595937550067902 (0.22423011523026687) + | > loss_disc_real_5: 0.2174256592988968 (0.22952705805118268) + | > loss_0: 2.4396023750305176 (2.4141623166891244) + | > grad_norm_0: tensor(17.4282, device='cuda:0') (tensor(8.2591, device='cuda:0')) + | > loss_gen: 2.333719253540039 (2.3920184979071974) + | > loss_kl: 1.8873945474624634 (1.840211142026461) + | > loss_feat: 7.6668009757995605 (7.370034027099609) + | > loss_mel: 17.666683197021484 (17.694594016441933) + | > loss_duration: 1.4461612701416016 (1.4308325327359712) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.00075912475586 (30.727690271230845) + | > grad_norm_1: tensor(257.2246, device='cuda:0') (tensor(186.5540, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.3929 (0.3565124878516564) + | > loader_time: 0.0067 (0.005359844061044546) + + + --> TIME: 2025-04-19 05:42:39 -- STEP: 90/142 -- GLOBAL_STEP: 437450 + | > loss_disc: 2.3903796672821045 (2.4379522191153633) + | > loss_disc_real_0: 0.14910896122455597 (0.11775495579673184) + | > loss_disc_real_1: 0.19087576866149902 (0.20580987102455559) + | > loss_disc_real_2: 0.21882574260234833 (0.22567604068252775) + | > loss_disc_real_3: 0.18737202882766724 (0.22790965636571248) + | > loss_disc_real_4: 0.22235016524791718 (0.22533830934100682) + | > loss_disc_real_5: 0.22114749252796173 (0.232270485162735) + | > loss_0: 2.3903796672821045 (2.4379522191153633) + | > grad_norm_0: tensor(5.2226, device='cuda:0') (tensor(8.2862, device='cuda:0')) + | > loss_gen: 2.4127755165100098 (2.3700379000769707) + | > loss_kl: 1.9003093242645264 (1.8545235395431519) + | > loss_feat: 7.111391067504883 (7.271007251739502) + | > loss_mel: 18.522804260253906 (17.71227638456557) + | > loss_duration: 1.4873160123825073 (1.4430794225798715) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.434595108032227 (30.65092459784614) + | > grad_norm_1: tensor(52.4556, device='cuda:0') (tensor(181.3091, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.4005 (0.36707755459679514) + | > loader_time: 0.0062 (0.0056817584567599826) + + + --> TIME: 2025-04-19 05:42:49 -- STEP: 115/142 -- GLOBAL_STEP: 437475 + | > loss_disc: 2.3595564365386963 (2.4406579763992973) + | > loss_disc_real_0: 0.12335076183080673 (0.11957217944057091) + | > loss_disc_real_1: 0.1777157038450241 (0.20524141892142914) + | > loss_disc_real_2: 0.20294155180454254 (0.22491772252580394) + | > loss_disc_real_3: 0.22342868149280548 (0.2274626927531284) + | > loss_disc_real_4: 0.21516859531402588 (0.22668613324994627) + | > loss_disc_real_5: 0.18803104758262634 (0.23137030769949374) + | > loss_0: 2.3595564365386963 (2.4406579763992973) + | > grad_norm_0: tensor(6.6831, device='cuda:0') (tensor(8.2125, device='cuda:0')) + | > loss_gen: 2.4614651203155518 (2.3645872655122155) + | > loss_kl: 1.853963851928711 (1.8557513143705284) + | > loss_feat: 7.797003269195557 (7.250868909255318) + | > loss_mel: 18.45577621459961 (17.71670163196067) + | > loss_duration: 1.5022249221801758 (1.4511648354323017) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.0704345703125 (30.63907409336256) + | > grad_norm_1: tensor(308.6848, device='cuda:0') (tensor(195.3004, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.4232 (0.37737790605296273) + | > loader_time: 0.0069 (0.005976799260015073) + + + --> TIME: 2025-04-19 05:43:00 -- STEP: 140/142 -- GLOBAL_STEP: 437500 + | > loss_disc: 2.626129388809204 (2.4478460686547407) + | > loss_disc_real_0: 0.15922527015209198 (0.12128627090049642) + | > loss_disc_real_1: 0.1818549782037735 (0.20666533825652939) + | > loss_disc_real_2: 0.25230297446250916 (0.22575521341391971) + | > loss_disc_real_3: 0.21530242264270782 (0.2273245102592877) + | > loss_disc_real_4: 0.20480580627918243 (0.22698420201029096) + | > loss_disc_real_5: 0.19776928424835205 (0.23193417393735477) + | > loss_0: 2.626129388809204 (2.4478460686547407) + | > grad_norm_0: tensor(8.2509, device='cuda:0') (tensor(8.5288, device='cuda:0')) + | > loss_gen: 2.026371717453003 (2.36238407577787) + | > loss_kl: 1.878068447113037 (1.8581813190664564) + | > loss_feat: 6.678114891052246 (7.206975364685059) + | > loss_mel: 16.248401641845703 (17.65927204404559) + | > loss_duration: 1.503330945968628 (1.4580318110329769) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.334285736083984 (30.544844709123883) + | > grad_norm_1: tensor(326.0299, device='cuda:0') (tensor(205.2254, device='cuda:0')) + | > current_lr_0: 0.00019003137136327548 + | > current_lr_1: 0.00019003137136327548 + | > step_time: 0.4532 (0.38917318752833785) + | > loader_time: 0.0074 (0.006240839617592947) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.381643533706665 (2.381643533706665) + | > loss_disc_real_0: 0.1369272619485855 (0.1369272619485855) + | > loss_disc_real_1: 0.20405232906341553 (0.20405232906341553) + | > loss_disc_real_2: 0.2225358486175537 (0.2225358486175537) + | > loss_disc_real_3: 0.21792711317539215 (0.21792711317539215) + | > loss_disc_real_4: 0.18896785378456116 (0.18896785378456116) + | > loss_disc_real_5: 0.2610265612602234 (0.2610265612602234) + | > loss_0: 2.381643533706665 (2.381643533706665) + | > loss_gen: 2.376826763153076 (2.376826763153076) + | > loss_kl: 2.529242753982544 (2.529242753982544) + | > loss_feat: 8.01024341583252 (8.01024341583252) + | > loss_mel: 20.901458740234375 (20.901458740234375) + | > loss_duration: 1.7479146718978882 (1.7479146718978882) + | > loss_1: 35.5656852722168 (35.5656852722168) + + --> STEP: 1 + | > loss_disc: 2.5651564598083496 (2.5651564598083496) + | > loss_disc_real_0: 0.22176747024059296 (0.22176747024059296) + | > loss_disc_real_1: 0.24653859436511993 (0.24653859436511993) + | > loss_disc_real_2: 0.2183363288640976 (0.2183363288640976) + | > loss_disc_real_3: 0.226103737950325 (0.226103737950325) + | > loss_disc_real_4: 0.25881028175354004 (0.25881028175354004) + | > loss_disc_real_5: 0.2396957129240036 (0.2396957129240036) + | > loss_0: 2.5651564598083496 (2.5651564598083496) + | > loss_gen: 2.290907382965088 (2.290907382965088) + | > loss_kl: 2.782405376434326 (2.782405376434326) + | > loss_feat: 5.611788749694824 (5.611788749694824) + | > loss_mel: 18.457040786743164 (18.457040786743164) + | > loss_duration: 1.786557912826538 (1.786557912826538) + | > loss_1: 30.928699493408203 (30.928699493408203) + + --> STEP: 2 + | > loss_disc: 2.7055845260620117 (2.6353704929351807) + | > loss_disc_real_0: 0.2168051302433014 (0.21928630024194717) + | > loss_disc_real_1: 0.25655367970466614 (0.25154613703489304) + | > loss_disc_real_2: 0.24458637833595276 (0.23146135360002518) + | > loss_disc_real_3: 0.2667708098888397 (0.24643727391958237) + | > loss_disc_real_4: 0.24789433181285858 (0.2533523067831993) + | > loss_disc_real_5: 0.26436734199523926 (0.25203152745962143) + | > loss_0: 2.7055845260620117 (2.6353704929351807) + | > loss_gen: 2.150695562362671 (2.2208014726638794) + | > loss_kl: 2.304215669631958 (2.543310523033142) + | > loss_feat: 5.816315174102783 (5.714051961898804) + | > loss_mel: 15.62537670135498 (17.041208744049072) + | > loss_duration: 1.7316824197769165 (1.7591201663017273) + | > loss_1: 27.628284454345703 (29.278491973876953) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003844141960144043 (+0.0005712509155273438) + | > avg_loss_disc: 2.6353704929351807 (+0.08646285533905029) + | > avg_loss_disc_real_0: 0.21928630024194717 (-0.03493031859397888) + | > avg_loss_disc_real_1: 0.25154613703489304 (-0.014122150838375092) + | > avg_loss_disc_real_2: 0.23146135360002518 (+0.001809820532798767) + | > avg_loss_disc_real_3: 0.24643727391958237 (+0.0031609460711479187) + | > avg_loss_disc_real_4: 0.2533523067831993 (+0.02647903561592102) + | > avg_loss_disc_real_5: 0.25203152745962143 (-0.011276908218860626) + | > avg_loss_0: 2.6353704929351807 (+0.08646285533905029) + | > avg_loss_gen: 2.2208014726638794 (-0.3285698890686035) + | > avg_loss_kl: 2.543310523033142 (-0.1626739501953125) + | > avg_loss_feat: 5.714051961898804 (-0.9390759468078613) + | > avg_loss_mel: 17.041208744049072 (-0.05803489685058594) + | > avg_loss_duration: 1.7591201663017273 (+0.009449601173400879) + | > avg_loss_1: 29.278491973876953 (-1.4789056777954102) + + + > EPOCH: 410/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:43:03)  + + --> TIME: 2025-04-19 05:43:11 -- STEP: 23/142 -- GLOBAL_STEP: 437525 + | > loss_disc: 2.453075885772705 (2.4263776385265845) + | > loss_disc_real_0: 0.1291734278202057 (0.12519352008467138) + | > loss_disc_real_1: 0.17431484162807465 (0.20471390693084054) + | > loss_disc_real_2: 0.23444849252700806 (0.22101476011068924) + | > loss_disc_real_3: 0.24616552889347076 (0.22717577089434085) + | > loss_disc_real_4: 0.23032237589359283 (0.21958233480868133) + | > loss_disc_real_5: 0.243703231215477 (0.22891964212707852) + | > loss_0: 2.453075885772705 (2.4263776385265845) + | > grad_norm_0: tensor(18.6744, device='cuda:0') (tensor(14.8150, device='cuda:0')) + | > loss_gen: 2.4252262115478516 (2.4119674848473593) + | > loss_kl: 1.8138493299484253 (1.7768636527268782) + | > loss_feat: 7.662199974060059 (7.334408656410549) + | > loss_mel: 17.518056869506836 (17.46605578712795) + | > loss_duration: 1.415243148803711 (1.411161412363467) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.834575653076172 (30.40045680170474) + | > grad_norm_1: tensor(272.4001, device='cuda:0') (tensor(292.1377, device='cuda:0')) + | > current_lr_0: 0.00019000761744185506 + | > current_lr_1: 0.00019000761744185506 + | > step_time: 0.3471 (0.3321697608284328) + | > loader_time: 0.0052 (0.004699364952419116) + + + --> TIME: 2025-04-19 05:43:20 -- STEP: 48/142 -- GLOBAL_STEP: 437550 + | > loss_disc: 2.491502523422241 (2.449331368009249) + | > loss_disc_real_0: 0.09283390641212463 (0.1274127659077446) + | > loss_disc_real_1: 0.23148760199546814 (0.20681354713936648) + | > loss_disc_real_2: 0.22848322987556458 (0.22404095064848661) + | > loss_disc_real_3: 0.23633050918579102 (0.2275798193489512) + | > loss_disc_real_4: 0.25788143277168274 (0.2253600669403871) + | > loss_disc_real_5: 0.2283540666103363 (0.22860401837776104) + | > loss_0: 2.491502523422241 (2.449331368009249) + | > grad_norm_0: tensor(9.0633, device='cuda:0') (tensor(17.0209, device='cuda:0')) + | > loss_gen: 2.382431983947754 (2.424810335040093) + | > loss_kl: 1.8570382595062256 (1.8116807366410892) + | > loss_feat: 6.499139308929443 (7.381152222553889) + | > loss_mel: 17.230195999145508 (17.62546879053116) + | > loss_duration: 1.438509464263916 (1.422562283774217) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.407312393188477 (30.665674209594727) + | > grad_norm_1: tensor(175.2333, device='cuda:0') (tensor(336.4489, device='cuda:0')) + | > current_lr_0: 0.00019000761744185506 + | > current_lr_1: 0.00019000761744185506 + | > step_time: 0.3635 (0.3442752758661906) + | > loader_time: 0.0056 (0.005110363165537517) + + + --> TIME: 2025-04-19 05:43:30 -- STEP: 73/142 -- GLOBAL_STEP: 437575 + | > loss_disc: 2.49914813041687 (2.4540528695877275) + | > loss_disc_real_0: 0.06416274607181549 (0.1307321048456512) + | > loss_disc_real_1: 0.21964725852012634 (0.2068534495487605) + | > loss_disc_real_2: 0.23041582107543945 (0.22345969387113232) + | > loss_disc_real_3: 0.2633412182331085 (0.22740100242503702) + | > loss_disc_real_4: 0.24225246906280518 (0.22552476762092277) + | > loss_disc_real_5: 0.2359912097454071 (0.2308062829383432) + | > loss_0: 2.49914813041687 (2.4540528695877275) + | > grad_norm_0: tensor(9.1582, device='cuda:0') (tensor(18.9786, device='cuda:0')) + | > loss_gen: 2.3033480644226074 (2.4195250811642164) + | > loss_kl: 1.751892328262329 (1.828324801301303) + | > loss_feat: 7.084674835205078 (7.342899061229131) + | > loss_mel: 17.56904411315918 (17.66977301035842) + | > loss_duration: 1.4862322807312012 (1.4355432578962144) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.195192337036133 (30.69606514499612) + | > grad_norm_1: tensor(254.9717, device='cuda:0') (tensor(333.9022, device='cuda:0')) + | > current_lr_0: 0.00019000761744185506 + | > current_lr_1: 0.00019000761744185506 + | > step_time: 0.3973 (0.35669658935233334) + | > loader_time: 0.0061 (0.005399050777905609) + + + --> TIME: 2025-04-19 05:43:40 -- STEP: 98/142 -- GLOBAL_STEP: 437600 + | > loss_disc: 2.4650557041168213 (2.455248207462076) + | > loss_disc_real_0: 0.13718095421791077 (0.128055618315631) + | > loss_disc_real_1: 0.2148139923810959 (0.20522601172632102) + | > loss_disc_real_2: 0.23853354156017303 (0.2240908574693057) + | > loss_disc_real_3: 0.23845484852790833 (0.22719366164231786) + | > loss_disc_real_4: 0.21203802525997162 (0.22737551739021222) + | > loss_disc_real_5: 0.23341888189315796 (0.23060811919217206) + | > loss_0: 2.4650557041168213 (2.455248207462076) + | > grad_norm_0: tensor(5.4363, device='cuda:0') (tensor(16.9036, device='cuda:0')) + | > loss_gen: 2.4066221714019775 (2.3984642734332953) + | > loss_kl: 1.9459422826766968 (1.8416029920383377) + | > loss_feat: 7.6083807945251465 (7.266543792218578) + | > loss_mel: 18.022706985473633 (17.621450628553113) + | > loss_duration: 1.4542248249053955 (1.4458486109363788) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.437877655029297 (30.57391016823905) + | > grad_norm_1: tensor(80.3034, device='cuda:0') (tensor(304.8430, device='cuda:0')) + | > current_lr_0: 0.00019000761744185506 + | > current_lr_1: 0.00019000761744185506 + | > step_time: 0.4096 (0.3676427262169974) + | > loader_time: 0.0068 (0.0056542450067948325) + + + --> TIME: 2025-04-19 05:43:51 -- STEP: 123/142 -- GLOBAL_STEP: 437625 + | > loss_disc: 2.5020973682403564 (2.457018443239413) + | > loss_disc_real_0: 0.13430741429328918 (0.12708997329681868) + | > loss_disc_real_1: 0.22143802046775818 (0.20562081511427716) + | > loss_disc_real_2: 0.26568278670310974 (0.22472039737352512) + | > loss_disc_real_3: 0.24774658679962158 (0.2279586960387424) + | > loss_disc_real_4: 0.2360752671957016 (0.22791605133835863) + | > loss_disc_real_5: 0.23492276668548584 (0.23182040440842389) + | > loss_0: 2.5020973682403564 (2.457018443239413) + | > grad_norm_0: tensor(4.8396, device='cuda:0') (tensor(14.8567, device='cuda:0')) + | > loss_gen: 2.3948569297790527 (2.3801610179063752) + | > loss_kl: nan (nan) + | > loss_feat: 6.93222713470459 (7.180846822940238) + | > loss_mel: 17.711698532104492 (17.55768204510696) + | > loss_duration: nan (nan) + | > amp_scaler: 64.0 (127.47967479674797) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(263.3630, device='cuda:0')) + | > current_lr_0: 0.00019000761744185506 + | > current_lr_1: 0.00019000761744185506 + | > step_time: 0.4293 (0.38153542735712315) + | > loader_time: 0.0069 (0.005975886089045827) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.532355785369873 (2.532355785369873) + | > loss_disc_real_0: 0.20729222893714905 (0.20729222893714905) + | > loss_disc_real_1: 0.19933055341243744 (0.19933055341243744) + | > loss_disc_real_2: 0.2814904451370239 (0.2814904451370239) + | > loss_disc_real_3: 0.2428407520055771 (0.2428407520055771) + | > loss_disc_real_4: 0.19647477567195892 (0.19647477567195892) + | > loss_disc_real_5: 0.28199535608291626 (0.28199535608291626) + | > loss_0: 2.532355785369873 (2.532355785369873) + | > loss_gen: 2.581066608428955 (2.581066608428955) + | > loss_kl: 2.3301186561584473 (2.3301186561584473) + | > loss_feat: 7.260637283325195 (7.260637283325195) + | > loss_mel: 18.207651138305664 (18.207651138305664) + | > loss_duration: 1.747850775718689 (1.747850775718689) + | > loss_1: 32.127323150634766 (32.127323150634766) + + --> STEP: 1 + | > loss_disc: 2.498134136199951 (2.498134136199951) + | > loss_disc_real_0: 0.19949209690093994 (0.19949209690093994) + | > loss_disc_real_1: 0.22374671697616577 (0.22374671697616577) + | > loss_disc_real_2: 0.25193750858306885 (0.25193750858306885) + | > loss_disc_real_3: 0.22279424965381622 (0.22279424965381622) + | > loss_disc_real_4: 0.22663593292236328 (0.22663593292236328) + | > loss_disc_real_5: 0.22651538252830505 (0.22651538252830505) + | > loss_0: 2.498134136199951 (2.498134136199951) + | > loss_gen: 2.3545634746551514 (2.3545634746551514) + | > loss_kl: 2.6058309078216553 (2.6058309078216553) + | > loss_feat: 6.3169097900390625 (6.3169097900390625) + | > loss_mel: 17.47663116455078 (17.47663116455078) + | > loss_duration: 1.7970359325408936 (1.7970359325408936) + | > loss_1: 30.55097007751465 (30.55097007751465) + + --> STEP: 2 + | > loss_disc: 2.517855167388916 (2.5079946517944336) + | > loss_disc_real_0: 0.2132989466190338 (0.20639552175998688) + | > loss_disc_real_1: 0.21969681978225708 (0.22172176837921143) + | > loss_disc_real_2: 0.22663995623588562 (0.23928873240947723) + | > loss_disc_real_3: 0.21936330199241638 (0.2210787758231163) + | > loss_disc_real_4: 0.2170221507549286 (0.22182904183864594) + | > loss_disc_real_5: 0.23450390994548798 (0.23050964623689651) + | > loss_0: 2.517855167388916 (2.5079946517944336) + | > loss_gen: 2.244136095046997 (2.299349784851074) + | > loss_kl: 2.442579746246338 (2.5242053270339966) + | > loss_feat: 6.316784858703613 (6.316847324371338) + | > loss_mel: 14.88011646270752 (16.17837381362915) + | > loss_duration: 1.7239742279052734 (1.7605050802230835) + | > loss_1: 27.60759162902832 (29.079280853271484) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003448009490966797 (-0.0003961324691772461) + | > avg_loss_disc: 2.5079946517944336 (-0.12737584114074707) + | > avg_loss_disc_real_0: 0.20639552175998688 (-0.012890778481960297) + | > avg_loss_disc_real_1: 0.22172176837921143 (-0.02982436865568161) + | > avg_loss_disc_real_2: 0.23928873240947723 (+0.007827378809452057) + | > avg_loss_disc_real_3: 0.2210787758231163 (-0.025358498096466064) + | > avg_loss_disc_real_4: 0.22182904183864594 (-0.031523264944553375) + | > avg_loss_disc_real_5: 0.23050964623689651 (-0.021521881222724915) + | > avg_loss_0: 2.5079946517944336 (-0.12737584114074707) + | > avg_loss_gen: 2.299349784851074 (+0.07854831218719482) + | > avg_loss_kl: 2.5242053270339966 (-0.019105195999145508) + | > avg_loss_feat: 6.316847324371338 (+0.6027953624725342) + | > avg_loss_mel: 16.17837381362915 (-0.8628349304199219) + | > avg_loss_duration: 1.7605050802230835 (+0.0013849139213562012) + | > avg_loss_1: 29.079280853271484 (-0.19921112060546875) + + + > EPOCH: 411/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:44:01)  + + --> TIME: 2025-04-19 05:44:04 -- STEP: 6/142 -- GLOBAL_STEP: 437650 + | > loss_disc: 2.3312478065490723 (2.4005055030186973) + | > loss_disc_real_0: 0.15277352929115295 (0.13559389859437943) + | > loss_disc_real_1: 0.16243568062782288 (0.20106513301531473) + | > loss_disc_real_2: 0.1591750979423523 (0.21070066094398499) + | > loss_disc_real_3: 0.19518239796161652 (0.22200980534156164) + | > loss_disc_real_4: 0.18667106330394745 (0.22368783752123514) + | > loss_disc_real_5: 0.2059606909751892 (0.24366858353217444) + | > loss_0: 2.3312478065490723 (2.4005055030186973) + | > grad_norm_0: tensor(6.4621, device='cuda:0') (tensor(8.3567, device='cuda:0')) + | > loss_gen: 2.3584070205688477 (2.401423215866089) + | > loss_kl: 1.6802183389663696 (1.725387414296468) + | > loss_feat: 7.845263957977295 (7.3809293905893965) + | > loss_mel: 18.572534561157227 (17.632997194925945) + | > loss_duration: 1.4124162197113037 (1.4120472073554993) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.86884117126465 (30.55278491973877) + | > grad_norm_1: tensor(56.2593, device='cuda:0') (tensor(130.6024, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.3267 (0.3269560734430949) + | > loader_time: 0.0047 (0.004337986310323079) + + + --> TIME: 2025-04-19 05:44:13 -- STEP: 31/142 -- GLOBAL_STEP: 437675 + | > loss_disc: 2.4895942211151123 (2.445331019739951) + | > loss_disc_real_0: 0.15270544588565826 (0.1248989597924294) + | > loss_disc_real_1: 0.21809609234333038 (0.20573448990621873) + | > loss_disc_real_2: 0.21455930173397064 (0.22456102025124333) + | > loss_disc_real_3: 0.21835960447788239 (0.22977474427992298) + | > loss_disc_real_4: 0.21825364232063293 (0.22586966474210063) + | > loss_disc_real_5: 0.20875921845436096 (0.23419071878156356) + | > loss_0: 2.4895942211151123 (2.445331019739951) + | > grad_norm_0: tensor(10.1041, device='cuda:0') (tensor(6.9015, device='cuda:0')) + | > loss_gen: 2.2441964149475098 (2.352334422449912) + | > loss_kl: 1.7606796026229858 (1.769220636736962) + | > loss_feat: 6.779606342315674 (7.168537724402643) + | > loss_mel: 16.35865020751953 (17.56192441140452) + | > loss_duration: 1.401869297027588 (1.4170340837970856) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.545001983642578 (30.269051459527784) + | > grad_norm_1: tensor(313.4987, device='cuda:0') (tensor(108.6365, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.3513 (0.33669951654249625) + | > loader_time: 0.0053 (0.004975872655068675) + + + --> TIME: 2025-04-19 05:44:22 -- STEP: 56/142 -- GLOBAL_STEP: 437700 + | > loss_disc: 2.5206499099731445 (2.4338982403278355) + | > loss_disc_real_0: 0.12669377028942108 (0.12115278214748416) + | > loss_disc_real_1: 0.21667592227458954 (0.20566390162067755) + | > loss_disc_real_2: 0.24678806960582733 (0.22324411677462713) + | > loss_disc_real_3: 0.248976930975914 (0.22608706541359425) + | > loss_disc_real_4: 0.25516095757484436 (0.22668636510414736) + | > loss_disc_real_5: 0.24520407617092133 (0.23529718523578985) + | > loss_0: 2.5206499099731445 (2.4338982403278355) + | > grad_norm_0: tensor(7.7511, device='cuda:0') (tensor(6.9259, device='cuda:0')) + | > loss_gen: 2.2221336364746094 (2.3763619363307953) + | > loss_kl: 1.7942899465560913 (1.7993489461285728) + | > loss_feat: 7.045501708984375 (7.317221564905984) + | > loss_mel: 17.992536544799805 (17.684095416750218) + | > loss_duration: 1.4674832820892334 (1.4279699751308983) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.52194595336914 (30.60499804360526) + | > grad_norm_1: tensor(106.7984, device='cuda:0') (tensor(124.4696, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.3753 (0.34821732555116924) + | > loader_time: 0.0062 (0.005318386214120048) + + + --> TIME: 2025-04-19 05:44:32 -- STEP: 81/142 -- GLOBAL_STEP: 437725 + | > loss_disc: 2.5694966316223145 (2.457037195747282) + | > loss_disc_real_0: 0.1423143744468689 (0.1237800673202232) + | > loss_disc_real_1: 0.22172056138515472 (0.2085188380730005) + | > loss_disc_real_2: 0.22796590626239777 (0.22535383811703436) + | > loss_disc_real_3: 0.21803030371665955 (0.22697054990279822) + | > loss_disc_real_4: 0.23243077099323273 (0.22839320405029956) + | > loss_disc_real_5: 0.25543200969696045 (0.23610457888355962) + | > loss_0: 2.5694966316223145 (2.457037195747282) + | > grad_norm_0: tensor(11.3135, device='cuda:0') (tensor(7.1914, device='cuda:0')) + | > loss_gen: 2.150158405303955 (2.3575583446173014) + | > loss_kl: 1.6420482397079468 (1.8196985368375425) + | > loss_feat: 5.603268623352051 (7.221063519701546) + | > loss_mel: 15.376367568969727 (17.65872126449773) + | > loss_duration: 1.4477660655975342 (1.4391307816093346) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 26.219608306884766 (30.496172634171852) + | > grad_norm_1: tensor(51.4514, device='cuda:0') (tensor(130.5049, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.3934 (0.36013725657521944) + | > loader_time: 0.0061 (0.005571491924332984) + + + --> TIME: 2025-04-19 05:44:42 -- STEP: 106/142 -- GLOBAL_STEP: 437750 + | > loss_disc: 2.4474480152130127 (2.466739611805611) + | > loss_disc_real_0: 0.16085031628608704 (0.12528695468351525) + | > loss_disc_real_1: 0.21501924097537994 (0.20746738210601626) + | > loss_disc_real_2: 0.22076794505119324 (0.22649023991148426) + | > loss_disc_real_3: 0.21840274333953857 (0.22797982155714394) + | > loss_disc_real_4: 0.21258527040481567 (0.22865055065672352) + | > loss_disc_real_5: 0.21077008545398712 (0.23617273358241567) + | > loss_0: 2.4474480152130127 (2.466739611805611) + | > grad_norm_0: tensor(8.9602, device='cuda:0') (tensor(7.4565, device='cuda:0')) + | > loss_gen: 2.409515380859375 (2.3480603042638513) + | > loss_kl: 1.9002535343170166 (1.8442953510104485) + | > loss_feat: 7.694888591766357 (7.150802454858456) + | > loss_mel: 18.752758026123047 (17.646749523450737) + | > loss_duration: 1.4819458723068237 (1.449751549171951) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.23936080932617 (30.439659280597038) + | > grad_norm_1: tensor(50.5390, device='cuda:0') (tensor(134.8416, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.4082 (0.37108851378818736) + | > loader_time: 0.0067 (0.005823011668223255) + + + --> TIME: 2025-04-19 05:44:53 -- STEP: 131/142 -- GLOBAL_STEP: 437775 + | > loss_disc: 2.356778383255005 (2.468729399542773) + | > loss_disc_real_0: 0.09951875358819962 (0.12474513264330289) + | > loss_disc_real_1: 0.20804724097251892 (0.20819452484600418) + | > loss_disc_real_2: 0.23039819300174713 (0.22574049223015327) + | > loss_disc_real_3: 0.19287407398223877 (0.2283865486392538) + | > loss_disc_real_4: 0.23295769095420837 (0.2296421951010027) + | > loss_disc_real_5: 0.22162961959838867 (0.23544006345381263) + | > loss_0: 2.356778383255005 (2.468729399542773) + | > grad_norm_0: tensor(5.0088, device='cuda:0') (tensor(7.1405, device='cuda:0')) + | > loss_gen: 2.3532159328460693 (2.340516159552653) + | > loss_kl: 2.016312837600708 (1.860629406594138) + | > loss_feat: 7.6313605308532715 (7.120385523060806) + | > loss_mel: 18.314424514770508 (17.654762493745054) + | > loss_duration: 1.4990894794464111 (1.4559568821928879) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.814403533935547 (30.432250510645275) + | > grad_norm_1: tensor(171.8590, device='cuda:0') (tensor(132.9311, device='cuda:0')) + | > current_lr_0: 0.0001899838664896748 + | > current_lr_1: 0.0001899838664896748 + | > step_time: 0.4466 (0.38288280072103026) + | > loader_time: 0.0071 (0.006100261484393636) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.540006399154663 (2.540006399154663) + | > loss_disc_real_0: 0.0996127650141716 (0.0996127650141716) + | > loss_disc_real_1: 0.2277623414993286 (0.2277623414993286) + | > loss_disc_real_2: 0.22253529727458954 (0.22253529727458954) + | > loss_disc_real_3: 0.2416902780532837 (0.2416902780532837) + | > loss_disc_real_4: 0.1795291006565094 (0.1795291006565094) + | > loss_disc_real_5: 0.24821040034294128 (0.24821040034294128) + | > loss_0: 2.540006399154663 (2.540006399154663) + | > loss_gen: 2.040269374847412 (2.040269374847412) + | > loss_kl: 2.4242024421691895 (2.4242024421691895) + | > loss_feat: 6.327359676361084 (6.327359676361084) + | > loss_mel: 18.307802200317383 (18.307802200317383) + | > loss_duration: 1.7177248001098633 (1.7177248001098633) + | > loss_1: 30.817359924316406 (30.817359924316406) + + --> STEP: 1 + | > loss_disc: 2.452725410461426 (2.452725410461426) + | > loss_disc_real_0: 0.1121712476015091 (0.1121712476015091) + | > loss_disc_real_1: 0.24464908242225647 (0.24464908242225647) + | > loss_disc_real_2: 0.22537823021411896 (0.22537823021411896) + | > loss_disc_real_3: 0.23504839837551117 (0.23504839837551117) + | > loss_disc_real_4: 0.22071152925491333 (0.22071152925491333) + | > loss_disc_real_5: 0.23747816681861877 (0.23747816681861877) + | > loss_0: 2.452725410461426 (2.452725410461426) + | > loss_gen: 2.2583799362182617 (2.2583799362182617) + | > loss_kl: 2.9234962463378906 (2.9234962463378906) + | > loss_feat: 6.405083656311035 (6.405083656311035) + | > loss_mel: 17.399232864379883 (17.399232864379883) + | > loss_duration: 1.7780368328094482 (1.7780368328094482) + | > loss_1: 30.764230728149414 (30.764230728149414) + + --> STEP: 2 + | > loss_disc: 2.508591413497925 (2.4806584119796753) + | > loss_disc_real_0: 0.17343319952487946 (0.14280222356319427) + | > loss_disc_real_1: 0.2341587245464325 (0.23940390348434448) + | > loss_disc_real_2: 0.21321667730808258 (0.21929745376110077) + | > loss_disc_real_3: 0.21746794879436493 (0.22625817358493805) + | > loss_disc_real_4: 0.20620135962963104 (0.2134564444422722) + | > loss_disc_real_5: 0.21096065640449524 (0.224219411611557) + | > loss_0: 2.508591413497925 (2.4806584119796753) + | > loss_gen: 2.138995409011841 (2.1986876726150513) + | > loss_kl: 2.3207316398620605 (2.6221139430999756) + | > loss_feat: 6.301680088043213 (6.353381872177124) + | > loss_mel: 16.633026123046875 (17.01612949371338) + | > loss_duration: 1.7085075378417969 (1.7432721853256226) + | > loss_1: 29.102941513061523 (29.93358612060547) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003451824188232422 (+3.814697265625e-06) + | > avg_loss_disc: 2.4806584119796753 (-0.0273362398147583) + | > avg_loss_disc_real_0: 0.14280222356319427 (-0.0635932981967926) + | > avg_loss_disc_real_1: 0.23940390348434448 (+0.017682135105133057) + | > avg_loss_disc_real_2: 0.21929745376110077 (-0.019991278648376465) + | > avg_loss_disc_real_3: 0.22625817358493805 (+0.005179397761821747) + | > avg_loss_disc_real_4: 0.2134564444422722 (-0.008372597396373749) + | > avg_loss_disc_real_5: 0.224219411611557 (-0.006290234625339508) + | > avg_loss_0: 2.4806584119796753 (-0.0273362398147583) + | > avg_loss_gen: 2.1986876726150513 (-0.10066211223602295) + | > avg_loss_kl: 2.6221139430999756 (+0.097908616065979) + | > avg_loss_feat: 6.353381872177124 (+0.03653454780578613) + | > avg_loss_mel: 17.01612949371338 (+0.8377556800842285) + | > avg_loss_duration: 1.7432721853256226 (-0.017232894897460938) + | > avg_loss_1: 29.93358612060547 (+0.8543052673339844) + + + > EPOCH: 412/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:45:00)  + + --> TIME: 2025-04-19 05:45:05 -- STEP: 14/142 -- GLOBAL_STEP: 437800 + | > loss_disc: 2.5081799030303955 (2.412614805357797) + | > loss_disc_real_0: 0.1064421534538269 (0.11434462932603699) + | > loss_disc_real_1: 0.1983996480703354 (0.20308793123279298) + | > loss_disc_real_2: 0.25590941309928894 (0.2277382058756692) + | > loss_disc_real_3: 0.245094433426857 (0.22425292006560735) + | > loss_disc_real_4: 0.25318238139152527 (0.2214823237487248) + | > loss_disc_real_5: 0.22584646940231323 (0.23077721574476787) + | > loss_0: 2.5081799030303955 (2.412614805357797) + | > grad_norm_0: tensor(3.9820, device='cuda:0') (tensor(6.9114, device='cuda:0')) + | > loss_gen: 2.2922396659851074 (2.3628764833722795) + | > loss_kl: 1.9082895517349243 (1.837080180644989) + | > loss_feat: 7.589051246643066 (7.2596432481493265) + | > loss_mel: 18.286069869995117 (17.40724842888968) + | > loss_duration: 1.444269061088562 (1.4166701521192278) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.519920349121094 (30.28351879119873) + | > grad_norm_1: tensor(97.6341, device='cuda:0') (tensor(113.5108, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.3329 (0.3265932287488665) + | > loader_time: 0.0051 (0.00457986763545445) + + + --> TIME: 2025-04-19 05:45:14 -- STEP: 39/142 -- GLOBAL_STEP: 437825 + | > loss_disc: 2.428781032562256 (2.4321767672514305) + | > loss_disc_real_0: 0.08896473795175552 (0.11654317550934278) + | > loss_disc_real_1: 0.22600771486759186 (0.20392263317719483) + | > loss_disc_real_2: 0.22520649433135986 (0.2277835718332193) + | > loss_disc_real_3: 0.24182166159152985 (0.22689550923995483) + | > loss_disc_real_4: 0.28512927889823914 (0.22746229324585351) + | > loss_disc_real_5: 0.2786092460155487 (0.23475133646757174) + | > loss_0: 2.428781032562256 (2.4321767672514305) + | > grad_norm_0: tensor(5.6118, device='cuda:0') (tensor(7.0638, device='cuda:0')) + | > loss_gen: 2.4620442390441895 (2.3682577854547744) + | > loss_kl: 1.8646451234817505 (1.8683440685272217) + | > loss_feat: 6.961544036865234 (7.261051520323142) + | > loss_mel: 17.973554611206055 (17.464373686374763) + | > loss_duration: 1.4446810483932495 (1.4257722390003693) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.70646858215332 (30.387799483079178) + | > grad_norm_1: tensor(70.6456, device='cuda:0') (tensor(153.7878, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.3653 (0.3442978981213692) + | > loader_time: 0.0059 (0.005185310657207782) + + + --> TIME: 2025-04-19 05:45:24 -- STEP: 64/142 -- GLOBAL_STEP: 437850 + | > loss_disc: 2.544321298599243 (2.438837371766567) + | > loss_disc_real_0: 0.18174044787883759 (0.11871615424752235) + | > loss_disc_real_1: 0.18684642016887665 (0.20683737704530358) + | > loss_disc_real_2: 0.22019056975841522 (0.2245852597989142) + | > loss_disc_real_3: 0.23017118871212006 (0.22686179354786873) + | > loss_disc_real_4: 0.24885506927967072 (0.22685590735636652) + | > loss_disc_real_5: 0.26966631412506104 (0.2328029798809439) + | > loss_0: 2.544321298599243 (2.438837371766567) + | > grad_norm_0: tensor(5.8453, device='cuda:0') (tensor(7.2410, device='cuda:0')) + | > loss_gen: 2.2529244422912598 (2.360720671713352) + | > loss_kl: 1.8974417448043823 (1.8839710503816605) + | > loss_feat: 6.464157581329346 (7.2333821058273315) + | > loss_mel: 16.470287322998047 (17.597965165972706) + | > loss_duration: 1.474992275238037 (1.4351612329483032) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.559803009033203 (30.511200338602066) + | > grad_norm_1: tensor(117.9875, device='cuda:0') (tensor(168.4995, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.3933 (0.3587853275239468) + | > loader_time: 0.0062 (0.005524300038814545) + + + --> TIME: 2025-04-19 05:45:34 -- STEP: 89/142 -- GLOBAL_STEP: 437875 + | > loss_disc: 2.61917781829834 (2.449964335795199) + | > loss_disc_real_0: 0.11187825351953506 (0.12068309768866957) + | > loss_disc_real_1: 0.25138047337532043 (0.20647854617472444) + | > loss_disc_real_2: 0.23167645931243896 (0.22507078878665238) + | > loss_disc_real_3: 0.23515066504478455 (0.2276824615644605) + | > loss_disc_real_4: 0.24400393664836884 (0.22860289205995837) + | > loss_disc_real_5: 0.23017875850200653 (0.2330847786048825) + | > loss_0: 2.61917781829834 (2.449964335795199) + | > grad_norm_0: tensor(4.3166, device='cuda:0') (tensor(7.4924, device='cuda:0')) + | > loss_gen: 2.2359440326690674 (2.35247528419066) + | > loss_kl: 1.8250175714492798 (1.8849153290973621) + | > loss_feat: 6.698578357696533 (7.174938801969035) + | > loss_mel: 16.987096786499023 (17.60076603192961) + | > loss_duration: 1.4350396394729614 (1.4454457344633809) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.181676864624023 (30.458541312914218) + | > grad_norm_1: tensor(94.9634, device='cuda:0') (tensor(171.6816, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.3991 (0.3723763830206369) + | > loader_time: 0.006 (0.0057690438259853415) + + + --> TIME: 2025-04-19 05:45:45 -- STEP: 114/142 -- GLOBAL_STEP: 437900 + | > loss_disc: 2.447697639465332 (2.456706563631694) + | > loss_disc_real_0: 0.09166109561920166 (0.12418450722307488) + | > loss_disc_real_1: 0.21800440549850464 (0.20574223537716949) + | > loss_disc_real_2: 0.2214990258216858 (0.22528585817730218) + | > loss_disc_real_3: 0.20767728984355927 (0.22769154935029515) + | > loss_disc_real_4: 0.15941472351551056 (0.22687803404895882) + | > loss_disc_real_5: 0.19666071236133575 (0.23289453303604796) + | > loss_0: 2.447697639465332 (2.456706563631694) + | > grad_norm_0: tensor(14.5844, device='cuda:0') (tensor(10.0299, device='cuda:0')) + | > loss_gen: 2.321352005004883 (2.3522501598324705) + | > loss_kl: 1.8470412492752075 (1.8811904735732496) + | > loss_feat: 7.468401908874512 (7.130108381572523) + | > loss_mel: 17.847875595092773 (17.608570835046596) + | > loss_duration: 1.4667364358901978 (1.4523288147491322) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.951406478881836 (30.424448799668696) + | > grad_norm_1: tensor(53.8976, device='cuda:0') (tensor(203.3496, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.4196 (0.38141575821659035) + | > loader_time: 0.0063 (0.006003068204511677) + + + --> TIME: 2025-04-19 05:45:56 -- STEP: 139/142 -- GLOBAL_STEP: 437925 + | > loss_disc: 2.467421770095825 (2.4668355211079556) + | > loss_disc_real_0: 0.1314147561788559 (0.12597579155251276) + | > loss_disc_real_1: 0.1774846315383911 (0.2071382934026581) + | > loss_disc_real_2: 0.2294609397649765 (0.22667908947244822) + | > loss_disc_real_3: 0.19470854103565216 (0.22789709227119417) + | > loss_disc_real_4: 0.23523586988449097 (0.22904895589077215) + | > loss_disc_real_5: 0.22970996797084808 (0.23340632793285865) + | > loss_0: 2.467421770095825 (2.4668355211079556) + | > grad_norm_0: tensor(7.4727, device='cuda:0') (tensor(11.0566, device='cuda:0')) + | > loss_gen: 2.5085816383361816 (2.3502790996496636) + | > loss_kl: 1.8611971139907837 (1.8844886709460251) + | > loss_feat: 8.156087875366211 (7.083734800489687) + | > loss_mel: 18.198368072509766 (17.563354149139183) + | > loss_duration: 1.4814566373825073 (1.4583453548897942) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.2056884765625 (30.340202153157843) + | > grad_norm_1: tensor(391.9565, device='cuda:0') (tensor(214.3987, device='cuda:0')) + | > current_lr_0: 0.00018996011850636358 + | > current_lr_1: 0.00018996011850636358 + | > step_time: 0.4526 (0.39324723216269525) + | > loader_time: 0.0071 (0.0062968971060334365) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.519815683364868 (2.519815683364868) + | > loss_disc_real_0: 0.13709519803524017 (0.13709519803524017) + | > loss_disc_real_1: 0.2517169415950775 (0.2517169415950775) + | > loss_disc_real_2: 0.23197205364704132 (0.23197205364704132) + | > loss_disc_real_3: 0.2642744481563568 (0.2642744481563568) + | > loss_disc_real_4: 0.2297331988811493 (0.2297331988811493) + | > loss_disc_real_5: 0.30002057552337646 (0.30002057552337646) + | > loss_0: 2.519815683364868 (2.519815683364868) + | > loss_gen: 2.4452011585235596 (2.4452011585235596) + | > loss_kl: 2.679203748703003 (2.679203748703003) + | > loss_feat: 6.73414945602417 (6.73414945602417) + | > loss_mel: 17.780733108520508 (17.780733108520508) + | > loss_duration: 1.7338359355926514 (1.7338359355926514) + | > loss_1: 31.373123168945312 (31.373123168945312) + + --> STEP: 1 + | > loss_disc: 2.4983506202697754 (2.4983506202697754) + | > loss_disc_real_0: 0.12060713768005371 (0.12060713768005371) + | > loss_disc_real_1: 0.245363250374794 (0.245363250374794) + | > loss_disc_real_2: 0.22125044465065002 (0.22125044465065002) + | > loss_disc_real_3: 0.27292364835739136 (0.27292364835739136) + | > loss_disc_real_4: 0.2184760719537735 (0.2184760719537735) + | > loss_disc_real_5: 0.2803073525428772 (0.2803073525428772) + | > loss_0: 2.4983506202697754 (2.4983506202697754) + | > loss_gen: 2.2996766567230225 (2.2996766567230225) + | > loss_kl: 2.7687368392944336 (2.7687368392944336) + | > loss_feat: 7.318847179412842 (7.318847179412842) + | > loss_mel: 18.284847259521484 (18.284847259521484) + | > loss_duration: 1.7909722328186035 (1.7909722328186035) + | > loss_1: 32.46308135986328 (32.46308135986328) + + --> STEP: 2 + | > loss_disc: 2.535358428955078 (2.5168545246124268) + | > loss_disc_real_0: 0.16047021746635437 (0.14053867757320404) + | > loss_disc_real_1: 0.2887450158596039 (0.26705413311719894) + | > loss_disc_real_2: 0.24427545070648193 (0.23276294767856598) + | > loss_disc_real_3: 0.2902734875679016 (0.2815985679626465) + | > loss_disc_real_4: 0.20492814481258392 (0.2117021083831787) + | > loss_disc_real_5: 0.27426910400390625 (0.2772882282733917) + | > loss_0: 2.535358428955078 (2.5168545246124268) + | > loss_gen: 2.4116857051849365 (2.3556811809539795) + | > loss_kl: 2.4445607662200928 (2.606648802757263) + | > loss_feat: 7.7335286140441895 (7.526187896728516) + | > loss_mel: 18.82400894165039 (18.554428100585938) + | > loss_duration: 1.7042756080627441 (1.7476239204406738) + | > loss_1: 33.11806106567383 (32.790571212768555) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033839941024780273 (-6.783008575439453e-05) + | > avg_loss_disc: 2.5168545246124268 (+0.036196112632751465) + | > avg_loss_disc_real_0: 0.14053867757320404 (-0.0022635459899902344) + | > avg_loss_disc_real_1: 0.26705413311719894 (+0.02765022963285446) + | > avg_loss_disc_real_2: 0.23276294767856598 (+0.01346549391746521) + | > avg_loss_disc_real_3: 0.2815985679626465 (+0.055340394377708435) + | > avg_loss_disc_real_4: 0.2117021083831787 (-0.0017543360590934753) + | > avg_loss_disc_real_5: 0.2772882282733917 (+0.05306881666183472) + | > avg_loss_0: 2.5168545246124268 (+0.036196112632751465) + | > avg_loss_gen: 2.3556811809539795 (+0.15699350833892822) + | > avg_loss_kl: 2.606648802757263 (-0.015465140342712402) + | > avg_loss_feat: 7.526187896728516 (+1.1728060245513916) + | > avg_loss_mel: 18.554428100585938 (+1.5382986068725586) + | > avg_loss_duration: 1.7476239204406738 (+0.0043517351150512695) + | > avg_loss_1: 32.790571212768555 (+2.856985092163086) + + + > EPOCH: 413/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:45:59)  + + --> TIME: 2025-04-19 05:46:07 -- STEP: 22/142 -- GLOBAL_STEP: 437950 + | > loss_disc: 2.3664395809173584 (2.4046105688268486) + | > loss_disc_real_0: 0.12297884374856949 (0.11579901487989859) + | > loss_disc_real_1: 0.2293590009212494 (0.20274619216268713) + | > loss_disc_real_2: 0.23599781095981598 (0.22519522769884628) + | > loss_disc_real_3: 0.23414205014705658 (0.2238248187032613) + | > loss_disc_real_4: 0.22367730736732483 (0.22328605299646204) + | > loss_disc_real_5: 0.23311196267604828 (0.2283023873513395) + | > loss_0: 2.3664395809173584 (2.4046105688268486) + | > grad_norm_0: tensor(6.5692, device='cuda:0') (tensor(8.0432, device='cuda:0')) + | > loss_gen: 2.467564105987549 (2.3616890907287598) + | > loss_kl: 1.813278079032898 (1.8063014420596035) + | > loss_feat: 7.488853931427002 (7.183314388448542) + | > loss_mel: 17.74285888671875 (17.46751104701649) + | > loss_duration: 1.4123268127441406 (1.4066211527044123) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.924880981445312 (30.225437251004305) + | > grad_norm_1: tensor(183.5087, device='cuda:0') (tensor(155.1851, device='cuda:0')) + | > current_lr_0: 0.00018993637349155027 + | > current_lr_1: 0.00018993637349155027 + | > step_time: 0.3436 (0.33096870509060944) + | > loader_time: 0.005 (0.004831335761330344) + + + --> TIME: 2025-04-19 05:46:16 -- STEP: 47/142 -- GLOBAL_STEP: 437975 + | > loss_disc: 2.457167387008667 (2.4214465364496762) + | > loss_disc_real_0: 0.139475017786026 (0.11751243614769996) + | > loss_disc_real_1: 0.24033689498901367 (0.20144358277320862) + | > loss_disc_real_2: 0.23177585005760193 (0.2249607630866639) + | > loss_disc_real_3: 0.21363787353038788 (0.22540102772256163) + | > loss_disc_real_4: 0.20967046916484833 (0.22605418524843582) + | > loss_disc_real_5: 0.22024111449718475 (0.22930173036899973) + | > loss_0: 2.457167387008667 (2.4214465364496762) + | > grad_norm_0: tensor(5.6044, device='cuda:0') (tensor(6.9615, device='cuda:0')) + | > loss_gen: 2.4365952014923096 (2.3798054228437717) + | > loss_kl: 1.8770464658737183 (1.821811021642482) + | > loss_feat: 7.547350883483887 (7.276943886533696) + | > loss_mel: 18.109800338745117 (17.696791222754946) + | > loss_duration: 1.3984498977661133 (1.4189211231596925) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.369243621826172 (30.594272857016705) + | > grad_norm_1: tensor(54.0819, device='cuda:0') (tensor(127.7013, device='cuda:0')) + | > current_lr_0: 0.00018993637349155027 + | > current_lr_1: 0.00018993637349155027 + | > step_time: 0.3637 (0.3434557001641456) + | > loader_time: 0.0061 (0.005254121536904194) + + + --> TIME: 2025-04-19 05:46:26 -- STEP: 72/142 -- GLOBAL_STEP: 438000 + | > loss_disc: 2.512568712234497 (2.4448578225241766) + | > loss_disc_real_0: 0.11728531122207642 (0.1213710577123695) + | > loss_disc_real_1: 0.2140975147485733 (0.20436620008614328) + | > loss_disc_real_2: 0.21315576136112213 (0.22495846119191912) + | > loss_disc_real_3: 0.217691570520401 (0.2275682133105066) + | > loss_disc_real_4: 0.26061132550239563 (0.2290086050828298) + | > loss_disc_real_5: 0.25209668278694153 (0.23166251762045753) + | > loss_0: 2.512568712234497 (2.4448578225241766) + | > grad_norm_0: tensor(12.1384, device='cuda:0') (tensor(7.2066, device='cuda:0')) + | > loss_gen: 2.243422508239746 (2.363781544897291) + | > loss_kl: 1.6392282247543335 (1.8295234110620286) + | > loss_feat: 7.109552383422852 (7.199368086126116) + | > loss_mel: 18.138050079345703 (17.708661993344624) + | > loss_duration: 1.459934949874878 (1.4324076175689697) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.590187072753906 (30.53374285168118) + | > grad_norm_1: tensor(101.8260, device='cuda:0') (tensor(122.2219, device='cuda:0')) + | > current_lr_0: 0.00018993637349155027 + | > current_lr_1: 0.00018993637349155027 + | > step_time: 0.3942 (0.3556056684917874) + | > loader_time: 0.0062 (0.0055275559425354) + + + --> TIME: 2025-04-19 05:46:36 -- STEP: 97/142 -- GLOBAL_STEP: 438025 + | > loss_disc: 2.5969603061676025 (2.4499061550061727) + | > loss_disc_real_0: 0.18370512127876282 (0.12226259347397028) + | > loss_disc_real_1: 0.21086452901363373 (0.203136359139816) + | > loss_disc_real_2: 0.24210786819458008 (0.22568260256162623) + | > loss_disc_real_3: 0.2421833574771881 (0.22822312979968554) + | > loss_disc_real_4: 0.25364214181900024 (0.2294218428048891) + | > loss_disc_real_5: 0.209689661860466 (0.23203371127241665) + | > loss_0: 2.5969603061676025 (2.4499061550061727) + | > grad_norm_0: tensor(16.2807, device='cuda:0') (tensor(7.6488, device='cuda:0')) + | > loss_gen: 2.2368531227111816 (2.3571501810526114) + | > loss_kl: 1.8589262962341309 (1.840461804694736) + | > loss_feat: 7.008790016174316 (7.177397457594724) + | > loss_mel: 17.309654235839844 (17.70733941461623) + | > loss_duration: 1.4720327854156494 (1.4436211377075039) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.886255264282227 (30.525970105043392) + | > grad_norm_1: tensor(364.4422, device='cuda:0') (tensor(132.9771, device='cuda:0')) + | > current_lr_0: 0.00018993637349155027 + | > current_lr_1: 0.00018993637349155027 + | > step_time: 0.4118 (0.3664385240102552) + | > loader_time: 0.0065 (0.005793269147577975) + + + --> TIME: 2025-04-19 05:46:47 -- STEP: 122/142 -- GLOBAL_STEP: 438050 + | > loss_disc: 2.454287052154541 (2.454272879928839) + | > loss_disc_real_0: 0.10968124866485596 (0.123494526218684) + | > loss_disc_real_1: 0.21388371288776398 (0.20502532859806152) + | > loss_disc_real_2: 0.21720004081726074 (0.22534525260084964) + | > loss_disc_real_3: 0.18317903578281403 (0.22766440544949204) + | > loss_disc_real_4: 0.21837978065013885 (0.2285904837924926) + | > loss_disc_real_5: 0.20651653409004211 (0.23310663123599817) + | > loss_0: 2.454287052154541 (2.454272879928839) + | > grad_norm_0: tensor(4.3368, device='cuda:0') (tensor(7.9978, device='cuda:0')) + | > loss_gen: 2.3831865787506104 (2.3507536024343776) + | > loss_kl: 1.8143807649612427 (1.8440563180407539) + | > loss_feat: 7.901010513305664 (7.148465926529931) + | > loss_mel: 19.35494041442871 (17.652721920951475) + | > loss_duration: 1.470350980758667 (1.4512777348033716) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.92387008666992 (30.447275583861305) + | > grad_norm_1: tensor(58.2930, device='cuda:0') (tensor(154.9521, device='cuda:0')) + | > current_lr_0: 0.00018993637349155027 + | > current_lr_1: 0.00018993637349155027 + | > step_time: 0.4486 (0.3779416592394719) + | > loader_time: 0.0072 (0.006050037556007261) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.573091506958008 (2.573091506958008) + | > loss_disc_real_0: 0.13256779313087463 (0.13256779313087463) + | > loss_disc_real_1: 0.273062527179718 (0.273062527179718) + | > loss_disc_real_2: 0.29452288150787354 (0.29452288150787354) + | > loss_disc_real_3: 0.2572596073150635 (0.2572596073150635) + | > loss_disc_real_4: 0.26772844791412354 (0.26772844791412354) + | > loss_disc_real_5: 0.28585344552993774 (0.28585344552993774) + | > loss_0: 2.573091506958008 (2.573091506958008) + | > loss_gen: 2.5795774459838867 (2.5795774459838867) + | > loss_kl: 2.41865611076355 (2.41865611076355) + | > loss_feat: 6.748075008392334 (6.748075008392334) + | > loss_mel: 17.63087272644043 (17.63087272644043) + | > loss_duration: 1.7669705152511597 (1.7669705152511597) + | > loss_1: 31.14415168762207 (31.14415168762207) + + --> STEP: 1 + | > loss_disc: 2.6323657035827637 (2.6323657035827637) + | > loss_disc_real_0: 0.1201426163315773 (0.1201426163315773) + | > loss_disc_real_1: 0.25837332010269165 (0.25837332010269165) + | > loss_disc_real_2: 0.26490306854248047 (0.26490306854248047) + | > loss_disc_real_3: 0.24452802538871765 (0.24452802538871765) + | > loss_disc_real_4: 0.2977406680583954 (0.2977406680583954) + | > loss_disc_real_5: 0.23616923391819 (0.23616923391819) + | > loss_0: 2.6323657035827637 (2.6323657035827637) + | > loss_gen: 2.260423183441162 (2.260423183441162) + | > loss_kl: 2.7230052947998047 (2.7230052947998047) + | > loss_feat: 6.801246643066406 (6.801246643066406) + | > loss_mel: 16.68863296508789 (16.68863296508789) + | > loss_duration: 1.7922985553741455 (1.7922985553741455) + | > loss_1: 30.265607833862305 (30.265607833862305) + + --> STEP: 2 + | > loss_disc: 2.553553819656372 (2.592959761619568) + | > loss_disc_real_0: 0.16953995823860168 (0.1448412872850895) + | > loss_disc_real_1: 0.2151983231306076 (0.23678582161664963) + | > loss_disc_real_2: 0.2698572874069214 (0.2673801779747009) + | > loss_disc_real_3: 0.2970602512359619 (0.2707941383123398) + | > loss_disc_real_4: 0.3005812168121338 (0.2991609424352646) + | > loss_disc_real_5: 0.2388221025466919 (0.23749566823244095) + | > loss_0: 2.553553819656372 (2.592959761619568) + | > loss_gen: 2.616875648498535 (2.4386494159698486) + | > loss_kl: 2.2376608848571777 (2.480333089828491) + | > loss_feat: 8.116554260253906 (7.458900451660156) + | > loss_mel: 18.892732620239258 (17.790682792663574) + | > loss_duration: 1.7450470924377441 (1.7686728239059448) + | > loss_1: 33.60887145996094 (31.93723964691162) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003857135772705078 (+0.0004731416702270508) + | > avg_loss_disc: 2.592959761619568 (+0.07610523700714111) + | > avg_loss_disc_real_0: 0.1448412872850895 (+0.004302609711885452) + | > avg_loss_disc_real_1: 0.23678582161664963 (-0.030268311500549316) + | > avg_loss_disc_real_2: 0.2673801779747009 (+0.03461723029613495) + | > avg_loss_disc_real_3: 0.2707941383123398 (-0.010804429650306702) + | > avg_loss_disc_real_4: 0.2991609424352646 (+0.08745883405208588) + | > avg_loss_disc_real_5: 0.23749566823244095 (-0.039792560040950775) + | > avg_loss_0: 2.592959761619568 (+0.07610523700714111) + | > avg_loss_gen: 2.4386494159698486 (+0.08296823501586914) + | > avg_loss_kl: 2.480333089828491 (-0.12631571292877197) + | > avg_loss_feat: 7.458900451660156 (-0.06728744506835938) + | > avg_loss_mel: 17.790682792663574 (-0.7637453079223633) + | > avg_loss_duration: 1.7686728239059448 (+0.021048903465270996) + | > avg_loss_1: 31.93723964691162 (-0.8533315658569336) + + + > EPOCH: 414/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:46:57)  + + --> TIME: 2025-04-19 05:47:00 -- STEP: 5/142 -- GLOBAL_STEP: 438075 + | > loss_disc: 2.5389513969421387 (2.447397994995117) + | > loss_disc_real_0: 0.11571981757879257 (0.1193105325102806) + | > loss_disc_real_1: 0.21745680272579193 (0.2023467093706131) + | > loss_disc_real_2: 0.21891433000564575 (0.22587425708770753) + | > loss_disc_real_3: 0.23293066024780273 (0.22079906463623047) + | > loss_disc_real_4: 0.20030836760997772 (0.237156543135643) + | > loss_disc_real_5: 0.26035499572753906 (0.2555751234292984) + | > loss_0: 2.5389513969421387 (2.447397994995117) + | > grad_norm_0: tensor(5.1694, device='cuda:0') (tensor(8.1156, device='cuda:0')) + | > loss_gen: 2.2676944732666016 (2.3637094497680664) + | > loss_kl: 1.8638794422149658 (1.7494043111801147) + | > loss_feat: 6.830168724060059 (7.126418590545654) + | > loss_mel: 17.464618682861328 (17.4435791015625) + | > loss_duration: 1.4206345081329346 (1.4063694715499877) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.846996307373047 (30.089480590820312) + | > grad_norm_1: tensor(66.3695, device='cuda:0') (tensor(163.9786, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.3248 (0.3231672286987305) + | > loader_time: 0.0041 (0.004326438903808594) + + + --> TIME: 2025-04-19 05:47:09 -- STEP: 30/142 -- GLOBAL_STEP: 438100 + | > loss_disc: 2.3163318634033203 (2.4047468741734823) + | > loss_disc_real_0: 0.09625358879566193 (0.1114930808544159) + | > loss_disc_real_1: 0.1937030404806137 (0.20130440841118494) + | > loss_disc_real_2: 0.2249755561351776 (0.22412620584170023) + | > loss_disc_real_3: 0.2146291881799698 (0.22706063389778136) + | > loss_disc_real_4: 0.20574985444545746 (0.22963642627000808) + | > loss_disc_real_5: 0.2180020660161972 (0.22763193845748902) + | > loss_0: 2.3163318634033203 (2.4047468741734823) + | > grad_norm_0: tensor(6.3813, device='cuda:0') (tensor(9.3628, device='cuda:0')) + | > loss_gen: 2.4770781993865967 (2.424962488810221) + | > loss_kl: 1.7665396928787231 (1.7635695020357767) + | > loss_feat: 7.4432902336120605 (7.458365837732951) + | > loss_mel: 17.234529495239258 (17.89718437194825) + | > loss_duration: 1.4325001239776611 (1.4171383261680603) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.35393714904785 (30.961220423380535) + | > grad_norm_1: tensor(304.3846, device='cuda:0') (tensor(253.8070, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.3502 (0.3428129752477009) + | > loader_time: 0.0055 (0.005057120323181153) + + + --> TIME: 2025-04-19 05:47:18 -- STEP: 55/142 -- GLOBAL_STEP: 438125 + | > loss_disc: 2.3281190395355225 (2.4205014445564963) + | > loss_disc_real_0: 0.09758347272872925 (0.11623351682316173) + | > loss_disc_real_1: 0.19059187173843384 (0.20423532128334046) + | > loss_disc_real_2: 0.1923232227563858 (0.2250491827726364) + | > loss_disc_real_3: 0.22513599693775177 (0.22486788576299493) + | > loss_disc_real_4: 0.18012186884880066 (0.22851622727784243) + | > loss_disc_real_5: 0.22787541151046753 (0.2286499110135165) + | > loss_0: 2.3281190395355225 (2.4205014445564963) + | > grad_norm_0: tensor(7.2113, device='cuda:0') (tensor(10.3211, device='cuda:0')) + | > loss_gen: 2.3658816814422607 (2.413560212742198) + | > loss_kl: 1.8011590242385864 (1.7854939850893887) + | > loss_feat: 7.438816547393799 (7.355801304903897) + | > loss_mel: 18.533863067626953 (17.892137076637965) + | > loss_duration: 1.495469331741333 (1.4263893994418058) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.635189056396484 (30.87338184009899) + | > grad_norm_1: tensor(100.1161, device='cuda:0') (tensor(266.4529, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.3732 (0.35160540667447165) + | > loader_time: 0.006 (0.0053498614918101916) + + + --> TIME: 2025-04-19 05:47:28 -- STEP: 80/142 -- GLOBAL_STEP: 438150 + | > loss_disc: 2.40840744972229 (2.43499592244625) + | > loss_disc_real_0: 0.11877476423978806 (0.118815734423697) + | > loss_disc_real_1: 0.2028653919696808 (0.20456115249544377) + | > loss_disc_real_2: 0.20697307586669922 (0.22722810618579387) + | > loss_disc_real_3: 0.21896179020404816 (0.22825829721987248) + | > loss_disc_real_4: 0.21281415224075317 (0.2281561942771077) + | > loss_disc_real_5: 0.22777913510799408 (0.23187757153064012) + | > loss_0: 2.40840744972229 (2.43499592244625) + | > grad_norm_0: tensor(12.1733, device='cuda:0') (tensor(10.5881, device='cuda:0')) + | > loss_gen: 2.343194007873535 (2.404651501774788) + | > loss_kl: 1.8357861042022705 (1.8098926931619643) + | > loss_feat: 7.482166290283203 (7.293808734416961) + | > loss_mel: 17.576396942138672 (17.825832962989814) + | > loss_duration: 1.4672696590423584 (1.4373366132378578) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.704811096191406 (30.77152225971222) + | > grad_norm_1: tensor(77.6245, device='cuda:0') (tensor(263.1609, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.4057 (0.36259609758853906) + | > loader_time: 0.0066 (0.00561068058013916) + + + --> TIME: 2025-04-19 05:47:38 -- STEP: 105/142 -- GLOBAL_STEP: 438175 + | > loss_disc: 2.3957056999206543 (2.44218277022952) + | > loss_disc_real_0: 0.16822031140327454 (0.1199544411329996) + | > loss_disc_real_1: 0.18101173639297485 (0.20551557753767283) + | > loss_disc_real_2: 0.19518586993217468 (0.22712168792883555) + | > loss_disc_real_3: 0.2140527069568634 (0.22721773627258482) + | > loss_disc_real_4: 0.1937766820192337 (0.2302016053880964) + | > loss_disc_real_5: 0.2229466736316681 (0.2320775234983081) + | > loss_0: 2.3957056999206543 (2.44218277022952) + | > grad_norm_0: tensor(19.2707, device='cuda:0') (tensor(11.8377, device='cuda:0')) + | > loss_gen: 2.349928379058838 (2.403428066344488) + | > loss_kl: 2.042797088623047 (1.8188034057617188) + | > loss_feat: 7.3920745849609375 (7.263558501289005) + | > loss_mel: 18.888427734375 (17.824325398036418) + | > loss_duration: 1.5094037055969238 (1.4482687723068968) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.18263244628906 (30.758383923485166) + | > grad_norm_1: tensor(500.2614, device='cuda:0') (tensor(288.7181, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.4159 (0.3725780691419329) + | > loader_time: 0.0065 (0.005859745116460891) + + + --> TIME: 2025-04-19 05:47:49 -- STEP: 130/142 -- GLOBAL_STEP: 438200 + | > loss_disc: 2.3790478706359863 (2.4422257019923275) + | > loss_disc_real_0: 0.1460321992635727 (0.12128478151101332) + | > loss_disc_real_1: 0.17646217346191406 (0.20436475804218876) + | > loss_disc_real_2: 0.21480166912078857 (0.22600942895962642) + | > loss_disc_real_3: 0.21484306454658508 (0.22684666159061284) + | > loss_disc_real_4: 0.2227523773908615 (0.22969563248065802) + | > loss_disc_real_5: 0.23078769445419312 (0.23087653804283875) + | > loss_0: 2.3790478706359863 (2.4422257019923275) + | > grad_norm_0: tensor(18.3018, device='cuda:0') (tensor(12.6174, device='cuda:0')) + | > loss_gen: 2.5199103355407715 (2.3966971910916843) + | > loss_kl: 1.9383245706558228 (1.8239873519310585) + | > loss_feat: 8.23984146118164 (7.191367292404175) + | > loss_mel: 19.022281646728516 (17.679322176713214) + | > loss_duration: 1.4650497436523438 (1.4547384867301356) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 33.18540954589844 (30.546112353985126) + | > grad_norm_1: tensor(425.7821, device='cuda:0') (tensor(303.4270, device='cuda:0')) + | > current_lr_0: 0.00018991263144486382 + | > current_lr_1: 0.00018991263144486382 + | > step_time: 0.4479 (0.3841362458008986) + | > loader_time: 0.0074 (0.0061057145778949445) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.598722457885742 (2.598722457885742) + | > loss_disc_real_0: 0.28070464730262756 (0.28070464730262756) + | > loss_disc_real_1: 0.2242380529642105 (0.2242380529642105) + | > loss_disc_real_2: 0.19520816206932068 (0.19520816206932068) + | > loss_disc_real_3: 0.25942909717559814 (0.25942909717559814) + | > loss_disc_real_4: 0.227496936917305 (0.227496936917305) + | > loss_disc_real_5: 0.23063717782497406 (0.23063717782497406) + | > loss_0: 2.598722457885742 (2.598722457885742) + | > loss_gen: 2.4361166954040527 (2.4361166954040527) + | > loss_kl: 2.324145555496216 (2.324145555496216) + | > loss_feat: 6.286402225494385 (6.286402225494385) + | > loss_mel: 18.381227493286133 (18.381227493286133) + | > loss_duration: 1.7255141735076904 (1.7255141735076904) + | > loss_1: 31.153406143188477 (31.153406143188477) + + --> STEP: 1 + | > loss_disc: 2.6184897422790527 (2.6184897422790527) + | > loss_disc_real_0: 0.22950248420238495 (0.22950248420238495) + | > loss_disc_real_1: 0.22295816242694855 (0.22295816242694855) + | > loss_disc_real_2: 0.23224997520446777 (0.23224997520446777) + | > loss_disc_real_3: 0.25015005469322205 (0.25015005469322205) + | > loss_disc_real_4: 0.26962581276893616 (0.26962581276893616) + | > loss_disc_real_5: 0.24988891184329987 (0.24988891184329987) + | > loss_0: 2.6184897422790527 (2.6184897422790527) + | > loss_gen: 2.4280166625976562 (2.4280166625976562) + | > loss_kl: 2.907611608505249 (2.907611608505249) + | > loss_feat: 6.8068013191223145 (6.8068013191223145) + | > loss_mel: 16.980266571044922 (16.980266571044922) + | > loss_duration: 1.7876317501068115 (1.7876317501068115) + | > loss_1: 30.910327911376953 (30.910327911376953) + + --> STEP: 2 + | > loss_disc: 2.580073595046997 (2.599281668663025) + | > loss_disc_real_0: 0.23103365302085876 (0.23026806861162186) + | > loss_disc_real_1: 0.23246075212955475 (0.22770945727825165) + | > loss_disc_real_2: 0.19197356700897217 (0.21211177110671997) + | > loss_disc_real_3: 0.2532578706741333 (0.2517039626836777) + | > loss_disc_real_4: 0.2498788982629776 (0.2597523555159569) + | > loss_disc_real_5: 0.32046762108802795 (0.2851782664656639) + | > loss_0: 2.580073595046997 (2.599281668663025) + | > loss_gen: 2.6230111122131348 (2.5255138874053955) + | > loss_kl: 2.289065361022949 (2.598338484764099) + | > loss_feat: 7.100815773010254 (6.953808546066284) + | > loss_mel: 18.3427791595459 (17.66152286529541) + | > loss_duration: 1.7034525871276855 (1.7455421686172485) + | > loss_1: 32.05912399291992 (31.484725952148438) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034017562866210938 (-0.0004553794860839844) + | > avg_loss_disc: 2.599281668663025 (+0.006321907043457031) + | > avg_loss_disc_real_0: 0.23026806861162186 (+0.08542678132653236) + | > avg_loss_disc_real_1: 0.22770945727825165 (-0.00907636433839798) + | > avg_loss_disc_real_2: 0.21211177110671997 (-0.05526840686798096) + | > avg_loss_disc_real_3: 0.2517039626836777 (-0.01909017562866211) + | > avg_loss_disc_real_4: 0.2597523555159569 (-0.03940858691930771) + | > avg_loss_disc_real_5: 0.2851782664656639 (+0.04768259823322296) + | > avg_loss_0: 2.599281668663025 (+0.006321907043457031) + | > avg_loss_gen: 2.5255138874053955 (+0.08686447143554688) + | > avg_loss_kl: 2.598338484764099 (+0.11800539493560791) + | > avg_loss_feat: 6.953808546066284 (-0.5050919055938721) + | > avg_loss_mel: 17.66152286529541 (-0.12915992736816406) + | > avg_loss_duration: 1.7455421686172485 (-0.02313065528869629) + | > avg_loss_1: 31.484725952148438 (-0.4525136947631836) + + + > EPOCH: 415/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:47:56)  + + --> TIME: 2025-04-19 05:48:01 -- STEP: 13/142 -- GLOBAL_STEP: 438225 + | > loss_disc: 2.2993111610412598 (2.4211226793435903) + | > loss_disc_real_0: 0.1340653896331787 (0.11630771767634612) + | > loss_disc_real_1: 0.19334626197814941 (0.20349891369159406) + | > loss_disc_real_2: 0.19221125543117523 (0.2253233973796551) + | > loss_disc_real_3: 0.20278377830982208 (0.2183986374965081) + | > loss_disc_real_4: 0.17029500007629395 (0.2223854110791133) + | > loss_disc_real_5: 0.2254004180431366 (0.2353054445523482) + | > loss_0: 2.2993111610412598 (2.4211226793435903) + | > grad_norm_0: tensor(22.4108, device='cuda:0') (tensor(11.4270, device='cuda:0')) + | > loss_gen: 2.4274790287017822 (2.3822871354910045) + | > loss_kl: 1.7058056592941284 (1.7458761471968431) + | > loss_feat: 7.367405414581299 (7.409508301661565) + | > loss_mel: 18.10077476501465 (17.940637881939228) + | > loss_duration: 1.4308052062988281 (1.4121304475344145) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.032268524169922 (30.890439693744366) + | > grad_norm_1: tensor(509.4767, device='cuda:0') (tensor(313.5363, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.3304 (0.32877100431002104) + | > loader_time: 0.0051 (0.004621175619272085) + + + --> TIME: 2025-04-19 05:48:10 -- STEP: 38/142 -- GLOBAL_STEP: 438250 + | > loss_disc: 2.481369972229004 (2.420749018066808) + | > loss_disc_real_0: 0.13129381835460663 (0.11532125935742729) + | > loss_disc_real_1: 0.25252795219421387 (0.20420711722813153) + | > loss_disc_real_2: 0.27194133400917053 (0.22596416622400284) + | > loss_disc_real_3: 0.254056453704834 (0.22379393483463086) + | > loss_disc_real_4: 0.21388424932956696 (0.22773020636094243) + | > loss_disc_real_5: 0.2721785008907318 (0.22893708160049037) + | > loss_0: 2.481369972229004 (2.420749018066808) + | > grad_norm_0: tensor(18.4065, device='cuda:0') (tensor(11.5138, device='cuda:0')) + | > loss_gen: 2.4131178855895996 (2.4080994944823417) + | > loss_kl: 1.8742777109146118 (1.8105936897428412) + | > loss_feat: 7.689718246459961 (7.350924680107518) + | > loss_mel: 17.9650821685791 (17.68212064943815) + | > loss_duration: 1.4436088800430298 (1.4210601699979681) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.385807037353516 (30.672798909639056) + | > grad_norm_1: tensor(318.0414, device='cuda:0') (tensor(311.8328, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.3517 (0.3402315817381206) + | > loader_time: 0.0057 (0.005160513677095113) + + + --> TIME: 2025-04-19 05:48:20 -- STEP: 63/142 -- GLOBAL_STEP: 438275 + | > loss_disc: 2.44464373588562 (2.4322669808826745) + | > loss_disc_real_0: 0.13661663234233856 (0.11815582901712447) + | > loss_disc_real_1: 0.21815291047096252 (0.2049060889652797) + | > loss_disc_real_2: 0.25129958987236023 (0.22420317218417213) + | > loss_disc_real_3: 0.23755522072315216 (0.2254915748323713) + | > loss_disc_real_4: 0.23303885757923126 (0.22730131541925763) + | > loss_disc_real_5: 0.2362208217382431 (0.22971980108155143) + | > loss_0: 2.44464373588562 (2.4322669808826745) + | > grad_norm_0: tensor(13.8587, device='cuda:0') (tensor(13.1602, device='cuda:0')) + | > loss_gen: 2.495551109313965 (2.3941316074795194) + | > loss_kl: 2.107335329055786 (1.829307995145283) + | > loss_feat: 5.954052925109863 (7.212762908330039) + | > loss_mel: 17.460983276367188 (17.650766781398225) + | > loss_duration: 1.4891200065612793 (1.4300897991846475) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.507041931152344 (30.517059053693497) + | > grad_norm_1: tensor(237.7150, device='cuda:0') (tensor(320.0150, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.3867 (0.3519065947759719) + | > loader_time: 0.0061 (0.005484089018806579) + + + --> TIME: 2025-04-19 05:48:30 -- STEP: 88/142 -- GLOBAL_STEP: 438300 + | > loss_disc: 2.441632032394409 (2.4500741308385674) + | > loss_disc_real_0: 0.10485434532165527 (0.12254989879544485) + | > loss_disc_real_1: 0.20651400089263916 (0.20631716518916868) + | > loss_disc_real_2: 0.24738559126853943 (0.22479096715423194) + | > loss_disc_real_3: 0.23931492865085602 (0.22747174972160297) + | > loss_disc_real_4: 0.2232893705368042 (0.22997119345448233) + | > loss_disc_real_5: 0.250376433134079 (0.23209599202329462) + | > loss_0: 2.441632032394409 (2.4500741308385674) + | > grad_norm_0: tensor(22.5052, device='cuda:0') (tensor(15.1933, device='cuda:0')) + | > loss_gen: 2.251086711883545 (2.3940390971573917) + | > loss_kl: 1.964297890663147 (1.8416073593226345) + | > loss_feat: 6.676280498504639 (7.17663422497836) + | > loss_mel: 17.134441375732422 (17.611146471717138) + | > loss_duration: 1.515749216079712 (1.4426585313948719) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.54185676574707 (30.466085607355293) + | > grad_norm_1: tensor(303.2413, device='cuda:0') (tensor(330.6632, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.395 (0.3632373728535392) + | > loader_time: 0.0073 (0.005764544010162354) + + + --> TIME: 2025-04-19 05:48:40 -- STEP: 113/142 -- GLOBAL_STEP: 438325 + | > loss_disc: 2.458646297454834 (2.453974839860359) + | > loss_disc_real_0: 0.18855872750282288 (0.12425423604724682) + | > loss_disc_real_1: 0.1810734122991562 (0.20690014589149341) + | > loss_disc_real_2: 0.18366888165473938 (0.22474900059467923) + | > loss_disc_real_3: 0.19070176780223846 (0.22574812582636303) + | > loss_disc_real_4: 0.22244927287101746 (0.22896135473145848) + | > loss_disc_real_5: 0.1844969242811203 (0.231063067253712) + | > loss_0: 2.458646297454834 (2.453974839860359) + | > grad_norm_0: tensor(41.5574, device='cuda:0') (tensor(16.4653, device='cuda:0')) + | > loss_gen: 2.433802366256714 (2.3922983713909587) + | > loss_kl: 1.7777074575424194 (1.8440058009814373) + | > loss_feat: 7.774765968322754 (7.162520028848563) + | > loss_mel: 19.517276763916016 (17.633600108391413) + | > loss_duration: 1.4786494970321655 (1.4505488397800816) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.982200622558594 (30.482973132513266) + | > grad_norm_1: tensor(359.6639, device='cuda:0') (tensor(332.0380, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.424 (0.3738123526615379) + | > loader_time: 0.0073 (0.006016872625435349) + + + --> TIME: 2025-04-19 05:48:51 -- STEP: 138/142 -- GLOBAL_STEP: 438350 + | > loss_disc: 2.4353177547454834 (2.4611728104992188) + | > loss_disc_real_0: 0.15801800787448883 (0.1260684734561305) + | > loss_disc_real_1: 0.22721153497695923 (0.20806837664998093) + | > loss_disc_real_2: 0.2090044766664505 (0.22597759195427963) + | > loss_disc_real_3: 0.2140914350748062 (0.22732947324065195) + | > loss_disc_real_4: 0.2327195107936859 (0.22920845859292627) + | > loss_disc_real_5: 0.19264206290245056 (0.2306600103991619) + | > loss_0: 2.4353177547454834 (2.4611728104992188) + | > grad_norm_0: tensor(3.7208, device='cuda:0') (tensor(16.3277, device='cuda:0')) + | > loss_gen: 2.1439104080200195 (2.376330211542655) + | > loss_kl: 1.881081223487854 (1.8551841939705005) + | > loss_feat: 5.858988285064697 (7.1017493717912314) + | > loss_mel: 17.16927146911621 (17.617872452390362) + | > loss_duration: 1.5231746435165405 (1.4574427751527315) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.57642364501953 (30.408578955608867) + | > grad_norm_1: tensor(96.0625, device='cuda:0') (tensor(329.2433, device='cuda:0')) + | > current_lr_0: 0.00018988889236593322 + | > current_lr_1: 0.00018988889236593322 + | > step_time: 0.4562 (0.38582603654999664) + | > loader_time: 0.0077 (0.0062781365021415386) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.399869918823242 (2.399869918823242) + | > loss_disc_real_0: 0.18863525986671448 (0.18863525986671448) + | > loss_disc_real_1: 0.2507455050945282 (0.2507455050945282) + | > loss_disc_real_2: 0.22255203127861023 (0.22255203127861023) + | > loss_disc_real_3: 0.20354245603084564 (0.20354245603084564) + | > loss_disc_real_4: 0.2017419934272766 (0.2017419934272766) + | > loss_disc_real_5: 0.20914436876773834 (0.20914436876773834) + | > loss_0: 2.399869918823242 (2.399869918823242) + | > loss_gen: 2.5012199878692627 (2.5012199878692627) + | > loss_kl: 2.496971845626831 (2.496971845626831) + | > loss_feat: 7.1422014236450195 (7.1422014236450195) + | > loss_mel: 17.706541061401367 (17.706541061401367) + | > loss_duration: 1.7351303100585938 (1.7351303100585938) + | > loss_1: 31.58206558227539 (31.58206558227539) + + --> STEP: 1 + | > loss_disc: 2.5732555389404297 (2.5732555389404297) + | > loss_disc_real_0: 0.21195700764656067 (0.21195700764656067) + | > loss_disc_real_1: 0.2629033923149109 (0.2629033923149109) + | > loss_disc_real_2: 0.24491290748119354 (0.24491290748119354) + | > loss_disc_real_3: 0.21746771037578583 (0.21746771037578583) + | > loss_disc_real_4: 0.2281152606010437 (0.2281152606010437) + | > loss_disc_real_5: 0.21393510699272156 (0.21393510699272156) + | > loss_0: 2.5732555389404297 (2.5732555389404297) + | > loss_gen: 2.3232951164245605 (2.3232951164245605) + | > loss_kl: 2.796588659286499 (2.796588659286499) + | > loss_feat: 6.348217964172363 (6.348217964172363) + | > loss_mel: 16.553707122802734 (16.553707122802734) + | > loss_duration: 1.787160873413086 (1.787160873413086) + | > loss_1: 29.808969497680664 (29.808969497680664) + + --> STEP: 2 + | > loss_disc: 2.2764596939086914 (2.4248576164245605) + | > loss_disc_real_0: 0.17146728932857513 (0.1917121484875679) + | > loss_disc_real_1: 0.23071981966495514 (0.24681160598993301) + | > loss_disc_real_2: 0.18455876410007477 (0.21473583579063416) + | > loss_disc_real_3: 0.16493147611618042 (0.19119959324598312) + | > loss_disc_real_4: 0.19168628752231598 (0.20990077406167984) + | > loss_disc_real_5: 0.1963050663471222 (0.20512008666992188) + | > loss_0: 2.2764596939086914 (2.4248576164245605) + | > loss_gen: 2.5910725593566895 (2.457183837890625) + | > loss_kl: 2.591740369796753 (2.694164514541626) + | > loss_feat: 9.434905052185059 (7.891561508178711) + | > loss_mel: 19.2265682220459 (17.890137672424316) + | > loss_duration: 1.71656334400177 (1.751862108707428) + | > loss_1: 35.56085205078125 (32.68491077423096) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034711360931396484 (+6.937980651855469e-05) + | > avg_loss_disc: 2.4248576164245605 (-0.17442405223846436) + | > avg_loss_disc_real_0: 0.1917121484875679 (-0.038555920124053955) + | > avg_loss_disc_real_1: 0.24681160598993301 (+0.019102148711681366) + | > avg_loss_disc_real_2: 0.21473583579063416 (+0.0026240646839141846) + | > avg_loss_disc_real_3: 0.19119959324598312 (-0.06050436943769455) + | > avg_loss_disc_real_4: 0.20990077406167984 (-0.04985158145427704) + | > avg_loss_disc_real_5: 0.20512008666992188 (-0.08005817979574203) + | > avg_loss_0: 2.4248576164245605 (-0.17442405223846436) + | > avg_loss_gen: 2.457183837890625 (-0.06833004951477051) + | > avg_loss_kl: 2.694164514541626 (+0.09582602977752686) + | > avg_loss_feat: 7.891561508178711 (+0.9377529621124268) + | > avg_loss_mel: 17.890137672424316 (+0.22861480712890625) + | > avg_loss_duration: 1.751862108707428 (+0.006319940090179443) + | > avg_loss_1: 32.68491077423096 (+1.2001848220825195) + + + > EPOCH: 416/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:48:55)  + + --> TIME: 2025-04-19 05:49:03 -- STEP: 21/142 -- GLOBAL_STEP: 438375 + | > loss_disc: 2.4022624492645264 (2.4589161532265797) + | > loss_disc_real_0: 0.08657383173704147 (0.11970995969715573) + | > loss_disc_real_1: 0.179879829287529 (0.2076916034732546) + | > loss_disc_real_2: 0.21391963958740234 (0.2263772033509754) + | > loss_disc_real_3: 0.21283923089504242 (0.23089913811002458) + | > loss_disc_real_4: 0.2019878774881363 (0.2290007933264687) + | > loss_disc_real_5: 0.2287452071905136 (0.23596032247656867) + | > loss_0: 2.4022624492645264 (2.4589161532265797) + | > grad_norm_0: tensor(9.1662, device='cuda:0') (tensor(9.3969, device='cuda:0')) + | > loss_gen: 2.4125354290008545 (2.382574297132946) + | > loss_kl: 1.7503775358200073 (1.7580701566877819) + | > loss_feat: 6.580985069274902 (7.36439391544887) + | > loss_mel: 17.600980758666992 (17.87463960193452) + | > loss_duration: 1.4286949634552002 (1.4081591424487887) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.77357292175293 (30.787836983090354) + | > grad_norm_1: tensor(75.0256, device='cuda:0') (tensor(210.5585, device='cuda:0')) + | > current_lr_0: 0.00018986515625438747 + | > current_lr_1: 0.00018986515625438747 + | > step_time: 0.3453 (0.33080401874723886) + | > loader_time: 0.0052 (0.004999274299258278) + + + --> TIME: 2025-04-19 05:49:12 -- STEP: 46/142 -- GLOBAL_STEP: 438400 + | > loss_disc: 2.3104324340820312 (2.457915430483611) + | > loss_disc_real_0: 0.11306222528219223 (0.11933707348678423) + | > loss_disc_real_1: 0.16786515712738037 (0.2080796370687692) + | > loss_disc_real_2: 0.21052208542823792 (0.22939267624979434) + | > loss_disc_real_3: 0.21894733607769012 (0.22874023020267487) + | > loss_disc_real_4: 0.19649900496006012 (0.22708306785510934) + | > loss_disc_real_5: 0.2166702002286911 (0.23636215018189471) + | > loss_0: 2.3104324340820312 (2.457915430483611) + | > grad_norm_0: tensor(6.7288, device='cuda:0') (tensor(8.1206, device='cuda:0')) + | > loss_gen: 2.539992570877075 (2.367984351904497) + | > loss_kl: 1.787610411643982 (1.7843930177066638) + | > loss_feat: 7.584444999694824 (7.255488126174264) + | > loss_mel: 17.804964065551758 (17.707239254661232) + | > loss_duration: 1.4281351566314697 (1.4196661710739136) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.1451473236084 (30.534770758255668) + | > grad_norm_1: tensor(305.0161, device='cuda:0') (tensor(192.5001, device='cuda:0')) + | > current_lr_0: 0.00018986515625438747 + | > current_lr_1: 0.00018986515625438747 + | > step_time: 0.3655 (0.3435221858646559) + | > loader_time: 0.006 (0.00530881985374119) + + + --> TIME: 2025-04-19 05:49:21 -- STEP: 71/142 -- GLOBAL_STEP: 438425 + | > loss_disc: 2.5043318271636963 (2.4498154143212547) + | > loss_disc_real_0: 0.12314052879810333 (0.120235537034525) + | > loss_disc_real_1: 0.2673482596874237 (0.20679515579216917) + | > loss_disc_real_2: 0.2422270029783249 (0.22760487652160752) + | > loss_disc_real_3: 0.2611328363418579 (0.22876191244158947) + | > loss_disc_real_4: 0.2714264690876007 (0.22657689431183775) + | > loss_disc_real_5: 0.2282446026802063 (0.23530482490297774) + | > loss_0: 2.5043318271636963 (2.4498154143212547) + | > grad_norm_0: tensor(11.2629, device='cuda:0') (tensor(8.1397, device='cuda:0')) + | > loss_gen: 2.4216248989105225 (2.376359456022022) + | > loss_kl: 1.9732683897018433 (1.805011829859774) + | > loss_feat: 6.659299373626709 (7.263720250465501) + | > loss_mel: 17.053823471069336 (17.74929133939072) + | > loss_duration: 1.4528582096099854 (1.4317388433805653) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.56087303161621 (30.62612154786016) + | > grad_norm_1: tensor(211.0706, device='cuda:0') (tensor(181.1741, device='cuda:0')) + | > current_lr_0: 0.00018986515625438747 + | > current_lr_1: 0.00018986515625438747 + | > step_time: 0.3867 (0.35578958081527495) + | > loader_time: 0.0062 (0.005613689691248074) + + + --> TIME: 2025-04-19 05:49:31 -- STEP: 96/142 -- GLOBAL_STEP: 438450 + | > loss_disc: 2.4940006732940674 (2.4534939428170524) + | > loss_disc_real_0: 0.12156901508569717 (0.12097716797143221) + | > loss_disc_real_1: 0.20420396327972412 (0.20666180178523064) + | > loss_disc_real_2: 0.19582240283489227 (0.22674103422711292) + | > loss_disc_real_3: 0.2366175800561905 (0.22896916853884855) + | > loss_disc_real_4: 0.22425714135169983 (0.2283840673044324) + | > loss_disc_real_5: 0.24396546185016632 (0.23420902217427889) + | > loss_0: 2.4940006732940674 (2.4534939428170524) + | > grad_norm_0: tensor(8.5927, device='cuda:0') (tensor(8.1713, device='cuda:0')) + | > loss_gen: 2.2229580879211426 (2.3650681028763465) + | > loss_kl: 1.9091510772705078 (1.8157431197663148) + | > loss_feat: 7.074383735656738 (7.255030329028766) + | > loss_mel: 17.491249084472656 (17.840542227029804) + | > loss_duration: 1.469714879989624 (1.44402069101731) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.167457580566406 (30.720404346783955) + | > grad_norm_1: tensor(328.5408, device='cuda:0') (tensor(181.5742, device='cuda:0')) + | > current_lr_0: 0.00018986515625438747 + | > current_lr_1: 0.00018986515625438747 + | > step_time: 0.4031 (0.36649688333272934) + | > loader_time: 0.0063 (0.005851181844870248) + + + --> TIME: 2025-04-19 05:49:42 -- STEP: 121/142 -- GLOBAL_STEP: 438475 + | > loss_disc: 2.515669822692871 (2.452534622397304) + | > loss_disc_real_0: 0.18796506524085999 (0.12273974094755394) + | > loss_disc_real_1: 0.2125025987625122 (0.2070619893714416) + | > loss_disc_real_2: 0.20245361328125 (0.22681387039748105) + | > loss_disc_real_3: 0.19143500924110413 (0.22785310397955996) + | > loss_disc_real_4: 0.2146390825510025 (0.2282866955049767) + | > loss_disc_real_5: 0.2359492927789688 (0.2326765928632957) + | > loss_0: 2.515669822692871 (2.452534622397304) + | > grad_norm_0: tensor(11.5500, device='cuda:0') (tensor(8.2384, device='cuda:0')) + | > loss_gen: 2.272472620010376 (2.3660298359295555) + | > loss_kl: 1.9804233312606812 (1.8270337709710618) + | > loss_feat: 6.9598541259765625 (7.228493781129191) + | > loss_mel: 18.56354331970215 (17.81290209588926) + | > loss_duration: 1.520664095878601 (1.4517081148368267) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.29695701599121 (30.68616748841341) + | > grad_norm_1: tensor(134.8346, device='cuda:0') (tensor(187.2405, device='cuda:0')) + | > current_lr_0: 0.00018986515625438747 + | > current_lr_1: 0.00018986515625438747 + | > step_time: 0.441 (0.3776442649935888) + | > loader_time: 0.0067 (0.006063445540499095) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5365989208221436 (2.5365989208221436) + | > loss_disc_real_0: 0.1772155612707138 (0.1772155612707138) + | > loss_disc_real_1: 0.2593754827976227 (0.2593754827976227) + | > loss_disc_real_2: 0.2553667426109314 (0.2553667426109314) + | > loss_disc_real_3: 0.23542428016662598 (0.23542428016662598) + | > loss_disc_real_4: 0.26759782433509827 (0.26759782433509827) + | > loss_disc_real_5: 0.24241670966148376 (0.24241670966148376) + | > loss_0: 2.5365989208221436 (2.5365989208221436) + | > loss_gen: 2.476715326309204 (2.476715326309204) + | > loss_kl: 2.481870174407959 (2.481870174407959) + | > loss_feat: 6.666567802429199 (6.666567802429199) + | > loss_mel: 16.931758880615234 (16.931758880615234) + | > loss_duration: 1.7583290338516235 (1.7583290338516235) + | > loss_1: 30.31524085998535 (30.31524085998535) + + --> STEP: 1 + | > loss_disc: 2.4809818267822266 (2.4809818267822266) + | > loss_disc_real_0: 0.1593848019838333 (0.1593848019838333) + | > loss_disc_real_1: 0.2508542835712433 (0.2508542835712433) + | > loss_disc_real_2: 0.24085544049739838 (0.24085544049739838) + | > loss_disc_real_3: 0.24369476735591888 (0.24369476735591888) + | > loss_disc_real_4: 0.24321390688419342 (0.24321390688419342) + | > loss_disc_real_5: 0.20111669600009918 (0.20111669600009918) + | > loss_0: 2.4809818267822266 (2.4809818267822266) + | > loss_gen: 2.3492188453674316 (2.3492188453674316) + | > loss_kl: 2.79248309135437 (2.79248309135437) + | > loss_feat: 7.042054653167725 (7.042054653167725) + | > loss_mel: 17.080917358398438 (17.080917358398438) + | > loss_duration: 1.8055349588394165 (1.8055349588394165) + | > loss_1: 31.070207595825195 (31.070207595825195) + + --> STEP: 2 + | > loss_disc: 2.5298473834991455 (2.505414605140686) + | > loss_disc_real_0: 0.15273231267929077 (0.15605855733156204) + | > loss_disc_real_1: 0.2836894392967224 (0.26727186143398285) + | > loss_disc_real_2: 0.23569242656230927 (0.23827393352985382) + | > loss_disc_real_3: 0.2226475030183792 (0.23317113518714905) + | > loss_disc_real_4: 0.2544008791446686 (0.248807393014431) + | > loss_disc_real_5: 0.25079748034477234 (0.22595708817243576) + | > loss_0: 2.5298473834991455 (2.505414605140686) + | > loss_gen: 2.435894250869751 (2.3925565481185913) + | > loss_kl: 2.3082525730133057 (2.550367832183838) + | > loss_feat: 7.3350701332092285 (7.188562393188477) + | > loss_mel: 18.531997680664062 (17.80645751953125) + | > loss_duration: 1.741542100906372 (1.7735385298728943) + | > loss_1: 32.35275650024414 (31.711482048034668) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033783912658691406 (-9.274482727050781e-05) + | > avg_loss_disc: 2.505414605140686 (+0.08055698871612549) + | > avg_loss_disc_real_0: 0.15605855733156204 (-0.03565359115600586) + | > avg_loss_disc_real_1: 0.26727186143398285 (+0.020460255444049835) + | > avg_loss_disc_real_2: 0.23827393352985382 (+0.023538097739219666) + | > avg_loss_disc_real_3: 0.23317113518714905 (+0.041971541941165924) + | > avg_loss_disc_real_4: 0.248807393014431 (+0.03890661895275116) + | > avg_loss_disc_real_5: 0.22595708817243576 (+0.020837001502513885) + | > avg_loss_0: 2.505414605140686 (+0.08055698871612549) + | > avg_loss_gen: 2.3925565481185913 (-0.06462728977203369) + | > avg_loss_kl: 2.550367832183838 (-0.14379668235778809) + | > avg_loss_feat: 7.188562393188477 (-0.7029991149902344) + | > avg_loss_mel: 17.80645751953125 (-0.0836801528930664) + | > avg_loss_duration: 1.7735385298728943 (+0.02167642116546631) + | > avg_loss_1: 31.711482048034668 (-0.9734287261962891) + + + > EPOCH: 417/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:49:53)  + + --> TIME: 2025-04-19 05:49:55 -- STEP: 4/142 -- GLOBAL_STEP: 438500 + | > loss_disc: 2.428618907928467 (2.3624756932258606) + | > loss_disc_real_0: 0.10234451293945312 (0.10941250622272491) + | > loss_disc_real_1: 0.19504126906394958 (0.1860385648906231) + | > loss_disc_real_2: 0.1970822662115097 (0.23137003555893898) + | > loss_disc_real_3: 0.23188559710979462 (0.22918668016791344) + | > loss_disc_real_4: 0.20658746361732483 (0.22949937731027603) + | > loss_disc_real_5: 0.2598343789577484 (0.24736788123846054) + | > loss_0: 2.428618907928467 (2.3624756932258606) + | > grad_norm_0: tensor(5.0977, device='cuda:0') (tensor(8.0475, device='cuda:0')) + | > loss_gen: 2.384483575820923 (2.4293148517608643) + | > loss_kl: 1.8544673919677734 (1.8181728720664978) + | > loss_feat: 6.477437973022461 (7.3272703886032104) + | > loss_mel: 15.223864555358887 (16.86428141593933) + | > loss_duration: 1.405792474746704 (1.4134680330753326) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.346046447753906 (29.852508068084717) + | > grad_norm_1: tensor(184.7072, device='cuda:0') (tensor(258.7215, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.3206 (0.32137060165405273) + | > loader_time: 0.0054 (0.004488706588745117) + + + --> TIME: 2025-04-19 05:50:04 -- STEP: 29/142 -- GLOBAL_STEP: 438525 + | > loss_disc: 2.4421379566192627 (2.4028767635082366) + | > loss_disc_real_0: 0.1057649478316307 (0.11589514027381766) + | > loss_disc_real_1: 0.2023208737373352 (0.20239972811320733) + | > loss_disc_real_2: 0.24043430387973785 (0.2249881188417303) + | > loss_disc_real_3: 0.22417187690734863 (0.222889496848501) + | > loss_disc_real_4: 0.22051824629306793 (0.2235683084561907) + | > loss_disc_real_5: 0.2583995461463928 (0.22900214739914598) + | > loss_0: 2.4421379566192627 (2.4028767635082366) + | > grad_norm_0: tensor(7.2308, device='cuda:0') (tensor(7.7653, device='cuda:0')) + | > loss_gen: 2.395674467086792 (2.413040169354143) + | > loss_kl: 1.8488216400146484 (1.7628165072408215) + | > loss_feat: 7.661766052246094 (7.300407656307878) + | > loss_mel: 18.275493621826172 (17.491613552488133) + | > loss_duration: 1.4218294620513916 (1.417407409898166) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.60358428955078 (30.385285278846478) + | > grad_norm_1: tensor(121.4141, device='cuda:0') (tensor(195.2520, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.3562 (0.3409083876116522) + | > loader_time: 0.0054 (0.005163801127466662) + + + --> TIME: 2025-04-19 05:50:13 -- STEP: 54/142 -- GLOBAL_STEP: 438550 + | > loss_disc: 2.4270381927490234 (2.4342107552069203) + | > loss_disc_real_0: 0.1354731172323227 (0.12093787902483234) + | > loss_disc_real_1: 0.2156493067741394 (0.20639318227767944) + | > loss_disc_real_2: 0.2778213322162628 (0.22597120315940292) + | > loss_disc_real_3: 0.2392217367887497 (0.22455238937227814) + | > loss_disc_real_4: 0.2511633336544037 (0.22619855845416034) + | > loss_disc_real_5: 0.24710127711296082 (0.23014505952596664) + | > loss_0: 2.4270381927490234 (2.4342107552069203) + | > grad_norm_0: tensor(7.9464, device='cuda:0') (tensor(8.1193, device='cuda:0')) + | > loss_gen: 2.4303483963012695 (2.3824175728691936) + | > loss_kl: 1.8974311351776123 (1.7932998074425592) + | > loss_feat: 7.294031620025635 (7.281119187672933) + | > loss_mel: 17.611324310302734 (17.65259751567135) + | > loss_duration: 1.429430365562439 (1.4278542730543349) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.662565231323242 (30.53728855980767) + | > grad_norm_1: tensor(114.1288, device='cuda:0') (tensor(199.8652, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.3795 (0.3544867303636338) + | > loader_time: 0.0067 (0.00552486048804389) + + + --> TIME: 2025-04-19 05:50:24 -- STEP: 79/142 -- GLOBAL_STEP: 438575 + | > loss_disc: 2.5285847187042236 (2.437940168984328) + | > loss_disc_real_0: 0.24550345540046692 (0.12310172570279881) + | > loss_disc_real_1: 0.21856726706027985 (0.2062970741262919) + | > loss_disc_real_2: 0.27126872539520264 (0.22461682350575168) + | > loss_disc_real_3: 0.26062753796577454 (0.22370054054109356) + | > loss_disc_real_4: 0.22368568181991577 (0.22556690890577774) + | > loss_disc_real_5: 0.2513163685798645 (0.2299177410859096) + | > loss_0: 2.5285847187042236 (2.437940168984328) + | > grad_norm_0: tensor(12.5676, device='cuda:0') (tensor(7.6728, device='cuda:0')) + | > loss_gen: 2.413479804992676 (2.3818702275239967) + | > loss_kl: 1.840190052986145 (1.8081206475632101) + | > loss_feat: 5.950578689575195 (7.244541258751592) + | > loss_mel: 18.202096939086914 (17.746773261058184) + | > loss_duration: 1.5167033672332764 (1.439439369153373) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.92304801940918 (30.620744825918464) + | > grad_norm_1: tensor(53.3661, device='cuda:0') (tensor(178.4669, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.3973 (0.3677233231218555) + | > loader_time: 0.0068 (0.0058582945715022995) + + + --> TIME: 2025-04-19 05:50:34 -- STEP: 104/142 -- GLOBAL_STEP: 438600 + | > loss_disc: 2.3961541652679443 (2.455851346254348) + | > loss_disc_real_0: 0.148866206407547 (0.1262594235058014) + | > loss_disc_real_1: 0.20250579714775085 (0.20806504370501408) + | > loss_disc_real_2: 0.20994910597801208 (0.22606008155987814) + | > loss_disc_real_3: 0.2254614531993866 (0.22576967899042827) + | > loss_disc_real_4: 0.24016477167606354 (0.22732175385149625) + | > loss_disc_real_5: 0.20449624955654144 (0.23073883555256403) + | > loss_0: 2.3961541652679443 (2.455851346254348) + | > grad_norm_0: tensor(7.7869, device='cuda:0') (tensor(8.1037, device='cuda:0')) + | > loss_gen: 2.257448673248291 (2.355466317671995) + | > loss_kl: 1.9988789558410645 (1.821308923455385) + | > loss_feat: 6.433427333831787 (7.140678625840407) + | > loss_mel: 17.017086029052734 (17.702649593353282) + | > loss_duration: 1.5035324096679688 (1.4489301190926476) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.210372924804688 (30.46903366308946) + | > grad_norm_1: tensor(178.5657, device='cuda:0') (tensor(166.1100, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.4217 (0.380112703029926) + | > loader_time: 0.0071 (0.006123109505726741) + + + --> TIME: 2025-04-19 05:50:46 -- STEP: 129/142 -- GLOBAL_STEP: 438625 + | > loss_disc: 2.3800792694091797 (2.4532288673312155) + | > loss_disc_real_0: 0.08865866810083389 (0.12613463089909666) + | > loss_disc_real_1: 0.16605572402477264 (0.2077179513005323) + | > loss_disc_real_2: 0.1742701530456543 (0.22568972408771515) + | > loss_disc_real_3: 0.1890771985054016 (0.2257102689770765) + | > loss_disc_real_4: 0.22024983167648315 (0.22649476960185885) + | > loss_disc_real_5: 0.17992638051509857 (0.22989521135193433) + | > loss_0: 2.3800792694091797 (2.4532288673312155) + | > grad_norm_0: tensor(6.0388, device='cuda:0') (tensor(8.2341, device='cuda:0')) + | > loss_gen: 2.2939682006835938 (2.3548924479373667) + | > loss_kl: 1.8697351217269897 (1.830012181932612) + | > loss_feat: 8.0673828125 (7.131492758906165) + | > loss_mel: 18.40186882019043 (17.649728641953587) + | > loss_duration: 1.478790521621704 (1.455814013185427) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.11174392700195 (30.421940115995184) + | > grad_norm_1: tensor(61.8976, device='cuda:0') (tensor(175.6835, device='cuda:0')) + | > current_lr_0: 0.00018984142310985566 + | > current_lr_1: 0.00018984142310985566 + | > step_time: 0.4441 (0.39280628418737606) + | > loader_time: 0.0078 (0.00636373194613198) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5741963386535645 (2.5741963386535645) + | > loss_disc_real_0: 0.20374904572963715 (0.20374904572963715) + | > loss_disc_real_1: 0.19469019770622253 (0.19469019770622253) + | > loss_disc_real_2: 0.24982976913452148 (0.24982976913452148) + | > loss_disc_real_3: 0.2871316969394684 (0.2871316969394684) + | > loss_disc_real_4: 0.23958124220371246 (0.23958124220371246) + | > loss_disc_real_5: 0.2772180736064911 (0.2772180736064911) + | > loss_0: 2.5741963386535645 (2.5741963386535645) + | > loss_gen: 2.353044033050537 (2.353044033050537) + | > loss_kl: 2.5156354904174805 (2.5156354904174805) + | > loss_feat: 6.218286037445068 (6.218286037445068) + | > loss_mel: 18.02367401123047 (18.02367401123047) + | > loss_duration: 1.756094217300415 (1.756094217300415) + | > loss_1: 30.86673355102539 (30.86673355102539) + + --> STEP: 1 + | > loss_disc: 2.597104549407959 (2.597104549407959) + | > loss_disc_real_0: 0.1696215122938156 (0.1696215122938156) + | > loss_disc_real_1: 0.22501546144485474 (0.22501546144485474) + | > loss_disc_real_2: 0.26337558031082153 (0.26337558031082153) + | > loss_disc_real_3: 0.2638923227787018 (0.2638923227787018) + | > loss_disc_real_4: 0.2538345158100128 (0.2538345158100128) + | > loss_disc_real_5: 0.293925404548645 (0.293925404548645) + | > loss_0: 2.597104549407959 (2.597104549407959) + | > loss_gen: 2.360835552215576 (2.360835552215576) + | > loss_kl: 2.7867653369903564 (2.7867653369903564) + | > loss_feat: 7.359810829162598 (7.359810829162598) + | > loss_mel: 18.857181549072266 (18.857181549072266) + | > loss_duration: 1.7895405292510986 (1.7895405292510986) + | > loss_1: 33.15413284301758 (33.15413284301758) + + --> STEP: 2 + | > loss_disc: 2.2795944213867188 (2.438349485397339) + | > loss_disc_real_0: 0.11479363590478897 (0.1422075740993023) + | > loss_disc_real_1: 0.22553201019763947 (0.2252737358212471) + | > loss_disc_real_2: 0.22919641435146332 (0.24628599733114243) + | > loss_disc_real_3: 0.25022459030151367 (0.2570584565401077) + | > loss_disc_real_4: 0.24128735065460205 (0.24756093323230743) + | > loss_disc_real_5: 0.18802447617053986 (0.24097494035959244) + | > loss_0: 2.2795944213867188 (2.438349485397339) + | > loss_gen: 2.5877888202667236 (2.47431218624115) + | > loss_kl: 2.286346435546875 (2.5365558862686157) + | > loss_feat: 8.384230613708496 (7.872020721435547) + | > loss_mel: 18.125438690185547 (18.491310119628906) + | > loss_duration: 1.7313754558563232 (1.760457992553711) + | > loss_1: 33.11518096923828 (33.13465690612793) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035545825958251953 (+0.0001761913299560547) + | > avg_loss_disc: 2.438349485397339 (-0.06706511974334717) + | > avg_loss_disc_real_0: 0.1422075740993023 (-0.01385098323225975) + | > avg_loss_disc_real_1: 0.2252737358212471 (-0.04199812561273575) + | > avg_loss_disc_real_2: 0.24628599733114243 (+0.008012063801288605) + | > avg_loss_disc_real_3: 0.2570584565401077 (+0.02388732135295868) + | > avg_loss_disc_real_4: 0.24756093323230743 (-0.0012464597821235657) + | > avg_loss_disc_real_5: 0.24097494035959244 (+0.015017852187156677) + | > avg_loss_0: 2.438349485397339 (-0.06706511974334717) + | > avg_loss_gen: 2.47431218624115 (+0.0817556381225586) + | > avg_loss_kl: 2.5365558862686157 (-0.013811945915222168) + | > avg_loss_feat: 7.872020721435547 (+0.6834583282470703) + | > avg_loss_mel: 18.491310119628906 (+0.6848526000976562) + | > avg_loss_duration: 1.760457992553711 (-0.01308053731918335) + | > avg_loss_1: 33.13465690612793 (+1.4231748580932617) + + + > EPOCH: 418/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:50:53)  + + --> TIME: 2025-04-19 05:50:58 -- STEP: 12/142 -- GLOBAL_STEP: 438650 + | > loss_disc: 2.483154058456421 (2.4616417487462363) + | > loss_disc_real_0: 0.1231931671500206 (0.12426826792458694) + | > loss_disc_real_1: 0.224326953291893 (0.2069252716998259) + | > loss_disc_real_2: 0.25194865465164185 (0.23239890982707342) + | > loss_disc_real_3: 0.22862760722637177 (0.23236048594117165) + | > loss_disc_real_4: 0.23450061678886414 (0.22622314964731535) + | > loss_disc_real_5: 0.24539512395858765 (0.22650206834077835) + | > loss_0: 2.483154058456421 (2.4616417487462363) + | > grad_norm_0: tensor(6.7651, device='cuda:0') (tensor(6.1853, device='cuda:0')) + | > loss_gen: 2.227654457092285 (2.3350110252698264) + | > loss_kl: 1.7294492721557617 (1.7400508324305217) + | > loss_feat: 6.6127190589904785 (7.2249674797058105) + | > loss_mel: 17.317819595336914 (17.587887128194172) + | > loss_duration: 1.4153037071228027 (1.4081219832102458) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.302946090698242 (30.29603846867879) + | > grad_norm_1: tensor(88.8835, device='cuda:0') (tensor(111.0292, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.3306 (0.32720353205998737) + | > loader_time: 0.0049 (0.004561086495717366) + + + --> TIME: 2025-04-19 05:51:07 -- STEP: 37/142 -- GLOBAL_STEP: 438675 + | > loss_disc: 2.4362075328826904 (2.4590869465389766) + | > loss_disc_real_0: 0.0771419033408165 (0.12260763125645148) + | > loss_disc_real_1: 0.22744566202163696 (0.20671365994053917) + | > loss_disc_real_2: 0.2042357623577118 (0.2297989270977072) + | > loss_disc_real_3: 0.21759027242660522 (0.2293869357656788) + | > loss_disc_real_4: 0.2132723033428192 (0.23426501533469637) + | > loss_disc_real_5: 0.24505005776882172 (0.23044583724962697) + | > loss_0: 2.4362075328826904 (2.4590869465389766) + | > grad_norm_0: tensor(5.2856, device='cuda:0') (tensor(6.5850, device='cuda:0')) + | > loss_gen: 2.3612470626831055 (2.359223359339946) + | > loss_kl: 1.8161656856536865 (1.7829915285110474) + | > loss_feat: 7.1844587326049805 (7.212879515982963) + | > loss_mel: 18.214242935180664 (17.522012066196755) + | > loss_duration: 1.43815016746521 (1.4197480227496173) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.014265060424805 (30.296854586214632) + | > grad_norm_1: tensor(73.9112, device='cuda:0') (tensor(167.6072, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.3525 (0.33925112518104344) + | > loader_time: 0.005 (0.004996712143356737) + + + --> TIME: 2025-04-19 05:51:16 -- STEP: 62/142 -- GLOBAL_STEP: 438700 + | > loss_disc: 2.3404650688171387 (2.4538853399215217) + | > loss_disc_real_0: 0.11112181097269058 (0.12442365529075745) + | > loss_disc_real_1: 0.21283088624477386 (0.20602792189005883) + | > loss_disc_real_2: 0.2154167890548706 (0.22541204431364614) + | > loss_disc_real_3: 0.19430014491081238 (0.22689097807292016) + | > loss_disc_real_4: 0.18821343779563904 (0.23115182019049121) + | > loss_disc_real_5: 0.22239553928375244 (0.22857722808276454) + | > loss_0: 2.3404650688171387 (2.4538853399215217) + | > grad_norm_0: tensor(12.0235, device='cuda:0') (tensor(7.4101, device='cuda:0')) + | > loss_gen: 2.4475646018981934 (2.355268032320083) + | > loss_kl: 1.8660826683044434 (1.801183106437806) + | > loss_feat: 7.614838600158691 (7.1996584553872385) + | > loss_mel: 17.682104110717773 (17.481154349542443) + | > loss_duration: 1.4893169403076172 (1.4303394729091274) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.09990692138672 (30.267603320460164) + | > grad_norm_1: tensor(143.1671, device='cuda:0') (tensor(163.8015, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.3841 (0.35121128251475675) + | > loader_time: 0.0062 (0.005347928693217616) + + + --> TIME: 2025-04-19 05:51:26 -- STEP: 87/142 -- GLOBAL_STEP: 438725 + | > loss_disc: 2.410562515258789 (2.4559808544728954) + | > loss_disc_real_0: 0.12324743717908859 (0.12491570704284757) + | > loss_disc_real_1: 0.20212385058403015 (0.2057892796636998) + | > loss_disc_real_2: 0.24622602760791779 (0.22753612902657738) + | > loss_disc_real_3: 0.2217559814453125 (0.22787743224494758) + | > loss_disc_real_4: 0.20006345212459564 (0.23024694690074043) + | > loss_disc_real_5: 0.2393309473991394 (0.230532747233051) + | > loss_0: 2.410562515258789 (2.4559808544728954) + | > grad_norm_0: tensor(11.5540, device='cuda:0') (tensor(7.6995, device='cuda:0')) + | > loss_gen: 2.592029571533203 (2.3561003180755957) + | > loss_kl: 1.8541150093078613 (1.8220417403626716) + | > loss_feat: 7.699782371520996 (7.176615588966457) + | > loss_mel: 19.30266761779785 (17.525757910191327) + | > loss_duration: 1.4867476224899292 (1.4420526411341528) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.935340881347656 (30.322568191879096) + | > grad_norm_1: tensor(341.9831, device='cuda:0') (tensor(170.5189, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.4012 (0.36274226506551116) + | > loader_time: 0.0063 (0.005594291906247194) + + + --> TIME: 2025-04-19 05:51:36 -- STEP: 112/142 -- GLOBAL_STEP: 438750 + | > loss_disc: 2.4976613521575928 (2.4612178291593296) + | > loss_disc_real_0: 0.1176021620631218 (0.1254971218960626) + | > loss_disc_real_1: 0.2405022829771042 (0.20698258826243027) + | > loss_disc_real_2: 0.2339906096458435 (0.2273477522124137) + | > loss_disc_real_3: 0.2611285150051117 (0.22839622784938132) + | > loss_disc_real_4: 0.2703115940093994 (0.23128822539001703) + | > loss_disc_real_5: 0.22941601276397705 (0.23082685457276447) + | > loss_0: 2.4976613521575928 (2.4612178291593296) + | > grad_norm_0: tensor(8.8266, device='cuda:0') (tensor(8.3696, device='cuda:0')) + | > loss_gen: 2.303873062133789 (2.354927028928484) + | > loss_kl: 1.8768209218978882 (1.8278934157320432) + | > loss_feat: 6.671839714050293 (7.137835634606225) + | > loss_mel: 16.509864807128906 (17.587042450904857) + | > loss_duration: 1.46865975856781 (1.4505030949200903) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.831058502197266 (30.35820162296295) + | > grad_norm_1: tensor(338.2082, device='cuda:0') (tensor(181.9751, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.4258 (0.37354110394205364) + | > loader_time: 0.0071 (0.005826002785137722) + + + --> TIME: 2025-04-19 05:51:48 -- STEP: 137/142 -- GLOBAL_STEP: 438775 + | > loss_disc: 2.5116124153137207 (2.4672694797933548) + | > loss_disc_real_0: 0.15611159801483154 (0.12628254272641926) + | > loss_disc_real_1: 0.22900761663913727 (0.20832527912881252) + | > loss_disc_real_2: 0.24024690687656403 (0.22685702123346121) + | > loss_disc_real_3: 0.24413025379180908 (0.22905118719939768) + | > loss_disc_real_4: 0.23206475377082825 (0.23143970585652512) + | > loss_disc_real_5: 0.24606367945671082 (0.23044367591394996) + | > loss_0: 2.5116124153137207 (2.4672694797933548) + | > grad_norm_0: tensor(5.3888, device='cuda:0') (tensor(9.2625, device='cuda:0')) + | > loss_gen: 2.356879949569702 (2.3552313947329555) + | > loss_kl: 1.7563965320587158 (1.8311222295691496) + | > loss_feat: 6.930974006652832 (7.1260229479657475) + | > loss_mel: 17.442930221557617 (17.548521640526992) + | > loss_duration: 1.463071584701538 (1.456948018422092) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.950252532958984 (30.317846200762006) + | > grad_norm_1: tensor(250.6976, device='cuda:0') (tensor(205.1659, device='cuda:0')) + | > current_lr_0: 0.00018981769293196693 + | > current_lr_1: 0.00018981769293196693 + | > step_time: 0.4546 (0.3859497926531047) + | > loader_time: 0.0074 (0.006091363238592218) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6723380088806152 (2.6723380088806152) + | > loss_disc_real_0: 0.22404980659484863 (0.22404980659484863) + | > loss_disc_real_1: 0.28830066323280334 (0.28830066323280334) + | > loss_disc_real_2: 0.2507682740688324 (0.2507682740688324) + | > loss_disc_real_3: 0.2697521448135376 (0.2697521448135376) + | > loss_disc_real_4: 0.24237418174743652 (0.24237418174743652) + | > loss_disc_real_5: 0.2577573359012604 (0.2577573359012604) + | > loss_0: 2.6723380088806152 (2.6723380088806152) + | > loss_gen: 2.453198194503784 (2.453198194503784) + | > loss_kl: 2.491997241973877 (2.491997241973877) + | > loss_feat: 6.103156566619873 (6.103156566619873) + | > loss_mel: 17.264114379882812 (17.264114379882812) + | > loss_duration: 1.735704779624939 (1.735704779624939) + | > loss_1: 30.04817008972168 (30.04817008972168) + + --> STEP: 1 + | > loss_disc: 2.6262731552124023 (2.6262731552124023) + | > loss_disc_real_0: 0.23229356110095978 (0.23229356110095978) + | > loss_disc_real_1: 0.2934625744819641 (0.2934625744819641) + | > loss_disc_real_2: 0.22785568237304688 (0.22785568237304688) + | > loss_disc_real_3: 0.25121599435806274 (0.25121599435806274) + | > loss_disc_real_4: 0.21321479976177216 (0.21321479976177216) + | > loss_disc_real_5: 0.25819534063339233 (0.25819534063339233) + | > loss_0: 2.6262731552124023 (2.6262731552124023) + | > loss_gen: 2.349937915802002 (2.349937915802002) + | > loss_kl: 2.667652130126953 (2.667652130126953) + | > loss_feat: 6.139829635620117 (6.139829635620117) + | > loss_mel: 17.029909133911133 (17.029909133911133) + | > loss_duration: 1.821865200996399 (1.821865200996399) + | > loss_1: 30.009193420410156 (30.009193420410156) + + --> STEP: 2 + | > loss_disc: 2.5797946453094482 (2.6030339002609253) + | > loss_disc_real_0: 0.17481116950511932 (0.20355236530303955) + | > loss_disc_real_1: 0.29567578434944153 (0.2945691794157028) + | > loss_disc_real_2: 0.2325512021780014 (0.23020344227552414) + | > loss_disc_real_3: 0.25403544306755066 (0.2526257187128067) + | > loss_disc_real_4: 0.2880633473396301 (0.25063907355070114) + | > loss_disc_real_5: 0.26910293102264404 (0.2636491358280182) + | > loss_0: 2.5797946453094482 (2.6030339002609253) + | > loss_gen: 2.457637310028076 (2.403787612915039) + | > loss_kl: 2.0736470222473145 (2.370649576187134) + | > loss_feat: 7.158615589141846 (6.6492226123809814) + | > loss_mel: 17.903766632080078 (17.466837882995605) + | > loss_duration: 1.7539620399475098 (1.7879136204719543) + | > loss_1: 31.347627639770508 (30.678410530090332) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032699108123779297 (-0.0002846717834472656) + | > avg_loss_disc: 2.6030339002609253 (+0.16468441486358643) + | > avg_loss_disc_real_0: 0.20355236530303955 (+0.06134479120373726) + | > avg_loss_disc_real_1: 0.2945691794157028 (+0.06929544359445572) + | > avg_loss_disc_real_2: 0.23020344227552414 (-0.016082555055618286) + | > avg_loss_disc_real_3: 0.2526257187128067 (-0.004432737827301025) + | > avg_loss_disc_real_4: 0.25063907355070114 (+0.0030781403183937073) + | > avg_loss_disc_real_5: 0.2636491358280182 (+0.02267419546842575) + | > avg_loss_0: 2.6030339002609253 (+0.16468441486358643) + | > avg_loss_gen: 2.403787612915039 (-0.07052457332611084) + | > avg_loss_kl: 2.370649576187134 (-0.16590631008148193) + | > avg_loss_feat: 6.6492226123809814 (-1.2227981090545654) + | > avg_loss_mel: 17.466837882995605 (-1.0244722366333008) + | > avg_loss_duration: 1.7879136204719543 (+0.027455627918243408) + | > avg_loss_1: 30.678410530090332 (-2.4562463760375977) + + + > EPOCH: 419/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:51:51)  + + --> TIME: 2025-04-19 05:51:59 -- STEP: 20/142 -- GLOBAL_STEP: 438800 + | > loss_disc: 2.5064492225646973 (2.4130648970603943) + | > loss_disc_real_0: 0.14005134999752045 (0.1130710881203413) + | > loss_disc_real_1: 0.2246403992176056 (0.20521073788404465) + | > loss_disc_real_2: 0.2170010507106781 (0.22594215124845504) + | > loss_disc_real_3: 0.22747544944286346 (0.22328636646270753) + | > loss_disc_real_4: 0.18812885880470276 (0.22013043016195297) + | > loss_disc_real_5: 0.259508341550827 (0.2234452873468399) + | > loss_0: 2.5064492225646973 (2.4130648970603943) + | > grad_norm_0: tensor(6.2660, device='cuda:0') (tensor(9.8963, device='cuda:0')) + | > loss_gen: 2.4550869464874268 (2.3897565007209782) + | > loss_kl: 1.8170433044433594 (1.7820729553699493) + | > loss_feat: 6.848550319671631 (7.44227294921875) + | > loss_mel: 18.126798629760742 (17.979053592681886) + | > loss_duration: 1.4725092649459839 (1.4142339527606966) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.719989776611328 (31.007390213012695) + | > grad_norm_1: tensor(69.2314, device='cuda:0') (tensor(242.5411, device='cuda:0')) + | > current_lr_0: 0.00018979396572035043 + | > current_lr_1: 0.00018979396572035043 + | > step_time: 0.342 (0.32991005182266236) + | > loader_time: 0.0052 (0.004648816585540771) + + + --> TIME: 2025-04-19 05:52:08 -- STEP: 45/142 -- GLOBAL_STEP: 438825 + | > loss_disc: 2.5717763900756836 (2.4287800735897482) + | > loss_disc_real_0: 0.10089629143476486 (0.11773699339893129) + | > loss_disc_real_1: 0.20606277883052826 (0.20433559682634142) + | > loss_disc_real_2: 0.2241741418838501 (0.2254586163494322) + | > loss_disc_real_3: 0.21645426750183105 (0.2233470125330819) + | > loss_disc_real_4: 0.232465922832489 (0.2237415572007497) + | > loss_disc_real_5: 0.23698537051677704 (0.23046045502026877) + | > loss_0: 2.5717763900756836 (2.4287800735897482) + | > grad_norm_0: tensor(8.8479, device='cuda:0') (tensor(7.9538, device='cuda:0')) + | > loss_gen: 2.2373504638671875 (2.3737409114837646) + | > loss_kl: 1.749211311340332 (1.7930573489930894) + | > loss_feat: 6.581417560577393 (7.343848705291748) + | > loss_mel: 16.602191925048828 (17.709367752075195) + | > loss_duration: 1.4386169910430908 (1.4262218024995592) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.608789443969727 (30.64623654683431) + | > grad_norm_1: tensor(189.9550, device='cuda:0') (tensor(185.1978, device='cuda:0')) + | > current_lr_0: 0.00018979396572035043 + | > current_lr_1: 0.00018979396572035043 + | > step_time: 0.3791 (0.3493066628774007) + | > loader_time: 0.0062 (0.005222002665201822) + + + --> TIME: 2025-04-19 05:52:18 -- STEP: 70/142 -- GLOBAL_STEP: 438850 + | > loss_disc: 2.4309229850769043 (2.4321165255137847) + | > loss_disc_real_0: 0.12377510219812393 (0.12067224266273635) + | > loss_disc_real_1: 0.17066358029842377 (0.203632952272892) + | > loss_disc_real_2: 0.21420788764953613 (0.2253708916051047) + | > loss_disc_real_3: 0.23206229507923126 (0.22633743307420184) + | > loss_disc_real_4: 0.2473972886800766 (0.2278093431677137) + | > loss_disc_real_5: 0.28678372502326965 (0.23016011459486826) + | > loss_0: 2.4309229850769043 (2.4321165255137847) + | > grad_norm_0: tensor(8.9046, device='cuda:0') (tensor(7.9641, device='cuda:0')) + | > loss_gen: 2.384993553161621 (2.3782640899930687) + | > loss_kl: 1.850820779800415 (1.8141331434249879) + | > loss_feat: 7.308199882507324 (7.325355536597115) + | > loss_mel: 17.756654739379883 (17.732064301627027) + | > loss_duration: 1.4715533256530762 (1.4361761042049952) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.77222442626953 (30.68599327632359) + | > grad_norm_1: tensor(231.4912, device='cuda:0') (tensor(188.1803, device='cuda:0')) + | > current_lr_0: 0.00018979396572035043 + | > current_lr_1: 0.00018979396572035043 + | > step_time: 0.3888 (0.36509737627846856) + | > loader_time: 0.0061 (0.005594594138009208) + + + --> TIME: 2025-04-19 05:52:28 -- STEP: 95/142 -- GLOBAL_STEP: 438875 + | > loss_disc: 2.5269041061401367 (2.445980481097574) + | > loss_disc_real_0: 0.13186919689178467 (0.12196059595597418) + | > loss_disc_real_1: 0.185550719499588 (0.20504791579748455) + | > loss_disc_real_2: 0.20955757796764374 (0.22531352529400273) + | > loss_disc_real_3: 0.23657333850860596 (0.22839618340918894) + | > loss_disc_real_4: 0.24835161864757538 (0.22916218967814195) + | > loss_disc_real_5: 0.24143119156360626 (0.23119686136120243) + | > loss_0: 2.5269041061401367 (2.445980481097574) + | > grad_norm_0: tensor(11.0260, device='cuda:0') (tensor(9.5900, device='cuda:0')) + | > loss_gen: 2.287440776824951 (2.3733843878695837) + | > loss_kl: 1.8475757837295532 (1.823013250451339) + | > loss_feat: 6.003161907196045 (7.236436783640008) + | > loss_mel: 15.687366485595703 (17.63628625367818) + | > loss_duration: 1.4772576093673706 (1.4462551393006977) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 27.30280303955078 (30.515375840036494) + | > grad_norm_1: tensor(302.4525, device='cuda:0') (tensor(222.5417, device='cuda:0')) + | > current_lr_0: 0.00018979396572035043 + | > current_lr_1: 0.00018979396572035043 + | > step_time: 0.4054 (0.373423917669999) + | > loader_time: 0.0065 (0.005868071003964074) + + + --> TIME: 2025-04-19 05:52:39 -- STEP: 120/142 -- GLOBAL_STEP: 438900 + | > loss_disc: 2.441178321838379 (2.4503186881542214) + | > loss_disc_real_0: 0.10455236583948135 (0.12284849360585212) + | > loss_disc_real_1: 0.20475566387176514 (0.20526833981275558) + | > loss_disc_real_2: 0.2305193990468979 (0.2260623394201199) + | > loss_disc_real_3: 0.23175789415836334 (0.22772201485931873) + | > loss_disc_real_4: 0.24173839390277863 (0.22908372605840366) + | > loss_disc_real_5: 0.227332204580307 (0.2316638274739186) + | > loss_0: 2.441178321838379 (2.4503186881542214) + | > grad_norm_0: tensor(24.5059, device='cuda:0') (tensor(10.9975, device='cuda:0')) + | > loss_gen: 2.2380971908569336 (2.3754903316497793) + | > loss_kl: 1.911145567893982 (1.8346703678369523) + | > loss_feat: 7.077430248260498 (7.2364009340604145) + | > loss_mel: 18.05169677734375 (17.68733882904054) + | > loss_duration: 1.4948465824127197 (1.4530211259921393) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.773216247558594 (30.586921596527098) + | > grad_norm_1: tensor(468.0837, device='cuda:0') (tensor(241.4790, device='cuda:0')) + | > current_lr_0: 0.00018979396572035043 + | > current_lr_1: 0.00018979396572035043 + | > step_time: 0.4356 (0.3832057535648346) + | > loader_time: 0.0069 (0.006148771444956462) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4085888862609863 (2.4085888862609863) + | > loss_disc_real_0: 0.08259343355894089 (0.08259343355894089) + | > loss_disc_real_1: 0.23711001873016357 (0.23711001873016357) + | > loss_disc_real_2: 0.23656152188777924 (0.23656152188777924) + | > loss_disc_real_3: 0.22194361686706543 (0.22194361686706543) + | > loss_disc_real_4: 0.23488396406173706 (0.23488396406173706) + | > loss_disc_real_5: 0.2759566307067871 (0.2759566307067871) + | > loss_0: 2.4085888862609863 (2.4085888862609863) + | > loss_gen: 2.3154139518737793 (2.3154139518737793) + | > loss_kl: 2.394545316696167 (2.394545316696167) + | > loss_feat: 6.609136581420898 (6.609136581420898) + | > loss_mel: 17.202640533447266 (17.202640533447266) + | > loss_duration: 1.7604682445526123 (1.7604682445526123) + | > loss_1: 30.282203674316406 (30.282203674316406) + + --> STEP: 1 + | > loss_disc: 2.609712600708008 (2.609712600708008) + | > loss_disc_real_0: 0.08421454578638077 (0.08421454578638077) + | > loss_disc_real_1: 0.2337285429239273 (0.2337285429239273) + | > loss_disc_real_2: 0.20891153812408447 (0.20891153812408447) + | > loss_disc_real_3: 0.25431767106056213 (0.25431767106056213) + | > loss_disc_real_4: 0.22725631296634674 (0.22725631296634674) + | > loss_disc_real_5: 0.2579447031021118 (0.2579447031021118) + | > loss_0: 2.609712600708008 (2.609712600708008) + | > loss_gen: 2.038860559463501 (2.038860559463501) + | > loss_kl: 2.8027801513671875 (2.8027801513671875) + | > loss_feat: 6.992803573608398 (6.992803573608398) + | > loss_mel: 17.78972625732422 (17.78972625732422) + | > loss_duration: 1.8123126029968262 (1.8123126029968262) + | > loss_1: 31.43648338317871 (31.43648338317871) + + --> STEP: 2 + | > loss_disc: 2.390669822692871 (2.5001912117004395) + | > loss_disc_real_0: 0.0805579349398613 (0.08238624036312103) + | > loss_disc_real_1: 0.21622198820114136 (0.22497526556253433) + | > loss_disc_real_2: 0.2310212105512619 (0.2199663743376732) + | > loss_disc_real_3: 0.2538989186286926 (0.2541082948446274) + | > loss_disc_real_4: 0.23914563655853271 (0.23320097476243973) + | > loss_disc_real_5: 0.23008352518081665 (0.24401411414146423) + | > loss_0: 2.390669822692871 (2.5001912117004395) + | > loss_gen: 2.4037399291992188 (2.22130024433136) + | > loss_kl: 2.5441622734069824 (2.673471212387085) + | > loss_feat: 7.2857794761657715 (7.139291524887085) + | > loss_mel: 17.529394149780273 (17.659560203552246) + | > loss_duration: 1.7379149198532104 (1.7751137614250183) + | > loss_1: 31.500991821289062 (31.468737602233887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003624558448791504 (+0.0003546476364135742) + | > avg_loss_disc: 2.5001912117004395 (-0.10284268856048584) + | > avg_loss_disc_real_0: 0.08238624036312103 (-0.12116612493991852) + | > avg_loss_disc_real_1: 0.22497526556253433 (-0.06959391385316849) + | > avg_loss_disc_real_2: 0.2199663743376732 (-0.010237067937850952) + | > avg_loss_disc_real_3: 0.2541082948446274 (+0.0014825761318206787) + | > avg_loss_disc_real_4: 0.23320097476243973 (-0.017438098788261414) + | > avg_loss_disc_real_5: 0.24401411414146423 (-0.019635021686553955) + | > avg_loss_0: 2.5001912117004395 (-0.10284268856048584) + | > avg_loss_gen: 2.22130024433136 (-0.1824873685836792) + | > avg_loss_kl: 2.673471212387085 (+0.30282163619995117) + | > avg_loss_feat: 7.139291524887085 (+0.4900689125061035) + | > avg_loss_mel: 17.659560203552246 (+0.19272232055664062) + | > avg_loss_duration: 1.7751137614250183 (-0.012799859046936035) + | > avg_loss_1: 31.468737602233887 (+0.7903270721435547) + + + > EPOCH: 420/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:52:50)  + + --> TIME: 2025-04-19 05:52:52 -- STEP: 3/142 -- GLOBAL_STEP: 438925 + | > loss_disc: 2.389570951461792 (2.421217918395996) + | > loss_disc_real_0: 0.11754750460386276 (0.10633728404839833) + | > loss_disc_real_1: 0.21698974072933197 (0.21928873161474863) + | > loss_disc_real_2: 0.2166212648153305 (0.23094747960567474) + | > loss_disc_real_3: 0.27683472633361816 (0.2629074553648631) + | > loss_disc_real_4: 0.2258307784795761 (0.22874275346597037) + | > loss_disc_real_5: 0.24342429637908936 (0.2470575471719106) + | > loss_0: 2.389570951461792 (2.421217918395996) + | > grad_norm_0: tensor(3.3994, device='cuda:0') (tensor(5.1714, device='cuda:0')) + | > loss_gen: 2.424344539642334 (2.4542812506357827) + | > loss_kl: 1.8100591897964478 (1.8611738681793213) + | > loss_feat: 7.4803385734558105 (7.354817231496175) + | > loss_mel: 17.489749908447266 (17.029062271118164) + | > loss_duration: 1.4267206192016602 (1.4083051284154255) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.631214141845703 (30.107641220092773) + | > grad_norm_1: tensor(66.5347, device='cuda:0') (tensor(158.9297, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.3213 (0.3227882385253906) + | > loader_time: 0.0039 (0.004473129908243815) + + + --> TIME: 2025-04-19 05:53:01 -- STEP: 28/142 -- GLOBAL_STEP: 438950 + | > loss_disc: 2.376978874206543 (2.4276364360536844) + | > loss_disc_real_0: 0.11823053658008575 (0.1143981572240591) + | > loss_disc_real_1: 0.19513583183288574 (0.20408054973397935) + | > loss_disc_real_2: 0.24593941867351532 (0.22385468121085847) + | > loss_disc_real_3: 0.21631361544132233 (0.22825083136558533) + | > loss_disc_real_4: 0.2496883124113083 (0.22957843701754296) + | > loss_disc_real_5: 0.2234998494386673 (0.23422792074935778) + | > loss_0: 2.376978874206543 (2.4276364360536844) + | > grad_norm_0: tensor(4.6251, device='cuda:0') (tensor(6.9418, device='cuda:0')) + | > loss_gen: 2.397765636444092 (2.398536707673754) + | > loss_kl: 1.805829405784607 (1.8148116980280196) + | > loss_feat: 6.611495494842529 (7.555410640580313) + | > loss_mel: 17.782155990600586 (17.873776367732457) + | > loss_duration: 1.435725450515747 (1.418138563632965) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.032974243164062 (31.060674258640834) + | > grad_norm_1: tensor(171.1366, device='cuda:0') (tensor(178.7795, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.3458 (0.33511810643332346) + | > loader_time: 0.0051 (0.005123155457632882) + + + --> TIME: 2025-04-19 05:53:10 -- STEP: 53/142 -- GLOBAL_STEP: 438975 + | > loss_disc: 2.4702744483947754 (2.4129586354741512) + | > loss_disc_real_0: 0.1035006195306778 (0.11438812905887388) + | > loss_disc_real_1: 0.18253952264785767 (0.20426130997684766) + | > loss_disc_real_2: 0.19966207444667816 (0.22247370290306379) + | > loss_disc_real_3: 0.23461155593395233 (0.2281869523367792) + | > loss_disc_real_4: 0.19512006640434265 (0.22686111252262908) + | > loss_disc_real_5: 0.25397396087646484 (0.22879803489963962) + | > loss_0: 2.4702744483947754 (2.4129586354741512) + | > grad_norm_0: tensor(6.2873, device='cuda:0') (tensor(7.1241, device='cuda:0')) + | > loss_gen: 2.3795225620269775 (2.4003801570748386) + | > loss_kl: 1.9091994762420654 (1.8350571214028124) + | > loss_feat: 7.2952423095703125 (7.412111669216516) + | > loss_mel: 17.295013427734375 (17.74213830480036) + | > loss_duration: 1.4354248046875 (1.4253539386785254) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.314401626586914 (30.815041308133107) + | > grad_norm_1: tensor(235.8624, device='cuda:0') (tensor(190.3471, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.3676 (0.34689964438384435) + | > loader_time: 0.0056 (0.005442488868281526) + + + --> TIME: 2025-04-19 05:53:20 -- STEP: 78/142 -- GLOBAL_STEP: 439000 + | > loss_disc: 2.570089340209961 (2.442870766688616) + | > loss_disc_real_0: 0.06481887400150299 (0.11819457320066598) + | > loss_disc_real_1: 0.17235518991947174 (0.20799014335259414) + | > loss_disc_real_2: 0.20231591165065765 (0.22511350134244332) + | > loss_disc_real_3: 0.2270437479019165 (0.22888996241948542) + | > loss_disc_real_4: 0.20842671394348145 (0.22914752612511316) + | > loss_disc_real_5: 0.2354835718870163 (0.22997718094251093) + | > loss_0: 2.570089340209961 (2.442870766688616) + | > grad_norm_0: tensor(15.0444, device='cuda:0') (tensor(8.1544, device='cuda:0')) + | > loss_gen: 2.176589250564575 (2.382621312752748) + | > loss_kl: 1.927873134613037 (1.8464743846502059) + | > loss_feat: 6.520274639129639 (7.319147030512492) + | > loss_mel: 16.477073669433594 (17.798810665424053) + | > loss_duration: 1.4726999998092651 (1.4366701214741437) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.57451057434082 (30.783723586644882) + | > grad_norm_1: tensor(319.1135, device='cuda:0') (tensor(216.8444, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.3925 (0.35904409029544926) + | > loader_time: 0.0062 (0.005707670480777055) + + + --> TIME: 2025-04-19 05:53:30 -- STEP: 103/142 -- GLOBAL_STEP: 439025 + | > loss_disc: 2.522448778152466 (2.446786121257301) + | > loss_disc_real_0: 0.1557757407426834 (0.12052122923066315) + | > loss_disc_real_1: 0.17566314339637756 (0.20810924586161827) + | > loss_disc_real_2: 0.19932492077350616 (0.22566438111865406) + | > loss_disc_real_3: 0.21725259721279144 (0.23005166230271162) + | > loss_disc_real_4: 0.20070596039295197 (0.22852354634155347) + | > loss_disc_real_5: 0.2475503534078598 (0.2299091348659645) + | > loss_0: 2.522448778152466 (2.446786121257301) + | > grad_norm_0: tensor(11.8722, device='cuda:0') (tensor(8.8920, device='cuda:0')) + | > loss_gen: 2.136653423309326 (2.375493334335031) + | > loss_kl: 1.8751473426818848 (1.8588911341232004) + | > loss_feat: 6.297086238861084 (7.272597877724657) + | > loss_mel: 16.90180015563965 (17.819768294547362) + | > loss_duration: 1.5020718574523926 (1.4473347721747982) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.712759017944336 (30.774085405960825) + | > grad_norm_1: tensor(147.9069, device='cuda:0') (tensor(235.6951, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.4112 (0.3694999657788323) + | > loader_time: 0.0065 (0.0059240535624976295) + + + --> TIME: 2025-04-19 05:53:41 -- STEP: 128/142 -- GLOBAL_STEP: 439050 + | > loss_disc: 2.5033211708068848 (2.45105422474444) + | > loss_disc_real_0: 0.1192563995718956 (0.12159705464728177) + | > loss_disc_real_1: 0.23122723400592804 (0.20833212160505354) + | > loss_disc_real_2: 0.2445307970046997 (0.2256508293794468) + | > loss_disc_real_3: 0.2653590142726898 (0.22980101115535945) + | > loss_disc_real_4: 0.22345243394374847 (0.22807507100515068) + | > loss_disc_real_5: 0.25721168518066406 (0.23052634042687714) + | > loss_0: 2.5033211708068848 (2.45105422474444) + | > grad_norm_0: tensor(14.7107, device='cuda:0') (tensor(9.5350, device='cuda:0')) + | > loss_gen: 2.438666343688965 (2.3725766763091083) + | > loss_kl: 1.8033466339111328 (1.866706303320825) + | > loss_feat: 7.97036075592041 (7.235283777117729) + | > loss_mel: 19.477767944335938 (17.786025866866105) + | > loss_duration: 1.4996819496154785 (1.4542912868782878) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 33.18982696533203 (30.714883908629417) + | > grad_norm_1: tensor(503.9899, device='cuda:0') (tensor(247.0732, device='cuda:0')) + | > current_lr_0: 0.00018977024147463537 + | > current_lr_1: 0.00018977024147463537 + | > step_time: 0.4358 (0.38122925348579884) + | > loader_time: 0.0074 (0.0061525385826826096) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.391526699066162 (2.391526699066162) + | > loss_disc_real_0: 0.08453166484832764 (0.08453166484832764) + | > loss_disc_real_1: 0.2559921145439148 (0.2559921145439148) + | > loss_disc_real_2: 0.2564254701137543 (0.2564254701137543) + | > loss_disc_real_3: 0.24533909559249878 (0.24533909559249878) + | > loss_disc_real_4: 0.205924853682518 (0.205924853682518) + | > loss_disc_real_5: 0.25575414299964905 (0.25575414299964905) + | > loss_0: 2.391526699066162 (2.391526699066162) + | > loss_gen: 2.3490078449249268 (2.3490078449249268) + | > loss_kl: 2.5774786472320557 (2.5774786472320557) + | > loss_feat: 7.4848151206970215 (7.4848151206970215) + | > loss_mel: 19.426786422729492 (19.426786422729492) + | > loss_duration: 1.7300907373428345 (1.7300907373428345) + | > loss_1: 33.568180084228516 (33.568180084228516) + + --> STEP: 1 + | > loss_disc: 2.576111078262329 (2.576111078262329) + | > loss_disc_real_0: 0.13178269565105438 (0.13178269565105438) + | > loss_disc_real_1: 0.22969800233840942 (0.22969800233840942) + | > loss_disc_real_2: 0.23607495427131653 (0.23607495427131653) + | > loss_disc_real_3: 0.19786155223846436 (0.19786155223846436) + | > loss_disc_real_4: 0.2788853347301483 (0.2788853347301483) + | > loss_disc_real_5: 0.23012226819992065 (0.23012226819992065) + | > loss_0: 2.576111078262329 (2.576111078262329) + | > loss_gen: 2.1338133811950684 (2.1338133811950684) + | > loss_kl: 2.849177837371826 (2.849177837371826) + | > loss_feat: 6.323335647583008 (6.323335647583008) + | > loss_mel: 17.192304611206055 (17.192304611206055) + | > loss_duration: 1.78667151927948 (1.78667151927948) + | > loss_1: 30.285303115844727 (30.285303115844727) + + --> STEP: 2 + | > loss_disc: 2.6397628784179688 (2.607936978340149) + | > loss_disc_real_0: 0.15514953434467316 (0.14346611499786377) + | > loss_disc_real_1: 0.2439257949590683 (0.23681189864873886) + | > loss_disc_real_2: 0.2660062313079834 (0.25104059278964996) + | > loss_disc_real_3: 0.2228870987892151 (0.21037432551383972) + | > loss_disc_real_4: 0.2652655839920044 (0.27207545936107635) + | > loss_disc_real_5: 0.273799329996109 (0.25196079909801483) + | > loss_0: 2.6397628784179688 (2.607936978340149) + | > loss_gen: 2.201693296432495 (2.1677533388137817) + | > loss_kl: 2.4376604557037354 (2.6434191465377808) + | > loss_feat: 7.926988124847412 (7.12516188621521) + | > loss_mel: 18.5068302154541 (17.849567413330078) + | > loss_duration: 1.7288305759429932 (1.7577510476112366) + | > loss_1: 32.802001953125 (31.543652534484863) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034608840942382812 (-0.00016367435455322266) + | > avg_loss_disc: 2.607936978340149 (+0.10774576663970947) + | > avg_loss_disc_real_0: 0.14346611499786377 (+0.06107987463474274) + | > avg_loss_disc_real_1: 0.23681189864873886 (+0.011836633086204529) + | > avg_loss_disc_real_2: 0.25104059278964996 (+0.031074218451976776) + | > avg_loss_disc_real_3: 0.21037432551383972 (-0.04373396933078766) + | > avg_loss_disc_real_4: 0.27207545936107635 (+0.03887448459863663) + | > avg_loss_disc_real_5: 0.25196079909801483 (+0.007946684956550598) + | > avg_loss_0: 2.607936978340149 (+0.10774576663970947) + | > avg_loss_gen: 2.1677533388137817 (-0.053546905517578125) + | > avg_loss_kl: 2.6434191465377808 (-0.0300520658493042) + | > avg_loss_feat: 7.12516188621521 (-0.014129638671875) + | > avg_loss_mel: 17.849567413330078 (+0.19000720977783203) + | > avg_loss_duration: 1.7577510476112366 (-0.01736271381378174) + | > avg_loss_1: 31.543652534484863 (+0.07491493225097656) + + + > EPOCH: 421/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:53:49)  + + --> TIME: 2025-04-19 05:53:53 -- STEP: 11/142 -- GLOBAL_STEP: 439075 + | > loss_disc: 2.401247978210449 (2.424148841337724) + | > loss_disc_real_0: 0.09583240747451782 (0.11215313998135654) + | > loss_disc_real_1: 0.21631458401679993 (0.2116863659837029) + | > loss_disc_real_2: 0.2537018060684204 (0.23253971473737198) + | > loss_disc_real_3: 0.25960931181907654 (0.23501065102490512) + | > loss_disc_real_4: 0.2545856535434723 (0.2318895377896049) + | > loss_disc_real_5: 0.2593008577823639 (0.23101164942437952) + | > loss_0: 2.401247978210449 (2.424148841337724) + | > grad_norm_0: tensor(7.4322, device='cuda:0') (tensor(12.3186, device='cuda:0')) + | > loss_gen: 2.5769059658050537 (2.4429114298387007) + | > loss_kl: 1.8230161666870117 (1.83196345242587) + | > loss_feat: 7.203569412231445 (7.381217826496471) + | > loss_mel: 17.570632934570312 (17.90097106586803) + | > loss_duration: 1.400545597076416 (1.4103871258822354) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.574668884277344 (30.967450401999734) + | > grad_norm_1: tensor(309.8400, device='cuda:0') (tensor(275.2159, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.3316 (0.3270923007618297) + | > loader_time: 0.0046 (0.00465215336192738) + + + --> TIME: 2025-04-19 05:54:02 -- STEP: 36/142 -- GLOBAL_STEP: 439100 + | > loss_disc: 2.5768370628356934 (2.4215596119562783) + | > loss_disc_real_0: 0.10558151453733444 (0.11436381253103416) + | > loss_disc_real_1: 0.22743858397006989 (0.20470687126119932) + | > loss_disc_real_2: 0.20190341770648956 (0.2246220020784272) + | > loss_disc_real_3: 0.21062208712100983 (0.22831075597140524) + | > loss_disc_real_4: 0.21770095825195312 (0.22547554100553194) + | > loss_disc_real_5: 0.28568917512893677 (0.23140955923332107) + | > loss_0: 2.5768370628356934 (2.4215596119562783) + | > grad_norm_0: tensor(12.5666, device='cuda:0') (tensor(12.0980, device='cuda:0')) + | > loss_gen: 2.3304800987243652 (2.4134437176916332) + | > loss_kl: 1.7674307823181152 (1.809122857120302) + | > loss_feat: 7.272902488708496 (7.26950216293335) + | > loss_mel: 18.143056869506836 (17.741005712085293) + | > loss_duration: 1.4177286624908447 (1.4198757708072662) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.931598663330078 (30.652949968973797) + | > grad_norm_1: tensor(293.2056, device='cuda:0') (tensor(277.8401, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.354 (0.33939478132459855) + | > loader_time: 0.0053 (0.0050675206714206255) + + + --> TIME: 2025-04-19 05:54:12 -- STEP: 61/142 -- GLOBAL_STEP: 439125 + | > loss_disc: 2.4132373332977295 (2.417908336295456) + | > loss_disc_real_0: 0.1638360470533371 (0.1165176826666613) + | > loss_disc_real_1: 0.1727987825870514 (0.20417464341296523) + | > loss_disc_real_2: 0.20471686124801636 (0.22427937486132637) + | > loss_disc_real_3: 0.2028690129518509 (0.22820922439215613) + | > loss_disc_real_4: 0.1932799369096756 (0.2261239566275331) + | > loss_disc_real_5: 0.1952332705259323 (0.22917831847902204) + | > loss_0: 2.4132373332977295 (2.417908336295456) + | > grad_norm_0: tensor(6.9225, device='cuda:0') (tensor(10.1722, device='cuda:0')) + | > loss_gen: 2.212820529937744 (2.410533752597747) + | > loss_kl: 1.8317116498947144 (1.8151635146531901) + | > loss_feat: 7.209794998168945 (7.405486052153541) + | > loss_mel: 17.372711181640625 (17.907554673366853) + | > loss_duration: 1.453926682472229 (1.4310744922669205) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.080965042114258 (30.969812424456485) + | > grad_norm_1: tensor(226.7786, device='cuda:0') (tensor(245.0186, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.3854 (0.3511475305088231) + | > loader_time: 0.006 (0.005414028636744763) + + + --> TIME: 2025-04-19 05:54:22 -- STEP: 86/142 -- GLOBAL_STEP: 439150 + | > loss_disc: 2.4525198936462402 (2.433737153230712) + | > loss_disc_real_0: 0.09287024289369583 (0.11892147804068964) + | > loss_disc_real_1: 0.18254081904888153 (0.2067237704299217) + | > loss_disc_real_2: 0.2048160284757614 (0.2244743891926699) + | > loss_disc_real_3: 0.24140490591526031 (0.22860443851975507) + | > loss_disc_real_4: 0.21806541085243225 (0.22599381260400594) + | > loss_disc_real_5: 0.22556063532829285 (0.22980833382800567) + | > loss_0: 2.4525198936462402 (2.433737153230712) + | > grad_norm_0: tensor(22.5990, device='cuda:0') (tensor(9.9423, device='cuda:0')) + | > loss_gen: 2.2399091720581055 (2.3907265746316253) + | > loss_kl: 1.9306042194366455 (1.8327413215193638) + | > loss_feat: 7.752688884735107 (7.309397414673207) + | > loss_mel: 18.703760147094727 (17.799282029617665) + | > loss_duration: 1.5036556720733643 (1.4420832672784494) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.130619049072266 (30.774230557818747) + | > grad_norm_1: tensor(382.2895, device='cuda:0') (tensor(234.1497, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.4023 (0.3642587107281352) + | > loader_time: 0.0079 (0.0057882985403371395) + + + --> TIME: 2025-04-19 05:54:32 -- STEP: 111/142 -- GLOBAL_STEP: 439175 + | > loss_disc: 2.4568119049072266 (2.436409544300389) + | > loss_disc_real_0: 0.1408700793981552 (0.12005175307795808) + | > loss_disc_real_1: 0.18920089304447174 (0.20634043189856383) + | > loss_disc_real_2: 0.23812128603458405 (0.22488013003860507) + | > loss_disc_real_3: 0.236095130443573 (0.22896418042548067) + | > loss_disc_real_4: 0.24623200297355652 (0.22685411894643628) + | > loss_disc_real_5: 0.22864867746829987 (0.22917085630936665) + | > loss_0: 2.4568119049072266 (2.436409544300389) + | > grad_norm_0: tensor(5.9171, device='cuda:0') (tensor(9.8830, device='cuda:0')) + | > loss_gen: 2.455477237701416 (2.387850742082338) + | > loss_kl: 1.89884614944458 (1.8411619341051257) + | > loss_feat: 7.684064865112305 (7.2692145528020085) + | > loss_mel: 18.880043029785156 (17.78273227622918) + | > loss_duration: 1.499045491218567 (1.4501242906123666) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.417476654052734 (30.731083706692534) + | > grad_norm_1: tensor(334.0480, device='cuda:0') (tensor(226.7042, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.4282 (0.3754973862622235) + | > loader_time: 0.0073 (0.006052240595087272) + + + --> TIME: 2025-04-19 05:54:44 -- STEP: 136/142 -- GLOBAL_STEP: 439200 + | > loss_disc: 2.4424424171447754 (2.4411895993877866) + | > loss_disc_real_0: 0.10182910412549973 (0.12144859337850529) + | > loss_disc_real_1: 0.19906006753444672 (0.20691977748099497) + | > loss_disc_real_2: 0.24609826505184174 (0.22442593732300928) + | > loss_disc_real_3: 0.24520666897296906 (0.22825809798258195) + | > loss_disc_real_4: 0.22451944649219513 (0.22688224758295453) + | > loss_disc_real_5: 0.21459679305553436 (0.2303667089299244) + | > loss_0: 2.4424424171447754 (2.4411895993877866) + | > grad_norm_0: tensor(5.6492, device='cuda:0') (tensor(9.5908, device='cuda:0')) + | > loss_gen: 2.405891180038452 (2.3800865972743317) + | > loss_kl: 1.9786285161972046 (1.8470288664102554) + | > loss_feat: 7.060695648193359 (7.2288508274975944) + | > loss_mel: 17.17131233215332 (17.725554473259876) + | > loss_duration: 1.488248348236084 (1.4559704529888489) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.10477638244629 (30.63749111399931) + | > grad_norm_1: tensor(194.1807, device='cuda:0') (tensor(221.8084, device='cuda:0')) + | > current_lr_0: 0.00018974652019445104 + | > current_lr_1: 0.00018974652019445104 + | > step_time: 0.4501 (0.38739174429108114) + | > loader_time: 0.0076 (0.00628767644657808) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6106934547424316 (2.6106934547424316) + | > loss_disc_real_0: 0.1522274911403656 (0.1522274911403656) + | > loss_disc_real_1: 0.24997445940971375 (0.24997445940971375) + | > loss_disc_real_2: 0.24678148329257965 (0.24678148329257965) + | > loss_disc_real_3: 0.2541190981864929 (0.2541190981864929) + | > loss_disc_real_4: 0.2854372560977936 (0.2854372560977936) + | > loss_disc_real_5: 0.2498627007007599 (0.2498627007007599) + | > loss_0: 2.6106934547424316 (2.6106934547424316) + | > loss_gen: 2.292689085006714 (2.292689085006714) + | > loss_kl: 2.3347065448760986 (2.3347065448760986) + | > loss_feat: 6.398904800415039 (6.398904800415039) + | > loss_mel: 17.89653205871582 (17.89653205871582) + | > loss_duration: 1.7362303733825684 (1.7362303733825684) + | > loss_1: 30.659061431884766 (30.659061431884766) + + --> STEP: 1 + | > loss_disc: 2.628448486328125 (2.628448486328125) + | > loss_disc_real_0: 0.14961227774620056 (0.14961227774620056) + | > loss_disc_real_1: 0.227242112159729 (0.227242112159729) + | > loss_disc_real_2: 0.26582014560699463 (0.26582014560699463) + | > loss_disc_real_3: 0.2544020414352417 (0.2544020414352417) + | > loss_disc_real_4: 0.25832146406173706 (0.25832146406173706) + | > loss_disc_real_5: 0.24744784832000732 (0.24744784832000732) + | > loss_0: 2.628448486328125 (2.628448486328125) + | > loss_gen: 2.322110176086426 (2.322110176086426) + | > loss_kl: 2.6824262142181396 (2.6824262142181396) + | > loss_feat: 7.25089168548584 (7.25089168548584) + | > loss_mel: 18.507177352905273 (18.507177352905273) + | > loss_duration: 1.8156453371047974 (1.8156453371047974) + | > loss_1: 32.578250885009766 (32.578250885009766) + + --> STEP: 2 + | > loss_disc: 2.389065742492676 (2.5087571144104004) + | > loss_disc_real_0: 0.13990746438503265 (0.1447598710656166) + | > loss_disc_real_1: 0.17436353862285614 (0.20080282539129257) + | > loss_disc_real_2: 0.2539427876472473 (0.25988146662712097) + | > loss_disc_real_3: 0.24045708775520325 (0.24742956459522247) + | > loss_disc_real_4: 0.20565463602542877 (0.23198805004358292) + | > loss_disc_real_5: 0.17989405989646912 (0.21367095410823822) + | > loss_0: 2.389065742492676 (2.5087571144104004) + | > loss_gen: 2.3445985317230225 (2.333354353904724) + | > loss_kl: 2.393791437149048 (2.5381088256835938) + | > loss_feat: 9.052724838256836 (8.151808261871338) + | > loss_mel: 19.74774932861328 (19.127463340759277) + | > loss_duration: 1.747746467590332 (1.7816959023475647) + | > loss_1: 35.2866096496582 (33.932430267333984) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003262162208557129 (-0.00019872188568115234) + | > avg_loss_disc: 2.5087571144104004 (-0.09917986392974854) + | > avg_loss_disc_real_0: 0.1447598710656166 (+0.0012937560677528381) + | > avg_loss_disc_real_1: 0.20080282539129257 (-0.03600907325744629) + | > avg_loss_disc_real_2: 0.25988146662712097 (+0.008840873837471008) + | > avg_loss_disc_real_3: 0.24742956459522247 (+0.03705523908138275) + | > avg_loss_disc_real_4: 0.23198805004358292 (-0.04008740931749344) + | > avg_loss_disc_real_5: 0.21367095410823822 (-0.03828984498977661) + | > avg_loss_0: 2.5087571144104004 (-0.09917986392974854) + | > avg_loss_gen: 2.333354353904724 (+0.16560101509094238) + | > avg_loss_kl: 2.5381088256835938 (-0.10531032085418701) + | > avg_loss_feat: 8.151808261871338 (+1.026646375656128) + | > avg_loss_mel: 19.127463340759277 (+1.2778959274291992) + | > avg_loss_duration: 1.7816959023475647 (+0.023944854736328125) + | > avg_loss_1: 33.932430267333984 (+2.388777732849121) + + + > EPOCH: 422/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:54:48)  + + --> TIME: 2025-04-19 05:54:55 -- STEP: 19/142 -- GLOBAL_STEP: 439225 + | > loss_disc: 2.434034824371338 (2.443621723275436) + | > loss_disc_real_0: 0.09655126184225082 (0.11413266352917019) + | > loss_disc_real_1: 0.20134873688220978 (0.20838479540849986) + | > loss_disc_real_2: 0.21622015535831451 (0.22758627173147702) + | > loss_disc_real_3: 0.22011636197566986 (0.22593585989977183) + | > loss_disc_real_4: 0.22152549028396606 (0.22789443950904043) + | > loss_disc_real_5: 0.1920628547668457 (0.22046712433037005) + | > loss_0: 2.434034824371338 (2.443621723275436) + | > grad_norm_0: tensor(4.9438, device='cuda:0') (tensor(7.1065, device='cuda:0')) + | > loss_gen: 2.4913899898529053 (2.3518817048323784) + | > loss_kl: 1.8736308813095093 (1.7566556240382947) + | > loss_feat: 7.9618821144104 (7.271893425991661) + | > loss_mel: 18.1925106048584 (17.63590034685637) + | > loss_duration: 1.422572374343872 (1.405817006763659) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.941986083984375 (30.422147951628034) + | > grad_norm_1: tensor(103.3185, device='cuda:0') (tensor(111.5356, device='cuda:0')) + | > current_lr_0: 0.00018972280187942673 + | > current_lr_1: 0.00018972280187942673 + | > step_time: 0.3375 (0.3294922678094161) + | > loader_time: 0.0047 (0.004715681076049805) + + + --> TIME: 2025-04-19 05:55:04 -- STEP: 44/142 -- GLOBAL_STEP: 439250 + | > loss_disc: 2.5195860862731934 (2.4290857802737844) + | > loss_disc_real_0: 0.1849905103445053 (0.11783905005590482) + | > loss_disc_real_1: 0.20921087265014648 (0.20587308332324028) + | > loss_disc_real_2: 0.25415346026420593 (0.2267525406046347) + | > loss_disc_real_3: 0.24848812818527222 (0.22709362344308334) + | > loss_disc_real_4: 0.26229456067085266 (0.2263204584067518) + | > loss_disc_real_5: 0.2498358190059662 (0.23151286860758608) + | > loss_0: 2.5195860862731934 (2.4290857802737844) + | > grad_norm_0: tensor(8.2597, device='cuda:0') (tensor(8.2738, device='cuda:0')) + | > loss_gen: 2.5059142112731934 (2.397459095174616) + | > loss_kl: 1.8940101861953735 (1.802614832466299) + | > loss_feat: 7.25712776184082 (7.391979867761785) + | > loss_mel: 18.033979415893555 (17.786240317604754) + | > loss_duration: 1.4491558074951172 (1.4216054732149301) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.140188217163086 (30.799899274652656) + | > grad_norm_1: tensor(56.9910, device='cuda:0') (tensor(152.9061, device='cuda:0')) + | > current_lr_0: 0.00018972280187942673 + | > current_lr_1: 0.00018972280187942673 + | > step_time: 0.3593 (0.34213896231217816) + | > loader_time: 0.0057 (0.005124617706645621) + + + --> TIME: 2025-04-19 05:55:13 -- STEP: 69/142 -- GLOBAL_STEP: 439275 + | > loss_disc: 2.4873249530792236 (2.44836039474045) + | > loss_disc_real_0: 0.13007627427577972 (0.12234717531912569) + | > loss_disc_real_1: 0.1759643703699112 (0.20804361675096594) + | > loss_disc_real_2: 0.2412172555923462 (0.22777784302614737) + | > loss_disc_real_3: 0.2653808891773224 (0.22810097470663596) + | > loss_disc_real_4: 0.31159430742263794 (0.22900036150130673) + | > loss_disc_real_5: 0.31556108593940735 (0.22885210812091827) + | > loss_0: 2.4873249530792236 (2.44836039474045) + | > grad_norm_0: tensor(6.4073, device='cuda:0') (tensor(9.2993, device='cuda:0')) + | > loss_gen: 2.540426731109619 (2.381400326023931) + | > loss_kl: 1.8173657655715942 (1.8212396010108616) + | > loss_feat: 7.886605262756348 (7.275096914042598) + | > loss_mel: 18.97859764099121 (17.754845895628986) + | > loss_duration: 1.4943830966949463 (1.4322495771491013) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.717376708984375 (30.66483207370924) + | > grad_norm_1: tensor(253.9332, device='cuda:0') (tensor(185.4570, device='cuda:0')) + | > current_lr_0: 0.00018972280187942673 + | > current_lr_1: 0.00018972280187942673 + | > step_time: 0.398 (0.35435204575027246) + | > loader_time: 0.0063 (0.00549194087152896) + + + --> TIME: 2025-04-19 05:55:24 -- STEP: 94/142 -- GLOBAL_STEP: 439300 + | > loss_disc: 2.524812698364258 (2.44785402176228) + | > loss_disc_real_0: 0.1045168787240982 (0.1226638228969371) + | > loss_disc_real_1: 0.23746724426746368 (0.2066103518009186) + | > loss_disc_real_2: 0.25613248348236084 (0.22659299950650397) + | > loss_disc_real_3: 0.24036608636379242 (0.2281305878720385) + | > loss_disc_real_4: 0.24055415391921997 (0.22959819975051474) + | > loss_disc_real_5: 0.24252553284168243 (0.23128301000341456) + | > loss_0: 2.524812698364258 (2.44785402176228) + | > grad_norm_0: tensor(7.2895, device='cuda:0') (tensor(9.3407, device='cuda:0')) + | > loss_gen: 2.2498912811279297 (2.377530577334952) + | > loss_kl: 1.7801785469055176 (1.8287200686779428) + | > loss_feat: 6.828615665435791 (7.195150142020368) + | > loss_mel: 17.505329132080078 (17.73044378199475) + | > loss_duration: 1.5016015768051147 (1.4447934995306304) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.865617752075195 (30.57663797824941) + | > grad_norm_1: tensor(118.4454, device='cuda:0') (tensor(188.9771, device='cuda:0')) + | > current_lr_0: 0.00018972280187942673 + | > current_lr_1: 0.00018972280187942673 + | > step_time: 0.3991 (0.3661476703400308) + | > loader_time: 0.0065 (0.005790236148428411) + + + --> TIME: 2025-04-19 05:55:34 -- STEP: 119/142 -- GLOBAL_STEP: 439325 + | > loss_disc: 2.4403433799743652 (2.452998886589243) + | > loss_disc_real_0: 0.11105316132307053 (0.12494456004194851) + | > loss_disc_real_1: 0.2081126868724823 (0.2063226718612078) + | > loss_disc_real_2: 0.21259306371212006 (0.2273445809338273) + | > loss_disc_real_3: 0.2127193808555603 (0.2276460470772591) + | > loss_disc_real_4: 0.242097869515419 (0.22940135014658214) + | > loss_disc_real_5: 0.23329497873783112 (0.23061446933185353) + | > loss_0: 2.4403433799743652 (2.452998886589243) + | > grad_norm_0: tensor(13.2377, device='cuda:0') (tensor(9.8331, device='cuda:0')) + | > loss_gen: 2.3237221240997314 (2.3699857627644256) + | > loss_kl: 1.9031126499176025 (1.8421159331538097) + | > loss_feat: 6.965588569641113 (7.12921384202332) + | > loss_mel: 18.205026626586914 (17.67786249593526) + | > loss_duration: 1.5157275199890137 (1.4530200577583638) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.913177490234375 (30.47219802151207) + | > grad_norm_1: tensor(305.2780, device='cuda:0') (tensor(211.8666, device='cuda:0')) + | > current_lr_0: 0.00018972280187942673 + | > current_lr_1: 0.00018972280187942673 + | > step_time: 0.426 (0.3777661864497081) + | > loader_time: 0.0078 (0.0060436905933027515) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.512040138244629 (2.512040138244629) + | > loss_disc_real_0: 0.2236347645521164 (0.2236347645521164) + | > loss_disc_real_1: 0.20962753891944885 (0.20962753891944885) + | > loss_disc_real_2: 0.22148118913173676 (0.22148118913173676) + | > loss_disc_real_3: 0.22624152898788452 (0.22624152898788452) + | > loss_disc_real_4: 0.2504230737686157 (0.2504230737686157) + | > loss_disc_real_5: 0.25383010506629944 (0.25383010506629944) + | > loss_0: 2.512040138244629 (2.512040138244629) + | > loss_gen: 2.427639961242676 (2.427639961242676) + | > loss_kl: 2.5735182762145996 (2.5735182762145996) + | > loss_feat: 6.282995223999023 (6.282995223999023) + | > loss_mel: 18.2960147857666 (18.2960147857666) + | > loss_duration: 1.7502766847610474 (1.7502766847610474) + | > loss_1: 31.330446243286133 (31.330446243286133) + + --> STEP: 1 + | > loss_disc: 2.3875489234924316 (2.3875489234924316) + | > loss_disc_real_0: 0.20211338996887207 (0.20211338996887207) + | > loss_disc_real_1: 0.1771220564842224 (0.1771220564842224) + | > loss_disc_real_2: 0.16509132087230682 (0.16509132087230682) + | > loss_disc_real_3: 0.2526329457759857 (0.2526329457759857) + | > loss_disc_real_4: 0.21826128661632538 (0.21826128661632538) + | > loss_disc_real_5: 0.20887182652950287 (0.20887182652950287) + | > loss_0: 2.3875489234924316 (2.3875489234924316) + | > loss_gen: 2.380904197692871 (2.380904197692871) + | > loss_kl: 2.823680877685547 (2.823680877685547) + | > loss_feat: 7.749197483062744 (7.749197483062744) + | > loss_mel: 18.28860092163086 (18.28860092163086) + | > loss_duration: 1.8075215816497803 (1.8075215816497803) + | > loss_1: 33.049903869628906 (33.049903869628906) + + --> STEP: 2 + | > loss_disc: 2.4364466667175293 (2.4119977951049805) + | > loss_disc_real_0: 0.22813886404037476 (0.2151261270046234) + | > loss_disc_real_1: 0.22066591680049896 (0.1988939866423607) + | > loss_disc_real_2: 0.18330423533916473 (0.17419777810573578) + | > loss_disc_real_3: 0.2242719531059265 (0.23845244944095612) + | > loss_disc_real_4: 0.2725062072277069 (0.24538374692201614) + | > loss_disc_real_5: 0.22376002371311188 (0.21631592512130737) + | > loss_0: 2.4364466667175293 (2.4119977951049805) + | > loss_gen: 2.4103689193725586 (2.395636558532715) + | > loss_kl: 2.5928497314453125 (2.7082653045654297) + | > loss_feat: 7.065882682800293 (7.4075400829315186) + | > loss_mel: 18.92488670349121 (18.606743812561035) + | > loss_duration: 1.7460074424743652 (1.7767645120620728) + | > loss_1: 32.739994049072266 (32.894948959350586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031697750091552734 (-9.238719940185547e-05) + | > avg_loss_disc: 2.4119977951049805 (-0.09675931930541992) + | > avg_loss_disc_real_0: 0.2151261270046234 (+0.0703662559390068) + | > avg_loss_disc_real_1: 0.1988939866423607 (-0.0019088387489318848) + | > avg_loss_disc_real_2: 0.17419777810573578 (-0.08568368852138519) + | > avg_loss_disc_real_3: 0.23845244944095612 (-0.008977115154266357) + | > avg_loss_disc_real_4: 0.24538374692201614 (+0.013395696878433228) + | > avg_loss_disc_real_5: 0.21631592512130737 (+0.002644971013069153) + | > avg_loss_0: 2.4119977951049805 (-0.09675931930541992) + | > avg_loss_gen: 2.395636558532715 (+0.06228220462799072) + | > avg_loss_kl: 2.7082653045654297 (+0.17015647888183594) + | > avg_loss_feat: 7.4075400829315186 (-0.7442681789398193) + | > avg_loss_mel: 18.606743812561035 (-0.5207195281982422) + | > avg_loss_duration: 1.7767645120620728 (-0.004931390285491943) + | > avg_loss_1: 32.894948959350586 (-1.0374813079833984) + + + > EPOCH: 423/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:55:46)  + + --> TIME: 2025-04-19 05:55:48 -- STEP: 2/142 -- GLOBAL_STEP: 439350 + | > loss_disc: 2.3197600841522217 (2.343450427055359) + | > loss_disc_real_0: 0.11581702530384064 (0.12218451499938965) + | > loss_disc_real_1: 0.1958121657371521 (0.18261893093585968) + | > loss_disc_real_2: 0.1912190467119217 (0.19842825084924698) + | > loss_disc_real_3: 0.21311326324939728 (0.2238636016845703) + | > loss_disc_real_4: 0.21436044573783875 (0.2213694378733635) + | > loss_disc_real_5: 0.21920005977153778 (0.21201103180646896) + | > loss_0: 2.3197600841522217 (2.343450427055359) + | > grad_norm_0: tensor(14.1329, device='cuda:0') (tensor(18.4810, device='cuda:0')) + | > loss_gen: 2.395185708999634 (2.410479187965393) + | > loss_kl: 1.757028579711914 (1.853839099407196) + | > loss_feat: 7.731734752655029 (7.7746052742004395) + | > loss_mel: 18.046123504638672 (18.608135223388672) + | > loss_duration: 1.4185181856155396 (1.4066380858421326) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.348590850830078 (32.05369567871094) + | > grad_norm_1: tensor(417.4001, device='cuda:0') (tensor(426.0569, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.3241 (0.32415008544921875) + | > loader_time: 0.0038 (0.0045893192291259766) + + + --> TIME: 2025-04-19 05:55:56 -- STEP: 27/142 -- GLOBAL_STEP: 439375 + | > loss_disc: 2.4816548824310303 (2.4138599148503057) + | > loss_disc_real_0: 0.09283679723739624 (0.11294948530417902) + | > loss_disc_real_1: 0.18419580161571503 (0.20044028924571144) + | > loss_disc_real_2: 0.20638889074325562 (0.2229736546675364) + | > loss_disc_real_3: 0.22118042409420013 (0.22634556061691707) + | > loss_disc_real_4: 0.2514230012893677 (0.22618967846587854) + | > loss_disc_real_5: 0.23533469438552856 (0.22770264799948092) + | > loss_0: 2.4816548824310303 (2.4138599148503057) + | > grad_norm_0: tensor(22.0032, device='cuda:0') (tensor(15.1983, device='cuda:0')) + | > loss_gen: 2.276827812194824 (2.4096838191703513) + | > loss_kl: 2.01004695892334 (1.8155337042278714) + | > loss_feat: 7.568049430847168 (7.341448042127821) + | > loss_mel: 17.716644287109375 (17.71403348004377) + | > loss_duration: 1.416161298751831 (1.4235307508044772) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.987728118896484 (30.704229637428565) + | > grad_norm_1: tensor(417.5671, device='cuda:0') (tensor(331.9063, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.3487 (0.3356835488919859) + | > loader_time: 0.0053 (0.004865310810230396) + + + --> TIME: 2025-04-19 05:56:06 -- STEP: 52/142 -- GLOBAL_STEP: 439400 + | > loss_disc: 2.3746116161346436 (2.4337636782572813) + | > loss_disc_real_0: 0.09290645271539688 (0.1166265165576568) + | > loss_disc_real_1: 0.2097744345664978 (0.20605471185766733) + | > loss_disc_real_2: 0.23135337233543396 (0.22434761231908432) + | > loss_disc_real_3: 0.23246575891971588 (0.22605317974319825) + | > loss_disc_real_4: 0.22641406953334808 (0.226755973811333) + | > loss_disc_real_5: 0.207530215382576 (0.22842062379305178) + | > loss_0: 2.3746116161346436 (2.4337636782572813) + | > grad_norm_0: tensor(10.2160, device='cuda:0') (tensor(13.5627, device='cuda:0')) + | > loss_gen: 2.545081615447998 (2.384350166870997) + | > loss_kl: 1.7397693395614624 (1.8325305512318244) + | > loss_feat: 7.474282264709473 (7.242087208307707) + | > loss_mel: 18.689485549926758 (17.602945841275734) + | > loss_duration: 1.4799436330795288 (1.430235622020868) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.928564071655273 (30.49214931634756) + | > grad_norm_1: tensor(278.8729, device='cuda:0') (tensor(321.7445, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.3652 (0.34771471757155187) + | > loader_time: 0.0057 (0.005239573808816763) + + + --> TIME: 2025-04-19 05:56:15 -- STEP: 77/142 -- GLOBAL_STEP: 439425 + | > loss_disc: 2.462956666946411 (2.4255257457881774) + | > loss_disc_real_0: 0.09982682019472122 (0.11613270292034397) + | > loss_disc_real_1: 0.22438547015190125 (0.20485886370206807) + | > loss_disc_real_2: 0.22269274294376373 (0.22409704463048416) + | > loss_disc_real_3: 0.24049369990825653 (0.22728783569552682) + | > loss_disc_real_4: 0.21364910900592804 (0.22744987885673323) + | > loss_disc_real_5: 0.25800755620002747 (0.22954032424982493) + | > loss_0: 2.462956666946411 (2.4255257457881774) + | > grad_norm_0: tensor(9.9835, device='cuda:0') (tensor(12.1132, device='cuda:0')) + | > loss_gen: 2.3020427227020264 (2.3931882814927525) + | > loss_kl: 1.774015188217163 (1.8428885054278683) + | > loss_feat: 7.234406471252441 (7.2996695258400655) + | > loss_mel: 17.344905853271484 (17.62540309460132) + | > loss_duration: 1.4583255052566528 (1.442261488406689) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.113697052001953 (30.603410918991287) + | > grad_norm_1: tensor(296.7400, device='cuda:0') (tensor(296.5024, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.3911 (0.35964253041651334) + | > loader_time: 0.0064 (0.005551390833668894) + + + --> TIME: 2025-04-19 05:56:26 -- STEP: 102/142 -- GLOBAL_STEP: 439450 + | > loss_disc: 2.3939051628112793 (2.432225152557971) + | > loss_disc_real_0: 0.14936105906963348 (0.11652852302672816) + | > loss_disc_real_1: 0.18098540604114532 (0.20496379744772816) + | > loss_disc_real_2: 0.21675197780132294 (0.22474139768118953) + | > loss_disc_real_3: 0.2607917785644531 (0.22643742155210644) + | > loss_disc_real_4: 0.24462983012199402 (0.22739435001915576) + | > loss_disc_real_5: 0.25699371099472046 (0.22975950030719533) + | > loss_0: 2.3939051628112793 (2.432225152557971) + | > grad_norm_0: tensor(5.9812, device='cuda:0') (tensor(11.2688, device='cuda:0')) + | > loss_gen: 2.6534736156463623 (2.3875553607940656) + | > loss_kl: 1.8033416271209717 (1.8498900006799137) + | > loss_feat: 7.19256067276001 (7.2697753579008815) + | > loss_mel: 17.38971519470215 (17.673860194636326) + | > loss_duration: 1.4684321880340576 (1.4521771926505893) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.50752067565918 (30.63325809029972) + | > grad_norm_1: tensor(375.3626, device='cuda:0') (tensor(280.2484, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.4184 (0.3711465036167818) + | > loader_time: 0.0068 (0.005863304231681073) + + + --> TIME: 2025-04-19 05:56:37 -- STEP: 127/142 -- GLOBAL_STEP: 439475 + | > loss_disc: 2.5264649391174316 (2.443067959913118) + | > loss_disc_real_0: 0.2071944773197174 (0.11998711502927495) + | > loss_disc_real_1: 0.20385301113128662 (0.20528928829929022) + | > loss_disc_real_2: 0.21640589833259583 (0.22549638738782388) + | > loss_disc_real_3: 0.2093023657798767 (0.22699761484551617) + | > loss_disc_real_4: 0.22808706760406494 (0.2281234829913913) + | > loss_disc_real_5: 0.26146361231803894 (0.23055776402236908) + | > loss_0: 2.5264649391174316 (2.443067959913118) + | > grad_norm_0: tensor(22.1731, device='cuda:0') (tensor(11.3595, device='cuda:0')) + | > loss_gen: 2.4758310317993164 (2.379437696276686) + | > loss_kl: 1.854794979095459 (1.8537478484506682) + | > loss_feat: 6.9619035720825195 (7.210120253675566) + | > loss_mel: 16.840972900390625 (17.57655631838821) + | > loss_duration: 1.4848113059997559 (1.4589466053669846) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.618314743041992 (30.478808740931232) + | > grad_norm_1: tensor(500.7094, device='cuda:0') (tensor(273.5545, device='cuda:0')) + | > current_lr_0: 0.00018969908652919179 + | > current_lr_1: 0.00018969908652919179 + | > step_time: 0.4369 (0.3822412509617843) + | > loader_time: 0.0073 (0.0061604131863811805) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4925198554992676 (2.4925198554992676) + | > loss_disc_real_0: 0.2441944181919098 (0.2441944181919098) + | > loss_disc_real_1: 0.22415389120578766 (0.22415389120578766) + | > loss_disc_real_2: 0.2545757591724396 (0.2545757591724396) + | > loss_disc_real_3: 0.21817998588085175 (0.21817998588085175) + | > loss_disc_real_4: 0.26526275277137756 (0.26526275277137756) + | > loss_disc_real_5: 0.22674331068992615 (0.22674331068992615) + | > loss_0: 2.4925198554992676 (2.4925198554992676) + | > loss_gen: 2.606081008911133 (2.606081008911133) + | > loss_kl: 2.636580228805542 (2.636580228805542) + | > loss_feat: 7.080663681030273 (7.080663681030273) + | > loss_mel: 18.449254989624023 (18.449254989624023) + | > loss_duration: 1.7480499744415283 (1.7480499744415283) + | > loss_1: 32.5206298828125 (32.5206298828125) + + --> STEP: 1 + | > loss_disc: 2.5152740478515625 (2.5152740478515625) + | > loss_disc_real_0: 0.2509170472621918 (0.2509170472621918) + | > loss_disc_real_1: 0.18501286208629608 (0.18501286208629608) + | > loss_disc_real_2: 0.2563859224319458 (0.2563859224319458) + | > loss_disc_real_3: 0.22141586244106293 (0.22141586244106293) + | > loss_disc_real_4: 0.24866215884685516 (0.24866215884685516) + | > loss_disc_real_5: 0.2256571352481842 (0.2256571352481842) + | > loss_0: 2.5152740478515625 (2.5152740478515625) + | > loss_gen: 2.443885326385498 (2.443885326385498) + | > loss_kl: 2.9301421642303467 (2.9301421642303467) + | > loss_feat: 7.630389213562012 (7.630389213562012) + | > loss_mel: 19.283023834228516 (19.283023834228516) + | > loss_duration: 1.8160381317138672 (1.8160381317138672) + | > loss_1: 34.103477478027344 (34.103477478027344) + + --> STEP: 2 + | > loss_disc: 2.3758175373077393 (2.445545792579651) + | > loss_disc_real_0: 0.21604008972644806 (0.23347856849431992) + | > loss_disc_real_1: 0.17715023458003998 (0.18108154833316803) + | > loss_disc_real_2: 0.21170194447040558 (0.2340439334511757) + | > loss_disc_real_3: 0.17844340205192566 (0.1999296322464943) + | > loss_disc_real_4: 0.19713857769966125 (0.2229003682732582) + | > loss_disc_real_5: 0.1677107959985733 (0.19668396562337875) + | > loss_0: 2.3758175373077393 (2.445545792579651) + | > loss_gen: 2.3011741638183594 (2.3725297451019287) + | > loss_kl: 2.4878814220428467 (2.7090117931365967) + | > loss_feat: 7.296273231506348 (7.46333122253418) + | > loss_mel: 16.90435218811035 (18.093688011169434) + | > loss_duration: 1.7366706132888794 (1.7763543725013733) + | > loss_1: 30.72635269165039 (32.41491508483887) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003252744674682617 (+8.296966552734375e-05) + | > avg_loss_disc: 2.445545792579651 (+0.03354799747467041) + | > avg_loss_disc_real_0: 0.23347856849431992 (+0.018352441489696503) + | > avg_loss_disc_real_1: 0.18108154833316803 (-0.017812438309192657) + | > avg_loss_disc_real_2: 0.2340439334511757 (+0.05984615534543991) + | > avg_loss_disc_real_3: 0.1999296322464943 (-0.03852281719446182) + | > avg_loss_disc_real_4: 0.2229003682732582 (-0.022483378648757935) + | > avg_loss_disc_real_5: 0.19668396562337875 (-0.01963195949792862) + | > avg_loss_0: 2.445545792579651 (+0.03354799747467041) + | > avg_loss_gen: 2.3725297451019287 (-0.023106813430786133) + | > avg_loss_kl: 2.7090117931365967 (+0.0007464885711669922) + | > avg_loss_feat: 7.46333122253418 (+0.05579113960266113) + | > avg_loss_mel: 18.093688011169434 (-0.5130558013916016) + | > avg_loss_duration: 1.7763543725013733 (-0.0004101395606994629) + | > avg_loss_1: 32.41491508483887 (-0.48003387451171875) + + + > EPOCH: 424/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:56:45)  + + --> TIME: 2025-04-19 05:56:49 -- STEP: 10/142 -- GLOBAL_STEP: 439500 + | > loss_disc: 2.3906006813049316 (2.4124327898025513) + | > loss_disc_real_0: 0.10687172412872314 (0.11182007268071174) + | > loss_disc_real_1: 0.17605328559875488 (0.20907244831323624) + | > loss_disc_real_2: 0.2039526104927063 (0.21959131807088852) + | > loss_disc_real_3: 0.21257062256336212 (0.22985998094081878) + | > loss_disc_real_4: 0.2093396782875061 (0.23091692328453065) + | > loss_disc_real_5: 0.19338423013687134 (0.24417171031236648) + | > loss_0: 2.3906006813049316 (2.4124327898025513) + | > grad_norm_0: tensor(3.5211, device='cuda:0') (tensor(9.6494, device='cuda:0')) + | > loss_gen: 2.3811299800872803 (2.4391043186187744) + | > loss_kl: 1.8407100439071655 (1.781409752368927) + | > loss_feat: 7.508199691772461 (7.440855169296265) + | > loss_mel: 17.51030731201172 (17.552594566345213) + | > loss_duration: 1.376631498336792 (1.4033405900001525) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.61697769165039 (30.617304039001464) + | > grad_norm_1: tensor(39.7299, device='cuda:0') (tensor(306.6462, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.3324 (0.32571437358856203) + | > loader_time: 0.0049 (0.004577469825744629) + + + --> TIME: 2025-04-19 05:56:58 -- STEP: 35/142 -- GLOBAL_STEP: 439525 + | > loss_disc: 2.337580442428589 (2.439659343447004) + | > loss_disc_real_0: 0.09019890427589417 (0.11240946458918707) + | > loss_disc_real_1: 0.2033793181180954 (0.20905787391322) + | > loss_disc_real_2: 0.22455468773841858 (0.22420624451977866) + | > loss_disc_real_3: 0.1995096504688263 (0.2281711207968848) + | > loss_disc_real_4: 0.19752679765224457 (0.23039258718490602) + | > loss_disc_real_5: 0.1736113429069519 (0.23343207240104674) + | > loss_0: 2.337580442428589 (2.439659343447004) + | > grad_norm_0: tensor(7.1723, device='cuda:0') (tensor(8.6764, device='cuda:0')) + | > loss_gen: 2.2541189193725586 (2.3816739559173583) + | > loss_kl: 1.8994547128677368 (1.8206984519958496) + | > loss_feat: 7.76950740814209 (7.285537011282784) + | > loss_mel: 17.09050178527832 (17.55180252620152) + | > loss_duration: 1.4340431690216064 (1.418531894683838) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.447628021240234 (30.458243669782366) + | > grad_norm_1: tensor(50.4217, device='cuda:0') (tensor(254.7636, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.3654 (0.34171669142586847) + | > loader_time: 0.0055 (0.00509486198425293) + + + --> TIME: 2025-04-19 05:57:07 -- STEP: 60/142 -- GLOBAL_STEP: 439550 + | > loss_disc: 2.5086135864257812 (2.4332697431246446) + | > loss_disc_real_0: 0.11287248134613037 (0.11432764617105325) + | > loss_disc_real_1: 0.23689135909080505 (0.20804884533087412) + | > loss_disc_real_2: 0.2309805452823639 (0.22532085527976353) + | > loss_disc_real_3: 0.24866269528865814 (0.2285124974946181) + | > loss_disc_real_4: 0.24939048290252686 (0.22772342363993328) + | > loss_disc_real_5: 0.25559476017951965 (0.23155073126157125) + | > loss_0: 2.5086135864257812 (2.4332697431246446) + | > grad_norm_0: tensor(12.7567, device='cuda:0') (tensor(9.6201, device='cuda:0')) + | > loss_gen: 2.4467222690582275 (2.3829395373662314) + | > loss_kl: 1.7799596786499023 (1.8194208939870198) + | > loss_feat: 7.268128871917725 (7.25841027100881) + | > loss_mel: 18.195287704467773 (17.582283099492376) + | > loss_duration: 1.4632019996643066 (1.4298733870188396) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.15329933166504 (30.472927061716714) + | > grad_norm_1: tensor(364.4662, device='cuda:0') (tensor(262.7231, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.3852 (0.3555794954299927) + | > loader_time: 0.0065 (0.005495377381642659) + + + --> TIME: 2025-04-19 05:57:18 -- STEP: 85/142 -- GLOBAL_STEP: 439575 + | > loss_disc: 2.3103537559509277 (2.444542738970589) + | > loss_disc_real_0: 0.07065768539905548 (0.11697687129763996) + | > loss_disc_real_1: 0.23860378563404083 (0.20759321275879355) + | > loss_disc_real_2: 0.22062619030475616 (0.22622517505112816) + | > loss_disc_real_3: 0.1947629302740097 (0.22944558788748348) + | > loss_disc_real_4: 0.20894820988178253 (0.22900767133516423) + | > loss_disc_real_5: 0.21884924173355103 (0.23271785869317896) + | > loss_0: 2.3103537559509277 (2.444542738970589) + | > grad_norm_0: tensor(13.7030, device='cuda:0') (tensor(10.9742, device='cuda:0')) + | > loss_gen: 2.3780357837677 (2.37314494637882) + | > loss_kl: 2.011838436126709 (1.8298848558874692) + | > loss_feat: 6.794797420501709 (7.185713560440961) + | > loss_mel: 17.420686721801758 (17.662818314047403) + | > loss_duration: 1.4727017879486084 (1.4417351694668041) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.078060150146484 (30.493296858843635) + | > grad_norm_1: tensor(406.5699, device='cuda:0') (tensor(277.6832, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.4075 (0.3693412135629094) + | > loader_time: 0.0067 (0.005794415754430434) + + + --> TIME: 2025-04-19 05:57:29 -- STEP: 110/142 -- GLOBAL_STEP: 439600 + | > loss_disc: 2.468773126602173 (2.4409000049937863) + | > loss_disc_real_0: 0.11302900314331055 (0.11833520426668905) + | > loss_disc_real_1: 0.19070610404014587 (0.20576894459399311) + | > loss_disc_real_2: 0.2222815603017807 (0.22518859830769625) + | > loss_disc_real_3: 0.22032570838928223 (0.22771779217503288) + | > loss_disc_real_4: 0.22934648394584656 (0.2272254383022135) + | > loss_disc_real_5: 0.20365716516971588 (0.2306078466502103) + | > loss_0: 2.468773126602173 (2.4409000049937863) + | > grad_norm_0: tensor(6.4285, device='cuda:0') (tensor(11.3687, device='cuda:0')) + | > loss_gen: 2.4132142066955566 (2.3752312876961437) + | > loss_kl: 1.953912377357483 (1.8422644853591919) + | > loss_feat: 8.071449279785156 (7.190089871666649) + | > loss_mel: 18.616703033447266 (17.72754205356943) + | > loss_duration: 1.4737763404846191 (1.4499214865944603) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.529056549072266 (30.585049161044033) + | > grad_norm_1: tensor(395.8285, device='cuda:0') (tensor(281.4961, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.4288 (0.382701139016585) + | > loader_time: 0.0073 (0.0060726967724886805) + + + --> TIME: 2025-04-19 05:57:40 -- STEP: 135/142 -- GLOBAL_STEP: 439625 + | > loss_disc: 2.411924362182617 (2.454038210268375) + | > loss_disc_real_0: 0.13961349427700043 (0.12090614937521793) + | > loss_disc_real_1: 0.22696462273597717 (0.20771156163127333) + | > loss_disc_real_2: 0.20814868807792664 (0.22672622347319568) + | > loss_disc_real_3: 0.25034138560295105 (0.22843398495956704) + | > loss_disc_real_4: 0.20575685799121857 (0.22779665346498842) + | > loss_disc_real_5: 0.21533873677253723 (0.2315971275170644) + | > loss_0: 2.411924362182617 (2.454038210268375) + | > grad_norm_0: tensor(9.1340, device='cuda:0') (tensor(10.9029, device='cuda:0')) + | > loss_gen: 2.5030364990234375 (2.366191456052991) + | > loss_kl: 1.7781661748886108 (1.8472762743631999) + | > loss_feat: 7.400151252746582 (7.149349982650191) + | > loss_mel: 17.692331314086914 (17.66288729067201) + | > loss_duration: 1.480571985244751 (1.4566578997506037) + | > amp_scaler: 128.0 (64.47407407407407) + | > loss_1: 30.85425567626953 (30.482362916734484) + | > grad_norm_1: tensor(339.8800, device='cuda:0') (tensor(269.8195, device='cuda:0')) + | > current_lr_0: 0.00018967537414337564 + | > current_lr_1: 0.00018967537414337564 + | > step_time: 0.4554 (0.39274737570020896) + | > loader_time: 0.0074 (0.0063243230183919275) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.488389492034912 (2.488389492034912) + | > loss_disc_real_0: 0.20371416211128235 (0.20371416211128235) + | > loss_disc_real_1: 0.29072174429893494 (0.29072174429893494) + | > loss_disc_real_2: 0.26030007004737854 (0.26030007004737854) + | > loss_disc_real_3: 0.2166004329919815 (0.2166004329919815) + | > loss_disc_real_4: 0.2039124071598053 (0.2039124071598053) + | > loss_disc_real_5: 0.23404338955879211 (0.23404338955879211) + | > loss_0: 2.488389492034912 (2.488389492034912) + | > loss_gen: 2.513599157333374 (2.513599157333374) + | > loss_kl: 2.4499261379241943 (2.4499261379241943) + | > loss_feat: 7.165637493133545 (7.165637493133545) + | > loss_mel: 18.30624771118164 (18.30624771118164) + | > loss_duration: 1.7519185543060303 (1.7519185543060303) + | > loss_1: 32.18732833862305 (32.18732833862305) + + --> STEP: 1 + | > loss_disc: 2.456890106201172 (2.456890106201172) + | > loss_disc_real_0: 0.17088830471038818 (0.17088830471038818) + | > loss_disc_real_1: 0.24693350493907928 (0.24693350493907928) + | > loss_disc_real_2: 0.2655053734779358 (0.2655053734779358) + | > loss_disc_real_3: 0.2611977756023407 (0.2611977756023407) + | > loss_disc_real_4: 0.25012508034706116 (0.25012508034706116) + | > loss_disc_real_5: 0.220587357878685 (0.220587357878685) + | > loss_0: 2.456890106201172 (2.456890106201172) + | > loss_gen: 2.648970603942871 (2.648970603942871) + | > loss_kl: 2.5965075492858887 (2.5965075492858887) + | > loss_feat: 7.3360795974731445 (7.3360795974731445) + | > loss_mel: 19.425827026367188 (19.425827026367188) + | > loss_duration: 1.7941296100616455 (1.7941296100616455) + | > loss_1: 33.801513671875 (33.801513671875) + + --> STEP: 2 + | > loss_disc: 2.4342312812805176 (2.4455606937408447) + | > loss_disc_real_0: 0.1732623279094696 (0.1720753163099289) + | > loss_disc_real_1: 0.2527408003807068 (0.24983715265989304) + | > loss_disc_real_2: 0.27329298853874207 (0.26939918100833893) + | > loss_disc_real_3: 0.211050346493721 (0.23612406104803085) + | > loss_disc_real_4: 0.22808577120304108 (0.23910542577505112) + | > loss_disc_real_5: 0.20492814481258392 (0.21275775134563446) + | > loss_0: 2.4342312812805176 (2.4455606937408447) + | > loss_gen: 2.5486297607421875 (2.5988001823425293) + | > loss_kl: 2.448654890060425 (2.5225812196731567) + | > loss_feat: 8.022314071655273 (7.679196834564209) + | > loss_mel: 18.70782470703125 (19.06682586669922) + | > loss_duration: 1.757002353668213 (1.7755659818649292) + | > loss_1: 33.48442840576172 (33.64297103881836) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036499500274658203 (+0.0003972053527832031) + | > avg_loss_disc: 2.4455606937408447 (+1.4901161193847656e-05) + | > avg_loss_disc_real_0: 0.1720753163099289 (-0.06140325218439102) + | > avg_loss_disc_real_1: 0.24983715265989304 (+0.068755604326725) + | > avg_loss_disc_real_2: 0.26939918100833893 (+0.03535524755716324) + | > avg_loss_disc_real_3: 0.23612406104803085 (+0.03619442880153656) + | > avg_loss_disc_real_4: 0.23910542577505112 (+0.016205057501792908) + | > avg_loss_disc_real_5: 0.21275775134563446 (+0.016073785722255707) + | > avg_loss_0: 2.4455606937408447 (+1.4901161193847656e-05) + | > avg_loss_gen: 2.5988001823425293 (+0.22627043724060059) + | > avg_loss_kl: 2.5225812196731567 (-0.18643057346343994) + | > avg_loss_feat: 7.679196834564209 (+0.2158656120300293) + | > avg_loss_mel: 19.06682586669922 (+0.9731378555297852) + | > avg_loss_duration: 1.7755659818649292 (-0.0007883906364440918) + | > avg_loss_1: 33.64297103881836 (+1.2280559539794922) + + + > EPOCH: 425/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:57:44)  + + --> TIME: 2025-04-19 05:57:51 -- STEP: 18/142 -- GLOBAL_STEP: 439650 + | > loss_disc: 2.4603757858276367 (2.442142062717014) + | > loss_disc_real_0: 0.10915012657642365 (0.12276913887924618) + | > loss_disc_real_1: 0.22963224351406097 (0.20349176559183332) + | > loss_disc_real_2: 0.2186369001865387 (0.22927340120077133) + | > loss_disc_real_3: 0.23945845663547516 (0.2290303102797932) + | > loss_disc_real_4: 0.22026845812797546 (0.23059482706917656) + | > loss_disc_real_5: 0.2310098558664322 (0.23133506294753817) + | > loss_0: 2.4603757858276367 (2.442142062717014) + | > grad_norm_0: tensor(10.0742, device='cuda:0') (tensor(9.6021, device='cuda:0')) + | > loss_gen: 2.3304283618927 (2.3760583930545383) + | > loss_kl: 1.7603230476379395 (1.786284327507019) + | > loss_feat: 7.379772186279297 (7.58348282178243) + | > loss_mel: 18.247325897216797 (18.04774538675944) + | > loss_duration: 1.3953723907470703 (1.4052797158559163) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.113222122192383 (31.198850525750053) + | > grad_norm_1: tensor(199.9553, device='cuda:0') (tensor(237.8514, device='cuda:0')) + | > current_lr_0: 0.00018965166472160772 + | > current_lr_1: 0.00018965166472160772 + | > step_time: 0.339 (0.32915867699517143) + | > loader_time: 0.0049 (0.0047588348388671875) + + + --> TIME: 2025-04-19 05:58:00 -- STEP: 43/142 -- GLOBAL_STEP: 439675 + | > loss_disc: 2.4439563751220703 (2.436582931252414) + | > loss_disc_real_0: 0.10356654226779938 (0.12026904610007308) + | > loss_disc_real_1: 0.22402015328407288 (0.20367400521455808) + | > loss_disc_real_2: 0.225042924284935 (0.22689514000748479) + | > loss_disc_real_3: 0.18363875150680542 (0.22683141051336778) + | > loss_disc_real_4: 0.24000683426856995 (0.2290222974017609) + | > loss_disc_real_5: 0.23016047477722168 (0.23167603480261426) + | > loss_0: 2.4439563751220703 (2.436582931252414) + | > grad_norm_0: tensor(7.0146, device='cuda:0') (tensor(9.4034, device='cuda:0')) + | > loss_gen: 2.30191969871521 (2.3799200889676113) + | > loss_kl: 1.9137747287750244 (1.805807354838349) + | > loss_feat: 7.2158002853393555 (7.344391235085421) + | > loss_mel: 17.21438217163086 (17.71673734797988) + | > loss_duration: 1.4584662914276123 (1.4219854848329412) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.104341506958008 (30.668841472891874) + | > grad_norm_1: tensor(225.1546, device='cuda:0') (tensor(245.7711, device='cuda:0')) + | > current_lr_0: 0.00018965166472160772 + | > current_lr_1: 0.00018965166472160772 + | > step_time: 0.3605 (0.34187802048616633) + | > loader_time: 0.0054 (0.0051670407139977744) + + + --> TIME: 2025-04-19 05:58:10 -- STEP: 68/142 -- GLOBAL_STEP: 439700 + | > loss_disc: 2.5379042625427246 (2.4596155916943276) + | > loss_disc_real_0: 0.12726938724517822 (0.12378088231472407) + | > loss_disc_real_1: 0.1895275115966797 (0.2073226873488987) + | > loss_disc_real_2: 0.2039727419614792 (0.22545978887116208) + | > loss_disc_real_3: 0.21438975632190704 (0.22937307178097613) + | > loss_disc_real_4: 0.21784596145153046 (0.2299268276375883) + | > loss_disc_real_5: 0.22012852132320404 (0.22892513652058208) + | > loss_0: 2.5379042625427246 (2.4596155916943276) + | > grad_norm_0: tensor(10.4811, device='cuda:0') (tensor(9.4479, device='cuda:0')) + | > loss_gen: 2.2088160514831543 (2.369389460367314) + | > loss_kl: 1.9827276468276978 (1.8258199726834017) + | > loss_feat: 6.852330684661865 (7.279087459339815) + | > loss_mel: 16.556774139404297 (17.72543486426858) + | > loss_duration: 1.4509847164154053 (1.4340363863636463) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.051633834838867 (30.633768249960507) + | > grad_norm_1: tensor(210.5762, device='cuda:0') (tensor(234.3241, device='cuda:0')) + | > current_lr_0: 0.00018965166472160772 + | > current_lr_1: 0.00018965166472160772 + | > step_time: 0.3886 (0.3540156623896431) + | > loader_time: 0.0063 (0.005502223968505859) + + + --> TIME: 2025-04-19 05:58:20 -- STEP: 93/142 -- GLOBAL_STEP: 439725 + | > loss_disc: 2.5680582523345947 (2.4574424630852163) + | > loss_disc_real_0: 0.1246788278222084 (0.12447451479652877) + | > loss_disc_real_1: 0.24362172186374664 (0.20794016283045533) + | > loss_disc_real_2: 0.28088995814323425 (0.22643702693523898) + | > loss_disc_real_3: 0.26282447576522827 (0.22948907700277144) + | > loss_disc_real_4: 0.24176689982414246 (0.22805334259104984) + | > loss_disc_real_5: 0.22007052600383759 (0.2302804590873821) + | > loss_0: 2.5680582523345947 (2.4574424630852163) + | > grad_norm_0: tensor(5.0807, device='cuda:0') (tensor(8.6600, device='cuda:0')) + | > loss_gen: 2.4520466327667236 (2.365904256861697) + | > loss_kl: 1.6215553283691406 (1.8413539137891544) + | > loss_feat: 6.44148588180542 (7.226707417477844) + | > loss_mel: 17.830474853515625 (17.713633721874597) + | > loss_duration: 1.4685683250427246 (1.4446743675457536) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.814132690429688 (30.592273691649076) + | > grad_norm_1: tensor(55.5791, device='cuda:0') (tensor(212.9588, device='cuda:0')) + | > current_lr_0: 0.00018965166472160772 + | > current_lr_1: 0.00018965166472160772 + | > step_time: 0.3982 (0.3652903649114793) + | > loader_time: 0.0065 (0.0057269245065668575) + + + --> TIME: 2025-04-19 05:58:31 -- STEP: 118/142 -- GLOBAL_STEP: 439750 + | > loss_disc: 2.5472559928894043 (2.456563189878303) + | > loss_disc_real_0: 0.1417730301618576 (0.12443210923318138) + | > loss_disc_real_1: 0.2537541091442108 (0.20736651892884303) + | > loss_disc_real_2: 0.238078773021698 (0.22684828343532853) + | > loss_disc_real_3: 0.21461355686187744 (0.22958923989938476) + | > loss_disc_real_4: 0.22094392776489258 (0.22887823490773218) + | > loss_disc_real_5: 0.21727998554706573 (0.22999436029438244) + | > loss_0: 2.5472559928894043 (2.456563189878303) + | > grad_norm_0: tensor(9.5181, device='cuda:0') (tensor(8.4692, device='cuda:0')) + | > loss_gen: 2.2053723335266113 (2.36009901054835) + | > loss_kl: 1.8311097621917725 (1.8422541466809936) + | > loss_feat: 6.2348833084106445 (7.192373263633858) + | > loss_mel: 15.852813720703125 (17.71055788104816) + | > loss_duration: 1.4507418870925903 (1.4521033834602877) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.574922561645508 (30.55738770759712) + | > grad_norm_1: tensor(74.9825, device='cuda:0') (tensor(204.7008, device='cuda:0')) + | > current_lr_0: 0.00018965166472160772 + | > current_lr_1: 0.00018965166472160772 + | > step_time: 0.4409 (0.3773566241991722) + | > loader_time: 0.0068 (0.006021576412653519) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4135634899139404 (2.4135634899139404) + | > loss_disc_real_0: 0.12840251624584198 (0.12840251624584198) + | > loss_disc_real_1: 0.24745230376720428 (0.24745230376720428) + | > loss_disc_real_2: 0.24070963263511658 (0.24070963263511658) + | > loss_disc_real_3: 0.2765868604183197 (0.2765868604183197) + | > loss_disc_real_4: 0.22595694661140442 (0.22595694661140442) + | > loss_disc_real_5: 0.2582848370075226 (0.2582848370075226) + | > loss_0: 2.4135634899139404 (2.4135634899139404) + | > loss_gen: 2.548696279525757 (2.548696279525757) + | > loss_kl: 2.4966399669647217 (2.4966399669647217) + | > loss_feat: 7.284962177276611 (7.284962177276611) + | > loss_mel: 19.46691131591797 (19.46691131591797) + | > loss_duration: 1.7653696537017822 (1.7653696537017822) + | > loss_1: 33.56258010864258 (33.56258010864258) + + --> STEP: 1 + | > loss_disc: 2.4775774478912354 (2.4775774478912354) + | > loss_disc_real_0: 0.18454787135124207 (0.18454787135124207) + | > loss_disc_real_1: 0.25021860003471375 (0.25021860003471375) + | > loss_disc_real_2: 0.2137179672718048 (0.2137179672718048) + | > loss_disc_real_3: 0.25221753120422363 (0.25221753120422363) + | > loss_disc_real_4: 0.22481568157672882 (0.22481568157672882) + | > loss_disc_real_5: 0.22360315918922424 (0.22360315918922424) + | > loss_0: 2.4775774478912354 (2.4775774478912354) + | > loss_gen: 2.395413875579834 (2.395413875579834) + | > loss_kl: 2.6316983699798584 (2.6316983699798584) + | > loss_feat: 6.613283634185791 (6.613283634185791) + | > loss_mel: 16.848358154296875 (16.848358154296875) + | > loss_duration: 1.8218766450881958 (1.8218766450881958) + | > loss_1: 30.310630798339844 (30.310630798339844) + + --> STEP: 2 + | > loss_disc: 2.495631694793701 (2.4866045713424683) + | > loss_disc_real_0: 0.1591968983411789 (0.17187238484621048) + | > loss_disc_real_1: 0.2524273097515106 (0.2513229548931122) + | > loss_disc_real_2: 0.24092933535575867 (0.22732365131378174) + | > loss_disc_real_3: 0.23897702991962433 (0.24559728056192398) + | > loss_disc_real_4: 0.24499964714050293 (0.23490766435861588) + | > loss_disc_real_5: 0.23444022238254547 (0.22902169078588486) + | > loss_0: 2.495631694793701 (2.4866045713424683) + | > loss_gen: 2.286163330078125 (2.3407886028289795) + | > loss_kl: 2.170440196990967 (2.4010692834854126) + | > loss_feat: 6.149054527282715 (6.381169080734253) + | > loss_mel: 16.10565948486328 (16.477008819580078) + | > loss_duration: 1.718637228012085 (1.7702569365501404) + | > loss_1: 28.429954528808594 (29.37029266357422) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035048723220825195 (-0.00014507770538330078) + | > avg_loss_disc: 2.4866045713424683 (+0.041043877601623535) + | > avg_loss_disc_real_0: 0.17187238484621048 (-0.0002029314637184143) + | > avg_loss_disc_real_1: 0.2513229548931122 (+0.0014858022332191467) + | > avg_loss_disc_real_2: 0.22732365131378174 (-0.04207552969455719) + | > avg_loss_disc_real_3: 0.24559728056192398 (+0.009473219513893127) + | > avg_loss_disc_real_4: 0.23490766435861588 (-0.004197761416435242) + | > avg_loss_disc_real_5: 0.22902169078588486 (+0.016263939440250397) + | > avg_loss_0: 2.4866045713424683 (+0.041043877601623535) + | > avg_loss_gen: 2.3407886028289795 (-0.2580115795135498) + | > avg_loss_kl: 2.4010692834854126 (-0.12151193618774414) + | > avg_loss_feat: 6.381169080734253 (-1.298027753829956) + | > avg_loss_mel: 16.477008819580078 (-2.5898170471191406) + | > avg_loss_duration: 1.7702569365501404 (-0.005309045314788818) + | > avg_loss_1: 29.37029266357422 (-4.272678375244141) + + + > EPOCH: 426/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:58:43)  + + --> TIME: 2025-04-19 05:58:44 -- STEP: 1/142 -- GLOBAL_STEP: 439775 + | > loss_disc: 2.3857545852661133 (2.3857545852661133) + | > loss_disc_real_0: 0.1253771185874939 (0.1253771185874939) + | > loss_disc_real_1: 0.24401432275772095 (0.24401432275772095) + | > loss_disc_real_2: 0.22053277492523193 (0.22053277492523193) + | > loss_disc_real_3: 0.23001164197921753 (0.23001164197921753) + | > loss_disc_real_4: 0.25425511598587036 (0.25425511598587036) + | > loss_disc_real_5: 0.22312991321086884 (0.22312991321086884) + | > loss_0: 2.3857545852661133 (2.3857545852661133) + | > grad_norm_0: tensor(8.8774, device='cuda:0') (tensor(8.8774, device='cuda:0')) + | > loss_gen: 2.5534281730651855 (2.5534281730651855) + | > loss_kl: 1.8621220588684082 (1.8621220588684082) + | > loss_feat: 7.679311752319336 (7.679311752319336) + | > loss_mel: 17.203174591064453 (17.203174591064453) + | > loss_duration: 1.392332673072815 (1.392332673072815) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.69036865234375 (30.69036865234375) + | > grad_norm_1: tensor(378.1392, device='cuda:0') (tensor(378.1392, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.3331 (0.33314990997314453) + | > loader_time: 0.0041 (0.004099369049072266) + + + --> TIME: 2025-04-19 05:58:53 -- STEP: 26/142 -- GLOBAL_STEP: 439800 + | > loss_disc: 2.286959409713745 (2.402924262560331) + | > loss_disc_real_0: 0.07975131273269653 (0.11523379643376057) + | > loss_disc_real_1: 0.2135012000799179 (0.20072659448935434) + | > loss_disc_real_2: 0.2436937689781189 (0.22377279916634926) + | > loss_disc_real_3: 0.24726440012454987 (0.22661094826001388) + | > loss_disc_real_4: 0.20534102618694305 (0.2226391669649344) + | > loss_disc_real_5: 0.24963758885860443 (0.2305365611727421) + | > loss_0: 2.286959409713745 (2.402924262560331) + | > grad_norm_0: tensor(3.7648, device='cuda:0') (tensor(8.4731, device='cuda:0')) + | > loss_gen: 2.6312484741210938 (2.410672279504629) + | > loss_kl: 1.6205525398254395 (1.786617636680603) + | > loss_feat: 8.478099822998047 (7.288504031988291) + | > loss_mel: 18.957042694091797 (17.67124656530527) + | > loss_duration: 1.4404377937316895 (1.4191642632851234) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.12738037109375 (30.57620466672457) + | > grad_norm_1: tensor(130.4312, device='cuda:0') (tensor(235.4351, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.3494 (0.33648016819587123) + | > loader_time: 0.0053 (0.004869378530062162) + + + --> TIME: 2025-04-19 05:59:02 -- STEP: 51/142 -- GLOBAL_STEP: 439825 + | > loss_disc: 2.5033891201019287 (2.4263103195265217) + | > loss_disc_real_0: 0.18990373611450195 (0.11898901021363688) + | > loss_disc_real_1: 0.21153128147125244 (0.20515670057605295) + | > loss_disc_real_2: 0.23087605834007263 (0.22483470714559742) + | > loss_disc_real_3: 0.23447224497795105 (0.22748384376366934) + | > loss_disc_real_4: 0.2584465742111206 (0.22520100252301084) + | > loss_disc_real_5: 0.22458906471729279 (0.23035589766268635) + | > loss_0: 2.5033891201019287 (2.4263103195265217) + | > grad_norm_0: tensor(16.5078, device='cuda:0') (tensor(9.9465, device='cuda:0')) + | > loss_gen: 2.307819128036499 (2.402863628724041) + | > loss_kl: 1.7261658906936646 (1.794182305242501) + | > loss_feat: 6.537235736846924 (7.275382088679893) + | > loss_mel: 16.938156127929688 (17.707148103153003) + | > loss_duration: 1.4362170696258545 (1.4267088198194318) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.945592880249023 (30.60628494561887) + | > grad_norm_1: tensor(383.4755, device='cuda:0') (tensor(246.7428, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.3689 (0.34721100096609087) + | > loader_time: 0.0062 (0.005280373143214805) + + + --> TIME: 2025-04-19 05:59:12 -- STEP: 76/142 -- GLOBAL_STEP: 439850 + | > loss_disc: 2.4118502140045166 (2.43324002152995) + | > loss_disc_real_0: 0.12614919245243073 (0.12007446905696079) + | > loss_disc_real_1: 0.18341588973999023 (0.20387912188705648) + | > loss_disc_real_2: 0.23199467360973358 (0.2252190809108709) + | > loss_disc_real_3: 0.23771032691001892 (0.2271762492233201) + | > loss_disc_real_4: 0.24616745114326477 (0.22486155460539617) + | > loss_disc_real_5: 0.24284210801124573 (0.2313673872696726) + | > loss_0: 2.4118502140045166 (2.43324002152995) + | > grad_norm_0: tensor(4.0558, device='cuda:0') (tensor(10.6334, device='cuda:0')) + | > loss_gen: 2.3860440254211426 (2.3806804889126822) + | > loss_kl: 1.852310299873352 (1.8045776298171596) + | > loss_feat: 6.783014297485352 (7.172502172620673) + | > loss_mel: 16.768205642700195 (17.61349754584463) + | > loss_duration: 1.483957052230835 (1.4373302396975065) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.273530960083008 (30.408587982780055) + | > grad_norm_1: tensor(50.3765, device='cuda:0') (tensor(251.3424, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.3928 (0.35911810711810466) + | > loader_time: 0.0066 (0.005603667936827007) + + + --> TIME: 2025-04-19 05:59:22 -- STEP: 101/142 -- GLOBAL_STEP: 439875 + | > loss_disc: 2.3500733375549316 (2.4385459706334784) + | > loss_disc_real_0: 0.11481155455112457 (0.12199531336969668) + | > loss_disc_real_1: 0.22080567479133606 (0.20512696864581348) + | > loss_disc_real_2: 0.23704411089420319 (0.2256205346029584) + | > loss_disc_real_3: 0.23865588009357452 (0.2275930165064217) + | > loss_disc_real_4: 0.2088862806558609 (0.22572338787635954) + | > loss_disc_real_5: 0.21863475441932678 (0.23077892962068614) + | > loss_0: 2.3500733375549316 (2.4385459706334784) + | > grad_norm_0: tensor(11.1607, device='cuda:0') (tensor(10.3574, device='cuda:0')) + | > loss_gen: 2.499891519546509 (2.377602194795514) + | > loss_kl: 2.057313919067383 (1.8203323123478654) + | > loss_feat: 7.7312750816345215 (7.143095786028569) + | > loss_mel: 18.15369415283203 (17.532899790471145) + | > loss_duration: 1.4902565479278564 (1.4475119668658423) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.932432174682617 (30.321442103621983) + | > grad_norm_1: tensor(368.0859, device='cuda:0') (tensor(256.2068, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.4144 (0.3693913705278151) + | > loader_time: 0.007 (0.005882881655551419) + + + --> TIME: 2025-04-19 05:59:33 -- STEP: 126/142 -- GLOBAL_STEP: 439900 + | > loss_disc: 2.567779541015625 (2.4454886780844793) + | > loss_disc_real_0: 0.12472102791070938 (0.12306801084842947) + | > loss_disc_real_1: 0.23801277577877045 (0.20524440788560447) + | > loss_disc_real_2: 0.208776593208313 (0.22597513265079922) + | > loss_disc_real_3: 0.2147335708141327 (0.2279553016026815) + | > loss_disc_real_4: 0.23254181444644928 (0.22651149261565434) + | > loss_disc_real_5: 0.2656109631061554 (0.23114909267141706) + | > loss_0: 2.567779541015625 (2.4454886780844793) + | > grad_norm_0: tensor(12.3837, device='cuda:0') (tensor(10.2518, device='cuda:0')) + | > loss_gen: 2.367161273956299 (2.368413166394309) + | > loss_kl: 1.7598729133605957 (1.825835062397851) + | > loss_feat: 7.633358001708984 (7.107832117686196) + | > loss_mel: 18.100889205932617 (17.47714956979903) + | > loss_duration: 1.482985496520996 (1.4543801706934736) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.34426498413086 (30.23361016833593) + | > grad_norm_1: tensor(287.9781, device='cuda:0') (tensor(248.7267, device='cuda:0')) + | > current_lr_0: 0.00018962795826351752 + | > current_lr_1: 0.00018962795826351752 + | > step_time: 0.4385 (0.38132460533626494) + | > loader_time: 0.0076 (0.006147068644326831) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3828272819519043 (2.3828272819519043) + | > loss_disc_real_0: 0.11795399338006973 (0.11795399338006973) + | > loss_disc_real_1: 0.2048659473657608 (0.2048659473657608) + | > loss_disc_real_2: 0.20704537630081177 (0.20704537630081177) + | > loss_disc_real_3: 0.22498726844787598 (0.22498726844787598) + | > loss_disc_real_4: 0.24535664916038513 (0.24535664916038513) + | > loss_disc_real_5: 0.20746898651123047 (0.20746898651123047) + | > loss_0: 2.3828272819519043 (2.3828272819519043) + | > loss_gen: 2.3551082611083984 (2.3551082611083984) + | > loss_kl: 2.3652918338775635 (2.3652918338775635) + | > loss_feat: 7.843194961547852 (7.843194961547852) + | > loss_mel: 20.253501892089844 (20.253501892089844) + | > loss_duration: 1.7536301612854004 (1.7536301612854004) + | > loss_1: 34.57072830200195 (34.57072830200195) + + --> STEP: 1 + | > loss_disc: 2.387396812438965 (2.387396812438965) + | > loss_disc_real_0: 0.14306913316249847 (0.14306913316249847) + | > loss_disc_real_1: 0.1783173680305481 (0.1783173680305481) + | > loss_disc_real_2: 0.24695269763469696 (0.24695269763469696) + | > loss_disc_real_3: 0.20913070440292358 (0.20913070440292358) + | > loss_disc_real_4: 0.23414044082164764 (0.23414044082164764) + | > loss_disc_real_5: 0.2042686939239502 (0.2042686939239502) + | > loss_0: 2.387396812438965 (2.387396812438965) + | > loss_gen: 2.2789855003356934 (2.2789855003356934) + | > loss_kl: 2.7118403911590576 (2.7118403911590576) + | > loss_feat: 7.112532138824463 (7.112532138824463) + | > loss_mel: 19.06951141357422 (19.06951141357422) + | > loss_duration: 1.7979280948638916 (1.7979280948638916) + | > loss_1: 32.97079849243164 (32.97079849243164) + + --> STEP: 2 + | > loss_disc: 2.507518768310547 (2.447457790374756) + | > loss_disc_real_0: 0.21482238173484802 (0.17894575744867325) + | > loss_disc_real_1: 0.2563067078590393 (0.2173120379447937) + | > loss_disc_real_2: 0.21251904964447021 (0.2297358736395836) + | > loss_disc_real_3: 0.20968596637248993 (0.20940833538770676) + | > loss_disc_real_4: 0.2678612470626831 (0.2510008439421654) + | > loss_disc_real_5: 0.19365358352661133 (0.19896113872528076) + | > loss_0: 2.507518768310547 (2.447457790374756) + | > loss_gen: 2.425740957260132 (2.3523632287979126) + | > loss_kl: 2.408958673477173 (2.5603995323181152) + | > loss_feat: 7.9946746826171875 (7.553603410720825) + | > loss_mel: 18.00832176208496 (18.53891658782959) + | > loss_duration: 1.7513577938079834 (1.7746429443359375) + | > loss_1: 32.589054107666016 (32.77992630004883) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035511255264282227 (+4.6253204345703125e-05) + | > avg_loss_disc: 2.447457790374756 (-0.0391467809677124) + | > avg_loss_disc_real_0: 0.17894575744867325 (+0.0070733726024627686) + | > avg_loss_disc_real_1: 0.2173120379447937 (-0.03401091694831848) + | > avg_loss_disc_real_2: 0.2297358736395836 (+0.0024122223258018494) + | > avg_loss_disc_real_3: 0.20940833538770676 (-0.036188945174217224) + | > avg_loss_disc_real_4: 0.2510008439421654 (+0.0160931795835495) + | > avg_loss_disc_real_5: 0.19896113872528076 (-0.030060552060604095) + | > avg_loss_0: 2.447457790374756 (-0.0391467809677124) + | > avg_loss_gen: 2.3523632287979126 (+0.011574625968933105) + | > avg_loss_kl: 2.5603995323181152 (+0.15933024883270264) + | > avg_loss_feat: 7.553603410720825 (+1.1724343299865723) + | > avg_loss_mel: 18.53891658782959 (+2.0619077682495117) + | > avg_loss_duration: 1.7746429443359375 (+0.004386007785797119) + | > avg_loss_1: 32.77992630004883 (+3.4096336364746094) + + + > EPOCH: 427/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 05:59:42)  + + --> TIME: 2025-04-19 05:59:45 -- STEP: 9/142 -- GLOBAL_STEP: 439925 + | > loss_disc: 2.4007959365844727 (2.371192773183187) + | > loss_disc_real_0: 0.13338342308998108 (0.11813275929954317) + | > loss_disc_real_1: 0.21143729984760284 (0.2088758879237705) + | > loss_disc_real_2: 0.23780864477157593 (0.22298728426297507) + | > loss_disc_real_3: 0.24830856919288635 (0.227897047996521) + | > loss_disc_real_4: 0.1995057463645935 (0.22291480004787445) + | > loss_disc_real_5: 0.2850582003593445 (0.23153011169698504) + | > loss_0: 2.4007959365844727 (2.371192773183187) + | > grad_norm_0: tensor(10.3568, device='cuda:0') (tensor(10.7593, device='cuda:0')) + | > loss_gen: 2.531850814819336 (2.4908674822913275) + | > loss_kl: 1.7949455976486206 (1.7738181087705824) + | > loss_feat: 7.436387062072754 (7.781725618574354) + | > loss_mel: 18.11306381225586 (18.30239020453559) + | > loss_duration: 1.3724958896636963 (1.4051977660920885) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.248743057250977 (31.753998862372505) + | > grad_norm_1: tensor(319.2368, device='cuda:0') (tensor(301.2101, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.3222 (0.32575390073988175) + | > loader_time: 0.0049 (0.004513872994316949) + + + --> TIME: 2025-04-19 05:59:54 -- STEP: 34/142 -- GLOBAL_STEP: 439950 + | > loss_disc: 2.4071788787841797 (2.4022658712723675) + | > loss_disc_real_0: 0.1462470293045044 (0.11518152562134407) + | > loss_disc_real_1: 0.1933649480342865 (0.20590642532881567) + | > loss_disc_real_2: 0.2573159337043762 (0.22359702034908183) + | > loss_disc_real_3: 0.23618385195732117 (0.22488311196074767) + | > loss_disc_real_4: 0.2638396620750427 (0.2257224877967554) + | > loss_disc_real_5: 0.21630741655826569 (0.22690319751991944) + | > loss_0: 2.4071788787841797 (2.4022658712723675) + | > grad_norm_0: tensor(4.7292, device='cuda:0') (tensor(9.0130, device='cuda:0')) + | > loss_gen: 2.407823085784912 (2.423370852189905) + | > loss_kl: 1.833229899406433 (1.7680419753579533) + | > loss_feat: 7.776213645935059 (7.395261988920324) + | > loss_mel: 17.6201229095459 (17.563325966105737) + | > loss_duration: 1.4410972595214844 (1.4155170496772318) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.078487396240234 (30.565517818226535) + | > grad_norm_1: tensor(74.5139, device='cuda:0') (tensor(248.4266, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.357 (0.33937758557936726) + | > loader_time: 0.0062 (0.005110824809354895) + + + --> TIME: 2025-04-19 06:00:04 -- STEP: 59/142 -- GLOBAL_STEP: 439975 + | > loss_disc: 2.4509060382843018 (2.414581593820604) + | > loss_disc_real_0: 0.12448316812515259 (0.11747900459725978) + | > loss_disc_real_1: 0.21405957639217377 (0.20637546833288872) + | > loss_disc_real_2: 0.22708354890346527 (0.22263099126896616) + | > loss_disc_real_3: 0.2150365561246872 (0.2260553015995834) + | > loss_disc_real_4: 0.2430621087551117 (0.22821315062248101) + | > loss_disc_real_5: 0.19824542105197906 (0.22606868233721136) + | > loss_0: 2.4509060382843018 (2.414581593820604) + | > grad_norm_0: tensor(6.9598, device='cuda:0') (tensor(9.1726, device='cuda:0')) + | > loss_gen: 2.2808432579040527 (2.405703285993155) + | > loss_kl: 1.7823059558868408 (1.7961847155781117) + | > loss_feat: 6.877370357513428 (7.359670655202057) + | > loss_mel: 17.261001586914062 (17.623828936431373) + | > loss_duration: 1.4427502155303955 (1.4290836362515462) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.644271850585938 (30.61447104761156) + | > grad_norm_1: tensor(368.4877, device='cuda:0') (tensor(248.3177, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.3737 (0.35022280175807113) + | > loader_time: 0.0064 (0.00549893460031283) + + + --> TIME: 2025-04-19 06:00:14 -- STEP: 84/142 -- GLOBAL_STEP: 440000 + | > loss_disc: 2.2872698307037354 (2.430795368694124) + | > loss_disc_real_0: 0.06903886795043945 (0.11899293311649844) + | > loss_disc_real_1: 0.19069281220436096 (0.20804885455540248) + | > loss_disc_real_2: 0.22103963792324066 (0.22366073284120785) + | > loss_disc_real_3: 0.19295282661914825 (0.22670808381267957) + | > loss_disc_real_4: 0.1864759474992752 (0.2281217257536593) + | > loss_disc_real_5: 0.23083485662937164 (0.22648139138306891) + | > loss_0: 2.2872698307037354 (2.430795368694124) + | > grad_norm_0: tensor(6.3450, device='cuda:0') (tensor(8.8716, device='cuda:0')) + | > loss_gen: 2.4896068572998047 (2.393978448141188) + | > loss_kl: 2.013338565826416 (1.8221132187616258) + | > loss_feat: 8.12926959991455 (7.2939625183741255) + | > loss_mel: 18.765172958374023 (17.608945437840053) + | > loss_duration: 1.4843547344207764 (1.4403866245633072) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.881744384765625 (30.559386230650404) + | > grad_norm_1: tensor(99.1917, device='cuda:0') (tensor(234.5532, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.3963 (0.36245971350442807) + | > loader_time: 0.0063 (0.005786208879379998) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_440000.pth + + --> TIME: 2025-04-19 06:00:25 -- STEP: 109/142 -- GLOBAL_STEP: 440025 + | > loss_disc: 2.482264995574951 (2.4421740256318247) + | > loss_disc_real_0: 0.10891934484243393 (0.12104102545375124) + | > loss_disc_real_1: 0.2324378937482834 (0.20818662096601015) + | > loss_disc_real_2: 0.25689077377319336 (0.22516081188250026) + | > loss_disc_real_3: 0.2243291586637497 (0.22726030292314128) + | > loss_disc_real_4: 0.20520861446857452 (0.2298255811590667) + | > loss_disc_real_5: 0.2351914346218109 (0.22791579379401075) + | > loss_0: 2.482264995574951 (2.4421740256318247) + | > grad_norm_0: tensor(15.8882, device='cuda:0') (tensor(9.1853, device='cuda:0')) + | > loss_gen: 2.2960221767425537 (2.375868022988694) + | > loss_kl: 1.9120726585388184 (1.8405015709203318) + | > loss_feat: 5.900623798370361 (7.20377467969142) + | > loss_mel: 15.756509780883789 (17.498182401744586) + | > loss_duration: 1.494591236114502 (1.44886444139918) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.359819412231445 (30.36719110471393) + | > grad_norm_1: tensor(338.8033, device='cuda:0') (tensor(238.5572, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.4129 (0.37500577891638526) + | > loader_time: 0.007 (0.006026167388356059) + + + --> TIME: 2025-04-19 06:00:37 -- STEP: 134/142 -- GLOBAL_STEP: 440050 + | > loss_disc: 2.539417266845703 (2.4466043408237277) + | > loss_disc_real_0: 0.1401004046201706 (0.1224577107202651) + | > loss_disc_real_1: 0.2189289927482605 (0.20765478909015656) + | > loss_disc_real_2: 0.2588607668876648 (0.22579088395656044) + | > loss_disc_real_3: 0.21398033201694489 (0.22813245487302097) + | > loss_disc_real_4: 0.27741798758506775 (0.22975133192628178) + | > loss_disc_real_5: 0.25380805134773254 (0.22917093770272695) + | > loss_0: 2.539417266845703 (2.4466043408237277) + | > grad_norm_0: tensor(6.8776, device='cuda:0') (tensor(9.4769, device='cuda:0')) + | > loss_gen: 2.3402938842773438 (2.370497838774722) + | > loss_kl: 1.845948338508606 (1.848787166289429) + | > loss_feat: 6.6391777992248535 (7.1587431857835) + | > loss_mel: 16.131879806518555 (17.426670515715173) + | > loss_duration: 1.5146632194519043 (1.4555153846740718) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.47196388244629 (30.260214136607612) + | > grad_norm_1: tensor(382.7695, device='cuda:0') (tensor(238.5764, device='cuda:0')) + | > current_lr_0: 0.00018960425476873457 + | > current_lr_1: 0.00018960425476873457 + | > step_time: 0.4531 (0.3867815092428407) + | > loader_time: 0.0076 (0.006253313662400885) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3636059761047363 (2.3636059761047363) + | > loss_disc_real_0: 0.1126699224114418 (0.1126699224114418) + | > loss_disc_real_1: 0.23605550825595856 (0.23605550825595856) + | > loss_disc_real_2: 0.24871088564395905 (0.24871088564395905) + | > loss_disc_real_3: 0.20075932145118713 (0.20075932145118713) + | > loss_disc_real_4: 0.26823320984840393 (0.26823320984840393) + | > loss_disc_real_5: 0.27992621064186096 (0.27992621064186096) + | > loss_0: 2.3636059761047363 (2.3636059761047363) + | > loss_gen: 2.6102519035339355 (2.6102519035339355) + | > loss_kl: 2.497999668121338 (2.497999668121338) + | > loss_feat: 7.959463119506836 (7.959463119506836) + | > loss_mel: 19.77345085144043 (19.77345085144043) + | > loss_duration: 1.7453105449676514 (1.7453105449676514) + | > loss_1: 34.58647537231445 (34.58647537231445) + + --> STEP: 1 + | > loss_disc: 2.680739164352417 (2.680739164352417) + | > loss_disc_real_0: 0.13317446410655975 (0.13317446410655975) + | > loss_disc_real_1: 0.25768834352493286 (0.25768834352493286) + | > loss_disc_real_2: 0.2690601646900177 (0.2690601646900177) + | > loss_disc_real_3: 0.22656990587711334 (0.22656990587711334) + | > loss_disc_real_4: 0.2658006250858307 (0.2658006250858307) + | > loss_disc_real_5: 0.27151012420654297 (0.27151012420654297) + | > loss_0: 2.680739164352417 (2.680739164352417) + | > loss_gen: 2.2651724815368652 (2.2651724815368652) + | > loss_kl: 2.7772140502929688 (2.7772140502929688) + | > loss_feat: 8.068859100341797 (8.068859100341797) + | > loss_mel: 19.9111328125 (19.9111328125) + | > loss_duration: 1.803483486175537 (1.803483486175537) + | > loss_1: 34.82585906982422 (34.82585906982422) + + --> STEP: 2 + | > loss_disc: 2.575277805328369 (2.628008484840393) + | > loss_disc_real_0: 0.1886606365442276 (0.16091755032539368) + | > loss_disc_real_1: 0.2151646465063095 (0.23642649501562119) + | > loss_disc_real_2: 0.26241087913513184 (0.26573552191257477) + | > loss_disc_real_3: 0.2268654704093933 (0.22671768814325333) + | > loss_disc_real_4: 0.24516098201274872 (0.2554808035492897) + | > loss_disc_real_5: 0.3137242794036865 (0.29261720180511475) + | > loss_0: 2.575277805328369 (2.628008484840393) + | > loss_gen: 2.249058246612549 (2.257115364074707) + | > loss_kl: 2.288911819458008 (2.5330629348754883) + | > loss_feat: 6.346095085144043 (7.20747709274292) + | > loss_mel: 15.987064361572266 (17.949098587036133) + | > loss_duration: 1.7233644723892212 (1.7634239792823792) + | > loss_1: 28.594491958618164 (31.71017551422119) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032312870025634766 (-0.0003198385238647461) + | > avg_loss_disc: 2.628008484840393 (+0.1805506944656372) + | > avg_loss_disc_real_0: 0.16091755032539368 (-0.01802820712327957) + | > avg_loss_disc_real_1: 0.23642649501562119 (+0.019114457070827484) + | > avg_loss_disc_real_2: 0.26573552191257477 (+0.03599964827299118) + | > avg_loss_disc_real_3: 0.22671768814325333 (+0.01730935275554657) + | > avg_loss_disc_real_4: 0.2554808035492897 (+0.004479959607124329) + | > avg_loss_disc_real_5: 0.29261720180511475 (+0.09365606307983398) + | > avg_loss_0: 2.628008484840393 (+0.1805506944656372) + | > avg_loss_gen: 2.257115364074707 (-0.09524786472320557) + | > avg_loss_kl: 2.5330629348754883 (-0.027336597442626953) + | > avg_loss_feat: 7.20747709274292 (-0.3461263179779053) + | > avg_loss_mel: 17.949098587036133 (-0.589818000793457) + | > avg_loss_duration: 1.7634239792823792 (-0.01121896505355835) + | > avg_loss_1: 31.71017551422119 (-1.0697507858276367) + + + > EPOCH: 428/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:00:42)  + + --> TIME: 2025-04-19 06:00:48 -- STEP: 17/142 -- GLOBAL_STEP: 440075 + | > loss_disc: 2.4227466583251953 (2.4845860565409943) + | > loss_disc_real_0: 0.1897604763507843 (0.12759663296096466) + | > loss_disc_real_1: 0.2429683655500412 (0.20929786650573506) + | > loss_disc_real_2: 0.2316310703754425 (0.2326425199999529) + | > loss_disc_real_3: 0.19441655278205872 (0.2294518877478207) + | > loss_disc_real_4: 0.22728583216667175 (0.22810826494413264) + | > loss_disc_real_5: 0.1681688278913498 (0.22664522774079265) + | > loss_0: 2.4227466583251953 (2.4845860565409943) + | > grad_norm_0: tensor(8.6442, device='cuda:0') (tensor(13.3518, device='cuda:0')) + | > loss_gen: 2.467604398727417 (2.3483199372011074) + | > loss_kl: 1.8579968214035034 (1.78110250304727) + | > loss_feat: 7.1230926513671875 (7.107099224539364) + | > loss_mel: 17.07395362854004 (17.429290715385886) + | > loss_duration: 1.4017060995101929 (1.4056082122466143) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.924352645874023 (30.07142022076775) + | > grad_norm_1: tensor(231.7632, device='cuda:0') (tensor(282.0452, device='cuda:0')) + | > current_lr_0: 0.00018958055423688846 + | > current_lr_1: 0.00018958055423688846 + | > step_time: 0.3361 (0.329590544981115) + | > loader_time: 0.0051 (0.005017210455501781) + + + --> TIME: 2025-04-19 06:00:57 -- STEP: 42/142 -- GLOBAL_STEP: 440100 + | > loss_disc: 2.4749302864074707 (2.4342483736219864) + | > loss_disc_real_0: 0.08781197667121887 (0.11932244648536046) + | > loss_disc_real_1: 0.1990366131067276 (0.2039651462719554) + | > loss_disc_real_2: 0.2062552124261856 (0.22510498655693872) + | > loss_disc_real_3: 0.2408323884010315 (0.2275148066026824) + | > loss_disc_real_4: 0.21383526921272278 (0.22567538775148846) + | > loss_disc_real_5: 0.24921689927577972 (0.22722877916835604) + | > loss_0: 2.4749302864074707 (2.4342483736219864) + | > grad_norm_0: tensor(11.4802, device='cuda:0') (tensor(11.2049, device='cuda:0')) + | > loss_gen: 2.463718891143799 (2.3700248740968255) + | > loss_kl: 1.804666519165039 (1.7965268804913475) + | > loss_feat: 7.116886615753174 (7.224624191011701) + | > loss_mel: 16.53807258605957 (17.436989216577437) + | > loss_duration: 1.4672904014587402 (1.4203429505938578) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.390634536743164 (30.248507999238512) + | > grad_norm_1: tensor(176.9461, device='cuda:0') (tensor(271.5651, device='cuda:0')) + | > current_lr_0: 0.00018958055423688846 + | > current_lr_1: 0.00018958055423688846 + | > step_time: 0.3672 (0.34289184638432096) + | > loader_time: 0.0059 (0.005288992609296527) + + + --> TIME: 2025-04-19 06:01:07 -- STEP: 67/142 -- GLOBAL_STEP: 440125 + | > loss_disc: 2.4102628231048584 (2.443599469626128) + | > loss_disc_real_0: 0.13516049087047577 (0.12257737099234738) + | > loss_disc_real_1: 0.19581849873065948 (0.20715725933437917) + | > loss_disc_real_2: 0.18918131291866302 (0.22447475635293704) + | > loss_disc_real_3: 0.19654305279254913 (0.22799515857625363) + | > loss_disc_real_4: 0.16852721571922302 (0.22543652667038477) + | > loss_disc_real_5: 0.2368624359369278 (0.22785598239791927) + | > loss_0: 2.4102628231048584 (2.443599469626128) + | > grad_norm_0: tensor(14.6817, device='cuda:0') (tensor(10.7205, device='cuda:0')) + | > loss_gen: 2.3773529529571533 (2.3669451172672105) + | > loss_kl: 1.7582601308822632 (1.813133060042538) + | > loss_feat: 7.357808589935303 (7.2168760015003715) + | > loss_mel: 18.3988094329834 (17.559866649001385) + | > loss_duration: 1.410314679145813 (1.431078951750229) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.30254554748535 (30.38789974041839) + | > grad_norm_1: tensor(437.2322, device='cuda:0') (tensor(249.9431, device='cuda:0')) + | > current_lr_0: 0.00018958055423688846 + | > current_lr_1: 0.00018958055423688846 + | > step_time: 0.381 (0.35415341249152793) + | > loader_time: 0.0063 (0.0055890545916201465) + + + --> TIME: 2025-04-19 06:01:17 -- STEP: 92/142 -- GLOBAL_STEP: 440150 + | > loss_disc: 2.562882661819458 (2.4545736908912668) + | > loss_disc_real_0: 0.12739895284175873 (0.12383972756240678) + | > loss_disc_real_1: 0.22077101469039917 (0.20820463691716604) + | > loss_disc_real_2: 0.2543698847293854 (0.226729616522789) + | > loss_disc_real_3: 0.2471887171268463 (0.22968327869539676) + | > loss_disc_real_4: 0.26093268394470215 (0.22721404630852782) + | > loss_disc_real_5: 0.29440900683403015 (0.23011898443750714) + | > loss_0: 2.562882661819458 (2.4545736908912668) + | > grad_norm_0: tensor(11.1566, device='cuda:0') (tensor(10.6299, device='cuda:0')) + | > loss_gen: 2.3538312911987305 (2.3699357302292525) + | > loss_kl: 1.8403592109680176 (1.8277346621388975) + | > loss_feat: 6.869308948516846 (7.205178877581721) + | > loss_mel: 17.837112426757812 (17.617599311082248) + | > loss_duration: 1.4345567226409912 (1.442636787891388) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.335168838500977 (30.4630852782208) + | > grad_norm_1: tensor(444.2215, device='cuda:0') (tensor(256.0938, device='cuda:0')) + | > current_lr_0: 0.00018958055423688846 + | > current_lr_1: 0.00018958055423688846 + | > step_time: 0.4112 (0.3658464550971985) + | > loader_time: 0.0065 (0.00588071346282959) + + + --> TIME: 2025-04-19 06:01:27 -- STEP: 117/142 -- GLOBAL_STEP: 440175 + | > loss_disc: 2.4681174755096436 (2.457370572619969) + | > loss_disc_real_0: 0.14528435468673706 (0.12454499253350446) + | > loss_disc_real_1: 0.22048181295394897 (0.2068035522332558) + | > loss_disc_real_2: 0.24038517475128174 (0.22726839526086792) + | > loss_disc_real_3: 0.26019445061683655 (0.229011293914583) + | > loss_disc_real_4: 0.22659508883953094 (0.22836604956378284) + | > loss_disc_real_5: 0.23588845133781433 (0.22936076346116188) + | > loss_0: 2.4681174755096436 (2.457370572619969) + | > grad_norm_0: tensor(5.2621, device='cuda:0') (tensor(10.4305, device='cuda:0')) + | > loss_gen: 2.298727035522461 (2.3601515924828673) + | > loss_kl: 1.742223858833313 (1.8327991493746765) + | > loss_feat: 6.810959339141846 (7.172667161012307) + | > loss_mel: 16.744367599487305 (17.61164741027048) + | > loss_duration: 1.4679560661315918 (1.4497258765065772) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.064233779907227 (30.42699117742033) + | > grad_norm_1: tensor(211.8056, device='cuda:0') (tensor(255.9563, device='cuda:0')) + | > current_lr_0: 0.00018958055423688846 + | > current_lr_1: 0.00018958055423688846 + | > step_time: 0.4331 (0.3763203152224549) + | > loader_time: 0.007 (0.006116893556382921) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5553197860717773 (2.5553197860717773) + | > loss_disc_real_0: 0.21497857570648193 (0.21497857570648193) + | > loss_disc_real_1: 0.1813352257013321 (0.1813352257013321) + | > loss_disc_real_2: 0.20101049542427063 (0.20101049542427063) + | > loss_disc_real_3: 0.19241221249103546 (0.19241221249103546) + | > loss_disc_real_4: 0.1758577674627304 (0.1758577674627304) + | > loss_disc_real_5: 0.21560092270374298 (0.21560092270374298) + | > loss_0: 2.5553197860717773 (2.5553197860717773) + | > loss_gen: 2.1500284671783447 (2.1500284671783447) + | > loss_kl: 2.701293468475342 (2.701293468475342) + | > loss_feat: 6.938486099243164 (6.938486099243164) + | > loss_mel: 17.349292755126953 (17.349292755126953) + | > loss_duration: 1.7451419830322266 (1.7451419830322266) + | > loss_1: 30.88424301147461 (30.88424301147461) + + --> STEP: 1 + | > loss_disc: 2.538433790206909 (2.538433790206909) + | > loss_disc_real_0: 0.22004742920398712 (0.22004742920398712) + | > loss_disc_real_1: 0.19424724578857422 (0.19424724578857422) + | > loss_disc_real_2: 0.21403446793556213 (0.21403446793556213) + | > loss_disc_real_3: 0.22740964591503143 (0.22740964591503143) + | > loss_disc_real_4: 0.19726699590682983 (0.19726699590682983) + | > loss_disc_real_5: 0.23920311033725739 (0.23920311033725739) + | > loss_0: 2.538433790206909 (2.538433790206909) + | > loss_gen: 2.186980962753296 (2.186980962753296) + | > loss_kl: 2.5406081676483154 (2.5406081676483154) + | > loss_feat: 5.724590301513672 (5.724590301513672) + | > loss_mel: 16.986574172973633 (16.986574172973633) + | > loss_duration: 1.817319631576538 (1.817319631576538) + | > loss_1: 29.256074905395508 (29.256074905395508) + + --> STEP: 2 + | > loss_disc: 2.5056638717651367 (2.522048830986023) + | > loss_disc_real_0: 0.14669349789619446 (0.1833704635500908) + | > loss_disc_real_1: 0.16172783076763153 (0.17798753827810287) + | > loss_disc_real_2: 0.21383963525295258 (0.21393705159425735) + | > loss_disc_real_3: 0.20598290860652924 (0.21669627726078033) + | > loss_disc_real_4: 0.24323177337646484 (0.22024938464164734) + | > loss_disc_real_5: 0.30990585684776306 (0.2745544835925102) + | > loss_0: 2.5056638717651367 (2.522048830986023) + | > loss_gen: 2.287094831466675 (2.2370378971099854) + | > loss_kl: 2.3076841831207275 (2.4241461753845215) + | > loss_feat: 7.271271228790283 (6.4979307651519775) + | > loss_mel: 18.804338455200195 (17.895456314086914) + | > loss_duration: 1.7043983936309814 (1.7608590126037598) + | > loss_1: 32.374786376953125 (30.815430641174316) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003478527069091797 (+0.0002472400665283203) + | > avg_loss_disc: 2.522048830986023 (-0.10595965385437012) + | > avg_loss_disc_real_0: 0.1833704635500908 (+0.022452913224697113) + | > avg_loss_disc_real_1: 0.17798753827810287 (-0.05843895673751831) + | > avg_loss_disc_real_2: 0.21393705159425735 (-0.05179847031831741) + | > avg_loss_disc_real_3: 0.21669627726078033 (-0.010021410882472992) + | > avg_loss_disc_real_4: 0.22024938464164734 (-0.035231418907642365) + | > avg_loss_disc_real_5: 0.2745544835925102 (-0.018062718212604523) + | > avg_loss_0: 2.522048830986023 (-0.10595965385437012) + | > avg_loss_gen: 2.2370378971099854 (-0.02007746696472168) + | > avg_loss_kl: 2.4241461753845215 (-0.1089167594909668) + | > avg_loss_feat: 6.4979307651519775 (-0.7095463275909424) + | > avg_loss_mel: 17.895456314086914 (-0.05364227294921875) + | > avg_loss_duration: 1.7608590126037598 (-0.0025649666786193848) + | > avg_loss_1: 30.815430641174316 (-0.894744873046875) + + + > EPOCH: 429/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:01:40)  + + --> TIME: 2025-04-19 06:01:41 -- STEP: 0/142 -- GLOBAL_STEP: 440200 + | > loss_disc: 2.3416366577148438 (2.3416366577148438) + | > loss_disc_real_0: 0.12338703125715256 (0.12338703125715256) + | > loss_disc_real_1: 0.1527511030435562 (0.1527511030435562) + | > loss_disc_real_2: 0.20753686130046844 (0.20753686130046844) + | > loss_disc_real_3: 0.22619807720184326 (0.22619807720184326) + | > loss_disc_real_4: 0.20537826418876648 (0.20537826418876648) + | > loss_disc_real_5: 0.21808308362960815 (0.21808308362960815) + | > loss_0: 2.3416366577148438 (2.3416366577148438) + | > grad_norm_0: tensor(6.2017, device='cuda:0') (tensor(6.2017, device='cuda:0')) + | > loss_gen: 2.4967713356018066 (2.4967713356018066) + | > loss_kl: 1.5572973489761353 (1.5572973489761353) + | > loss_feat: 7.862828731536865 (7.862828731536865) + | > loss_mel: 18.26906394958496 (18.26906394958496) + | > loss_duration: 1.4315245151519775 (1.4315245151519775) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.61748504638672 (31.61748504638672) + | > grad_norm_1: tensor(198.2320, device='cuda:0') (tensor(198.2320, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.3511 (0.3511011600494385) + | > loader_time: 0.3333 (0.3332991600036621) + + + --> TIME: 2025-04-19 06:01:50 -- STEP: 25/142 -- GLOBAL_STEP: 440225 + | > loss_disc: 2.3456225395202637 (2.4391948604583735) + | > loss_disc_real_0: 0.10278021544218063 (0.11564618140459061) + | > loss_disc_real_1: 0.17129269242286682 (0.20993022739887238) + | > loss_disc_real_2: 0.20932795107364655 (0.2295738285779953) + | > loss_disc_real_3: 0.20896978676319122 (0.22689801931381226) + | > loss_disc_real_4: 0.21092192828655243 (0.2261057513952255) + | > loss_disc_real_5: 0.24092768132686615 (0.2291312175989151) + | > loss_0: 2.3456225395202637 (2.4391948604583735) + | > grad_norm_0: tensor(7.3272, device='cuda:0') (tensor(9.3773, device='cuda:0')) + | > loss_gen: 2.4782304763793945 (2.4215867710113526) + | > loss_kl: 1.7703214883804321 (1.76047776222229) + | > loss_feat: 7.673058032989502 (7.399726467132568) + | > loss_mel: 17.67074966430664 (17.5693643951416) + | > loss_duration: 1.4473888874053955 (1.4154869318008423) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.03974723815918 (30.566642532348634) + | > grad_norm_1: tensor(426.2682, device='cuda:0') (tensor(231.1784, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.3532 (0.3369920825958252) + | > loader_time: 0.0055 (0.0050451183319091784) + + + --> TIME: 2025-04-19 06:01:59 -- STEP: 50/142 -- GLOBAL_STEP: 440250 + | > loss_disc: 2.484417676925659 (2.4441894245147697) + | > loss_disc_real_0: 0.10073584318161011 (0.11866487398743629) + | > loss_disc_real_1: 0.2271038144826889 (0.20876937180757524) + | > loss_disc_real_2: 0.23269030451774597 (0.22794032573699952) + | > loss_disc_real_3: 0.24885700643062592 (0.2276565870642662) + | > loss_disc_real_4: 0.21063609421253204 (0.22740367114543913) + | > loss_disc_real_5: 0.2227204144001007 (0.2304781624674797) + | > loss_0: 2.484417676925659 (2.4441894245147697) + | > grad_norm_0: tensor(22.1933, device='cuda:0') (tensor(12.2004, device='cuda:0')) + | > loss_gen: 2.297710657119751 (2.4014847278594966) + | > loss_kl: 1.8260971307754517 (1.7872966051101684) + | > loss_feat: 6.88595724105835 (7.311847324371338) + | > loss_mel: 17.41816520690918 (17.662849349975588) + | > loss_duration: 1.4481257200241089 (1.4225627613067626) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.876054763793945 (30.586040802001953) + | > grad_norm_1: tensor(386.4304, device='cuda:0') (tensor(271.6903, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.3796 (0.35191592693328855) + | > loader_time: 0.0064 (0.005407814979553223) + + + --> TIME: 2025-04-19 06:02:09 -- STEP: 75/142 -- GLOBAL_STEP: 440275 + | > loss_disc: 2.475356340408325 (2.4530522537231443) + | > loss_disc_real_0: 0.1670338362455368 (0.12121792783339819) + | > loss_disc_real_1: 0.20034117996692657 (0.20829979042212168) + | > loss_disc_real_2: 0.2477632611989975 (0.22733691136042278) + | > loss_disc_real_3: 0.22658638656139374 (0.22896099090576172) + | > loss_disc_real_4: 0.2227449268102646 (0.22872230013211567) + | > loss_disc_real_5: 0.24885813891887665 (0.23163818776607514) + | > loss_0: 2.475356340408325 (2.4530522537231443) + | > grad_norm_0: tensor(14.0078, device='cuda:0') (tensor(12.3982, device='cuda:0')) + | > loss_gen: 2.3630475997924805 (2.391975994110107) + | > loss_kl: 1.9325358867645264 (1.8082886775334677) + | > loss_feat: 6.396845817565918 (7.265727945963541) + | > loss_mel: 15.91865348815918 (17.60022664388021) + | > loss_duration: 1.4283649921417236 (1.4358539597193398) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.03944969177246 (30.502073186238608) + | > grad_norm_1: tensor(217.9746, device='cuda:0') (tensor(279.9571, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.3985 (0.36570506731669106) + | > loader_time: 0.0062 (0.005653998057047526) + + + --> TIME: 2025-04-19 06:02:19 -- STEP: 100/142 -- GLOBAL_STEP: 440300 + | > loss_disc: 2.4345345497131348 (2.452636535167695) + | > loss_disc_real_0: 0.08989774435758591 (0.12175991639494896) + | > loss_disc_real_1: 0.18752869963645935 (0.20721398249268533) + | > loss_disc_real_2: 0.20973742008209229 (0.22588808268308638) + | > loss_disc_real_3: 0.22430148720741272 (0.22936185345053672) + | > loss_disc_real_4: 0.2251591831445694 (0.22920494183897971) + | > loss_disc_real_5: 0.17675916850566864 (0.23085825085639955) + | > loss_0: 2.4345345497131348 (2.452636535167695) + | > grad_norm_0: tensor(9.3996, device='cuda:0') (tensor(11.8919, device='cuda:0')) + | > loss_gen: 2.2987735271453857 (2.3793827438354502) + | > loss_kl: 1.8331235647201538 (1.827507929801941) + | > loss_feat: 7.494880676269531 (7.225296201705933) + | > loss_mel: 18.728944778442383 (17.663198528289804) + | > loss_duration: 1.4862455129623413 (1.4465864694118493) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.841968536376953 (30.54197175979614) + | > grad_norm_1: tensor(58.2058, device='cuda:0') (tensor(277.5585, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.4113 (0.37433309078216553) + | > loader_time: 0.0072 (0.005867438316345216) + + + --> TIME: 2025-04-19 06:02:30 -- STEP: 125/142 -- GLOBAL_STEP: 440325 + | > loss_disc: 2.5155909061431885 (2.4531903800964368) + | > loss_disc_real_0: 0.14893090724945068 (0.12304903090000152) + | > loss_disc_real_1: 0.19988490641117096 (0.20749784827232362) + | > loss_disc_real_2: 0.23479533195495605 (0.22586522829532624) + | > loss_disc_real_3: 0.23838169872760773 (0.22871708071231842) + | > loss_disc_real_4: 0.2332092821598053 (0.2285341452360153) + | > loss_disc_real_5: 0.23715196549892426 (0.23119059717655183) + | > loss_0: 2.5155909061431885 (2.4531903800964368) + | > grad_norm_0: tensor(17.1539, device='cuda:0') (tensor(11.9870, device='cuda:0')) + | > loss_gen: 2.3887181282043457 (2.3763425483703617) + | > loss_kl: 1.7656986713409424 (1.833388566017151) + | > loss_feat: 6.565495491027832 (7.190951389312744) + | > loss_mel: 18.02363395690918 (17.64378553771973) + | > loss_duration: 1.5243868827819824 (1.4536334152221675) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.267932891845703 (30.49810137939453) + | > grad_norm_1: tensor(351.3167, device='cuda:0') (tensor(269.3913, device='cuda:0')) + | > current_lr_0: 0.00018955685666760885 + | > current_lr_1: 0.00018955685666760885 + | > step_time: 0.4358 (0.38527919578552244) + | > loader_time: 0.0077 (0.006128993988037111) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5503392219543457 (2.5503392219543457) + | > loss_disc_real_0: 0.19565658271312714 (0.19565658271312714) + | > loss_disc_real_1: 0.21673749387264252 (0.21673749387264252) + | > loss_disc_real_2: 0.2897033989429474 (0.2897033989429474) + | > loss_disc_real_3: 0.24885974824428558 (0.24885974824428558) + | > loss_disc_real_4: 0.22714358568191528 (0.22714358568191528) + | > loss_disc_real_5: 0.21265703439712524 (0.21265703439712524) + | > loss_0: 2.5503392219543457 (2.5503392219543457) + | > loss_gen: 2.7227041721343994 (2.7227041721343994) + | > loss_kl: 2.507730722427368 (2.507730722427368) + | > loss_feat: 7.536590576171875 (7.536590576171875) + | > loss_mel: 18.464256286621094 (18.464256286621094) + | > loss_duration: 1.7389005422592163 (1.7389005422592163) + | > loss_1: 32.97018051147461 (32.97018051147461) + + --> STEP: 1 + | > loss_disc: 2.601472854614258 (2.601472854614258) + | > loss_disc_real_0: 0.20134256780147552 (0.20134256780147552) + | > loss_disc_real_1: 0.22191981971263885 (0.22191981971263885) + | > loss_disc_real_2: 0.28646111488342285 (0.28646111488342285) + | > loss_disc_real_3: 0.22896039485931396 (0.22896039485931396) + | > loss_disc_real_4: 0.2157796174287796 (0.2157796174287796) + | > loss_disc_real_5: 0.26116424798965454 (0.26116424798965454) + | > loss_0: 2.601472854614258 (2.601472854614258) + | > loss_gen: 2.5709800720214844 (2.5709800720214844) + | > loss_kl: 2.692789316177368 (2.692789316177368) + | > loss_feat: 8.323173522949219 (8.323173522949219) + | > loss_mel: 18.898324966430664 (18.898324966430664) + | > loss_duration: 1.8284834623336792 (1.8284834623336792) + | > loss_1: 34.313751220703125 (34.313751220703125) + + --> STEP: 2 + | > loss_disc: 2.6685009002685547 (2.6349868774414062) + | > loss_disc_real_0: 0.22254277765750885 (0.2119426727294922) + | > loss_disc_real_1: 0.25302955508232117 (0.23747468739748) + | > loss_disc_real_2: 0.26309463381767273 (0.2747778743505478) + | > loss_disc_real_3: 0.25624024868011475 (0.24260032176971436) + | > loss_disc_real_4: 0.17193041741847992 (0.19385501742362976) + | > loss_disc_real_5: 0.2221175730228424 (0.24164091050624847) + | > loss_0: 2.6685009002685547 (2.6349868774414062) + | > loss_gen: 2.4470455646514893 (2.509012818336487) + | > loss_kl: 2.1685280799865723 (2.43065869808197) + | > loss_feat: 6.488224983215332 (7.405699253082275) + | > loss_mel: 18.77293586730957 (18.835630416870117) + | > loss_duration: 1.7388243675231934 (1.7836539149284363) + | > loss_1: 31.61556053161621 (32.96465587615967) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003424406051635742 (-5.412101745605469e-05) + | > avg_loss_disc: 2.6349868774414062 (+0.1129380464553833) + | > avg_loss_disc_real_0: 0.2119426727294922 (+0.028572209179401398) + | > avg_loss_disc_real_1: 0.23747468739748 (+0.059487149119377136) + | > avg_loss_disc_real_2: 0.2747778743505478 (+0.060840822756290436) + | > avg_loss_disc_real_3: 0.24260032176971436 (+0.02590404450893402) + | > avg_loss_disc_real_4: 0.19385501742362976 (-0.026394367218017578) + | > avg_loss_disc_real_5: 0.24164091050624847 (-0.03291357308626175) + | > avg_loss_0: 2.6349868774414062 (+0.1129380464553833) + | > avg_loss_gen: 2.509012818336487 (+0.27197492122650146) + | > avg_loss_kl: 2.43065869808197 (+0.0065125226974487305) + | > avg_loss_feat: 7.405699253082275 (+0.9077684879302979) + | > avg_loss_mel: 18.835630416870117 (+0.9401741027832031) + | > avg_loss_duration: 1.7836539149284363 (+0.022794902324676514) + | > avg_loss_1: 32.96465587615967 (+2.1492252349853516) + + + > EPOCH: 430/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:02:39)  + + --> TIME: 2025-04-19 06:02:43 -- STEP: 8/142 -- GLOBAL_STEP: 440350 + | > loss_disc: 2.4120354652404785 (2.4913049042224884) + | > loss_disc_real_0: 0.07550054788589478 (0.11131387762725353) + | > loss_disc_real_1: 0.20812156796455383 (0.21024048142135143) + | > loss_disc_real_2: 0.2039552628993988 (0.2362645138055086) + | > loss_disc_real_3: 0.19907215237617493 (0.2310872133821249) + | > loss_disc_real_4: 0.22821785509586334 (0.23415485210716724) + | > loss_disc_real_5: 0.20888297259807587 (0.23832488805055618) + | > loss_0: 2.4120354652404785 (2.4913049042224884) + | > grad_norm_0: tensor(10.8743, device='cuda:0') (tensor(14.6580, device='cuda:0')) + | > loss_gen: 2.500946521759033 (2.3953462541103363) + | > loss_kl: 1.7238513231277466 (1.72879159450531) + | > loss_feat: 7.768610000610352 (7.334862172603607) + | > loss_mel: 18.53118896484375 (17.833516836166382) + | > loss_duration: 1.4067885875701904 (1.4123068004846573) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.931385040283203 (30.70482349395752) + | > grad_norm_1: tensor(307.2152, device='cuda:0') (tensor(343.5223, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.3393 (0.34417369961738586) + | > loader_time: 0.0051 (0.004983186721801758) + + + --> TIME: 2025-04-19 06:02:52 -- STEP: 33/142 -- GLOBAL_STEP: 440375 + | > loss_disc: 2.4780468940734863 (2.448724277091749) + | > loss_disc_real_0: 0.14682042598724365 (0.12575383100545767) + | > loss_disc_real_1: 0.1947108656167984 (0.2056473231676853) + | > loss_disc_real_2: 0.23678362369537354 (0.22766445757764758) + | > loss_disc_real_3: 0.23388035595417023 (0.2263194756074385) + | > loss_disc_real_4: 0.27104315161705017 (0.2272919700904326) + | > loss_disc_real_5: 0.2102784812450409 (0.22394461252472617) + | > loss_0: 2.4780468940734863 (2.448724277091749) + | > grad_norm_0: tensor(13.4411, device='cuda:0') (tensor(17.9589, device='cuda:0')) + | > loss_gen: 2.292264223098755 (2.3855771584944296) + | > loss_kl: 1.7334309816360474 (1.7974128578648423) + | > loss_feat: 6.171182155609131 (7.240811535806367) + | > loss_mel: 16.444082260131836 (17.789883469090316) + | > loss_duration: 1.4385652542114258 (1.4182606068524448) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.079524993896484 (30.631945523348723) + | > grad_norm_1: tensor(354.1682, device='cuda:0') (tensor(335.3698, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.3588 (0.3445730715086966) + | > loader_time: 0.0056 (0.005171876965147077) + + + --> TIME: 2025-04-19 06:03:01 -- STEP: 58/142 -- GLOBAL_STEP: 440400 + | > loss_disc: 2.411734104156494 (2.4413584511855553) + | > loss_disc_real_0: 0.08475204557180405 (0.1237142008954081) + | > loss_disc_real_1: 0.20448936522006989 (0.20577912263829132) + | > loss_disc_real_2: 0.25475484132766724 (0.22765715163329553) + | > loss_disc_real_3: 0.2202203869819641 (0.22759318428820577) + | > loss_disc_real_4: 0.239188551902771 (0.22683827938704654) + | > loss_disc_real_5: 0.23542636632919312 (0.22817209234525418) + | > loss_0: 2.411734104156494 (2.4413584511855553) + | > grad_norm_0: tensor(7.0980, device='cuda:0') (tensor(15.3212, device='cuda:0')) + | > loss_gen: 2.3672473430633545 (2.4015758900806823) + | > loss_kl: 1.7540971040725708 (1.8135054892507092) + | > loss_feat: 6.717944622039795 (7.287898047217007) + | > loss_mel: 18.407163619995117 (17.769329334127484) + | > loss_duration: 1.466035008430481 (1.4293380153590236) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.712488174438477 (30.701646706153607) + | > grad_norm_1: tensor(135.9443, device='cuda:0') (tensor(320.2078, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.3743 (0.3531908495672818) + | > loader_time: 0.0068 (0.005555794156830886) + + + --> TIME: 2025-04-19 06:03:11 -- STEP: 83/142 -- GLOBAL_STEP: 440425 + | > loss_disc: 2.5047318935394287 (2.4556411404207528) + | > loss_disc_real_0: 0.10019737482070923 (0.12580821968345757) + | > loss_disc_real_1: 0.21440096199512482 (0.20756829521024084) + | > loss_disc_real_2: 0.2429283708333969 (0.2285002979887537) + | > loss_disc_real_3: 0.22864942252635956 (0.2273542545287006) + | > loss_disc_real_4: 0.22219052910804749 (0.2272961624415524) + | > loss_disc_real_5: 0.23365174233913422 (0.22967212853661503) + | > loss_0: 2.5047318935394287 (2.4556411404207528) + | > grad_norm_0: tensor(7.9423, device='cuda:0') (tensor(13.1948, device='cuda:0')) + | > loss_gen: 2.3271148204803467 (2.380513400916594) + | > loss_kl: 1.9243777990341187 (1.8346509143530605) + | > loss_feat: 6.619792461395264 (7.170097615345415) + | > loss_mel: 17.239635467529297 (17.697641901223044) + | > loss_duration: 1.5287758111953735 (1.4406542462038707) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.63969612121582 (30.523558122565948) + | > grad_norm_1: tensor(162.3742, device='cuda:0') (tensor(274.2192, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.399 (0.36465831837022167) + | > loader_time: 0.0068 (0.005815391080925264) + + + --> TIME: 2025-04-19 06:03:21 -- STEP: 108/142 -- GLOBAL_STEP: 440450 + | > loss_disc: 2.5093917846679688 (2.4590579668680825) + | > loss_disc_real_0: 0.13467679917812347 (0.12669833287320761) + | > loss_disc_real_1: 0.2109317034482956 (0.20690233467353714) + | > loss_disc_real_2: 0.23615762591362 (0.22872795619898373) + | > loss_disc_real_3: 0.23845212161540985 (0.22867277237000289) + | > loss_disc_real_4: 0.23122406005859375 (0.22829313396855636) + | > loss_disc_real_5: 0.2581501305103302 (0.22934708633908518) + | > loss_0: 2.5093917846679688 (2.4590579668680825) + | > grad_norm_0: tensor(4.2064, device='cuda:0') (tensor(11.7471, device='cuda:0')) + | > loss_gen: 2.3235340118408203 (2.36167033954903) + | > loss_kl: 1.754847526550293 (1.837673364966004) + | > loss_feat: 6.3541975021362305 (7.075519080515261) + | > loss_mel: 16.580242156982422 (17.57451411529823) + | > loss_duration: 1.4871790409088135 (1.4495469872598294) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.5 (30.298923863304985) + | > grad_norm_1: tensor(152.6446, device='cuda:0') (tensor(243.3341, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.4208 (0.374399964456205) + | > loader_time: 0.0066 (0.00608492339098895) + + + --> TIME: 2025-04-19 06:03:33 -- STEP: 133/142 -- GLOBAL_STEP: 440475 + | > loss_disc: 2.5495128631591797 (2.4582500027534655) + | > loss_disc_real_0: 0.10799234360456467 (0.12638513278916377) + | > loss_disc_real_1: 0.20372456312179565 (0.2064881335971947) + | > loss_disc_real_2: 0.2536390721797943 (0.22830823343947418) + | > loss_disc_real_3: 0.21814166009426117 (0.22847843551097957) + | > loss_disc_real_4: 0.2034011334180832 (0.22780071467833413) + | > loss_disc_real_5: 0.2043769210577011 (0.22892150596568459) + | > loss_0: 2.5495128631591797 (2.4582500027534655) + | > grad_norm_0: tensor(9.7748, device='cuda:0') (tensor(10.9736, device='cuda:0')) + | > loss_gen: 2.225306510925293 (2.3517928195178963) + | > loss_kl: 1.8008990287780762 (1.8380400829745414) + | > loss_feat: 6.977991104125977 (7.045790073566867) + | > loss_mel: 17.09610939025879 (17.525949714775365) + | > loss_duration: 1.5242791175842285 (1.456680485180446) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.62458610534668 (30.218253157192603) + | > grad_norm_1: tensor(143.1364, device='cuda:0') (tensor(231.5330, device='cuda:0')) + | > current_lr_0: 0.0001895331620605254 + | > current_lr_1: 0.0001895331620605254 + | > step_time: 0.4361 (0.38580010170327095) + | > loader_time: 0.0072 (0.006333618235767335) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5194454193115234 (2.5194454193115234) + | > loss_disc_real_0: 0.19376933574676514 (0.19376933574676514) + | > loss_disc_real_1: 0.25427019596099854 (0.25427019596099854) + | > loss_disc_real_2: 0.2887108623981476 (0.2887108623981476) + | > loss_disc_real_3: 0.2504566013813019 (0.2504566013813019) + | > loss_disc_real_4: 0.22895795106887817 (0.22895795106887817) + | > loss_disc_real_5: 0.26149076223373413 (0.26149076223373413) + | > loss_0: 2.5194454193115234 (2.5194454193115234) + | > loss_gen: 2.6909940242767334 (2.6909940242767334) + | > loss_kl: 2.3876922130584717 (2.3876922130584717) + | > loss_feat: 6.894815444946289 (6.894815444946289) + | > loss_mel: 18.488136291503906 (18.488136291503906) + | > loss_duration: 1.7673379182815552 (1.7673379182815552) + | > loss_1: 32.22897720336914 (32.22897720336914) + + --> STEP: 1 + | > loss_disc: 2.528653144836426 (2.528653144836426) + | > loss_disc_real_0: 0.19403080642223358 (0.19403080642223358) + | > loss_disc_real_1: 0.23358137905597687 (0.23358137905597687) + | > loss_disc_real_2: 0.2644778788089752 (0.2644778788089752) + | > loss_disc_real_3: 0.2597793936729431 (0.2597793936729431) + | > loss_disc_real_4: 0.26692718267440796 (0.26692718267440796) + | > loss_disc_real_5: 0.20042669773101807 (0.20042669773101807) + | > loss_0: 2.528653144836426 (2.528653144836426) + | > loss_gen: 2.663835287094116 (2.663835287094116) + | > loss_kl: 2.8106849193573 (2.8106849193573) + | > loss_feat: 8.463998794555664 (8.463998794555664) + | > loss_mel: 19.47977066040039 (19.47977066040039) + | > loss_duration: 1.808985710144043 (1.808985710144043) + | > loss_1: 35.22727584838867 (35.22727584838867) + + --> STEP: 2 + | > loss_disc: 2.601923704147339 (2.5652884244918823) + | > loss_disc_real_0: 0.216116800904274 (0.20507380366325378) + | > loss_disc_real_1: 0.19999736547470093 (0.2167893722653389) + | > loss_disc_real_2: 0.296059787273407 (0.2802688330411911) + | > loss_disc_real_3: 0.20613215863704681 (0.23295577615499496) + | > loss_disc_real_4: 0.28680160641670227 (0.2768643945455551) + | > loss_disc_real_5: 0.24696746468544006 (0.22369708120822906) + | > loss_0: 2.601923704147339 (2.5652884244918823) + | > loss_gen: 2.3956291675567627 (2.5297322273254395) + | > loss_kl: 2.171497344970703 (2.4910911321640015) + | > loss_feat: 6.871469020843506 (7.667733907699585) + | > loss_mel: 17.576068878173828 (18.52791976928711) + | > loss_duration: 1.7552196979522705 (1.7821027040481567) + | > loss_1: 30.769886016845703 (32.99858093261719) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004056692123413086 (+0.0006322860717773438) + | > avg_loss_disc: 2.5652884244918823 (-0.06969845294952393) + | > avg_loss_disc_real_0: 0.20507380366325378 (-0.006868869066238403) + | > avg_loss_disc_real_1: 0.2167893722653389 (-0.020685315132141113) + | > avg_loss_disc_real_2: 0.2802688330411911 (+0.0054909586906433105) + | > avg_loss_disc_real_3: 0.23295577615499496 (-0.009644545614719391) + | > avg_loss_disc_real_4: 0.2768643945455551 (+0.08300937712192535) + | > avg_loss_disc_real_5: 0.22369708120822906 (-0.01794382929801941) + | > avg_loss_0: 2.5652884244918823 (-0.06969845294952393) + | > avg_loss_gen: 2.5297322273254395 (+0.020719408988952637) + | > avg_loss_kl: 2.4910911321640015 (+0.06043243408203125) + | > avg_loss_feat: 7.667733907699585 (+0.26203465461730957) + | > avg_loss_mel: 18.52791976928711 (-0.3077106475830078) + | > avg_loss_duration: 1.7821027040481567 (-0.001551210880279541) + | > avg_loss_1: 32.99858093261719 (+0.03392505645751953) + + + > EPOCH: 431/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:03:38)  + + --> TIME: 2025-04-19 06:03:44 -- STEP: 16/142 -- GLOBAL_STEP: 440500 + | > loss_disc: 2.47979998588562 (2.474800556898117) + | > loss_disc_real_0: 0.12374788522720337 (0.12292883917689323) + | > loss_disc_real_1: 0.2085580974817276 (0.2039794521406293) + | > loss_disc_real_2: 0.24606473743915558 (0.2280290648341179) + | > loss_disc_real_3: 0.19673465192317963 (0.22963995579630136) + | > loss_disc_real_4: 0.228782519698143 (0.2368309535086155) + | > loss_disc_real_5: 0.22378461062908173 (0.24505623430013657) + | > loss_0: 2.47979998588562 (2.474800556898117) + | > grad_norm_0: tensor(8.2382, device='cuda:0') (tensor(8.4338, device='cuda:0')) + | > loss_gen: 2.2708964347839355 (2.383841961622238) + | > loss_kl: 1.8623028993606567 (1.786396861076355) + | > loss_feat: 7.235598564147949 (7.437526166439056) + | > loss_mel: 18.46028709411621 (17.778775572776794) + | > loss_duration: 1.3979053497314453 (1.4096714705228806) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.226991653442383 (30.796212315559387) + | > grad_norm_1: tensor(136.9541, device='cuda:0') (tensor(168.1378, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.3397 (0.32837292551994324) + | > loader_time: 0.005 (0.004770293831825256) + + + --> TIME: 2025-04-19 06:03:53 -- STEP: 41/142 -- GLOBAL_STEP: 440525 + | > loss_disc: 2.490676164627075 (2.449376408646746) + | > loss_disc_real_0: 0.09537353366613388 (0.1199163391095836) + | > loss_disc_real_1: 0.22357812523841858 (0.20500602773049983) + | > loss_disc_real_2: 0.23750470578670502 (0.2276738751225355) + | > loss_disc_real_3: 0.21589547395706177 (0.2284592397329284) + | > loss_disc_real_4: 0.22501002252101898 (0.22938412464246516) + | > loss_disc_real_5: 0.253945916891098 (0.23363900039254165) + | > loss_0: 2.490676164627075 (2.449376408646746) + | > grad_norm_0: tensor(10.3848, device='cuda:0') (tensor(7.9789, device='cuda:0')) + | > loss_gen: 2.238285541534424 (2.3730310347022074) + | > loss_kl: 1.7799088954925537 (1.8112932966976631) + | > loss_feat: 7.445542335510254 (7.358172300385266) + | > loss_mel: 18.363121032714844 (17.744628836468955) + | > loss_duration: 1.416912317276001 (1.4245633061339216) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.243770599365234 (30.711688855799235) + | > grad_norm_1: tensor(210.1661, device='cuda:0') (tensor(177.7487, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.36 (0.3414349497818365) + | > loader_time: 0.0052 (0.005158296445520912) + + + --> TIME: 2025-04-19 06:04:03 -- STEP: 66/142 -- GLOBAL_STEP: 440550 + | > loss_disc: 2.5034828186035156 (2.4542027459000093) + | > loss_disc_real_0: 0.1444154679775238 (0.12196742731964949) + | > loss_disc_real_1: 0.18769323825836182 (0.20525245052395444) + | > loss_disc_real_2: 0.19317029416561127 (0.22719218423872284) + | > loss_disc_real_3: 0.23409025371074677 (0.2287857128363667) + | > loss_disc_real_4: 0.17724746465682983 (0.2313208668069406) + | > loss_disc_real_5: 0.19021619856357574 (0.2320795820066423) + | > loss_0: 2.5034828186035156 (2.4542027459000093) + | > grad_norm_0: tensor(5.3478, device='cuda:0') (tensor(7.8280, device='cuda:0')) + | > loss_gen: 2.386457681655884 (2.3676579937790376) + | > loss_kl: 1.9070062637329102 (1.8193146026495732) + | > loss_feat: 8.041326522827148 (7.306797150409583) + | > loss_mel: 18.19074249267578 (17.73208769884977) + | > loss_duration: 1.4346258640289307 (1.4357594052950542) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.960159301757812 (30.661616816665187) + | > grad_norm_1: tensor(207.1675, device='cuda:0') (tensor(178.8109, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.3863 (0.35350871447360876) + | > loader_time: 0.0061 (0.005483956047982881) + + + --> TIME: 2025-04-19 06:04:13 -- STEP: 91/142 -- GLOBAL_STEP: 440575 + | > loss_disc: 2.4618759155273438 (2.4543626911037557) + | > loss_disc_real_0: 0.1406596601009369 (0.12424125751623741) + | > loss_disc_real_1: 0.22704821825027466 (0.20678491677556718) + | > loss_disc_real_2: 0.2068077027797699 (0.22596567361564426) + | > loss_disc_real_3: 0.19973130524158478 (0.2267543051269028) + | > loss_disc_real_4: 0.2381567358970642 (0.23028539808896872) + | > loss_disc_real_5: 0.17725087702274323 (0.23149922178996787) + | > loss_0: 2.4618759155273438 (2.4543626911037557) + | > grad_norm_0: tensor(16.6702, device='cuda:0') (tensor(8.5979, device='cuda:0')) + | > loss_gen: 2.3335795402526855 (2.363555544025295) + | > loss_kl: 1.8357062339782715 (1.832149332696265) + | > loss_feat: 7.231688499450684 (7.212669424958282) + | > loss_mel: 17.979703903198242 (17.734279024731986) + | > loss_duration: 1.482986569404602 (1.446733807469462) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.863664627075195 (30.58938695048238) + | > grad_norm_1: tensor(387.3233, device='cuda:0') (tensor(193.7647, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.4079 (0.3657813962999281) + | > loader_time: 0.0068 (0.005757379007863475) + + + --> TIME: 2025-04-19 06:04:24 -- STEP: 116/142 -- GLOBAL_STEP: 440600 + | > loss_disc: 2.370516300201416 (2.453990609481415) + | > loss_disc_real_0: 0.1350143700838089 (0.12424091962647849) + | > loss_disc_real_1: 0.1949104219675064 (0.20612338082543735) + | > loss_disc_real_2: 0.2399476170539856 (0.22673241344505343) + | > loss_disc_real_3: 0.23736269772052765 (0.22679079555231949) + | > loss_disc_real_4: 0.23438028991222382 (0.23120240250538135) + | > loss_disc_real_5: 0.2573956847190857 (0.2316989189591901) + | > loss_0: 2.370516300201416 (2.453990609481415) + | > grad_norm_0: tensor(5.7586, device='cuda:0') (tensor(9.2535, device='cuda:0')) + | > loss_gen: 2.4866693019866943 (2.371119597862507) + | > loss_kl: 1.9459654092788696 (1.8381605867681832) + | > loss_feat: 7.054882526397705 (7.230675093058882) + | > loss_mel: 16.084692001342773 (17.757667541503913) + | > loss_duration: 1.4719384908676147 (1.4536039705934194) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.04414939880371 (30.651226701407598) + | > grad_norm_1: tensor(399.5252, device='cuda:0') (tensor(212.0713, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.4325 (0.3772698250310174) + | > loader_time: 0.0068 (0.006012893956283043) + + + --> TIME: 2025-04-19 06:04:35 -- STEP: 141/142 -- GLOBAL_STEP: 440625 + | > loss_disc: 2.549221992492676 (2.4509707816103656) + | > loss_disc_real_0: 0.10230990499258041 (0.12364603890806225) + | > loss_disc_real_1: 0.21057851612567902 (0.20617403185113947) + | > loss_disc_real_2: 0.22923193871974945 (0.22650928858746877) + | > loss_disc_real_3: 0.17640025913715363 (0.226975494665457) + | > loss_disc_real_4: 0.20654699206352234 (0.22985574074670778) + | > loss_disc_real_5: 0.22386647760868073 (0.2305428276882104) + | > loss_0: 2.549221992492676 (2.4509707816103656) + | > grad_norm_0: tensor(7.8486, device='cuda:0') (tensor(9.4706, device='cuda:0')) + | > loss_gen: 2.2994534969329834 (2.371262158062441) + | > loss_kl: 1.8220653533935547 (1.8407316740522994) + | > loss_feat: 6.817241191864014 (7.178199697048106) + | > loss_mel: 14.848362922668457 (17.622858892941313) + | > loss_duration: 1.3349344730377197 (1.4588563907230998) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.122058868408203 (30.471908718136184) + | > grad_norm_1: tensor(166.2192, device='cuda:0') (tensor(220.2265, device='cuda:0')) + | > current_lr_0: 0.00018950947041526783 + | > current_lr_1: 0.00018950947041526783 + | > step_time: 0.2034 (0.38737872306336746) + | > loader_time: 0.0026 (0.006207379888980947) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6425764560699463 (2.6425764560699463) + | > loss_disc_real_0: 0.21198756992816925 (0.21198756992816925) + | > loss_disc_real_1: 0.18316367268562317 (0.18316367268562317) + | > loss_disc_real_2: 0.25425902009010315 (0.25425902009010315) + | > loss_disc_real_3: 0.2336454540491104 (0.2336454540491104) + | > loss_disc_real_4: 0.2574186325073242 (0.2574186325073242) + | > loss_disc_real_5: 0.2564133107662201 (0.2564133107662201) + | > loss_0: 2.6425764560699463 (2.6425764560699463) + | > loss_gen: 2.3331305980682373 (2.3331305980682373) + | > loss_kl: 2.3934497833251953 (2.3934497833251953) + | > loss_feat: 7.544951915740967 (7.544951915740967) + | > loss_mel: 19.654634475708008 (19.654634475708008) + | > loss_duration: 1.7364308834075928 (1.7364308834075928) + | > loss_1: 33.66259765625 (33.66259765625) + + --> STEP: 1 + | > loss_disc: 2.5995798110961914 (2.5995798110961914) + | > loss_disc_real_0: 0.18250501155853271 (0.18250501155853271) + | > loss_disc_real_1: 0.21212522685527802 (0.21212522685527802) + | > loss_disc_real_2: 0.23051124811172485 (0.23051124811172485) + | > loss_disc_real_3: 0.2174645960330963 (0.2174645960330963) + | > loss_disc_real_4: 0.23776178061962128 (0.23776178061962128) + | > loss_disc_real_5: 0.25350221991539 (0.25350221991539) + | > loss_0: 2.5995798110961914 (2.5995798110961914) + | > loss_gen: 2.2035560607910156 (2.2035560607910156) + | > loss_kl: 2.948070764541626 (2.948070764541626) + | > loss_feat: 7.226844787597656 (7.226844787597656) + | > loss_mel: 18.452850341796875 (18.452850341796875) + | > loss_duration: 1.8128221035003662 (1.8128221035003662) + | > loss_1: 32.644142150878906 (32.644142150878906) + + --> STEP: 2 + | > loss_disc: 2.508120059967041 (2.553849935531616) + | > loss_disc_real_0: 0.1758980005979538 (0.17920150607824326) + | > loss_disc_real_1: 0.21351049840450287 (0.21281786262989044) + | > loss_disc_real_2: 0.1931626796722412 (0.21183696389198303) + | > loss_disc_real_3: 0.21959316730499268 (0.2185288816690445) + | > loss_disc_real_4: 0.22540636360645294 (0.2315840721130371) + | > loss_disc_real_5: 0.28333503007888794 (0.268418624997139) + | > loss_0: 2.508120059967041 (2.553849935531616) + | > loss_gen: 2.4413676261901855 (2.3224618434906006) + | > loss_kl: 2.434109687805176 (2.691090226173401) + | > loss_feat: 7.501241683959961 (7.364043235778809) + | > loss_mel: 17.50454330444336 (17.978696823120117) + | > loss_duration: 1.7330820560455322 (1.7729520797729492) + | > loss_1: 31.61434555053711 (32.12924385070801) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036346912384033203 (-0.0004220008850097656) + | > avg_loss_disc: 2.553849935531616 (-0.011438488960266113) + | > avg_loss_disc_real_0: 0.17920150607824326 (-0.02587229758501053) + | > avg_loss_disc_real_1: 0.21281786262989044 (-0.003971509635448456) + | > avg_loss_disc_real_2: 0.21183696389198303 (-0.06843186914920807) + | > avg_loss_disc_real_3: 0.2185288816690445 (-0.01442689448595047) + | > avg_loss_disc_real_4: 0.2315840721130371 (-0.045280322432518005) + | > avg_loss_disc_real_5: 0.268418624997139 (+0.04472154378890991) + | > avg_loss_0: 2.553849935531616 (-0.011438488960266113) + | > avg_loss_gen: 2.3224618434906006 (-0.20727038383483887) + | > avg_loss_kl: 2.691090226173401 (+0.19999909400939941) + | > avg_loss_feat: 7.364043235778809 (-0.30369067192077637) + | > avg_loss_mel: 17.978696823120117 (-0.5492229461669922) + | > avg_loss_duration: 1.7729520797729492 (-0.00915062427520752) + | > avg_loss_1: 32.12924385070801 (-0.8693370819091797) + + + > EPOCH: 432/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:04:37)  + + --> TIME: 2025-04-19 06:04:46 -- STEP: 24/142 -- GLOBAL_STEP: 440650 + | > loss_disc: 2.4965803623199463 (2.4378252426783242) + | > loss_disc_real_0: 0.11467204242944717 (0.10959881109495957) + | > loss_disc_real_1: 0.18337002396583557 (0.2108953638623158) + | > loss_disc_real_2: 0.23829373717308044 (0.2289989391962687) + | > loss_disc_real_3: 0.2358189970254898 (0.2304355806360642) + | > loss_disc_real_4: 0.20904600620269775 (0.22418377610544363) + | > loss_disc_real_5: 0.24132469296455383 (0.2320266223202149) + | > loss_0: 2.4965803623199463 (2.4378252426783242) + | > grad_norm_0: tensor(11.5858, device='cuda:0') (tensor(9.4445, device='cuda:0')) + | > loss_gen: 2.3231523036956787 (2.3748741348584494) + | > loss_kl: 1.890045404434204 (1.7619945506254833) + | > loss_feat: 6.579962730407715 (7.278400659561157) + | > loss_mel: 18.07806396484375 (17.589839259783428) + | > loss_duration: 1.438380241394043 (1.413022110859553) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.30960464477539 (30.41813039779663) + | > grad_norm_1: tensor(227.7442, device='cuda:0') (tensor(279.4095, device='cuda:0')) + | > current_lr_0: 0.00018948578173146592 + | > current_lr_1: 0.00018948578173146592 + | > step_time: 0.3496 (0.33258132139841706) + | > loader_time: 0.0051 (0.0048474470774332685) + + + --> TIME: 2025-04-19 06:04:55 -- STEP: 49/142 -- GLOBAL_STEP: 440675 + | > loss_disc: 2.4317939281463623 (2.4243184693005624) + | > loss_disc_real_0: 0.09753844141960144 (0.11062297970056534) + | > loss_disc_real_1: 0.2146788239479065 (0.2064058014324733) + | > loss_disc_real_2: 0.227395161986351 (0.22623535047988502) + | > loss_disc_real_3: 0.22522449493408203 (0.22710003262879896) + | > loss_disc_real_4: 0.23976585268974304 (0.22532217478265568) + | > loss_disc_real_5: 0.2220163643360138 (0.22991938524100244) + | > loss_0: 2.4317939281463623 (2.4243184693005624) + | > grad_norm_0: tensor(7.9548, device='cuda:0') (tensor(9.4023, device='cuda:0')) + | > loss_gen: 2.388920545578003 (2.3814497675214494) + | > loss_kl: 1.8882911205291748 (1.7821383427600472) + | > loss_feat: 6.5988359451293945 (7.225301100283253) + | > loss_mel: 17.43227767944336 (17.536120434196633) + | > loss_duration: 1.433600902557373 (1.423449309504762) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.741926193237305 (30.34845881559411) + | > grad_norm_1: tensor(208.4756, device='cuda:0') (tensor(264.2227, device='cuda:0')) + | > current_lr_0: 0.00018948578173146592 + | > current_lr_1: 0.00018948578173146592 + | > step_time: 0.3692 (0.3477302570732271) + | > loader_time: 0.0055 (0.0052909705103660115) + + + --> TIME: 2025-04-19 06:05:05 -- STEP: 74/142 -- GLOBAL_STEP: 440700 + | > loss_disc: 2.434253215789795 (2.421259880065918) + | > loss_disc_real_0: 0.13300053775310516 (0.11317865872705304) + | > loss_disc_real_1: 0.19023920595645905 (0.20665416040936033) + | > loss_disc_real_2: 0.23243068158626556 (0.22515943485337334) + | > loss_disc_real_3: 0.18813861906528473 (0.22648901411810438) + | > loss_disc_real_4: 0.2036503702402115 (0.22449162441330986) + | > loss_disc_real_5: 0.2335486114025116 (0.22900952096726443) + | > loss_0: 2.434253215789795 (2.421259880065918) + | > grad_norm_0: tensor(10.5163, device='cuda:0') (tensor(10.4329, device='cuda:0')) + | > loss_gen: 2.5729973316192627 (2.3899809766460107) + | > loss_kl: 1.7421131134033203 (1.8069928787850045) + | > loss_feat: 6.840219497680664 (7.2692344059815275) + | > loss_mel: 17.56744384765625 (17.64882732082058) + | > loss_duration: 1.4684168100357056 (1.4366537574175244) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.191190719604492 (30.551689302599108) + | > grad_norm_1: tensor(196.3000, device='cuda:0') (tensor(275.6972, device='cuda:0')) + | > current_lr_0: 0.00018948578173146592 + | > current_lr_1: 0.00018948578173146592 + | > step_time: 0.3917 (0.35955327588158675) + | > loader_time: 0.0062 (0.005609641204009184) + + + --> TIME: 2025-04-19 06:05:15 -- STEP: 99/142 -- GLOBAL_STEP: 440725 + | > loss_disc: 2.5254745483398438 (2.434924865009809) + | > loss_disc_real_0: 0.10679149627685547 (0.11556189273945008) + | > loss_disc_real_1: 0.1951460987329483 (0.20733543402618837) + | > loss_disc_real_2: 0.2744879126548767 (0.22585146686043403) + | > loss_disc_real_3: 0.24633312225341797 (0.22806102308360013) + | > loss_disc_real_4: 0.24487026035785675 (0.22622919970690603) + | > loss_disc_real_5: 0.25767579674720764 (0.23031619085807992) + | > loss_0: 2.5254745483398438 (2.434924865009809) + | > grad_norm_0: tensor(8.5169, device='cuda:0') (tensor(11.4575, device='cuda:0')) + | > loss_gen: 2.414217710494995 (2.3999884802885734) + | > loss_kl: 1.730863094329834 (1.8198681371380585) + | > loss_feat: 7.11619758605957 (7.2668579515784675) + | > loss_mel: 18.699947357177734 (17.68532099868312) + | > loss_duration: 1.4942288398742676 (1.4468314141938186) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.455453872680664 (30.618866949370414) + | > grad_norm_1: tensor(265.2271, device='cuda:0') (tensor(294.2383, device='cuda:0')) + | > current_lr_0: 0.00018948578173146592 + | > current_lr_1: 0.00018948578173146592 + | > step_time: 0.4086 (0.3709938574318932) + | > loader_time: 0.0065 (0.005921900874436505) + + + --> TIME: 2025-04-19 06:05:26 -- STEP: 124/142 -- GLOBAL_STEP: 440750 + | > loss_disc: 2.4277849197387695 (2.4397645669598735) + | > loss_disc_real_0: 0.15373370051383972 (0.11745356852489133) + | > loss_disc_real_1: 0.225833922624588 (0.20827673411657735) + | > loss_disc_real_2: 0.2271936535835266 (0.22546437070254358) + | > loss_disc_real_3: 0.22041422128677368 (0.2279203514898977) + | > loss_disc_real_4: 0.19243375957012177 (0.22578098444688705) + | > loss_disc_real_5: 0.22426237165927887 (0.2293936965927001) + | > loss_0: 2.4277849197387695 (2.4397645669598735) + | > grad_norm_0: tensor(12.1955, device='cuda:0') (tensor(11.9979, device='cuda:0')) + | > loss_gen: 2.449730634689331 (2.3905734439050006) + | > loss_kl: 1.8318901062011719 (1.8274243723961614) + | > loss_feat: 7.985877513885498 (7.196060999747245) + | > loss_mel: 18.961469650268555 (17.585106326687725) + | > loss_duration: 1.5114253759384155 (1.4545313312161359) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.740394592285156 (30.453696497025028) + | > grad_norm_1: tensor(494.1906, device='cuda:0') (tensor(300.5572, device='cuda:0')) + | > current_lr_0: 0.00018948578173146592 + | > current_lr_1: 0.00018948578173146592 + | > step_time: 0.4378 (0.3828511853371896) + | > loader_time: 0.0101 (0.006213759222338277) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.460693120956421 (2.460693120956421) + | > loss_disc_real_0: 0.110570028424263 (0.110570028424263) + | > loss_disc_real_1: 0.24724328517913818 (0.24724328517913818) + | > loss_disc_real_2: 0.22801467776298523 (0.22801467776298523) + | > loss_disc_real_3: 0.2415560483932495 (0.2415560483932495) + | > loss_disc_real_4: 0.23178905248641968 (0.23178905248641968) + | > loss_disc_real_5: 0.2802134156227112 (0.2802134156227112) + | > loss_0: 2.460693120956421 (2.460693120956421) + | > loss_gen: 2.3126044273376465 (2.3126044273376465) + | > loss_kl: 2.538625478744507 (2.538625478744507) + | > loss_feat: 6.419317245483398 (6.419317245483398) + | > loss_mel: 17.144105911254883 (17.144105911254883) + | > loss_duration: 1.74981689453125 (1.74981689453125) + | > loss_1: 30.164470672607422 (30.164470672607422) + + --> STEP: 1 + | > loss_disc: 2.450150489807129 (2.450150489807129) + | > loss_disc_real_0: 0.10123290866613388 (0.10123290866613388) + | > loss_disc_real_1: 0.25230592489242554 (0.25230592489242554) + | > loss_disc_real_2: 0.2345036417245865 (0.2345036417245865) + | > loss_disc_real_3: 0.24165421724319458 (0.24165421724319458) + | > loss_disc_real_4: 0.242002472281456 (0.242002472281456) + | > loss_disc_real_5: 0.2417999804019928 (0.2417999804019928) + | > loss_0: 2.450150489807129 (2.450150489807129) + | > loss_gen: 2.2420918941497803 (2.2420918941497803) + | > loss_kl: 2.687974452972412 (2.687974452972412) + | > loss_feat: 7.102431297302246 (7.102431297302246) + | > loss_mel: 17.785720825195312 (17.785720825195312) + | > loss_duration: 1.8038043975830078 (1.8038043975830078) + | > loss_1: 31.62202262878418 (31.62202262878418) + + --> STEP: 2 + | > loss_disc: 2.464246988296509 (2.457198739051819) + | > loss_disc_real_0: 0.1774933934211731 (0.1393631510436535) + | > loss_disc_real_1: 0.2733817398548126 (0.2628438323736191) + | > loss_disc_real_2: 0.29529914259910583 (0.26490139216184616) + | > loss_disc_real_3: 0.2327670454978943 (0.23721063137054443) + | > loss_disc_real_4: 0.22935053706169128 (0.23567650467157364) + | > loss_disc_real_5: 0.23569872975349426 (0.23874935507774353) + | > loss_0: 2.464246988296509 (2.457198739051819) + | > loss_gen: 2.6029951572418213 (2.422543525695801) + | > loss_kl: 2.346006155014038 (2.516990303993225) + | > loss_feat: 8.108783721923828 (7.605607509613037) + | > loss_mel: 18.591991424560547 (18.18885612487793) + | > loss_duration: 1.7241660356521606 (1.7639852166175842) + | > loss_1: 33.37394332885742 (32.4979829788208) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037796497344970703 (+0.00014495849609375) + | > avg_loss_disc: 2.457198739051819 (-0.09665119647979736) + | > avg_loss_disc_real_0: 0.1393631510436535 (-0.03983835503458977) + | > avg_loss_disc_real_1: 0.2628438323736191 (+0.05002596974372864) + | > avg_loss_disc_real_2: 0.26490139216184616 (+0.05306442826986313) + | > avg_loss_disc_real_3: 0.23721063137054443 (+0.01868174970149994) + | > avg_loss_disc_real_4: 0.23567650467157364 (+0.0040924325585365295) + | > avg_loss_disc_real_5: 0.23874935507774353 (-0.029669269919395447) + | > avg_loss_0: 2.457198739051819 (-0.09665119647979736) + | > avg_loss_gen: 2.422543525695801 (+0.1000816822052002) + | > avg_loss_kl: 2.516990303993225 (-0.17409992218017578) + | > avg_loss_feat: 7.605607509613037 (+0.24156427383422852) + | > avg_loss_mel: 18.18885612487793 (+0.2101593017578125) + | > avg_loss_duration: 1.7639852166175842 (-0.00896686315536499) + | > avg_loss_1: 32.4979829788208 (+0.36873912811279297) + + + > EPOCH: 433/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:05:36)  + + --> TIME: 2025-04-19 06:05:39 -- STEP: 7/142 -- GLOBAL_STEP: 440775 + | > loss_disc: 2.3536369800567627 (2.335970265524728) + | > loss_disc_real_0: 0.07522130012512207 (0.11562453955411911) + | > loss_disc_real_1: 0.2294154018163681 (0.19459002571446554) + | > loss_disc_real_2: 0.21741941571235657 (0.21060950202601297) + | > loss_disc_real_3: 0.23452319204807281 (0.22286399347441538) + | > loss_disc_real_4: 0.22778764367103577 (0.2164279477936881) + | > loss_disc_real_5: 0.23172999918460846 (0.2295979516846793) + | > loss_0: 2.3536369800567627 (2.335970265524728) + | > grad_norm_0: tensor(7.3084, device='cuda:0') (tensor(9.5799, device='cuda:0')) + | > loss_gen: 2.444883108139038 (2.4702722345079695) + | > loss_kl: 1.739890456199646 (1.7733503920691354) + | > loss_feat: 7.578353404998779 (7.670963900429862) + | > loss_mel: 17.95724868774414 (17.92417962210519) + | > loss_duration: 1.3979820013046265 (1.4143454347337996) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.118358612060547 (31.25311197553362) + | > grad_norm_1: tensor(265.5255, device='cuda:0') (tensor(321.1691, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.3194 (0.3239143575940813) + | > loader_time: 0.0039 (0.004371166229248047) + + + --> TIME: 2025-04-19 06:05:47 -- STEP: 32/142 -- GLOBAL_STEP: 440800 + | > loss_disc: 2.4136061668395996 (2.4100659862160683) + | > loss_disc_real_0: 0.1400482952594757 (0.1121345431311056) + | > loss_disc_real_1: 0.2060682773590088 (0.20203816471621394) + | > loss_disc_real_2: 0.22705592215061188 (0.22188117215409875) + | > loss_disc_real_3: 0.22705867886543274 (0.227566409856081) + | > loss_disc_real_4: 0.2331446260213852 (0.22332618245854974) + | > loss_disc_real_5: 0.20378746092319489 (0.2327478053048253) + | > loss_0: 2.4136061668395996 (2.4100659862160683) + | > grad_norm_0: tensor(15.8945, device='cuda:0') (tensor(10.6996, device='cuda:0')) + | > loss_gen: 2.4399631023406982 (2.4263481497764583) + | > loss_kl: 1.7872262001037598 (1.7682204060256481) + | > loss_feat: 7.384066104888916 (7.584315598011017) + | > loss_mel: 16.37113380432129 (18.0541250705719) + | > loss_duration: 1.4312841892242432 (1.4205389767885208) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.413673400878906 (31.25354838371277) + | > grad_norm_1: tensor(352.8135, device='cuda:0') (tensor(289.6837, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.3538 (0.3367879167199135) + | > loader_time: 0.0056 (0.005072936415672302) + + + --> TIME: 2025-04-19 06:05:57 -- STEP: 57/142 -- GLOBAL_STEP: 440825 + | > loss_disc: 2.4567861557006836 (2.4258584808885018) + | > loss_disc_real_0: 0.09435679763555527 (0.11615977050703868) + | > loss_disc_real_1: 0.1814291924238205 (0.20495517551898956) + | > loss_disc_real_2: 0.23019716143608093 (0.22167413511820006) + | > loss_disc_real_3: 0.19839030504226685 (0.22599461517835917) + | > loss_disc_real_4: 0.2531121075153351 (0.22708964530836073) + | > loss_disc_real_5: 0.24027477204799652 (0.23154690417281368) + | > loss_0: 2.4567861557006836 (2.4258584808885018) + | > grad_norm_0: tensor(7.6116, device='cuda:0') (tensor(10.4325, device='cuda:0')) + | > loss_gen: 2.2265686988830566 (2.4073063992617425) + | > loss_kl: 1.8101321458816528 (1.7816670861160546) + | > loss_feat: 7.164300918579102 (7.413195668605336) + | > loss_mel: 17.498912811279297 (17.894458603440665) + | > loss_duration: 1.491147518157959 (1.4311180825819052) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.191062927246094 (30.927746120252106) + | > grad_norm_1: tensor(343.0113, device='cuda:0') (tensor(269.9229, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.376 (0.34843149519803235) + | > loader_time: 0.0061 (0.005371219233462685) + + + --> TIME: 2025-04-19 06:06:07 -- STEP: 82/142 -- GLOBAL_STEP: 440850 + | > loss_disc: 2.459768056869507 (2.443145437938411) + | > loss_disc_real_0: 0.1280701607465744 (0.1200372715276189) + | > loss_disc_real_1: 0.1787991225719452 (0.20506627094454882) + | > loss_disc_real_2: 0.20231664180755615 (0.22375103631397572) + | > loss_disc_real_3: 0.23271489143371582 (0.22714286224871147) + | > loss_disc_real_4: 0.23608875274658203 (0.22787563884403647) + | > loss_disc_real_5: 0.23388205468654633 (0.23068041536139278) + | > loss_0: 2.459768056869507 (2.443145437938411) + | > grad_norm_0: tensor(14.4212, device='cuda:0') (tensor(10.1332, device='cuda:0')) + | > loss_gen: 2.4358301162719727 (2.3836739993676908) + | > loss_kl: 1.9406406879425049 (1.799958233426257) + | > loss_feat: 7.1746039390563965 (7.2712502712156715) + | > loss_mel: 17.137283325195312 (17.762837212260177) + | > loss_duration: 1.527491569519043 (1.4410443727563071) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.215850830078125 (30.65876430418433) + | > grad_norm_1: tensor(312.0098, device='cuda:0') (tensor(256.2401, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.3956 (0.36066216375769644) + | > loader_time: 0.0066 (0.005636686232031845) + + + --> TIME: 2025-04-19 06:06:17 -- STEP: 107/142 -- GLOBAL_STEP: 440875 + | > loss_disc: 2.4787790775299072 (2.4541943429786475) + | > loss_disc_real_0: 0.1954294890165329 (0.1225527291334121) + | > loss_disc_real_1: 0.22195349633693695 (0.206214104718137) + | > loss_disc_real_2: 0.23254168033599854 (0.22499761988069408) + | > loss_disc_real_3: 0.23450203239917755 (0.22830676795723281) + | > loss_disc_real_4: 0.23014861345291138 (0.22925386988671026) + | > loss_disc_real_5: 0.23818737268447876 (0.23049507366719646) + | > loss_0: 2.4787790775299072 (2.4541943429786475) + | > grad_norm_0: tensor(6.7858, device='cuda:0') (tensor(10.8210, device='cuda:0')) + | > loss_gen: 2.3359668254852295 (2.3683993069925036) + | > loss_kl: 1.7803399562835693 (1.8080833749236347) + | > loss_feat: 7.087449073791504 (7.1867052194114045) + | > loss_mel: 17.20051383972168 (17.698212685986096) + | > loss_duration: 1.4623929262161255 (1.4500363084757442) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.866661071777344 (30.5114369793473) + | > grad_norm_1: tensor(104.7844, device='cuda:0') (tensor(263.1767, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.4157 (0.3712109405303671) + | > loader_time: 0.0068 (0.005864693739703882) + + + --> TIME: 2025-04-19 06:06:28 -- STEP: 132/142 -- GLOBAL_STEP: 440900 + | > loss_disc: 2.387596845626831 (2.4623414061286235) + | > loss_disc_real_0: 0.09073510020971298 (0.12550182129736198) + | > loss_disc_real_1: 0.18879766762256622 (0.20558585632931103) + | > loss_disc_real_2: 0.24156621098518372 (0.2250450559411988) + | > loss_disc_real_3: 0.22128725051879883 (0.22883560474623332) + | > loss_disc_real_4: 0.2449682652950287 (0.22957697047880202) + | > loss_disc_real_5: 0.22279834747314453 (0.23181395740671593) + | > loss_0: 2.387596845626831 (2.4623414061286235) + | > grad_norm_0: tensor(3.4270, device='cuda:0') (tensor(11.4915, device='cuda:0')) + | > loss_gen: 2.516620397567749 (2.3600585758686057) + | > loss_kl: 1.836637020111084 (1.817031791715911) + | > loss_feat: 6.519194602966309 (7.1325955607674345) + | > loss_mel: 16.89579200744629 (17.689277670600198) + | > loss_duration: 1.4632329940795898 (1.4564550184842313) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.231475830078125 (30.45541865897901) + | > grad_norm_1: tensor(331.9932, device='cuda:0') (tensor(259.1802, device='cuda:0')) + | > current_lr_0: 0.00018946209600874948 + | > current_lr_1: 0.00018946209600874948 + | > step_time: 0.4445 (0.3830213438380849) + | > loader_time: 0.0072 (0.006087377215876724) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.353405475616455 (2.353405475616455) + | > loss_disc_real_0: 0.1253812313079834 (0.1253812313079834) + | > loss_disc_real_1: 0.17521236836910248 (0.17521236836910248) + | > loss_disc_real_2: 0.2627866864204407 (0.2627866864204407) + | > loss_disc_real_3: 0.25801995396614075 (0.25801995396614075) + | > loss_disc_real_4: 0.20910169184207916 (0.20910169184207916) + | > loss_disc_real_5: 0.25518062710762024 (0.25518062710762024) + | > loss_0: 2.353405475616455 (2.353405475616455) + | > loss_gen: 2.5040488243103027 (2.5040488243103027) + | > loss_kl: 2.349803924560547 (2.349803924560547) + | > loss_feat: 7.017232894897461 (7.017232894897461) + | > loss_mel: 18.676740646362305 (18.676740646362305) + | > loss_duration: 1.7204463481903076 (1.7204463481903076) + | > loss_1: 32.268272399902344 (32.268272399902344) + + --> STEP: 1 + | > loss_disc: 2.5083811283111572 (2.5083811283111572) + | > loss_disc_real_0: 0.1437380015850067 (0.1437380015850067) + | > loss_disc_real_1: 0.20199114084243774 (0.20199114084243774) + | > loss_disc_real_2: 0.25026166439056396 (0.25026166439056396) + | > loss_disc_real_3: 0.24902956187725067 (0.24902956187725067) + | > loss_disc_real_4: 0.2328256517648697 (0.2328256517648697) + | > loss_disc_real_5: 0.2591521739959717 (0.2591521739959717) + | > loss_0: 2.5083811283111572 (2.5083811283111572) + | > loss_gen: 2.323434352874756 (2.323434352874756) + | > loss_kl: 2.70509934425354 (2.70509934425354) + | > loss_feat: 6.876916885375977 (6.876916885375977) + | > loss_mel: 18.628551483154297 (18.628551483154297) + | > loss_duration: 1.8210053443908691 (1.8210053443908691) + | > loss_1: 32.35500717163086 (32.35500717163086) + + --> STEP: 2 + | > loss_disc: 2.3770341873168945 (2.442707657814026) + | > loss_disc_real_0: 0.10684449970722198 (0.12529125064611435) + | > loss_disc_real_1: 0.21475091576576233 (0.20837102830410004) + | > loss_disc_real_2: 0.2199089676141739 (0.23508531600236893) + | > loss_disc_real_3: 0.19106332957744598 (0.22004644572734833) + | > loss_disc_real_4: 0.23239395022392273 (0.2326098009943962) + | > loss_disc_real_5: 0.253030002117157 (0.25609108805656433) + | > loss_0: 2.3770341873168945 (2.442707657814026) + | > loss_gen: 2.462656021118164 (2.39304518699646) + | > loss_kl: 2.170213222503662 (2.437656283378601) + | > loss_feat: 8.004772186279297 (7.440844535827637) + | > loss_mel: 16.30755043029785 (17.468050956726074) + | > loss_duration: 1.739856481552124 (1.7804309129714966) + | > loss_1: 30.685049057006836 (31.520028114318848) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034807920455932617 (-0.0002988576889038086) + | > avg_loss_disc: 2.442707657814026 (-0.014491081237792969) + | > avg_loss_disc_real_0: 0.12529125064611435 (-0.014071900397539139) + | > avg_loss_disc_real_1: 0.20837102830410004 (-0.05447280406951904) + | > avg_loss_disc_real_2: 0.23508531600236893 (-0.029816076159477234) + | > avg_loss_disc_real_3: 0.22004644572734833 (-0.017164185643196106) + | > avg_loss_disc_real_4: 0.2326098009943962 (-0.003066703677177429) + | > avg_loss_disc_real_5: 0.25609108805656433 (+0.0173417329788208) + | > avg_loss_0: 2.442707657814026 (-0.014491081237792969) + | > avg_loss_gen: 2.39304518699646 (-0.02949833869934082) + | > avg_loss_kl: 2.437656283378601 (-0.07933402061462402) + | > avg_loss_feat: 7.440844535827637 (-0.1647629737854004) + | > avg_loss_mel: 17.468050956726074 (-0.7208051681518555) + | > avg_loss_duration: 1.7804309129714966 (+0.016445696353912354) + | > avg_loss_1: 31.520028114318848 (-0.9779548645019531) + + + > EPOCH: 434/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:06:34)  + + --> TIME: 2025-04-19 06:06:40 -- STEP: 15/142 -- GLOBAL_STEP: 440925 + | > loss_disc: 2.526453971862793 (2.4256384690602624) + | > loss_disc_real_0: 0.07976095378398895 (0.10771798367301623) + | > loss_disc_real_1: 0.18032635748386383 (0.20696680049101512) + | > loss_disc_real_2: 0.2203918993473053 (0.22902152339617413) + | > loss_disc_real_3: 0.20450665056705475 (0.23202898601690927) + | > loss_disc_real_4: 0.23814885318279266 (0.23265946706136068) + | > loss_disc_real_5: 0.224495068192482 (0.22382214466730754) + | > loss_0: 2.526453971862793 (2.4256384690602624) + | > grad_norm_0: tensor(6.9754, device='cuda:0') (tensor(12.3340, device='cuda:0')) + | > loss_gen: 2.152479887008667 (2.422830867767334) + | > loss_kl: 1.9368523359298706 (1.8346614758173625) + | > loss_feat: 6.53801155090332 (7.411906051635742) + | > loss_mel: 16.431825637817383 (17.81753094991048) + | > loss_duration: 1.40200936794281 (1.4037211020787557) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.461177825927734 (30.890650431315105) + | > grad_norm_1: tensor(126.6919, device='cuda:0') (tensor(298.5282, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.3334 (0.32771242459615074) + | > loader_time: 0.0051 (0.004592116673787435) + + + --> TIME: 2025-04-19 06:06:49 -- STEP: 40/142 -- GLOBAL_STEP: 440950 + | > loss_disc: 2.5146868228912354 (2.413556098937988) + | > loss_disc_real_0: 0.10232057422399521 (0.11071702493354678) + | > loss_disc_real_1: 0.1856708973646164 (0.20279752127826214) + | > loss_disc_real_2: 0.2034197449684143 (0.2252618908882141) + | > loss_disc_real_3: 0.2246338129043579 (0.22784877009689808) + | > loss_disc_real_4: 0.22999528050422668 (0.22808386199176311) + | > loss_disc_real_5: 0.2460436224937439 (0.22789178974926472) + | > loss_0: 2.5146868228912354 (2.413556098937988) + | > grad_norm_0: tensor(5.6335, device='cuda:0') (tensor(10.6765, device='cuda:0')) + | > loss_gen: 2.3428096771240234 (2.4075600802898416) + | > loss_kl: 1.7508881092071533 (1.8358670055866242) + | > loss_feat: 7.15600061416626 (7.400015759468078) + | > loss_mel: 18.302474975585938 (17.822821664810185) + | > loss_duration: 1.4143563508987427 (1.4170369565486909) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.966529846191406 (30.88330121040344) + | > grad_norm_1: tensor(55.6278, device='cuda:0') (tensor(262.5938, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.365 (0.34861783981323247) + | > loader_time: 0.006 (0.005153208971023559) + + + --> TIME: 2025-04-19 06:06:59 -- STEP: 65/142 -- GLOBAL_STEP: 440975 + | > loss_disc: 2.462996006011963 (2.4232968440422638) + | > loss_disc_real_0: 0.13656306266784668 (0.11667104460872137) + | > loss_disc_real_1: 0.18962499499320984 (0.2044794261455536) + | > loss_disc_real_2: 0.22281862795352936 (0.22488717092917515) + | > loss_disc_real_3: 0.24531173706054688 (0.22572454718443064) + | > loss_disc_real_4: 0.2283056676387787 (0.22708817972586706) + | > loss_disc_real_5: 0.23309165239334106 (0.22791999601400814) + | > loss_0: 2.462996006011963 (2.4232968440422638) + | > grad_norm_0: tensor(8.8229, device='cuda:0') (tensor(9.8289, device='cuda:0')) + | > loss_gen: 2.4835257530212402 (2.39520208652203) + | > loss_kl: 1.767382264137268 (1.842377444413992) + | > loss_feat: 7.759564399719238 (7.3400143256554236) + | > loss_mel: 18.46120834350586 (17.833027795644902) + | > loss_duration: 1.438979148864746 (1.430421280860901) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.910659790039062 (30.841042797382062) + | > grad_norm_1: tensor(77.9993, device='cuda:0') (tensor(224.9017, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.3933 (0.3595246498401349) + | > loader_time: 0.0064 (0.005522474875816932) + + + --> TIME: 2025-04-19 06:07:09 -- STEP: 90/142 -- GLOBAL_STEP: 441000 + | > loss_disc: 2.3883485794067383 (2.4349118709564213) + | > loss_disc_real_0: 0.10540363937616348 (0.11980446001721753) + | > loss_disc_real_1: 0.19371476769447327 (0.20567211574978297) + | > loss_disc_real_2: 0.21648433804512024 (0.224747485253546) + | > loss_disc_real_3: 0.22292029857635498 (0.2272593821088473) + | > loss_disc_real_4: 0.19495555758476257 (0.2278584490219752) + | > loss_disc_real_5: 0.24036277830600739 (0.22908440315061146) + | > loss_0: 2.3883485794067383 (2.4349118709564213) + | > grad_norm_0: tensor(8.1787, device='cuda:0') (tensor(9.5951, device='cuda:0')) + | > loss_gen: 2.361830711364746 (2.3768241696887555) + | > loss_kl: 1.6783233880996704 (1.8443197740448847) + | > loss_feat: 7.2822723388671875 (7.263093593385484) + | > loss_mel: 19.016225814819336 (17.82484702004326) + | > loss_duration: 1.477363109588623 (1.4417651679780752) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.816017150878906 (30.750849617852104) + | > grad_norm_1: tensor(235.5422, device='cuda:0') (tensor(215.5942, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.4026 (0.3692730214860705) + | > loader_time: 0.0066 (0.0058180862002902576) + + + --> TIME: 2025-04-19 06:07:19 -- STEP: 115/142 -- GLOBAL_STEP: 441025 + | > loss_disc: 2.438034772872925 (2.438127559164297) + | > loss_disc_real_0: 0.12576483190059662 (0.12203101033101911) + | > loss_disc_real_1: 0.19666852056980133 (0.20613059855025748) + | > loss_disc_real_2: 0.21462999284267426 (0.22444175494753796) + | > loss_disc_real_3: 0.19702084362506866 (0.22725345738556074) + | > loss_disc_real_4: 0.1931295245885849 (0.22671619239060775) + | > loss_disc_real_5: 0.21356409788131714 (0.2290645964767622) + | > loss_0: 2.438034772872925 (2.438127559164297) + | > grad_norm_0: tensor(6.4826, device='cuda:0') (tensor(9.6245, device='cuda:0')) + | > loss_gen: 2.3192367553710938 (2.3678543318872878) + | > loss_kl: 1.8405870199203491 (1.8442623293918112) + | > loss_feat: 7.256191253662109 (7.203650677722433) + | > loss_mel: 16.215290069580078 (17.737166330088733) + | > loss_duration: 1.4851759672164917 (1.4489272148712824) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.11648178100586 (30.601860825911814) + | > grad_norm_1: tensor(239.9458, device='cuda:0') (tensor(214.1787, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.4232 (0.3792666684026304) + | > loader_time: 0.0084 (0.006024713101594345) + + + --> TIME: 2025-04-19 06:07:31 -- STEP: 140/142 -- GLOBAL_STEP: 441050 + | > loss_disc: 2.433264970779419 (2.4479726706232343) + | > loss_disc_real_0: 0.1322420835494995 (0.12390132155269384) + | > loss_disc_real_1: 0.2191419005393982 (0.20654611300144876) + | > loss_disc_real_2: 0.22963471710681915 (0.22611383061323848) + | > loss_disc_real_3: 0.2156008929014206 (0.22831492732678144) + | > loss_disc_real_4: 0.24475666880607605 (0.2274920436952795) + | > loss_disc_real_5: 0.22731642425060272 (0.2297117779297488) + | > loss_0: 2.433264970779419 (2.4479726706232343) + | > grad_norm_0: tensor(10.8682, device='cuda:0') (tensor(9.5287, device='cuda:0')) + | > loss_gen: 2.3174350261688232 (2.3628878218787066) + | > loss_kl: 1.8630993366241455 (1.844208208152226) + | > loss_feat: 6.2448530197143555 (7.173942238943917) + | > loss_mel: 16.416269302368164 (17.705685452052524) + | > loss_duration: 1.517930269241333 (1.4561560826642175) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.359586715698242 (30.54287977218628) + | > grad_norm_1: tensor(104.1706, device='cuda:0') (tensor(210.9121, device='cuda:0')) + | > current_lr_0: 0.00018943841324674837 + | > current_lr_1: 0.00018943841324674837 + | > step_time: 0.4577 (0.3908973506518773) + | > loader_time: 0.0077 (0.006272840499877931) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3658742904663086 (2.3658742904663086) + | > loss_disc_real_0: 0.12544478476047516 (0.12544478476047516) + | > loss_disc_real_1: 0.20957818627357483 (0.20957818627357483) + | > loss_disc_real_2: 0.21119879186153412 (0.21119879186153412) + | > loss_disc_real_3: 0.22422848641872406 (0.22422848641872406) + | > loss_disc_real_4: 0.20581667125225067 (0.20581667125225067) + | > loss_disc_real_5: 0.20694448053836823 (0.20694448053836823) + | > loss_0: 2.3658742904663086 (2.3658742904663086) + | > loss_gen: 2.2791192531585693 (2.2791192531585693) + | > loss_kl: 2.5232937335968018 (2.5232937335968018) + | > loss_feat: 6.284968852996826 (6.284968852996826) + | > loss_mel: 17.267845153808594 (17.267845153808594) + | > loss_duration: 1.7558541297912598 (1.7558541297912598) + | > loss_1: 30.111080169677734 (30.111080169677734) + + --> STEP: 1 + | > loss_disc: 2.475688934326172 (2.475688934326172) + | > loss_disc_real_0: 0.11349015682935715 (0.11349015682935715) + | > loss_disc_real_1: 0.24017733335494995 (0.24017733335494995) + | > loss_disc_real_2: 0.20763209462165833 (0.20763209462165833) + | > loss_disc_real_3: 0.21575607359409332 (0.21575607359409332) + | > loss_disc_real_4: 0.23488309979438782 (0.23488309979438782) + | > loss_disc_real_5: 0.23527754843235016 (0.23527754843235016) + | > loss_0: 2.475688934326172 (2.475688934326172) + | > loss_gen: 2.2301173210144043 (2.2301173210144043) + | > loss_kl: 2.7068421840667725 (2.7068421840667725) + | > loss_feat: 7.52312707901001 (7.52312707901001) + | > loss_mel: 18.20895004272461 (18.20895004272461) + | > loss_duration: 1.8088270425796509 (1.8088270425796509) + | > loss_1: 32.47786331176758 (32.47786331176758) + + --> STEP: 2 + | > loss_disc: 2.2847325801849365 (2.380210757255554) + | > loss_disc_real_0: 0.10948066413402557 (0.11148541048169136) + | > loss_disc_real_1: 0.24792993068695068 (0.24405363202095032) + | > loss_disc_real_2: 0.20421123504638672 (0.20592166483402252) + | > loss_disc_real_3: 0.2189633697271347 (0.217359721660614) + | > loss_disc_real_4: 0.19816221296787262 (0.21652265638113022) + | > loss_disc_real_5: 0.20359425246715546 (0.2194359004497528) + | > loss_0: 2.2847325801849365 (2.380210757255554) + | > loss_gen: 2.463804006576538 (2.346960663795471) + | > loss_kl: 2.374937057495117 (2.540889620780945) + | > loss_feat: 8.056328773498535 (7.7897279262542725) + | > loss_mel: 18.414691925048828 (18.31182098388672) + | > loss_duration: 1.7552292346954346 (1.7820281386375427) + | > loss_1: 33.06499099731445 (32.771427154541016) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004026055335998535 (+0.0005452632904052734) + | > avg_loss_disc: 2.380210757255554 (-0.06249690055847168) + | > avg_loss_disc_real_0: 0.11148541048169136 (-0.013805840164422989) + | > avg_loss_disc_real_1: 0.24405363202095032 (+0.03568260371685028) + | > avg_loss_disc_real_2: 0.20592166483402252 (-0.029163651168346405) + | > avg_loss_disc_real_3: 0.217359721660614 (-0.002686724066734314) + | > avg_loss_disc_real_4: 0.21652265638113022 (-0.01608714461326599) + | > avg_loss_disc_real_5: 0.2194359004497528 (-0.03665518760681152) + | > avg_loss_0: 2.380210757255554 (-0.06249690055847168) + | > avg_loss_gen: 2.346960663795471 (-0.04608452320098877) + | > avg_loss_kl: 2.540889620780945 (+0.10323333740234375) + | > avg_loss_feat: 7.7897279262542725 (+0.34888339042663574) + | > avg_loss_mel: 18.31182098388672 (+0.8437700271606445) + | > avg_loss_duration: 1.7820281386375427 (+0.0015972256660461426) + | > avg_loss_1: 32.771427154541016 (+1.251399040222168) + + + > EPOCH: 435/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:07:33)  + + --> TIME: 2025-04-19 06:07:41 -- STEP: 23/142 -- GLOBAL_STEP: 441075 + | > loss_disc: 2.501027822494507 (2.455328433409981) + | > loss_disc_real_0: 0.12143416702747345 (0.12447232105161833) + | > loss_disc_real_1: 0.2155892699956894 (0.209831354410752) + | > loss_disc_real_2: 0.2636314332485199 (0.22238041071788126) + | > loss_disc_real_3: 0.24100802838802338 (0.22799108404180277) + | > loss_disc_real_4: 0.23494991660118103 (0.2261742204427719) + | > loss_disc_real_5: 0.2100067436695099 (0.22769299343876218) + | > loss_0: 2.501027822494507 (2.455328433409981) + | > grad_norm_0: tensor(6.2578, device='cuda:0') (tensor(7.3086, device='cuda:0')) + | > loss_gen: 2.32739520072937 (2.368686613829239) + | > loss_kl: 1.8532068729400635 (1.7573886902435967) + | > loss_feat: 6.889660835266113 (7.147960289664891) + | > loss_mel: 18.547454833984375 (17.444079274716586) + | > loss_duration: 1.4473007917404175 (1.4104657743288123) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.06501579284668 (30.128580508024797) + | > grad_norm_1: tensor(121.5225, device='cuda:0') (tensor(107.4370, device='cuda:0')) + | > current_lr_0: 0.00018941473344509253 + | > current_lr_1: 0.00018941473344509253 + | > step_time: 0.3568 (0.33291439388109284) + | > loader_time: 0.0053 (0.004902093306831693) + + + --> TIME: 2025-04-19 06:07:51 -- STEP: 48/142 -- GLOBAL_STEP: 441100 + | > loss_disc: 2.5116543769836426 (2.462792113423347) + | > loss_disc_real_0: 0.09182896465063095 (0.12326766612629092) + | > loss_disc_real_1: 0.21821896731853485 (0.20979959362496933) + | > loss_disc_real_2: 0.235426664352417 (0.2249869778752327) + | > loss_disc_real_3: 0.2729879319667816 (0.2299464844788114) + | > loss_disc_real_4: 0.24796856939792633 (0.2286786831294497) + | > loss_disc_real_5: 0.27453964948654175 (0.2304681735113263) + | > loss_0: 2.5116543769836426 (2.462792113423347) + | > grad_norm_0: tensor(19.9971, device='cuda:0') (tensor(8.3866, device='cuda:0')) + | > loss_gen: 2.4131383895874023 (2.364893386761347) + | > loss_kl: 1.857062816619873 (1.792976476252079) + | > loss_feat: 7.15852165222168 (7.17528439561526) + | > loss_mel: 17.569229125976562 (17.6275535027186) + | > loss_duration: 1.4364138841629028 (1.4235403165221217) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.43436622619629 (30.38424801826477) + | > grad_norm_1: tensor(389.8672, device='cuda:0') (tensor(172.1844, device='cuda:0')) + | > current_lr_0: 0.00018941473344509253 + | > current_lr_1: 0.00018941473344509253 + | > step_time: 0.3628 (0.3448903858661651) + | > loader_time: 0.0068 (0.005311936140060427) + + + --> TIME: 2025-04-19 06:08:00 -- STEP: 73/142 -- GLOBAL_STEP: 441125 + | > loss_disc: 2.4892020225524902 (2.458557723319693) + | > loss_disc_real_0: 0.11540017277002335 (0.12346392543348544) + | > loss_disc_real_1: 0.2268865406513214 (0.2082832767947079) + | > loss_disc_real_2: 0.22926923632621765 (0.22488141549776677) + | > loss_disc_real_3: 0.22774657607078552 (0.22909456716008383) + | > loss_disc_real_4: 0.2201104760169983 (0.22879604444111865) + | > loss_disc_real_5: 0.25779008865356445 (0.2297558368068852) + | > loss_0: 2.4892020225524902 (2.458557723319693) + | > grad_norm_0: tensor(11.2231, device='cuda:0') (tensor(8.6143, device='cuda:0')) + | > loss_gen: 2.4300661087036133 (2.365497468269034) + | > loss_kl: 1.7352615594863892 (1.8046039408200407) + | > loss_feat: 7.068351745605469 (7.181325566278745) + | > loss_mel: 17.249526977539062 (17.620114104388513) + | > loss_duration: 1.4842164516448975 (1.4354044068349552) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.967424392700195 (30.406945633561644) + | > grad_norm_1: tensor(272.4095, device='cuda:0') (tensor(185.9777, device='cuda:0')) + | > current_lr_0: 0.00018941473344509253 + | > current_lr_1: 0.00018941473344509253 + | > step_time: 0.3953 (0.3571882313244963) + | > loader_time: 0.0065 (0.005648877522716786) + + + --> TIME: 2025-04-19 06:08:10 -- STEP: 98/142 -- GLOBAL_STEP: 441150 + | > loss_disc: 2.299929618835449 (2.4563884686450557) + | > loss_disc_real_0: 0.10413973778486252 (0.12445239274173366) + | > loss_disc_real_1: 0.1856663078069687 (0.20756358516459558) + | > loss_disc_real_2: 0.19249340891838074 (0.22439747881524416) + | > loss_disc_real_3: 0.21767856180667877 (0.22892858848279837) + | > loss_disc_real_4: 0.1963498443365097 (0.22864555232987113) + | > loss_disc_real_5: 0.20577773451805115 (0.22906523471584125) + | > loss_0: 2.299929618835449 (2.4563884686450557) + | > grad_norm_0: tensor(5.3619, device='cuda:0') (tensor(8.6378, device='cuda:0')) + | > loss_gen: 2.4425063133239746 (2.357875123315927) + | > loss_kl: 1.8971331119537354 (1.8169805504837815) + | > loss_feat: 8.272180557250977 (7.1301049310333875) + | > loss_mel: 17.388206481933594 (17.555265922935654) + | > loss_duration: 1.445387601852417 (1.446856548591536) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.44541358947754 (30.307083227196518) + | > grad_norm_1: tensor(85.3474, device='cuda:0') (tensor(199.3949, device='cuda:0')) + | > current_lr_0: 0.00018941473344509253 + | > current_lr_1: 0.00018941473344509253 + | > step_time: 0.4087 (0.36764781085812315) + | > loader_time: 0.0072 (0.005942296008674468) + + + --> TIME: 2025-04-19 06:08:21 -- STEP: 123/142 -- GLOBAL_STEP: 441175 + | > loss_disc: 2.6463747024536133 (2.463650145181795) + | > loss_disc_real_0: 0.10752727091312408 (0.12442600018367533) + | > loss_disc_real_1: 0.23010790348052979 (0.20890848193226788) + | > loss_disc_real_2: 0.23871557414531708 (0.2254600698143486) + | > loss_disc_real_3: 0.2660022974014282 (0.22899313501226223) + | > loss_disc_real_4: 0.265615314245224 (0.228904416648353) + | > loss_disc_real_5: 0.22294539213180542 (0.22994105820733357) + | > loss_0: 2.6463747024536133 (2.463650145181795) + | > grad_norm_0: tensor(16.2423, device='cuda:0') (tensor(9.2885, device='cuda:0')) + | > loss_gen: 2.3767430782318115 (2.3581422421990363) + | > loss_kl: 1.834961175918579 (1.8270906064568497) + | > loss_feat: 7.712953567504883 (7.112544075260318) + | > loss_mel: 18.683210372924805 (17.536472940832635) + | > loss_duration: 1.47536039352417 (1.4539593564785593) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.083229064941406 (30.28820944995415) + | > grad_norm_1: tensor(468.9385, device='cuda:0') (tensor(217.6260, device='cuda:0')) + | > current_lr_0: 0.00018941473344509253 + | > current_lr_1: 0.00018941473344509253 + | > step_time: 0.4397 (0.3788999697057212) + | > loader_time: 0.007 (0.006173081514312001) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.42425799369812 (2.42425799369812) + | > loss_disc_real_0: 0.11659562587738037 (0.11659562587738037) + | > loss_disc_real_1: 0.19623173773288727 (0.19623173773288727) + | > loss_disc_real_2: 0.21430368721485138 (0.21430368721485138) + | > loss_disc_real_3: 0.22437039017677307 (0.22437039017677307) + | > loss_disc_real_4: 0.25735804438591003 (0.25735804438591003) + | > loss_disc_real_5: 0.2531287670135498 (0.2531287670135498) + | > loss_0: 2.42425799369812 (2.42425799369812) + | > loss_gen: 2.322378158569336 (2.322378158569336) + | > loss_kl: 2.4235987663269043 (2.4235987663269043) + | > loss_feat: 6.691323757171631 (6.691323757171631) + | > loss_mel: 17.12795639038086 (17.12795639038086) + | > loss_duration: 1.7346386909484863 (1.7346386909484863) + | > loss_1: 30.299894332885742 (30.299894332885742) + + --> STEP: 1 + | > loss_disc: 2.592542886734009 (2.592542886734009) + | > loss_disc_real_0: 0.1271171122789383 (0.1271171122789383) + | > loss_disc_real_1: 0.21920885145664215 (0.21920885145664215) + | > loss_disc_real_2: 0.22082814574241638 (0.22082814574241638) + | > loss_disc_real_3: 0.2764388620853424 (0.2764388620853424) + | > loss_disc_real_4: 0.25514811277389526 (0.25514811277389526) + | > loss_disc_real_5: 0.23966164886951447 (0.23966164886951447) + | > loss_0: 2.592542886734009 (2.592542886734009) + | > loss_gen: 2.170583963394165 (2.170583963394165) + | > loss_kl: 2.9668631553649902 (2.9668631553649902) + | > loss_feat: 6.42505407333374 (6.42505407333374) + | > loss_mel: 18.483108520507812 (18.483108520507812) + | > loss_duration: 1.795979619026184 (1.795979619026184) + | > loss_1: 31.841590881347656 (31.841590881347656) + + --> STEP: 2 + | > loss_disc: 2.4940662384033203 (2.5433045625686646) + | > loss_disc_real_0: 0.14108487963676453 (0.1341009959578514) + | > loss_disc_real_1: 0.19654013216495514 (0.20787449181079865) + | > loss_disc_real_2: 0.22674864530563354 (0.22378839552402496) + | > loss_disc_real_3: 0.2638680934906006 (0.2701534777879715) + | > loss_disc_real_4: 0.20105291903018951 (0.2281005159020424) + | > loss_disc_real_5: 0.22163057327270508 (0.23064611107110977) + | > loss_0: 2.4940662384033203 (2.5433045625686646) + | > loss_gen: 2.2583813667297363 (2.2144826650619507) + | > loss_kl: 2.335577964782715 (2.6512205600738525) + | > loss_feat: 8.007596969604492 (7.216325521469116) + | > loss_mel: 19.698871612548828 (19.09099006652832) + | > loss_duration: 1.7324364185333252 (1.7642080187797546) + | > loss_1: 34.03286361694336 (32.93722724914551) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003171086311340332 (-0.0008549690246582031) + | > avg_loss_disc: 2.5433045625686646 (+0.16309380531311035) + | > avg_loss_disc_real_0: 0.1341009959578514 (+0.02261558547616005) + | > avg_loss_disc_real_1: 0.20787449181079865 (-0.03617914021015167) + | > avg_loss_disc_real_2: 0.22378839552402496 (+0.01786673069000244) + | > avg_loss_disc_real_3: 0.2701534777879715 (+0.05279375612735748) + | > avg_loss_disc_real_4: 0.2281005159020424 (+0.01157785952091217) + | > avg_loss_disc_real_5: 0.23064611107110977 (+0.011210210621356964) + | > avg_loss_0: 2.5433045625686646 (+0.16309380531311035) + | > avg_loss_gen: 2.2144826650619507 (-0.1324779987335205) + | > avg_loss_kl: 2.6512205600738525 (+0.11033093929290771) + | > avg_loss_feat: 7.216325521469116 (-0.5734024047851562) + | > avg_loss_mel: 19.09099006652832 (+0.7791690826416016) + | > avg_loss_duration: 1.7642080187797546 (-0.017820119857788086) + | > avg_loss_1: 32.93722724914551 (+0.1658000946044922) + + + > EPOCH: 436/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:08:31)  + + --> TIME: 2025-04-19 06:08:34 -- STEP: 6/142 -- GLOBAL_STEP: 441200 + | > loss_disc: 2.386169672012329 (2.4111809730529785) + | > loss_disc_real_0: 0.18146805465221405 (0.13937623177965483) + | > loss_disc_real_1: 0.19436104595661163 (0.2061670869588852) + | > loss_disc_real_2: 0.2168787270784378 (0.21562851717074713) + | > loss_disc_real_3: 0.18927212059497833 (0.21522908906141916) + | > loss_disc_real_4: 0.21777723729610443 (0.21978561828533807) + | > loss_disc_real_5: 0.18048056960105896 (0.20784605542818704) + | > loss_0: 2.386169672012329 (2.4111809730529785) + | > grad_norm_0: tensor(15.4770, device='cuda:0') (tensor(16.6034, device='cuda:0')) + | > loss_gen: 2.486830234527588 (2.452446699142456) + | > loss_kl: 1.837939977645874 (1.7735639413197835) + | > loss_feat: 7.644684791564941 (7.519784927368164) + | > loss_mel: 17.311365127563477 (17.879793167114258) + | > loss_duration: 1.4022613763809204 (1.4047332406044006) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.683082580566406 (31.030321439107258) + | > grad_norm_1: tensor(107.8995, device='cuda:0') (tensor(255.8695, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.3298 (0.32448017597198486) + | > loader_time: 0.0041 (0.004524628321329753) + + + --> TIME: 2025-04-19 06:08:43 -- STEP: 31/142 -- GLOBAL_STEP: 441225 + | > loss_disc: 2.3915276527404785 (2.4192644088499002) + | > loss_disc_real_0: 0.11966059356927872 (0.11604816250262721) + | > loss_disc_real_1: 0.2159029096364975 (0.20517953124738508) + | > loss_disc_real_2: 0.22747676074504852 (0.22613398586550065) + | > loss_disc_real_3: 0.18938183784484863 (0.22622422585564275) + | > loss_disc_real_4: 0.2347257137298584 (0.22439137774129067) + | > loss_disc_real_5: 0.2032652646303177 (0.22771987799675233) + | > loss_0: 2.3915276527404785 (2.4192644088499002) + | > grad_norm_0: tensor(14.5675, device='cuda:0') (tensor(14.5322, device='cuda:0')) + | > loss_gen: 2.4149863719940186 (2.436628880039339) + | > loss_kl: 1.7555385828018188 (1.7777657931850803) + | > loss_feat: 6.970236778259277 (7.521191027856642) + | > loss_mel: 16.798635482788086 (17.84141842011482) + | > loss_duration: 1.3880856037139893 (1.412353269515499) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.327482223510742 (30.989357179211034) + | > grad_norm_1: tensor(561.9365, device='cuda:0') (tensor(332.3831, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.3545 (0.33691411633645335) + | > loader_time: 0.0052 (0.005042406820481823) + + + --> TIME: 2025-04-19 06:08:52 -- STEP: 56/142 -- GLOBAL_STEP: 441250 + | > loss_disc: 2.439012050628662 (2.445646094424384) + | > loss_disc_real_0: 0.08197595924139023 (0.12298856156745128) + | > loss_disc_real_1: 0.20208409428596497 (0.21016185704086507) + | > loss_disc_real_2: 0.2115033119916916 (0.22723304826234067) + | > loss_disc_real_3: 0.22726722061634064 (0.22700815168874605) + | > loss_disc_real_4: 0.23113089799880981 (0.22485571328018392) + | > loss_disc_real_5: 0.28043580055236816 (0.22849375435284205) + | > loss_0: 2.439012050628662 (2.445646094424384) + | > grad_norm_0: tensor(10.4901, device='cuda:0') (tensor(16.7385, device='cuda:0')) + | > loss_gen: 2.287681818008423 (2.4315427115985324) + | > loss_kl: 1.9141514301300049 (1.7956904023885727) + | > loss_feat: 7.3362603187561035 (7.482230390821185) + | > loss_mel: 18.15503692626953 (17.863315820693973) + | > loss_duration: 1.448744773864746 (1.4233540807451515) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.141876220703125 (30.99613336154393) + | > grad_norm_1: tensor(393.3289, device='cuda:0') (tensor(343.2666, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.374 (0.3486479009900774) + | > loader_time: 0.0061 (0.005325479166848319) + + + --> TIME: 2025-04-19 06:09:02 -- STEP: 81/142 -- GLOBAL_STEP: 441275 + | > loss_disc: 2.520671844482422 (2.455335608235112) + | > loss_disc_real_0: 0.12017065286636353 (0.12495613052153293) + | > loss_disc_real_1: 0.18588566780090332 (0.208532217660068) + | > loss_disc_real_2: 0.2160550206899643 (0.22706976238592172) + | > loss_disc_real_3: 0.23107145726680756 (0.22767873292351948) + | > loss_disc_real_4: 0.23075710237026215 (0.22713935669557547) + | > loss_disc_real_5: 0.2173823118209839 (0.23038776384459603) + | > loss_0: 2.520671844482422 (2.455335608235112) + | > grad_norm_0: tensor(17.9033, device='cuda:0') (tensor(16.0492, device='cuda:0')) + | > loss_gen: 2.1631596088409424 (2.4073567213835534) + | > loss_kl: 1.7281063795089722 (1.8059342525623463) + | > loss_feat: 6.224021911621094 (7.345984559000274) + | > loss_mel: 16.770200729370117 (17.686714878788706) + | > loss_duration: 1.4482712745666504 (1.4344907925452712) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.333759307861328 (30.68048107476882) + | > grad_norm_1: tensor(217.5876, device='cuda:0') (tensor(328.6583, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.3911 (0.3625894534735032) + | > loader_time: 0.0065 (0.0056248711950985) + + + --> TIME: 2025-04-19 06:09:12 -- STEP: 106/142 -- GLOBAL_STEP: 441300 + | > loss_disc: 2.465049982070923 (2.460948674183972) + | > loss_disc_real_0: 0.15653511881828308 (0.1248082941714323) + | > loss_disc_real_1: 0.20251323282718658 (0.2089436526568431) + | > loss_disc_real_2: 0.23266074061393738 (0.22689360661326713) + | > loss_disc_real_3: 0.24620820581912994 (0.22961245463141855) + | > loss_disc_real_4: 0.23580926656723022 (0.2274094527622439) + | > loss_disc_real_5: 0.1796010136604309 (0.2302044209725452) + | > loss_0: 2.465049982070923 (2.460948674183972) + | > grad_norm_0: tensor(7.9685, device='cuda:0') (tensor(14.5167, device='cuda:0')) + | > loss_gen: 2.2240583896636963 (2.381172015981852) + | > loss_kl: 1.814199686050415 (1.8148087611738242) + | > loss_feat: 6.170157432556152 (7.221697910776678) + | > loss_mel: 16.057689666748047 (17.645277698084996) + | > loss_duration: 1.4816566705703735 (1.4458307241493795) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 27.747760772705078 (30.508786939225107) + | > grad_norm_1: tensor(110.0462, device='cuda:0') (tensor(300.7050, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.4107 (0.37261716149887947) + | > loader_time: 0.0073 (0.005830229453320774) + + + --> TIME: 2025-04-19 06:09:24 -- STEP: 131/142 -- GLOBAL_STEP: 441325 + | > loss_disc: 2.4057445526123047 (2.466969229792821) + | > loss_disc_real_0: 0.10214769840240479 (0.12534947016539463) + | > loss_disc_real_1: 0.2043493688106537 (0.20810449214382026) + | > loss_disc_real_2: 0.2066006064414978 (0.22763461352304648) + | > loss_disc_real_3: 0.22658510506153107 (0.23045904092661296) + | > loss_disc_real_4: 0.21029794216156006 (0.22867884958973367) + | > loss_disc_real_5: 0.2247822880744934 (0.23134053037821792) + | > loss_0: 2.4057445526123047 (2.466969229792821) + | > grad_norm_0: tensor(8.1601, device='cuda:0') (tensor(13.4021, device='cuda:0')) + | > loss_gen: 2.3990654945373535 (2.362081573209688) + | > loss_kl: 1.8652576208114624 (1.831030105816499) + | > loss_feat: 6.141233921051025 (7.132002000590317) + | > loss_mel: 16.333267211914062 (17.54917011552184) + | > loss_duration: 1.473730444908142 (1.452408002533075) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.212554931640625 (30.326691620222483) + | > grad_norm_1: tensor(219.7859, device='cuda:0') (tensor(280.3487, device='cuda:0')) + | > current_lr_0: 0.0001893910566034119 + | > current_lr_1: 0.0001893910566034119 + | > step_time: 0.4464 (0.3841870176883144) + | > loader_time: 0.0076 (0.006089818386631159) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.595177173614502 (2.595177173614502) + | > loss_disc_real_0: 0.18890680372714996 (0.18890680372714996) + | > loss_disc_real_1: 0.26386749744415283 (0.26386749744415283) + | > loss_disc_real_2: 0.23818519711494446 (0.23818519711494446) + | > loss_disc_real_3: 0.241594597697258 (0.241594597697258) + | > loss_disc_real_4: 0.26137131452560425 (0.26137131452560425) + | > loss_disc_real_5: 0.26544392108917236 (0.26544392108917236) + | > loss_0: 2.595177173614502 (2.595177173614502) + | > loss_gen: 2.305291175842285 (2.305291175842285) + | > loss_kl: 2.485011339187622 (2.485011339187622) + | > loss_feat: 5.556817531585693 (5.556817531585693) + | > loss_mel: 15.169814109802246 (15.169814109802246) + | > loss_duration: 1.740313172340393 (1.740313172340393) + | > loss_1: 27.257246017456055 (27.257246017456055) + + --> STEP: 1 + | > loss_disc: 2.527496814727783 (2.527496814727783) + | > loss_disc_real_0: 0.18794642388820648 (0.18794642388820648) + | > loss_disc_real_1: 0.20127128064632416 (0.20127128064632416) + | > loss_disc_real_2: 0.21701288223266602 (0.21701288223266602) + | > loss_disc_real_3: 0.23394609987735748 (0.23394609987735748) + | > loss_disc_real_4: 0.23544107377529144 (0.23544107377529144) + | > loss_disc_real_5: 0.20735026895999908 (0.20735026895999908) + | > loss_0: 2.527496814727783 (2.527496814727783) + | > loss_gen: 2.216024875640869 (2.216024875640869) + | > loss_kl: 2.8007664680480957 (2.8007664680480957) + | > loss_feat: 6.534532070159912 (6.534532070159912) + | > loss_mel: 18.300148010253906 (18.300148010253906) + | > loss_duration: 1.83011794090271 (1.83011794090271) + | > loss_1: 31.681589126586914 (31.681589126586914) + + --> STEP: 2 + | > loss_disc: 2.614434003829956 (2.5709654092788696) + | > loss_disc_real_0: 0.18218261003494263 (0.18506451696157455) + | > loss_disc_real_1: 0.26461830735206604 (0.2329447939991951) + | > loss_disc_real_2: 0.25713050365448 (0.237071692943573) + | > loss_disc_real_3: 0.2369743436574936 (0.23546022176742554) + | > loss_disc_real_4: 0.21561312675476074 (0.2255271002650261) + | > loss_disc_real_5: 0.20983827114105225 (0.20859427005052567) + | > loss_0: 2.614434003829956 (2.5709654092788696) + | > loss_gen: 2.1577587127685547 (2.186891794204712) + | > loss_kl: 2.355560302734375 (2.5781633853912354) + | > loss_feat: 5.648106575012207 (6.09131932258606) + | > loss_mel: 12.978065490722656 (15.639106750488281) + | > loss_duration: 1.7239398956298828 (1.7770289182662964) + | > loss_1: 24.86343002319336 (28.272509574890137) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031745433807373047 (+3.4570693969726562e-06) + | > avg_loss_disc: 2.5709654092788696 (+0.027660846710205078) + | > avg_loss_disc_real_0: 0.18506451696157455 (+0.050963521003723145) + | > avg_loss_disc_real_1: 0.2329447939991951 (+0.025070302188396454) + | > avg_loss_disc_real_2: 0.237071692943573 (+0.013283297419548035) + | > avg_loss_disc_real_3: 0.23546022176742554 (-0.03469325602054596) + | > avg_loss_disc_real_4: 0.2255271002650261 (-0.0025734156370162964) + | > avg_loss_disc_real_5: 0.20859427005052567 (-0.022051841020584106) + | > avg_loss_0: 2.5709654092788696 (+0.027660846710205078) + | > avg_loss_gen: 2.186891794204712 (-0.02759087085723877) + | > avg_loss_kl: 2.5781633853912354 (-0.07305717468261719) + | > avg_loss_feat: 6.09131932258606 (-1.1250061988830566) + | > avg_loss_mel: 15.639106750488281 (-3.451883316040039) + | > avg_loss_duration: 1.7770289182662964 (+0.012820899486541748) + | > avg_loss_1: 28.272509574890137 (-4.664717674255371) + + + > EPOCH: 437/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:09:30)  + + --> TIME: 2025-04-19 06:09:35 -- STEP: 14/142 -- GLOBAL_STEP: 441350 + | > loss_disc: 2.340733051300049 (2.432610034942627) + | > loss_disc_real_0: 0.1047004833817482 (0.1161402277648449) + | > loss_disc_real_1: 0.21675075590610504 (0.2029504222529275) + | > loss_disc_real_2: 0.22437775135040283 (0.2284537756017276) + | > loss_disc_real_3: 0.1652071177959442 (0.22662094341857092) + | > loss_disc_real_4: 0.17332406342029572 (0.22259262097733362) + | > loss_disc_real_5: 0.17443746328353882 (0.226644438292299) + | > loss_0: 2.340733051300049 (2.432610034942627) + | > grad_norm_0: tensor(4.5397, device='cuda:0') (tensor(5.8279, device='cuda:0')) + | > loss_gen: 2.3171157836914062 (2.3258573157446727) + | > loss_kl: 1.8469529151916504 (1.7705957974706377) + | > loss_feat: 8.213223457336426 (7.15355430330549) + | > loss_mel: 18.63392448425293 (17.555405344281876) + | > loss_duration: 1.4121931791305542 (1.4097472940172469) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.42340850830078 (30.21516009739467) + | > grad_norm_1: tensor(40.1753, device='cuda:0') (tensor(119.3560, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.3314 (0.32798549107142855) + | > loader_time: 0.0053 (0.0046509674617222375) + + + --> TIME: 2025-04-19 06:09:44 -- STEP: 39/142 -- GLOBAL_STEP: 441375 + | > loss_disc: 2.443148136138916 (2.4158861270317664) + | > loss_disc_real_0: 0.11121435463428497 (0.11610066202970651) + | > loss_disc_real_1: 0.23761621117591858 (0.20174137827677607) + | > loss_disc_real_2: 0.26906657218933105 (0.2268701256849827) + | > loss_disc_real_3: 0.25236937403678894 (0.22499980414525056) + | > loss_disc_real_4: 0.2465374767780304 (0.22263699502517015) + | > loss_disc_real_5: 0.20955485105514526 (0.23026616336443487) + | > loss_0: 2.443148136138916 (2.4158861270317664) + | > grad_norm_0: tensor(11.7190, device='cuda:0') (tensor(6.6990, device='cuda:0')) + | > loss_gen: 2.402998924255371 (2.3798231283823648) + | > loss_kl: 1.9188520908355713 (1.8082336065096734) + | > loss_feat: 7.5634846687316895 (7.303362222818228) + | > loss_mel: 17.561960220336914 (17.60989467914288) + | > loss_duration: 1.4363200664520264 (1.420484279974913) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.883615493774414 (30.521798256116035) + | > grad_norm_1: tensor(325.5415, device='cuda:0') (tensor(157.0908, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.3602 (0.34630831082661945) + | > loader_time: 0.0062 (0.005189003088535406) + + + --> TIME: 2025-04-19 06:09:54 -- STEP: 64/142 -- GLOBAL_STEP: 441400 + | > loss_disc: 2.461507797241211 (2.4284007735550404) + | > loss_disc_real_0: 0.14547543227672577 (0.11739513487555087) + | > loss_disc_real_1: 0.19679909944534302 (0.2051044951658696) + | > loss_disc_real_2: 0.2180129885673523 (0.2254558578133583) + | > loss_disc_real_3: 0.26482775807380676 (0.2234943793155253) + | > loss_disc_real_4: 0.2495519071817398 (0.22613860946148634) + | > loss_disc_real_5: 0.2520916163921356 (0.2333213542588055) + | > loss_0: 2.461507797241211 (2.4284007735550404) + | > grad_norm_0: tensor(7.2234, device='cuda:0') (tensor(6.8013, device='cuda:0')) + | > loss_gen: 2.3108434677124023 (2.378824841231108) + | > loss_kl: 1.8132481575012207 (1.8151110261678696) + | > loss_feat: 6.996446132659912 (7.267087891697884) + | > loss_mel: 18.535058975219727 (17.636925488710403) + | > loss_duration: 1.4623034000396729 (1.4324811473488808) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.117900848388672 (30.530430525541306) + | > grad_norm_1: tensor(139.6390, device='cuda:0') (tensor(164.3487, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.3837 (0.3565860688686371) + | > loader_time: 0.0066 (0.005553632974624634) + + + --> TIME: 2025-04-19 06:10:04 -- STEP: 89/142 -- GLOBAL_STEP: 441425 + | > loss_disc: 2.4534058570861816 (2.4485504895113825) + | > loss_disc_real_0: 0.12711000442504883 (0.12043620453456814) + | > loss_disc_real_1: 0.2583041489124298 (0.20755626612834716) + | > loss_disc_real_2: 0.245228111743927 (0.2260227318894997) + | > loss_disc_real_3: 0.274795264005661 (0.22806281688508023) + | > loss_disc_real_4: 0.22674806416034698 (0.2271447494793474) + | > loss_disc_real_5: 0.21464568376541138 (0.2330765744273582) + | > loss_0: 2.4534058570861816 (2.4485504895113825) + | > grad_norm_0: tensor(9.7328, device='cuda:0') (tensor(7.2836, device='cuda:0')) + | > loss_gen: 2.451413869857788 (2.3715849110249736) + | > loss_kl: 2.012216806411743 (1.8363371699043873) + | > loss_feat: 7.346022605895996 (7.220227332597368) + | > loss_mel: 17.671157836914062 (17.621648241964614) + | > loss_duration: 1.4588932991027832 (1.4419699652811115) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.9397029876709 (30.491767668991947) + | > grad_norm_1: tensor(50.6442, device='cuda:0') (tensor(171.4009, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.3984 (0.3669610612847832) + | > loader_time: 0.0067 (0.005855375461364062) + + + --> TIME: 2025-04-19 06:10:15 -- STEP: 114/142 -- GLOBAL_STEP: 441450 + | > loss_disc: 2.551866054534912 (2.4572981324112204) + | > loss_disc_real_0: 0.135102778673172 (0.12139439889997766) + | > loss_disc_real_1: 0.21916957199573517 (0.20745777090390524) + | > loss_disc_real_2: 0.23159897327423096 (0.22731501988151617) + | > loss_disc_real_3: 0.2128181904554367 (0.2282196285954693) + | > loss_disc_real_4: 0.2011016309261322 (0.2268566652609591) + | > loss_disc_real_5: 0.20258304476737976 (0.23311116938528262) + | > loss_0: 2.551866054534912 (2.4572981324112204) + | > grad_norm_0: tensor(10.5808, device='cuda:0') (tensor(7.2159, device='cuda:0')) + | > loss_gen: 2.1377477645874023 (2.3562134044212217) + | > loss_kl: 1.7708221673965454 (1.8445882661300792) + | > loss_feat: 6.394171714782715 (7.144648317705121) + | > loss_mel: 16.470396041870117 (17.623680825819044) + | > loss_duration: 1.4714804887771606 (1.449563443660736) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.24462127685547 (30.418694295381243) + | > grad_norm_1: tensor(237.2064, device='cuda:0') (tensor(167.0807, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.4216 (0.37701472692322313) + | > loader_time: 0.0071 (0.006099454143591095) + + + --> TIME: 2025-04-19 06:10:26 -- STEP: 139/142 -- GLOBAL_STEP: 441475 + | > loss_disc: 2.4368653297424316 (2.4662870914816) + | > loss_disc_real_0: 0.10077648609876633 (0.1238650746589942) + | > loss_disc_real_1: 0.22750253975391388 (0.20855217603899592) + | > loss_disc_real_2: 0.21077598631381989 (0.2280808835149669) + | > loss_disc_real_3: 0.1959063559770584 (0.22868007646618987) + | > loss_disc_real_4: 0.21152707934379578 (0.2278144940626707) + | > loss_disc_real_5: 0.21320705115795135 (0.23372774726624113) + | > loss_0: 2.4368653297424316 (2.4662870914816) + | > grad_norm_0: tensor(7.3605, device='cuda:0') (tensor(7.5407, device='cuda:0')) + | > loss_gen: 2.286557912826538 (2.346132847902587) + | > loss_kl: 1.916075348854065 (1.8535614022248084) + | > loss_feat: 7.238739013671875 (7.083715757877707) + | > loss_mel: 17.27074432373047 (17.56170062881579) + | > loss_duration: 1.4817349910736084 (1.4561761308917036) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.1938533782959 (30.301286834607023) + | > grad_norm_1: tensor(250.9876, device='cuda:0') (tensor(170.5128, device='cuda:0')) + | > current_lr_0: 0.00018936738272133647 + | > current_lr_1: 0.00018936738272133647 + | > step_time: 0.456 (0.3888053156489091) + | > loader_time: 0.0076 (0.0063615497067677895) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.550873041152954 (2.550873041152954) + | > loss_disc_real_0: 0.26548609137535095 (0.26548609137535095) + | > loss_disc_real_1: 0.17614929378032684 (0.17614929378032684) + | > loss_disc_real_2: 0.23203516006469727 (0.23203516006469727) + | > loss_disc_real_3: 0.2463589310646057 (0.2463589310646057) + | > loss_disc_real_4: 0.24597106873989105 (0.24597106873989105) + | > loss_disc_real_5: 0.23046816885471344 (0.23046816885471344) + | > loss_0: 2.550873041152954 (2.550873041152954) + | > loss_gen: 2.4876484870910645 (2.4876484870910645) + | > loss_kl: 2.277451753616333 (2.277451753616333) + | > loss_feat: 6.090975761413574 (6.090975761413574) + | > loss_mel: 16.411176681518555 (16.411176681518555) + | > loss_duration: 1.7465145587921143 (1.7465145587921143) + | > loss_1: 29.01376724243164 (29.01376724243164) + + --> STEP: 1 + | > loss_disc: 2.572634696960449 (2.572634696960449) + | > loss_disc_real_0: 0.2049337923526764 (0.2049337923526764) + | > loss_disc_real_1: 0.20642855763435364 (0.20642855763435364) + | > loss_disc_real_2: 0.24857290089130402 (0.24857290089130402) + | > loss_disc_real_3: 0.2611061632633209 (0.2611061632633209) + | > loss_disc_real_4: 0.2255934327840805 (0.2255934327840805) + | > loss_disc_real_5: 0.2545117139816284 (0.2545117139816284) + | > loss_0: 2.572634696960449 (2.572634696960449) + | > loss_gen: 2.3952929973602295 (2.3952929973602295) + | > loss_kl: 2.7204623222351074 (2.7204623222351074) + | > loss_feat: 6.379920482635498 (6.379920482635498) + | > loss_mel: 17.87171173095703 (17.87171173095703) + | > loss_duration: 1.783420205116272 (1.783420205116272) + | > loss_1: 31.150808334350586 (31.150808334350586) + + --> STEP: 2 + | > loss_disc: 2.448303699493408 (2.5104691982269287) + | > loss_disc_real_0: 0.20552191138267517 (0.20522785186767578) + | > loss_disc_real_1: 0.2041880339384079 (0.20530829578638077) + | > loss_disc_real_2: 0.24132968485355377 (0.2449512928724289) + | > loss_disc_real_3: 0.2390567660331726 (0.25008146464824677) + | > loss_disc_real_4: 0.20803675055503845 (0.21681509166955948) + | > loss_disc_real_5: 0.2342817634344101 (0.24439673870801926) + | > loss_0: 2.448303699493408 (2.5104691982269287) + | > loss_gen: 2.728055238723755 (2.561674118041992) + | > loss_kl: 2.4129180908203125 (2.56669020652771) + | > loss_feat: 7.705145359039307 (7.042532920837402) + | > loss_mel: 17.926694869995117 (17.899203300476074) + | > loss_duration: 1.7323129177093506 (1.7578665614128113) + | > loss_1: 32.505126953125 (31.827967643737793) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003529667854309082 (+0.00035512447357177734) + | > avg_loss_disc: 2.5104691982269287 (-0.06049621105194092) + | > avg_loss_disc_real_0: 0.20522785186767578 (+0.020163334906101227) + | > avg_loss_disc_real_1: 0.20530829578638077 (-0.02763649821281433) + | > avg_loss_disc_real_2: 0.2449512928724289 (+0.007879599928855896) + | > avg_loss_disc_real_3: 0.25008146464824677 (+0.014621242880821228) + | > avg_loss_disc_real_4: 0.21681509166955948 (-0.008712008595466614) + | > avg_loss_disc_real_5: 0.24439673870801926 (+0.03580246865749359) + | > avg_loss_0: 2.5104691982269287 (-0.06049621105194092) + | > avg_loss_gen: 2.561674118041992 (+0.3747823238372803) + | > avg_loss_kl: 2.56669020652771 (-0.01147317886352539) + | > avg_loss_feat: 7.042532920837402 (+0.9512135982513428) + | > avg_loss_mel: 17.899203300476074 (+2.260096549987793) + | > avg_loss_duration: 1.7578665614128113 (-0.019162356853485107) + | > avg_loss_1: 31.827967643737793 (+3.5554580688476562) + + + > EPOCH: 438/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:10:29)  + + --> TIME: 2025-04-19 06:10:37 -- STEP: 22/142 -- GLOBAL_STEP: 441500 + | > loss_disc: 2.3525302410125732 (2.4201689308339898) + | > loss_disc_real_0: 0.112399160861969 (0.11924879828637297) + | > loss_disc_real_1: 0.18604280054569244 (0.20244870470328766) + | > loss_disc_real_2: 0.21933592855930328 (0.22258615832437167) + | > loss_disc_real_3: 0.240407332777977 (0.2287722419608723) + | > loss_disc_real_4: 0.25423234701156616 (0.2258687987923622) + | > loss_disc_real_5: 0.23412062227725983 (0.23234675553711978) + | > loss_0: 2.3525302410125732 (2.4201689308339898) + | > grad_norm_0: tensor(5.4334, device='cuda:0') (tensor(6.9358, device='cuda:0')) + | > loss_gen: 2.5179405212402344 (2.3771523887460884) + | > loss_kl: 1.750677466392517 (1.770333089611747) + | > loss_feat: 7.768968105316162 (7.127299547195435) + | > loss_mel: 17.926544189453125 (17.34573888778687) + | > loss_duration: 1.3674671649932861 (1.4125581546263262) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.33159828186035 (30.033082268454812) + | > grad_norm_1: tensor(216.5742, device='cuda:0') (tensor(137.9590, device='cuda:0')) + | > current_lr_0: 0.0001893437117984963 + | > current_lr_1: 0.0001893437117984963 + | > step_time: 0.3427 (0.3309269168160178) + | > loader_time: 0.0057 (0.005035877227783203) + + + --> TIME: 2025-04-19 06:10:46 -- STEP: 47/142 -- GLOBAL_STEP: 441525 + | > loss_disc: 2.5139060020446777 (2.4432249170668583) + | > loss_disc_real_0: 0.1266200840473175 (0.12122576953248775) + | > loss_disc_real_1: 0.21450094878673553 (0.20406803139980803) + | > loss_disc_real_2: 0.21507178246974945 (0.22577655600740554) + | > loss_disc_real_3: 0.21976463496685028 (0.22761752377165126) + | > loss_disc_real_4: 0.22512488067150116 (0.22976300627627272) + | > loss_disc_real_5: 0.23483906686306 (0.23363270182558832) + | > loss_0: 2.5139060020446777 (2.4432249170668583) + | > grad_norm_0: tensor(6.0788, device='cuda:0') (tensor(6.7360, device='cuda:0')) + | > loss_gen: 2.3300154209136963 (2.384694023335234) + | > loss_kl: 1.856003999710083 (1.784492997412986) + | > loss_feat: 6.710103511810303 (7.244397315573185) + | > loss_mel: 16.937545776367188 (17.62929041842197) + | > loss_duration: 1.416778326034546 (1.425491748972142) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.25044822692871 (30.468366541761032) + | > grad_norm_1: tensor(139.7576, device='cuda:0') (tensor(131.4443, device='cuda:0')) + | > current_lr_0: 0.0001893437117984963 + | > current_lr_1: 0.0001893437117984963 + | > step_time: 0.364 (0.3436545564773235) + | > loader_time: 0.0059 (0.005382593641889857) + + + --> TIME: 2025-04-19 06:10:56 -- STEP: 72/142 -- GLOBAL_STEP: 441550 + | > loss_disc: 2.561264753341675 (2.4531325300534568) + | > loss_disc_real_0: 0.18000289797782898 (0.12348353159096506) + | > loss_disc_real_1: 0.17592813074588776 (0.2061419039964676) + | > loss_disc_real_2: 0.26505085825920105 (0.22570373593933052) + | > loss_disc_real_3: 0.2233029007911682 (0.22805879968735907) + | > loss_disc_real_4: 0.253935307264328 (0.22991416023837197) + | > loss_disc_real_5: 0.2496332973241806 (0.23348091542720795) + | > loss_0: 2.561264753341675 (2.4531325300534568) + | > grad_norm_0: tensor(19.0362, device='cuda:0') (tensor(7.3281, device='cuda:0')) + | > loss_gen: 2.354865550994873 (2.3712309830718574) + | > loss_kl: 1.9681389331817627 (1.797069971760114) + | > loss_feat: 7.685641288757324 (7.208423773447673) + | > loss_mel: 18.11405372619629 (17.61514237191942) + | > loss_duration: 1.4394195079803467 (1.4368584884537592) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.562118530273438 (30.42872561348809) + | > grad_norm_1: tensor(361.7996, device='cuda:0') (tensor(131.4048, device='cuda:0')) + | > current_lr_0: 0.0001893437117984963 + | > current_lr_1: 0.0001893437117984963 + | > step_time: 0.3951 (0.3560396101739671) + | > loader_time: 0.0069 (0.005683157179090712) + + + --> TIME: 2025-04-19 06:11:06 -- STEP: 97/142 -- GLOBAL_STEP: 441575 + | > loss_disc: 2.4607455730438232 (2.4532049439617043) + | > loss_disc_real_0: 0.1306561380624771 (0.12456782407981833) + | > loss_disc_real_1: 0.202244833111763 (0.20717421327669597) + | > loss_disc_real_2: 0.19144850969314575 (0.2237154438938062) + | > loss_disc_real_3: 0.2148221731185913 (0.22779016196727753) + | > loss_disc_real_4: 0.18225103616714478 (0.2285107280482951) + | > loss_disc_real_5: 0.19972103834152222 (0.23224130272865295) + | > loss_0: 2.4607455730438232 (2.4532049439617043) + | > grad_norm_0: tensor(12.9371, device='cuda:0') (tensor(7.9508, device='cuda:0')) + | > loss_gen: 2.3014657497406006 (2.359048912205647) + | > loss_kl: 1.9021075963974 (1.8100186635538476) + | > loss_feat: 7.127384662628174 (7.138189478018849) + | > loss_mel: 16.964130401611328 (17.541293163889453) + | > loss_duration: 1.4780840873718262 (1.447251557074871) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.77317237854004 (30.295801831274918) + | > grad_norm_1: tensor(254.2436, device='cuda:0') (tensor(145.6614, device='cuda:0')) + | > current_lr_0: 0.0001893437117984963 + | > current_lr_1: 0.0001893437117984963 + | > step_time: 0.4095 (0.3676769684270485) + | > loader_time: 0.0069 (0.005945881617437933) + + + --> TIME: 2025-04-19 06:11:17 -- STEP: 122/142 -- GLOBAL_STEP: 441600 + | > loss_disc: 2.4729409217834473 (2.461619068364628) + | > loss_disc_real_0: 0.14653438329696655 (0.12595658010390942) + | > loss_disc_real_1: 0.2168598771095276 (0.20849595255539066) + | > loss_disc_real_2: 0.21207647025585175 (0.22495006305761026) + | > loss_disc_real_3: 0.21699726581573486 (0.22907771084640846) + | > loss_disc_real_4: 0.21311011910438538 (0.22899423063289923) + | > loss_disc_real_5: 0.24620413780212402 (0.23207274934307473) + | > loss_0: 2.4729409217834473 (2.461619068364628) + | > grad_norm_0: tensor(21.2306, device='cuda:0') (tensor(8.6965, device='cuda:0')) + | > loss_gen: 2.4620392322540283 (2.3540450315006445) + | > loss_kl: 1.890552043914795 (1.8214843224306576) + | > loss_feat: 7.314321041107178 (7.095550482390357) + | > loss_mel: 17.782215118408203 (17.54132893828094) + | > loss_duration: 1.4699779748916626 (1.4543099637891423) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.919105529785156 (30.266718864440918) + | > grad_norm_1: tensor(421.5343, device='cuda:0') (tensor(169.4610, device='cuda:0')) + | > current_lr_0: 0.0001893437117984963 + | > current_lr_1: 0.0001893437117984963 + | > step_time: 0.4524 (0.37910611903081176) + | > loader_time: 0.0076 (0.006170829788583224) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.7340502738952637 (2.7340502738952637) + | > loss_disc_real_0: 0.22137677669525146 (0.22137677669525146) + | > loss_disc_real_1: 0.2808716595172882 (0.2808716595172882) + | > loss_disc_real_2: 0.2858319580554962 (0.2858319580554962) + | > loss_disc_real_3: 0.291646271944046 (0.291646271944046) + | > loss_disc_real_4: 0.2679898738861084 (0.2679898738861084) + | > loss_disc_real_5: 0.25023096799850464 (0.25023096799850464) + | > loss_0: 2.7340502738952637 (2.7340502738952637) + | > loss_gen: 2.43442440032959 (2.43442440032959) + | > loss_kl: 2.5553674697875977 (2.5553674697875977) + | > loss_feat: 6.0543646812438965 (6.0543646812438965) + | > loss_mel: 16.995161056518555 (16.995161056518555) + | > loss_duration: 1.7487657070159912 (1.7487657070159912) + | > loss_1: 29.788084030151367 (29.788084030151367) + + --> STEP: 1 + | > loss_disc: 2.4200475215911865 (2.4200475215911865) + | > loss_disc_real_0: 0.158797487616539 (0.158797487616539) + | > loss_disc_real_1: 0.26886481046676636 (0.26886481046676636) + | > loss_disc_real_2: 0.26645198464393616 (0.26645198464393616) + | > loss_disc_real_3: 0.23961658775806427 (0.23961658775806427) + | > loss_disc_real_4: 0.2503979802131653 (0.2503979802131653) + | > loss_disc_real_5: 0.2530508041381836 (0.2530508041381836) + | > loss_0: 2.4200475215911865 (2.4200475215911865) + | > loss_gen: 2.58298397064209 (2.58298397064209) + | > loss_kl: 2.716899871826172 (2.716899871826172) + | > loss_feat: 6.815126419067383 (6.815126419067383) + | > loss_mel: 18.49819564819336 (18.49819564819336) + | > loss_duration: 1.8041491508483887 (1.8041491508483887) + | > loss_1: 32.417354583740234 (32.417354583740234) + + --> STEP: 2 + | > loss_disc: 2.4157328605651855 (2.417890191078186) + | > loss_disc_real_0: 0.1840105652809143 (0.17140402644872665) + | > loss_disc_real_1: 0.31559568643569946 (0.2922302484512329) + | > loss_disc_real_2: 0.2790241837501526 (0.2727380841970444) + | > loss_disc_real_3: 0.2364208698272705 (0.2380187287926674) + | > loss_disc_real_4: 0.23452447354793549 (0.24246122688055038) + | > loss_disc_real_5: 0.22580094635486603 (0.2394258752465248) + | > loss_0: 2.4157328605651855 (2.417890191078186) + | > loss_gen: 2.76590633392334 (2.674445152282715) + | > loss_kl: 2.2741899490356445 (2.495544910430908) + | > loss_feat: 7.925661563873291 (7.370393991470337) + | > loss_mel: 18.69944953918457 (18.598822593688965) + | > loss_duration: 1.7227404117584229 (1.7634447813034058) + | > loss_1: 33.38794708251953 (32.90265083312988) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003537893295288086 (+8.225440979003906e-06) + | > avg_loss_disc: 2.417890191078186 (-0.09257900714874268) + | > avg_loss_disc_real_0: 0.17140402644872665 (-0.03382382541894913) + | > avg_loss_disc_real_1: 0.2922302484512329 (+0.08692195266485214) + | > avg_loss_disc_real_2: 0.2727380841970444 (+0.02778679132461548) + | > avg_loss_disc_real_3: 0.2380187287926674 (-0.012062735855579376) + | > avg_loss_disc_real_4: 0.24246122688055038 (+0.025646135210990906) + | > avg_loss_disc_real_5: 0.2394258752465248 (-0.004970863461494446) + | > avg_loss_0: 2.417890191078186 (-0.09257900714874268) + | > avg_loss_gen: 2.674445152282715 (+0.11277103424072266) + | > avg_loss_kl: 2.495544910430908 (-0.07114529609680176) + | > avg_loss_feat: 7.370393991470337 (+0.32786107063293457) + | > avg_loss_mel: 18.598822593688965 (+0.6996192932128906) + | > avg_loss_duration: 1.7634447813034058 (+0.005578219890594482) + | > avg_loss_1: 32.90265083312988 (+1.0746831893920898) + + + > EPOCH: 439/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:11:27)  + + --> TIME: 2025-04-19 06:11:30 -- STEP: 5/142 -- GLOBAL_STEP: 441625 + | > loss_disc: 2.495769739151001 (2.4968988418579103) + | > loss_disc_real_0: 0.10663946717977524 (0.11213721185922623) + | > loss_disc_real_1: 0.1810532510280609 (0.2126002013683319) + | > loss_disc_real_2: 0.19042839109897614 (0.22755993008613587) + | > loss_disc_real_3: 0.22499866783618927 (0.22440455257892608) + | > loss_disc_real_4: 0.19813582301139832 (0.23685213029384614) + | > loss_disc_real_5: 0.25232836604118347 (0.2502220213413239) + | > loss_0: 2.495769739151001 (2.4968988418579103) + | > grad_norm_0: tensor(9.6560, device='cuda:0') (tensor(7.2191, device='cuda:0')) + | > loss_gen: 2.4177350997924805 (2.324375057220459) + | > loss_kl: 1.6923381090164185 (1.737657403945923) + | > loss_feat: 7.3827738761901855 (6.958555698394775) + | > loss_mel: 18.094160079956055 (17.58879737854004) + | > loss_duration: 1.4111390113830566 (1.4137315034866333) + | > amp_scaler: 256.0 (153.6) + | > loss_1: 30.99814796447754 (30.023117446899413) + | > grad_norm_1: tensor(406.1086, device='cuda:0') (tensor(247.2159, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.3244 (0.3232434749603271) + | > loader_time: 0.0051 (0.004515314102172851) + + + --> TIME: 2025-04-19 06:11:38 -- STEP: 30/142 -- GLOBAL_STEP: 441650 + | > loss_disc: 2.4734694957733154 (2.465044291814168) + | > loss_disc_real_0: 0.11876440048217773 (0.12264169628421466) + | > loss_disc_real_1: 0.2293001115322113 (0.2087719589471817) + | > loss_disc_real_2: 0.21266934275627136 (0.22646814584732056) + | > loss_disc_real_3: 0.24051488935947418 (0.22577911068995793) + | > loss_disc_real_4: 0.20075123012065887 (0.22889595925807954) + | > loss_disc_real_5: 0.24367071688175201 (0.23306842843691508) + | > loss_0: 2.4734694957733154 (2.465044291814168) + | > grad_norm_0: tensor(8.3487, device='cuda:0') (tensor(9.8728, device='cuda:0')) + | > loss_gen: 2.5933470726013184 (2.3624115308125817) + | > loss_kl: 1.8039449453353882 (1.7803756197293599) + | > loss_feat: 7.8804826736450195 (7.201090335845947) + | > loss_mel: 18.602806091308594 (17.732919788360597) + | > loss_duration: 1.4264107942581177 (1.418621019522349) + | > amp_scaler: 256.0 (238.93333333333334) + | > loss_1: 32.30699157714844 (30.495418357849122) + | > grad_norm_1: tensor(104.2735, device='cuda:0') (tensor(232.1373, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.3494 (0.33565882047017415) + | > loader_time: 0.0058 (0.0051264127095540365) + + + --> TIME: 2025-04-19 06:11:48 -- STEP: 55/142 -- GLOBAL_STEP: 441675 + | > loss_disc: 2.4545257091522217 (2.4575625116174864) + | > loss_disc_real_0: 0.14486268162727356 (0.12202020531350916) + | > loss_disc_real_1: 0.255033403635025 (0.20921059088273483) + | > loss_disc_real_2: 0.27892202138900757 (0.22865509580482135) + | > loss_disc_real_3: 0.23645634949207306 (0.22858072952790695) + | > loss_disc_real_4: 0.22317777574062347 (0.22797196995128285) + | > loss_disc_real_5: 0.20814159512519836 (0.23004960417747497) + | > loss_0: 2.4545257091522217 (2.4575625116174864) + | > grad_norm_0: tensor(7.1849, device='cuda:0') (tensor(9.7322, device='cuda:0')) + | > loss_gen: 2.435542583465576 (2.3747216831554065) + | > loss_kl: 1.9094102382659912 (1.8077284010973844) + | > loss_feat: 6.696404933929443 (7.244762073863637) + | > loss_mel: 16.296947479248047 (17.661157573353165) + | > loss_duration: 1.4855091571807861 (1.4283024939623745) + | > amp_scaler: 256.0 (246.6909090909091) + | > loss_1: 28.823816299438477 (30.51667234247381) + | > grad_norm_1: tensor(229.4456, device='cuda:0') (tensor(228.1929, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.3743 (0.34804559187455625) + | > loader_time: 0.0062 (0.005466604232788086) + + + --> TIME: 2025-04-19 06:11:57 -- STEP: 80/142 -- GLOBAL_STEP: 441700 + | > loss_disc: 2.347087860107422 (2.454958334565162) + | > loss_disc_real_0: 0.08288741111755371 (0.12103075738996268) + | > loss_disc_real_1: 0.19733065366744995 (0.20712596625089646) + | > loss_disc_real_2: 0.23780415952205658 (0.2283160174265504) + | > loss_disc_real_3: 0.24124625325202942 (0.22812975104898214) + | > loss_disc_real_4: 0.2202220857143402 (0.22884714249521493) + | > loss_disc_real_5: 0.18294234573841095 (0.2320248045027256) + | > loss_0: 2.347087860107422 (2.454958334565162) + | > grad_norm_0: tensor(12.9079, device='cuda:0') (tensor(10.8722, device='cuda:0')) + | > loss_gen: 2.4215586185455322 (2.3858810573816305) + | > loss_kl: 1.891036033630371 (1.8223533287644387) + | > loss_feat: 7.759893894195557 (7.246228736639023) + | > loss_mel: 18.94487190246582 (17.71378821134568) + | > loss_duration: 1.4603856801986694 (1.4390227451920512) + | > amp_scaler: 256.0 (249.6) + | > loss_1: 32.477745056152344 (30.6072740316391) + | > grad_norm_1: tensor(321.0124, device='cuda:0') (tensor(251.9226, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.4055 (0.36019454896450054) + | > loader_time: 0.0063 (0.005747187137603761) + + + --> TIME: 2025-04-19 06:12:08 -- STEP: 105/142 -- GLOBAL_STEP: 441725 + | > loss_disc: 2.4583091735839844 (2.446029490516299) + | > loss_disc_real_0: 0.11129573732614517 (0.1219306190099035) + | > loss_disc_real_1: 0.18174898624420166 (0.20604040523370107) + | > loss_disc_real_2: 0.18133899569511414 (0.22655945789246332) + | > loss_disc_real_3: 0.20891977846622467 (0.2282690031187875) + | > loss_disc_real_4: 0.17819435894489288 (0.22687171342827026) + | > loss_disc_real_5: 0.19084730744361877 (0.23065019661471958) + | > loss_0: 2.4583091735839844 (2.446029490516299) + | > grad_norm_0: tensor(21.7032, device='cuda:0') (tensor(11.1743, device='cuda:0')) + | > loss_gen: 2.1815245151519775 (2.3862883976527636) + | > loss_kl: 1.9004509449005127 (1.833977054414295) + | > loss_feat: 7.863609790802002 (7.238346077146984) + | > loss_mel: 18.698108673095703 (17.681879615783693) + | > loss_duration: 1.4933156967163086 (1.449015269960676) + | > amp_scaler: 256.0 (251.1238095238095) + | > loss_1: 32.13700866699219 (30.589506367274694) + | > grad_norm_1: tensor(355.4859, device='cuda:0') (tensor(259.0125, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.4167 (0.3704849243164064) + | > loader_time: 0.007 (0.006004263105846588) + + + --> TIME: 2025-04-19 06:12:19 -- STEP: 130/142 -- GLOBAL_STEP: 441750 + | > loss_disc: 2.345294952392578 (2.4480218098713795) + | > loss_disc_real_0: 0.099884994328022 (0.1219418147435555) + | > loss_disc_real_1: 0.21440397202968597 (0.20654170008806083) + | > loss_disc_real_2: 0.21540789306163788 (0.22664846720603796) + | > loss_disc_real_3: 0.2078717201948166 (0.22777738066819997) + | > loss_disc_real_4: 0.19965282082557678 (0.22807611123873636) + | > loss_disc_real_5: 0.21394670009613037 (0.23164038681066954) + | > loss_0: 2.345294952392578 (2.4480218098713795) + | > grad_norm_0: tensor(8.8539, device='cuda:0') (tensor(11.1084, device='cuda:0')) + | > loss_gen: 2.3353517055511475 (2.3863355838335485) + | > loss_kl: 1.8439432382583618 (1.8390197726396413) + | > loss_feat: 7.5971221923828125 (7.206735123120821) + | > loss_mel: 18.83226776123047 (17.687282642951377) + | > loss_duration: 1.4575086832046509 (1.4547322474993192) + | > amp_scaler: 256.0 (252.06153846153848) + | > loss_1: 32.06619644165039 (30.57410535078782) + | > grad_norm_1: tensor(155.2516, device='cuda:0') (tensor(252.8879, device='cuda:0')) + | > current_lr_0: 0.00018932004383452147 + | > current_lr_1: 0.00018932004383452147 + | > step_time: 0.4493 (0.38231837566082305) + | > loader_time: 0.0076 (0.006274958757253794) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.398425579071045 (2.398425579071045) + | > loss_disc_real_0: 0.13514910638332367 (0.13514910638332367) + | > loss_disc_real_1: 0.23119981586933136 (0.23119981586933136) + | > loss_disc_real_2: 0.24070625007152557 (0.24070625007152557) + | > loss_disc_real_3: 0.26060694456100464 (0.26060694456100464) + | > loss_disc_real_4: 0.284056693315506 (0.284056693315506) + | > loss_disc_real_5: 0.25253450870513916 (0.25253450870513916) + | > loss_0: 2.398425579071045 (2.398425579071045) + | > loss_gen: 2.645775556564331 (2.645775556564331) + | > loss_kl: 2.376636266708374 (2.376636266708374) + | > loss_feat: 7.448175430297852 (7.448175430297852) + | > loss_mel: 19.216217041015625 (19.216217041015625) + | > loss_duration: 1.7554882764816284 (1.7554882764816284) + | > loss_1: 33.442291259765625 (33.442291259765625) + + --> STEP: 1 + | > loss_disc: 2.5468955039978027 (2.5468955039978027) + | > loss_disc_real_0: 0.14632248878479004 (0.14632248878479004) + | > loss_disc_real_1: 0.27352553606033325 (0.27352553606033325) + | > loss_disc_real_2: 0.25608760118484497 (0.25608760118484497) + | > loss_disc_real_3: 0.2527795135974884 (0.2527795135974884) + | > loss_disc_real_4: 0.3173893988132477 (0.3173893988132477) + | > loss_disc_real_5: 0.25979870557785034 (0.25979870557785034) + | > loss_0: 2.5468955039978027 (2.5468955039978027) + | > loss_gen: 2.4873807430267334 (2.4873807430267334) + | > loss_kl: 2.7623229026794434 (2.7623229026794434) + | > loss_feat: 5.657382011413574 (5.657382011413574) + | > loss_mel: 16.526201248168945 (16.526201248168945) + | > loss_duration: 1.8391389846801758 (1.8391389846801758) + | > loss_1: 29.27242660522461 (29.27242660522461) + + --> STEP: 2 + | > loss_disc: 2.543166160583496 (2.5450308322906494) + | > loss_disc_real_0: 0.1748061627149582 (0.16056432574987411) + | > loss_disc_real_1: 0.24393902719020844 (0.25873228162527084) + | > loss_disc_real_2: 0.24623969197273254 (0.25116364657878876) + | > loss_disc_real_3: 0.2552184760570526 (0.2539989948272705) + | > loss_disc_real_4: 0.2974969446659088 (0.30744317173957825) + | > loss_disc_real_5: 0.2819056808948517 (0.270852193236351) + | > loss_0: 2.543166160583496 (2.5450308322906494) + | > loss_gen: 2.436485528945923 (2.461933135986328) + | > loss_kl: 2.4883453845977783 (2.625334143638611) + | > loss_feat: 5.846920013427734 (5.752151012420654) + | > loss_mel: 16.300506591796875 (16.41335391998291) + | > loss_duration: 1.7437756061553955 (1.7914572954177856) + | > loss_1: 28.8160343170166 (29.044230461120605) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032502412796020508 (-0.00028765201568603516) + | > avg_loss_disc: 2.5450308322906494 (+0.12714064121246338) + | > avg_loss_disc_real_0: 0.16056432574987411 (-0.010839700698852539) + | > avg_loss_disc_real_1: 0.25873228162527084 (-0.03349796682596207) + | > avg_loss_disc_real_2: 0.25116364657878876 (-0.021574437618255615) + | > avg_loss_disc_real_3: 0.2539989948272705 (+0.01598026603460312) + | > avg_loss_disc_real_4: 0.30744317173957825 (+0.06498194485902786) + | > avg_loss_disc_real_5: 0.270852193236351 (+0.0314263179898262) + | > avg_loss_0: 2.5450308322906494 (+0.12714064121246338) + | > avg_loss_gen: 2.461933135986328 (-0.21251201629638672) + | > avg_loss_kl: 2.625334143638611 (+0.12978923320770264) + | > avg_loss_feat: 5.752151012420654 (-1.6182429790496826) + | > avg_loss_mel: 16.41335391998291 (-2.1854686737060547) + | > avg_loss_duration: 1.7914572954177856 (+0.028012514114379883) + | > avg_loss_1: 29.044230461120605 (-3.8584203720092773) + + + > EPOCH: 440/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:12:26)  + + --> TIME: 2025-04-19 06:12:31 -- STEP: 13/142 -- GLOBAL_STEP: 441775 + | > loss_disc: 2.3721742630004883 (2.382024911733774) + | > loss_disc_real_0: 0.0833590105175972 (0.11000306044633572) + | > loss_disc_real_1: 0.22387531399726868 (0.20307998817700607) + | > loss_disc_real_2: 0.2533982992172241 (0.22638403681608346) + | > loss_disc_real_3: 0.20190118253231049 (0.22355445073201105) + | > loss_disc_real_4: 0.25767752528190613 (0.23412138796769655) + | > loss_disc_real_5: 0.22557225823402405 (0.223147180217963) + | > loss_0: 2.3721742630004883 (2.382024911733774) + | > grad_norm_0: tensor(6.1977, device='cuda:0') (tensor(7.7762, device='cuda:0')) + | > loss_gen: 2.412719488143921 (2.4373007187476525) + | > loss_kl: 1.7840591669082642 (1.7746526919878447) + | > loss_feat: 7.073260307312012 (7.368304179264949) + | > loss_mel: 16.929128646850586 (17.44104312016414) + | > loss_duration: 1.4150638580322266 (1.4041641217011671) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.614233016967773 (30.42546536372258) + | > grad_norm_1: tensor(120.0264, device='cuda:0') (tensor(137.0146, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.332 (0.32877641457777756) + | > loader_time: 0.0046 (0.0045857246105487515) + + + --> TIME: 2025-04-19 06:12:40 -- STEP: 38/142 -- GLOBAL_STEP: 441800 + | > loss_disc: 2.416240692138672 (2.4011992467077157) + | > loss_disc_real_0: 0.1036144495010376 (0.11157601856087383) + | > loss_disc_real_1: 0.22330635786056519 (0.20361610894140444) + | > loss_disc_real_2: 0.25281277298927307 (0.2267277750529741) + | > loss_disc_real_3: 0.22861208021640778 (0.2275059474141974) + | > loss_disc_real_4: 0.2536601424217224 (0.22607841303474024) + | > loss_disc_real_5: 0.24651426076889038 (0.22737976830256612) + | > loss_0: 2.416240692138672 (2.4011992467077157) + | > grad_norm_0: tensor(10.1046, device='cuda:0') (tensor(7.8078, device='cuda:0')) + | > loss_gen: 2.3368701934814453 (2.4170973112708647) + | > loss_kl: 1.6751892566680908 (1.7836642233948958) + | > loss_feat: 7.410569190979004 (7.429246187210083) + | > loss_mel: 17.209184646606445 (17.74833347922878) + | > loss_duration: 1.450599193572998 (1.4173043621213812) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.082412719726562 (30.79564586438631) + | > grad_norm_1: tensor(343.7881, device='cuda:0') (tensor(203.7132, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.3529 (0.3401998281478882) + | > loader_time: 0.0054 (0.005139946937561033) + + + --> TIME: 2025-04-19 06:12:49 -- STEP: 63/142 -- GLOBAL_STEP: 441825 + | > loss_disc: 2.444284677505493 (2.405232940401351) + | > loss_disc_real_0: 0.10755089670419693 (0.11416308108776335) + | > loss_disc_real_1: 0.21275778114795685 (0.2046658458217742) + | > loss_disc_real_2: 0.22960254549980164 (0.2256637021662697) + | > loss_disc_real_3: 0.22895488142967224 (0.2267048562329913) + | > loss_disc_real_4: 0.24781940877437592 (0.22451125534753952) + | > loss_disc_real_5: 0.26719212532043457 (0.22596928240760925) + | > loss_0: 2.444284677505493 (2.405232940401351) + | > grad_norm_0: tensor(9.9249, device='cuda:0') (tensor(8.7276, device='cuda:0')) + | > loss_gen: 2.3756368160247803 (2.4096044964260526) + | > loss_kl: 1.9258564710617065 (1.8153426079522996) + | > loss_feat: 6.714113712310791 (7.374241791074238) + | > loss_mel: 16.415145874023438 (17.743278291490345) + | > loss_duration: 1.4808332920074463 (1.4302148610826522) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.911584854125977 (30.772682129390656) + | > grad_norm_1: tensor(254.8698, device='cuda:0') (tensor(218.7010, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.3886 (0.3550422910660032) + | > loader_time: 0.0055 (0.005456807121397957) + + + --> TIME: 2025-04-19 06:12:59 -- STEP: 88/142 -- GLOBAL_STEP: 441850 + | > loss_disc: 2.4505903720855713 (2.4097254953601155) + | > loss_disc_real_0: 0.1747240573167801 (0.115566729567945) + | > loss_disc_real_1: 0.2000606507062912 (0.2035994753241539) + | > loss_disc_real_2: 0.21753059327602386 (0.22485823353583162) + | > loss_disc_real_3: 0.22143809497356415 (0.22797647101635282) + | > loss_disc_real_4: 0.240683451294899 (0.22647484171796928) + | > loss_disc_real_5: 0.20568722486495972 (0.22814785452051597) + | > loss_0: 2.4505903720855713 (2.4097254953601155) + | > grad_norm_0: tensor(8.0418, device='cuda:0') (tensor(9.2870, device='cuda:0')) + | > loss_gen: 2.3425004482269287 (2.4149839715524153) + | > loss_kl: 1.9454824924468994 (1.8326171799139543) + | > loss_feat: 6.63914680480957 (7.367131347006017) + | > loss_mel: 16.99474334716797 (17.765177932652552) + | > loss_duration: 1.5164356231689453 (1.4419954473322083) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.438308715820312 (30.8219059380618) + | > grad_norm_1: tensor(204.6004, device='cuda:0') (tensor(243.4828, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.3971 (0.3659508607604287) + | > loader_time: 0.0071 (0.005688924681056629) + + + --> TIME: 2025-04-19 06:13:10 -- STEP: 113/142 -- GLOBAL_STEP: 441875 + | > loss_disc: 2.489009380340576 (2.424011760053382) + | > loss_disc_real_0: 0.08618701994419098 (0.1166621529027424) + | > loss_disc_real_1: 0.18682347238063812 (0.20529587197620258) + | > loss_disc_real_2: 0.23377211391925812 (0.226895705129193) + | > loss_disc_real_3: 0.24430131912231445 (0.22842257397364726) + | > loss_disc_real_4: 0.23265020549297333 (0.22595062284870485) + | > loss_disc_real_5: 0.2398802936077118 (0.22818597768787788) + | > loss_0: 2.489009380340576 (2.424011760053382) + | > grad_norm_0: tensor(12.0189, device='cuda:0') (tensor(10.9512, device='cuda:0')) + | > loss_gen: 2.2090470790863037 (2.410008316546415) + | > loss_kl: 1.9220401048660278 (1.8420926944344445) + | > loss_feat: 6.642606735229492 (7.347886148807222) + | > loss_mel: 16.34218406677246 (17.752143691071367) + | > loss_duration: 1.478883981704712 (1.4488360881805415) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.59476089477539 (30.800967005501807) + | > grad_norm_1: tensor(371.7671, device='cuda:0') (tensor(267.8866, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.4304 (0.3769992030827345) + | > loader_time: 0.0074 (0.005981506499568973) + + + --> TIME: 2025-04-19 06:13:21 -- STEP: 138/142 -- GLOBAL_STEP: 441900 + | > loss_disc: 2.3517329692840576 (2.4237102529276977) + | > loss_disc_real_0: 0.11258228123188019 (0.11725859613954157) + | > loss_disc_real_1: 0.19780540466308594 (0.20558461050192514) + | > loss_disc_real_2: 0.20674262940883636 (0.22644175697064053) + | > loss_disc_real_3: 0.21566887199878693 (0.22795059175594992) + | > loss_disc_real_4: 0.2006419152021408 (0.2265208395926849) + | > loss_disc_real_5: 0.2191430628299713 (0.2287009194277335) + | > loss_0: 2.3517329692840576 (2.4237102529276977) + | > grad_norm_0: tensor(13.7823, device='cuda:0') (tensor(12.1310, device='cuda:0')) + | > loss_gen: 2.3793206214904785 (2.4176089055296304) + | > loss_kl: 1.8465704917907715 (1.8514142174651658) + | > loss_feat: 7.165918350219727 (7.337853607924088) + | > loss_mel: 17.890750885009766 (17.77264391857645) + | > loss_duration: 1.527543067932129 (1.4554267292437344) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.810104370117188 (30.8349474063818) + | > grad_norm_1: tensor(369.6415, device='cuda:0') (tensor(283.2566, device='cuda:0')) + | > current_lr_0: 0.00018929637882904215 + | > current_lr_1: 0.00018929637882904215 + | > step_time: 0.4513 (0.38870812499004864) + | > loader_time: 0.0081 (0.0062357964723006535) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.50158429145813 (2.50158429145813) + | > loss_disc_real_0: 0.07069016993045807 (0.07069016993045807) + | > loss_disc_real_1: 0.2262052446603775 (0.2262052446603775) + | > loss_disc_real_2: 0.23265209794044495 (0.23265209794044495) + | > loss_disc_real_3: 0.27582284808158875 (0.27582284808158875) + | > loss_disc_real_4: 0.23486310243606567 (0.23486310243606567) + | > loss_disc_real_5: 0.21554823219776154 (0.21554823219776154) + | > loss_0: 2.50158429145813 (2.50158429145813) + | > loss_gen: 2.1268486976623535 (2.1268486976623535) + | > loss_kl: 2.622143507003784 (2.622143507003784) + | > loss_feat: 6.134864807128906 (6.134864807128906) + | > loss_mel: 16.587148666381836 (16.587148666381836) + | > loss_duration: 1.7549357414245605 (1.7549357414245605) + | > loss_1: 29.225942611694336 (29.225942611694336) + + --> STEP: 1 + | > loss_disc: 2.1914281845092773 (2.1914281845092773) + | > loss_disc_real_0: 0.0636841207742691 (0.0636841207742691) + | > loss_disc_real_1: 0.18377265334129333 (0.18377265334129333) + | > loss_disc_real_2: 0.22405710816383362 (0.22405710816383362) + | > loss_disc_real_3: 0.22076377272605896 (0.22076377272605896) + | > loss_disc_real_4: 0.23328648507595062 (0.23328648507595062) + | > loss_disc_real_5: 0.14508472383022308 (0.14508472383022308) + | > loss_0: 2.1914281845092773 (2.1914281845092773) + | > loss_gen: 2.4043188095092773 (2.4043188095092773) + | > loss_kl: 2.7713334560394287 (2.7713334560394287) + | > loss_feat: 8.353280067443848 (8.353280067443848) + | > loss_mel: 19.640975952148438 (19.640975952148438) + | > loss_duration: 1.8190165758132935 (1.8190165758132935) + | > loss_1: 34.98892593383789 (34.98892593383789) + + --> STEP: 2 + | > loss_disc: 2.315453052520752 (2.2534406185150146) + | > loss_disc_real_0: 0.05299192667007446 (0.05833802372217178) + | > loss_disc_real_1: 0.20500098168849945 (0.1943868175148964) + | > loss_disc_real_2: 0.22702226042747498 (0.2255396842956543) + | > loss_disc_real_3: 0.22894571721553802 (0.2248547449707985) + | > loss_disc_real_4: 0.2133783996105194 (0.22333244234323502) + | > loss_disc_real_5: 0.18811900913715363 (0.16660186648368835) + | > loss_0: 2.315453052520752 (2.2534406185150146) + | > loss_gen: 2.299318313598633 (2.351818561553955) + | > loss_kl: 2.454627275466919 (2.612980365753174) + | > loss_feat: 7.677604675292969 (8.015442371368408) + | > loss_mel: 17.192453384399414 (18.416714668273926) + | > loss_duration: 1.7364792823791504 (1.777747929096222) + | > loss_1: 31.36048126220703 (33.17470359802246) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034027099609375 (+0.00015246868133544922) + | > avg_loss_disc: 2.2534406185150146 (-0.29159021377563477) + | > avg_loss_disc_real_0: 0.05833802372217178 (-0.10222630202770233) + | > avg_loss_disc_real_1: 0.1943868175148964 (-0.06434546411037445) + | > avg_loss_disc_real_2: 0.2255396842956543 (-0.02562396228313446) + | > avg_loss_disc_real_3: 0.2248547449707985 (-0.029144249856472015) + | > avg_loss_disc_real_4: 0.22333244234323502 (-0.08411072939634323) + | > avg_loss_disc_real_5: 0.16660186648368835 (-0.10425032675266266) + | > avg_loss_0: 2.2534406185150146 (-0.29159021377563477) + | > avg_loss_gen: 2.351818561553955 (-0.11011457443237305) + | > avg_loss_kl: 2.612980365753174 (-0.012353777885437012) + | > avg_loss_feat: 8.015442371368408 (+2.263291358947754) + | > avg_loss_mel: 18.416714668273926 (+2.0033607482910156) + | > avg_loss_duration: 1.777747929096222 (-0.01370936632156372) + | > avg_loss_1: 33.17470359802246 (+4.1304731369018555) + + + > EPOCH: 441/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:13:24)  + + --> TIME: 2025-04-19 06:13:32 -- STEP: 21/142 -- GLOBAL_STEP: 441925 + | > loss_disc: 2.3995819091796875 (2.4234009697323753) + | > loss_disc_real_0: 0.11137130111455917 (0.11731869674154691) + | > loss_disc_real_1: 0.19618722796440125 (0.20620243322281612) + | > loss_disc_real_2: 0.2025291472673416 (0.22996067433130174) + | > loss_disc_real_3: 0.2330022156238556 (0.23031106662182582) + | > loss_disc_real_4: 0.21694262325763702 (0.22802238592079707) + | > loss_disc_real_5: 0.1865818053483963 (0.2277685339961733) + | > loss_0: 2.3995819091796875 (2.4234009697323753) + | > grad_norm_0: tensor(8.8888, device='cuda:0') (tensor(15.9250, device='cuda:0')) + | > loss_gen: 2.2534871101379395 (2.44636667342413) + | > loss_kl: 1.6396496295928955 (1.763622562090556) + | > loss_feat: 7.4354634284973145 (7.659957295372372) + | > loss_mel: 17.061634063720703 (17.836046945481073) + | > loss_duration: 1.4044413566589355 (1.4113943974177043) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.794675827026367 (31.11738768078032) + | > grad_norm_1: tensor(65.9659, device='cuda:0') (tensor(370.0713, device='cuda:0')) + | > current_lr_0: 0.00018927271678168851 + | > current_lr_1: 0.00018927271678168851 + | > step_time: 0.3428 (0.3321580659775507) + | > loader_time: 0.0055 (0.0048217205774216424) + + + --> TIME: 2025-04-19 06:13:41 -- STEP: 46/142 -- GLOBAL_STEP: 441950 + | > loss_disc: 2.4217936992645264 (2.427334749180338) + | > loss_disc_real_0: 0.09663039445877075 (0.11704680866197399) + | > loss_disc_real_1: 0.1752130538225174 (0.2022647653584895) + | > loss_disc_real_2: 0.2517456114292145 (0.22738567091848538) + | > loss_disc_real_3: 0.24180501699447632 (0.22967144911703857) + | > loss_disc_real_4: 0.2209068387746811 (0.23188057574241058) + | > loss_disc_real_5: 0.2025158405303955 (0.2274162668896758) + | > loss_0: 2.4217936992645264 (2.427334749180338) + | > grad_norm_0: tensor(4.3650, device='cuda:0') (tensor(11.9030, device='cuda:0')) + | > loss_gen: 2.4114575386047363 (2.4218197542688116) + | > loss_kl: 1.8178433179855347 (1.8048511510309966) + | > loss_feat: 7.5179123878479 (7.441978039948837) + | > loss_mel: 18.269474029541016 (17.699184293332305) + | > loss_duration: 1.4147541522979736 (1.42138780977415) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.431440353393555 (30.78922072700832) + | > grad_norm_1: tensor(104.9423, device='cuda:0') (tensor(310.3692, device='cuda:0')) + | > current_lr_0: 0.00018927271678168851 + | > current_lr_1: 0.00018927271678168851 + | > step_time: 0.3668 (0.34422070047129744) + | > loader_time: 0.0069 (0.0052859264871348505) + + + --> TIME: 2025-04-19 06:13:51 -- STEP: 71/142 -- GLOBAL_STEP: 441975 + | > loss_disc: 2.376424789428711 (2.4348588896469328) + | > loss_disc_real_0: 0.10316135734319687 (0.11993873177070014) + | > loss_disc_real_1: 0.17815226316452026 (0.2051965252194606) + | > loss_disc_real_2: 0.21506941318511963 (0.22631614329949232) + | > loss_disc_real_3: 0.25285831093788147 (0.22879749788364895) + | > loss_disc_real_4: 0.25825026631355286 (0.23220168893605891) + | > loss_disc_real_5: 0.25165432691574097 (0.2279760871135013) + | > loss_0: 2.376424789428711 (2.4348588896469328) + | > grad_norm_0: tensor(8.9407, device='cuda:0') (tensor(10.8774, device='cuda:0')) + | > loss_gen: 2.372246742248535 (2.3992179716137088) + | > loss_kl: 1.7606650590896606 (1.8278874628980395) + | > loss_feat: 7.0589399337768555 (7.312688686478306) + | > loss_mel: 17.725847244262695 (17.521533160142496) + | > loss_duration: 1.4616949558258057 (1.4322677901093388) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.379392623901367 (30.493594774058167) + | > grad_norm_1: tensor(273.4129, device='cuda:0') (tensor(260.1750, device='cuda:0')) + | > current_lr_0: 0.00018927271678168851 + | > current_lr_1: 0.00018927271678168851 + | > step_time: 0.388 (0.3560843568452646) + | > loader_time: 0.0066 (0.005599226750118632) + + + --> TIME: 2025-04-19 06:14:01 -- STEP: 96/142 -- GLOBAL_STEP: 442000 + | > loss_disc: 2.5407347679138184 (2.4368799651662516) + | > loss_disc_real_0: 0.1451546996831894 (0.12105873681139201) + | > loss_disc_real_1: 0.22631403803825378 (0.20514205594857535) + | > loss_disc_real_2: 0.22121767699718475 (0.22638628166168928) + | > loss_disc_real_3: 0.22804482281208038 (0.2272117535273234) + | > loss_disc_real_4: 0.24422554671764374 (0.23100705972562233) + | > loss_disc_real_5: 0.24042458832263947 (0.22740390291437507) + | > loss_0: 2.5407347679138184 (2.4368799651662516) + | > grad_norm_0: tensor(5.5923, device='cuda:0') (tensor(9.8963, device='cuda:0')) + | > loss_gen: 2.4055933952331543 (2.3857263599832845) + | > loss_kl: 1.777490258216858 (1.8404216679433982) + | > loss_feat: 7.312941551208496 (7.251963645219803) + | > loss_mel: 17.152921676635742 (17.55498047669729) + | > loss_duration: 1.4648687839508057 (1.443799930314223) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.11381721496582 (30.47689187526703) + | > grad_norm_1: tensor(29.1841, device='cuda:0') (tensor(225.5028, device='cuda:0')) + | > current_lr_0: 0.00018927271678168851 + | > current_lr_1: 0.00018927271678168851 + | > step_time: 0.4019 (0.3668421159187951) + | > loader_time: 0.0075 (0.0059081340829531355) + + + --> TIME: 2025-04-19 06:14:12 -- STEP: 121/142 -- GLOBAL_STEP: 442025 + | > loss_disc: 2.440840244293213 (2.4412124353992053) + | > loss_disc_real_0: 0.14309269189834595 (0.12329235552879404) + | > loss_disc_real_1: 0.2000795304775238 (0.20561427481410918) + | > loss_disc_real_2: 0.2185681015253067 (0.22608967248565895) + | > loss_disc_real_3: 0.2354598045349121 (0.2287123363372708) + | > loss_disc_real_4: 0.23395490646362305 (0.23026944253562895) + | > loss_disc_real_5: 0.20328892767429352 (0.22804821362672759) + | > loss_0: 2.440840244293213 (2.4412124353992053) + | > grad_norm_0: tensor(4.2219, device='cuda:0') (tensor(10.2304, device='cuda:0')) + | > loss_gen: 2.2917709350585938 (2.3796967731034453) + | > loss_kl: 2.025115728378296 (1.8490391132260158) + | > loss_feat: 7.478890419006348 (7.2397134441974735) + | > loss_mel: 18.144681930541992 (17.581941809536012) + | > loss_duration: 1.5095150470733643 (1.4512067944550318) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.449975967407227 (30.501597743389034) + | > grad_norm_1: tensor(169.9039, device='cuda:0') (tensor(228.8479, device='cuda:0')) + | > current_lr_0: 0.00018927271678168851 + | > current_lr_1: 0.00018927271678168851 + | > step_time: 0.4423 (0.3779240501813651) + | > loader_time: 0.0094 (0.0061856025506642235) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4384567737579346 (2.4384567737579346) + | > loss_disc_real_0: 0.19955746829509735 (0.19955746829509735) + | > loss_disc_real_1: 0.25322413444519043 (0.25322413444519043) + | > loss_disc_real_2: 0.24541346728801727 (0.24541346728801727) + | > loss_disc_real_3: 0.23597075045108795 (0.23597075045108795) + | > loss_disc_real_4: 0.18701981008052826 (0.18701981008052826) + | > loss_disc_real_5: 0.22735238075256348 (0.22735238075256348) + | > loss_0: 2.4384567737579346 (2.4384567737579346) + | > loss_gen: 2.4943318367004395 (2.4943318367004395) + | > loss_kl: 2.7212460041046143 (2.7212460041046143) + | > loss_feat: 6.824334621429443 (6.824334621429443) + | > loss_mel: 18.10102653503418 (18.10102653503418) + | > loss_duration: 1.7564008235931396 (1.7564008235931396) + | > loss_1: 31.8973388671875 (31.8973388671875) + + --> STEP: 1 + | > loss_disc: 2.422919988632202 (2.422919988632202) + | > loss_disc_real_0: 0.19780333340168 (0.19780333340168) + | > loss_disc_real_1: 0.24517503380775452 (0.24517503380775452) + | > loss_disc_real_2: 0.24198950827121735 (0.24198950827121735) + | > loss_disc_real_3: 0.21578936278820038 (0.21578936278820038) + | > loss_disc_real_4: 0.21025113761425018 (0.21025113761425018) + | > loss_disc_real_5: 0.2137567400932312 (0.2137567400932312) + | > loss_0: 2.422919988632202 (2.422919988632202) + | > loss_gen: 2.6107988357543945 (2.6107988357543945) + | > loss_kl: 2.8577826023101807 (2.8577826023101807) + | > loss_feat: 7.658580780029297 (7.658580780029297) + | > loss_mel: 19.271915435791016 (19.271915435791016) + | > loss_duration: 1.824121117591858 (1.824121117591858) + | > loss_1: 34.22319793701172 (34.22319793701172) + + --> STEP: 2 + | > loss_disc: 2.2759969234466553 (2.3494584560394287) + | > loss_disc_real_0: 0.22371765971183777 (0.21076049655675888) + | > loss_disc_real_1: 0.2959440350532532 (0.27055953443050385) + | > loss_disc_real_2: 0.22145318984985352 (0.23172134906053543) + | > loss_disc_real_3: 0.21419671177864075 (0.21499303728342056) + | > loss_disc_real_4: 0.22261390089988708 (0.21643251925706863) + | > loss_disc_real_5: 0.1723533421754837 (0.19305504113435745) + | > loss_0: 2.2759969234466553 (2.3494584560394287) + | > loss_gen: 2.7992851734161377 (2.705042004585266) + | > loss_kl: 2.261216640472412 (2.5594996213912964) + | > loss_feat: 8.131501197814941 (7.895040988922119) + | > loss_mel: 18.739782333374023 (19.00584888458252) + | > loss_duration: 1.7237005233764648 (1.7739108204841614) + | > loss_1: 33.655487060546875 (33.9393424987793) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035849809646606445 (+0.00018227100372314453) + | > avg_loss_disc: 2.3494584560394287 (+0.09601783752441406) + | > avg_loss_disc_real_0: 0.21076049655675888 (+0.1524224728345871) + | > avg_loss_disc_real_1: 0.27055953443050385 (+0.07617271691560745) + | > avg_loss_disc_real_2: 0.23172134906053543 (+0.006181664764881134) + | > avg_loss_disc_real_3: 0.21499303728342056 (-0.00986170768737793) + | > avg_loss_disc_real_4: 0.21643251925706863 (-0.006899923086166382) + | > avg_loss_disc_real_5: 0.19305504113435745 (+0.026453174650669098) + | > avg_loss_0: 2.3494584560394287 (+0.09601783752441406) + | > avg_loss_gen: 2.705042004585266 (+0.35322344303131104) + | > avg_loss_kl: 2.5594996213912964 (-0.05348074436187744) + | > avg_loss_feat: 7.895040988922119 (-0.12040138244628906) + | > avg_loss_mel: 19.00584888458252 (+0.5891342163085938) + | > avg_loss_duration: 1.7739108204841614 (-0.003837108612060547) + | > avg_loss_1: 33.9393424987793 (+0.7646389007568359) + + + > EPOCH: 442/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:14:23)  + + --> TIME: 2025-04-19 06:14:25 -- STEP: 4/142 -- GLOBAL_STEP: 442050 + | > loss_disc: 2.383451461791992 (2.3812726736068726) + | > loss_disc_real_0: 0.07762714475393295 (0.104251304641366) + | > loss_disc_real_1: 0.20385374128818512 (0.2088216468691826) + | > loss_disc_real_2: 0.2086247056722641 (0.23485947400331497) + | > loss_disc_real_3: 0.21078278124332428 (0.229639932513237) + | > loss_disc_real_4: 0.2006048560142517 (0.24112259224057198) + | > loss_disc_real_5: 0.22395706176757812 (0.22677389532327652) + | > loss_0: 2.383451461791992 (2.3812726736068726) + | > grad_norm_0: tensor(6.4312, device='cuda:0') (tensor(11.6936, device='cuda:0')) + | > loss_gen: 2.516688585281372 (2.444741129875183) + | > loss_kl: 1.8368252515792847 (1.8202097415924072) + | > loss_feat: 7.947294235229492 (7.508639812469482) + | > loss_mel: 19.23008918762207 (17.74316716194153) + | > loss_duration: 1.4153378009796143 (1.4151241183280945) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.94623565673828 (30.93188190460205) + | > grad_norm_1: tensor(335.2823, device='cuda:0') (tensor(325.8838, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.3267 (0.32620298862457275) + | > loader_time: 0.0052 (0.004487335681915283) + + + --> TIME: 2025-04-19 06:14:34 -- STEP: 29/142 -- GLOBAL_STEP: 442075 + | > loss_disc: 2.4254422187805176 (2.437011743413991) + | > loss_disc_real_0: 0.09784583002328873 (0.11909397627259123) + | > loss_disc_real_1: 0.2410772144794464 (0.20707188341124305) + | > loss_disc_real_2: 0.21897076070308685 (0.2280104206553821) + | > loss_disc_real_3: 0.2463720738887787 (0.22883091455903545) + | > loss_disc_real_4: 0.2147809863090515 (0.22973975282290887) + | > loss_disc_real_5: 0.2074512392282486 (0.22506925926126284) + | > loss_0: 2.4254422187805176 (2.437011743413991) + | > grad_norm_0: tensor(11.0054, device='cuda:0') (tensor(13.2858, device='cuda:0')) + | > loss_gen: 2.365192413330078 (2.4091371256729652) + | > loss_kl: 1.7667813301086426 (1.7761397032902158) + | > loss_feat: 6.711006164550781 (7.3159757811447665) + | > loss_mel: 17.10222053527832 (17.509010479368012) + | > loss_duration: 1.432396650314331 (1.4198159102735848) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.377595901489258 (30.430078966864222) + | > grad_norm_1: tensor(410.2725, device='cuda:0') (tensor(317.8641, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.3462 (0.3358638533230486) + | > loader_time: 0.0054 (0.00501635978961813) + + + --> TIME: 2025-04-19 06:14:43 -- STEP: 54/142 -- GLOBAL_STEP: 442100 + | > loss_disc: 2.482069730758667 (2.430241205074169) + | > loss_disc_real_0: 0.15370361506938934 (0.12119396893238579) + | > loss_disc_real_1: 0.2094673365354538 (0.2056956716157772) + | > loss_disc_real_2: 0.2283681035041809 (0.22572238991657892) + | > loss_disc_real_3: 0.2143714874982834 (0.22614867681706394) + | > loss_disc_real_4: 0.2181992530822754 (0.22560797604145827) + | > loss_disc_real_5: 0.192725270986557 (0.224139968002284) + | > loss_0: 2.482069730758667 (2.430241205074169) + | > grad_norm_0: tensor(4.9029, device='cuda:0') (tensor(11.5365, device='cuda:0')) + | > loss_gen: 2.3718581199645996 (2.4031161423082703) + | > loss_kl: 1.8897374868392944 (1.808243407143487) + | > loss_feat: 7.216457366943359 (7.369000629142478) + | > loss_mel: 16.68292236328125 (17.65156009462144) + | > loss_duration: 1.4517419338226318 (1.4275074115505926) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.612716674804688 (30.65942774878608) + | > grad_norm_1: tensor(338.2352, device='cuda:0') (tensor(293.5434, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.3686 (0.3473533082891394) + | > loader_time: 0.0062 (0.005406194263034396) + + + --> TIME: 2025-04-19 06:14:53 -- STEP: 79/142 -- GLOBAL_STEP: 442125 + | > loss_disc: 2.4715588092803955 (2.4356375857244563) + | > loss_disc_real_0: 0.1269344538450241 (0.12090528317833249) + | > loss_disc_real_1: 0.20556947588920593 (0.20649925459034835) + | > loss_disc_real_2: 0.23275348544120789 (0.22660263712647594) + | > loss_disc_real_3: 0.25396502017974854 (0.22720804338968253) + | > loss_disc_real_4: 0.2605544924736023 (0.22871725845940505) + | > loss_disc_real_5: 0.23364856839179993 (0.22554816381086276) + | > loss_0: 2.4715588092803955 (2.4356375857244563) + | > grad_norm_0: tensor(10.2974, device='cuda:0') (tensor(10.7243, device='cuda:0')) + | > loss_gen: 2.4439549446105957 (2.396111482306371) + | > loss_kl: 1.8612847328186035 (1.823893267897111) + | > loss_feat: 6.892698287963867 (7.321384617044956) + | > loss_mel: 17.620141983032227 (17.705197080781183) + | > loss_duration: 1.5064446926116943 (1.438175008266787) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.32452392578125 (30.6847615060927) + | > grad_norm_1: tensor(277.6566, device='cuda:0') (tensor(275.8657, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.3868 (0.36011837705781197) + | > loader_time: 0.0063 (0.005709267869780334) + + + --> TIME: 2025-04-19 06:15:03 -- STEP: 104/142 -- GLOBAL_STEP: 442150 + | > loss_disc: 2.5054280757904053 (2.44489507491772) + | > loss_disc_real_0: 0.12697874009609222 (0.12248314813209268) + | > loss_disc_real_1: 0.2290966957807541 (0.20716500984361538) + | > loss_disc_real_2: 0.2528745234012604 (0.22715514669051537) + | > loss_disc_real_3: 0.26639336347579956 (0.22741855007524675) + | > loss_disc_real_4: 0.2600564956665039 (0.2302729508911188) + | > loss_disc_real_5: 0.2852855324745178 (0.22649301101381963) + | > loss_0: 2.5054280757904053 (2.44489507491772) + | > grad_norm_0: tensor(22.4101, device='cuda:0') (tensor(11.4554, device='cuda:0')) + | > loss_gen: 2.305903673171997 (2.3893718283910013) + | > loss_kl: 1.941295862197876 (1.8341338370855038) + | > loss_feat: 6.577249526977539 (7.255601960879106) + | > loss_mel: 16.801284790039062 (17.664300065774185) + | > loss_duration: 1.4979398250579834 (1.4477222768160016) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.123674392700195 (30.591130018234253) + | > grad_norm_1: tensor(363.2347, device='cuda:0') (tensor(285.2720, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.4117 (0.37146605436618524) + | > loader_time: 0.0066 (0.005945723790388839) + + + --> TIME: 2025-04-19 06:15:14 -- STEP: 129/142 -- GLOBAL_STEP: 442175 + | > loss_disc: 2.3159902095794678 (2.44532971049464) + | > loss_disc_real_0: 0.09564963728189468 (0.12281393274202827) + | > loss_disc_real_1: 0.21849922835826874 (0.2070416629083397) + | > loss_disc_real_2: 0.22351093590259552 (0.22682401480138764) + | > loss_disc_real_3: 0.20301228761672974 (0.2275457558936851) + | > loss_disc_real_4: 0.1984899640083313 (0.22912189468394878) + | > loss_disc_real_5: 0.2336316555738449 (0.22684812095276144) + | > loss_0: 2.3159902095794678 (2.44532971049464) + | > grad_norm_0: tensor(4.7310, device='cuda:0') (tensor(11.8166, device='cuda:0')) + | > loss_gen: 2.4318838119506836 (2.3835727392241) + | > loss_kl: 1.9361848831176758 (1.842505502146344) + | > loss_feat: 7.393704414367676 (7.237743041312048) + | > loss_mel: 18.337154388427734 (17.67354130560114) + | > loss_duration: 1.467057228088379 (1.4542565419692404) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.565982818603516 (30.591619166292887) + | > grad_norm_1: tensor(300.4477, device='cuda:0') (tensor(288.7293, device='cuda:0')) + | > current_lr_0: 0.0001892490576920908 + | > current_lr_1: 0.0001892490576920908 + | > step_time: 0.4565 (0.3836605105289194) + | > loader_time: 0.0074 (0.0061855999998344) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.603822708129883 (2.603822708129883) + | > loss_disc_real_0: 0.19052818417549133 (0.19052818417549133) + | > loss_disc_real_1: 0.2460973858833313 (0.2460973858833313) + | > loss_disc_real_2: 0.2628774046897888 (0.2628774046897888) + | > loss_disc_real_3: 0.26428303122520447 (0.26428303122520447) + | > loss_disc_real_4: 0.2733841836452484 (0.2733841836452484) + | > loss_disc_real_5: 0.27673855423927307 (0.27673855423927307) + | > loss_0: 2.603822708129883 (2.603822708129883) + | > loss_gen: 2.369450569152832 (2.369450569152832) + | > loss_kl: 2.4432497024536133 (2.4432497024536133) + | > loss_feat: 5.592772483825684 (5.592772483825684) + | > loss_mel: 17.437904357910156 (17.437904357910156) + | > loss_duration: 1.7689907550811768 (1.7689907550811768) + | > loss_1: 29.612367630004883 (29.612367630004883) + + --> STEP: 1 + | > loss_disc: 2.451805591583252 (2.451805591583252) + | > loss_disc_real_0: 0.2048148214817047 (0.2048148214817047) + | > loss_disc_real_1: 0.2843016982078552 (0.2843016982078552) + | > loss_disc_real_2: 0.2501685619354248 (0.2501685619354248) + | > loss_disc_real_3: 0.23180745542049408 (0.23180745542049408) + | > loss_disc_real_4: 0.21128515899181366 (0.21128515899181366) + | > loss_disc_real_5: 0.19908125698566437 (0.19908125698566437) + | > loss_0: 2.451805591583252 (2.451805591583252) + | > loss_gen: 2.490253448486328 (2.490253448486328) + | > loss_kl: 2.8857014179229736 (2.8857014179229736) + | > loss_feat: 6.871029376983643 (6.871029376983643) + | > loss_mel: 15.914528846740723 (15.914528846740723) + | > loss_duration: 1.8264919519424438 (1.8264919519424438) + | > loss_1: 29.988006591796875 (29.988006591796875) + + --> STEP: 2 + | > loss_disc: 2.5107903480529785 (2.4812979698181152) + | > loss_disc_real_0: 0.23520933091640472 (0.22001207619905472) + | > loss_disc_real_1: 0.23142342269420624 (0.25786256045103073) + | > loss_disc_real_2: 0.25563234090805054 (0.25290045142173767) + | > loss_disc_real_3: 0.24395006895065308 (0.23787876218557358) + | > loss_disc_real_4: 0.23642055690288544 (0.22385285794734955) + | > loss_disc_real_5: 0.227150097489357 (0.21311567723751068) + | > loss_0: 2.5107903480529785 (2.4812979698181152) + | > loss_gen: 2.3812789916992188 (2.4357662200927734) + | > loss_kl: 2.1549928188323975 (2.5203471183776855) + | > loss_feat: 5.8921661376953125 (6.3815977573394775) + | > loss_mel: 17.301111221313477 (16.6078200340271) + | > loss_duration: 1.7257323265075684 (1.776112139225006) + | > loss_1: 29.45528221130371 (29.721644401550293) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003284454345703125 (-0.00030052661895751953) + | > avg_loss_disc: 2.4812979698181152 (+0.13183951377868652) + | > avg_loss_disc_real_0: 0.22001207619905472 (+0.009251579642295837) + | > avg_loss_disc_real_1: 0.25786256045103073 (-0.012696973979473114) + | > avg_loss_disc_real_2: 0.25290045142173767 (+0.02117910236120224) + | > avg_loss_disc_real_3: 0.23787876218557358 (+0.022885724902153015) + | > avg_loss_disc_real_4: 0.22385285794734955 (+0.007420338690280914) + | > avg_loss_disc_real_5: 0.21311567723751068 (+0.02006063610315323) + | > avg_loss_0: 2.4812979698181152 (+0.13183951377868652) + | > avg_loss_gen: 2.4357662200927734 (-0.2692757844924927) + | > avg_loss_kl: 2.5203471183776855 (-0.03915250301361084) + | > avg_loss_feat: 6.3815977573394775 (-1.5134432315826416) + | > avg_loss_mel: 16.6078200340271 (-2.39802885055542) + | > avg_loss_duration: 1.776112139225006 (+0.0022013187408447266) + | > avg_loss_1: 29.721644401550293 (-4.217698097229004) + + + > EPOCH: 443/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:15:22)  + + --> TIME: 2025-04-19 06:15:26 -- STEP: 12/142 -- GLOBAL_STEP: 442200 + | > loss_disc: 2.3817930221557617 (2.3824551900227866) + | > loss_disc_real_0: 0.10111057013273239 (0.11769724140564601) + | > loss_disc_real_1: 0.23405307531356812 (0.20730099454522133) + | > loss_disc_real_2: 0.19669733941555023 (0.22437814995646477) + | > loss_disc_real_3: 0.21659040451049805 (0.22135724251468977) + | > loss_disc_real_4: 0.2269640564918518 (0.21891189739108086) + | > loss_disc_real_5: 0.23251596093177795 (0.2124525991578897) + | > loss_0: 2.3817930221557617 (2.3824551900227866) + | > grad_norm_0: tensor(5.0553, device='cuda:0') (tensor(6.2172, device='cuda:0')) + | > loss_gen: 2.3927810192108154 (2.367019414901734) + | > loss_kl: 1.8485996723175049 (1.7782166202863057) + | > loss_feat: 7.493768692016602 (7.405788342158) + | > loss_mel: 17.28097152709961 (17.60542392730713) + | > loss_duration: 1.4215362071990967 (1.408382515112559) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.43765640258789 (30.56483046213786) + | > grad_norm_1: tensor(63.5874, device='cuda:0') (tensor(80.4795, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.3299 (0.325156311194102) + | > loader_time: 0.0047 (0.0046515266100565595) + + + --> TIME: 2025-04-19 06:15:35 -- STEP: 37/142 -- GLOBAL_STEP: 442225 + | > loss_disc: 2.431659698486328 (2.410346682007248) + | > loss_disc_real_0: 0.14316125214099884 (0.11923187608654434) + | > loss_disc_real_1: 0.20650753378868103 (0.20284206440319885) + | > loss_disc_real_2: 0.2608414888381958 (0.22298872793042981) + | > loss_disc_real_3: 0.24556535482406616 (0.22718307255087672) + | > loss_disc_real_4: 0.23521274328231812 (0.22504317559100487) + | > loss_disc_real_5: 0.20130173861980438 (0.22406903312012955) + | > loss_0: 2.431659698486328 (2.410346682007248) + | > grad_norm_0: tensor(13.8262, device='cuda:0') (tensor(6.7402, device='cuda:0')) + | > loss_gen: 2.520852565765381 (2.3825486415141346) + | > loss_kl: 1.7568150758743286 (1.7782146576288584) + | > loss_feat: 6.969540596008301 (7.385280480255952) + | > loss_mel: 17.871578216552734 (17.764425277709968) + | > loss_duration: 1.4619624614715576 (1.4179332675160587) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.58074951171875 (30.728402266631257) + | > grad_norm_1: tensor(220.0535, device='cuda:0') (tensor(126.7743, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.3525 (0.3384105128210944) + | > loader_time: 0.0059 (0.00527300061406316) + + + --> TIME: 2025-04-19 06:15:45 -- STEP: 62/142 -- GLOBAL_STEP: 442250 + | > loss_disc: 2.5334770679473877 (2.4290176283928657) + | > loss_disc_real_0: 0.14013369381427765 (0.12098710040651983) + | > loss_disc_real_1: 0.2058880478143692 (0.2055696258621831) + | > loss_disc_real_2: 0.21480761468410492 (0.22395944811644092) + | > loss_disc_real_3: 0.2047949880361557 (0.22516050502177207) + | > loss_disc_real_4: 0.26523375511169434 (0.2251606384111989) + | > loss_disc_real_5: 0.20082752406597137 (0.22693554071649427) + | > loss_0: 2.5334770679473877 (2.4290176283928657) + | > grad_norm_0: tensor(18.4175, device='cuda:0') (tensor(8.3775, device='cuda:0')) + | > loss_gen: 2.417503595352173 (2.382433891296387) + | > loss_kl: 1.792496919631958 (1.7952963690603934) + | > loss_feat: 7.883365154266357 (7.279846768225393) + | > loss_mel: 18.7172794342041 (17.695439430975146) + | > loss_duration: 1.468031406402588 (1.4292420360349842) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.27867889404297 (30.58225843983312) + | > grad_norm_1: tensor(278.9496, device='cuda:0') (tensor(178.0013, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.3822 (0.3500255038661341) + | > loader_time: 0.0063 (0.005621317894228043) + + + --> TIME: 2025-04-19 06:15:55 -- STEP: 87/142 -- GLOBAL_STEP: 442275 + | > loss_disc: 2.478487014770508 (2.439545132647986) + | > loss_disc_real_0: 0.11135835945606232 (0.12066689840164678) + | > loss_disc_real_1: 0.20301155745983124 (0.2067327828242861) + | > loss_disc_real_2: 0.23555848002433777 (0.22534401577779617) + | > loss_disc_real_3: 0.25827017426490784 (0.22764521460423523) + | > loss_disc_real_4: 0.2552506625652313 (0.22808448542123552) + | > loss_disc_real_5: 0.2376524806022644 (0.22900168426420497) + | > loss_0: 2.478487014770508 (2.439545132647986) + | > grad_norm_0: tensor(10.6184, device='cuda:0') (tensor(9.1076, device='cuda:0')) + | > loss_gen: 2.445873498916626 (2.377562259805615) + | > loss_kl: 1.9403712749481201 (1.810977446621862) + | > loss_feat: 7.484428405761719 (7.249915813577586) + | > loss_mel: 18.860008239746094 (17.74957571358517) + | > loss_duration: 1.4857558012008667 (1.4413618794802963) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.21643829345703 (30.62939312814296) + | > grad_norm_1: tensor(250.4906, device='cuda:0') (tensor(201.0300, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.4096 (0.3629383163890619) + | > loader_time: 0.0065 (0.005871630263054508) + + + --> TIME: 2025-04-19 06:16:05 -- STEP: 112/142 -- GLOBAL_STEP: 442300 + | > loss_disc: 2.2585628032684326 (2.438610672950744) + | > loss_disc_real_0: 0.09967634081840515 (0.1222552235038685) + | > loss_disc_real_1: 0.18383730947971344 (0.20602890024227755) + | > loss_disc_real_2: 0.2310243397951126 (0.22473841612892492) + | > loss_disc_real_3: 0.21193021535873413 (0.22711129167250224) + | > loss_disc_real_4: 0.19457246363162994 (0.2293620722900544) + | > loss_disc_real_5: 0.23075345158576965 (0.22847412220601523) + | > loss_0: 2.2585628032684326 (2.438610672950744) + | > grad_norm_0: tensor(6.9890, device='cuda:0') (tensor(9.0352, device='cuda:0')) + | > loss_gen: 2.5301644802093506 (2.372425717966899) + | > loss_kl: 1.9460573196411133 (1.8246117223586356) + | > loss_feat: 8.293469429016113 (7.220383388655526) + | > loss_mel: 18.959197998046875 (17.76493122747966) + | > loss_duration: 1.449458360671997 (1.4491669631430082) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 33.178348541259766 (30.63151890890939) + | > grad_norm_1: tensor(193.7706, device='cuda:0') (tensor(203.4411, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.4312 (0.37444605997630515) + | > loader_time: 0.0075 (0.006145651851381574) + + + --> TIME: 2025-04-19 06:16:16 -- STEP: 137/142 -- GLOBAL_STEP: 442325 + | > loss_disc: 2.4901010990142822 (2.436791564426282) + | > loss_disc_real_0: 0.18679790198802948 (0.12320495720436103) + | > loss_disc_real_1: 0.20599010586738586 (0.20589454613462851) + | > loss_disc_real_2: 0.21563522517681122 (0.2244522059268325) + | > loss_disc_real_3: 0.23121847212314606 (0.22714611518121983) + | > loss_disc_real_4: 0.24615783989429474 (0.22869670510727125) + | > loss_disc_real_5: 0.25798600912094116 (0.22794698755236437) + | > loss_0: 2.4901010990142822 (2.436791564426282) + | > grad_norm_0: tensor(5.8868, device='cuda:0') (tensor(9.2859, device='cuda:0')) + | > loss_gen: 2.215385675430298 (2.3745116943860594) + | > loss_kl: 1.8631727695465088 (1.834028390202209) + | > loss_feat: 6.741289138793945 (7.21828323211113) + | > loss_mel: 17.606388092041016 (17.722314855478107) + | > loss_duration: 1.4682338237762451 (1.4557729080645705) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.894468307495117 (30.60491096886405) + | > grad_norm_1: tensor(101.2179, device='cuda:0') (tensor(207.8047, device='cuda:0')) + | > current_lr_0: 0.0001892254015598793 + | > current_lr_1: 0.0001892254015598793 + | > step_time: 0.4588 (0.3863960861289587) + | > loader_time: 0.0076 (0.0063947204255709695) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5661439895629883 (2.5661439895629883) + | > loss_disc_real_0: 0.15535876154899597 (0.15535876154899597) + | > loss_disc_real_1: 0.24868856370449066 (0.24868856370449066) + | > loss_disc_real_2: 0.3006351888179779 (0.3006351888179779) + | > loss_disc_real_3: 0.21977375447750092 (0.21977375447750092) + | > loss_disc_real_4: 0.24229232966899872 (0.24229232966899872) + | > loss_disc_real_5: 0.22643999755382538 (0.22643999755382538) + | > loss_0: 2.5661439895629883 (2.5661439895629883) + | > loss_gen: 2.3069610595703125 (2.3069610595703125) + | > loss_kl: 2.42657470703125 (2.42657470703125) + | > loss_feat: 6.945616722106934 (6.945616722106934) + | > loss_mel: 17.55602264404297 (17.55602264404297) + | > loss_duration: 1.7873719930648804 (1.7873719930648804) + | > loss_1: 31.02254867553711 (31.02254867553711) + + --> STEP: 1 + | > loss_disc: 2.6651246547698975 (2.6651246547698975) + | > loss_disc_real_0: 0.1686692088842392 (0.1686692088842392) + | > loss_disc_real_1: 0.25967150926589966 (0.25967150926589966) + | > loss_disc_real_2: 0.2880082130432129 (0.2880082130432129) + | > loss_disc_real_3: 0.25989386439323425 (0.25989386439323425) + | > loss_disc_real_4: 0.2846919000148773 (0.2846919000148773) + | > loss_disc_real_5: 0.2505108714103699 (0.2505108714103699) + | > loss_0: 2.6651246547698975 (2.6651246547698975) + | > loss_gen: 2.2595839500427246 (2.2595839500427246) + | > loss_kl: 2.6855556964874268 (2.6855556964874268) + | > loss_feat: 5.852256774902344 (5.852256774902344) + | > loss_mel: 17.81460189819336 (17.81460189819336) + | > loss_duration: 1.8105477094650269 (1.8105477094650269) + | > loss_1: 30.422544479370117 (30.422544479370117) + + --> STEP: 2 + | > loss_disc: 2.6279735565185547 (2.646549105644226) + | > loss_disc_real_0: 0.1780305653810501 (0.17334988713264465) + | > loss_disc_real_1: 0.25097277760505676 (0.2553221434354782) + | > loss_disc_real_2: 0.28901734948158264 (0.28851278126239777) + | > loss_disc_real_3: 0.2573387324810028 (0.25861629843711853) + | > loss_disc_real_4: 0.2833057641983032 (0.28399883210659027) + | > loss_disc_real_5: 0.24882107973098755 (0.2496659755706787) + | > loss_0: 2.6279735565185547 (2.646549105644226) + | > loss_gen: 2.3618760108947754 (2.31072998046875) + | > loss_kl: 2.283003330230713 (2.48427951335907) + | > loss_feat: 5.337717056274414 (5.594986915588379) + | > loss_mel: 16.726303100585938 (17.27045249938965) + | > loss_duration: 1.7463085651397705 (1.7784281373023987) + | > loss_1: 28.45520782470703 (29.438876152038574) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003395557403564453 (+0.00011110305786132812) + | > avg_loss_disc: 2.646549105644226 (+0.16525113582611084) + | > avg_loss_disc_real_0: 0.17334988713264465 (-0.046662189066410065) + | > avg_loss_disc_real_1: 0.2553221434354782 (-0.0025404170155525208) + | > avg_loss_disc_real_2: 0.28851278126239777 (+0.035612329840660095) + | > avg_loss_disc_real_3: 0.25861629843711853 (+0.020737536251544952) + | > avg_loss_disc_real_4: 0.28399883210659027 (+0.06014597415924072) + | > avg_loss_disc_real_5: 0.2496659755706787 (+0.03655029833316803) + | > avg_loss_0: 2.646549105644226 (+0.16525113582611084) + | > avg_loss_gen: 2.31072998046875 (-0.12503623962402344) + | > avg_loss_kl: 2.48427951335907 (-0.03606760501861572) + | > avg_loss_feat: 5.594986915588379 (-0.7866108417510986) + | > avg_loss_mel: 17.27045249938965 (+0.6626324653625488) + | > avg_loss_duration: 1.7784281373023987 (+0.002315998077392578) + | > avg_loss_1: 29.438876152038574 (-0.28276824951171875) + + + > EPOCH: 444/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:16:20)  + + --> TIME: 2025-04-19 06:16:28 -- STEP: 20/142 -- GLOBAL_STEP: 442350 + | > loss_disc: 2.4500272274017334 (2.444161832332611) + | > loss_disc_real_0: 0.1097961887717247 (0.11255932934582233) + | > loss_disc_real_1: 0.19769009947776794 (0.207245621830225) + | > loss_disc_real_2: 0.24456042051315308 (0.2307103954255581) + | > loss_disc_real_3: 0.2292858213186264 (0.2298120692372322) + | > loss_disc_real_4: 0.24356204271316528 (0.2305929109454155) + | > loss_disc_real_5: 0.26892194151878357 (0.23295466154813765) + | > loss_0: 2.4500272274017334 (2.444161832332611) + | > grad_norm_0: tensor(3.5248, device='cuda:0') (tensor(8.1460, device='cuda:0')) + | > loss_gen: 2.3852767944335938 (2.3975618720054626) + | > loss_kl: 1.7707921266555786 (1.791403067111969) + | > loss_feat: 6.9299492835998535 (7.4686668634414675) + | > loss_mel: 16.777729034423828 (17.877557468414302) + | > loss_duration: 1.463794469833374 (1.4084515154361725) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.32754135131836 (30.94364070892334) + | > grad_norm_1: tensor(113.2620, device='cuda:0') (tensor(199.7733, device='cuda:0')) + | > current_lr_0: 0.0001892017483846843 + | > current_lr_1: 0.0001892017483846843 + | > step_time: 0.3412 (0.33006811141967773) + | > loader_time: 0.0056 (0.004869723320007324) + + + --> TIME: 2025-04-19 06:16:37 -- STEP: 45/142 -- GLOBAL_STEP: 442375 + | > loss_disc: 2.4321327209472656 (2.430345778995091) + | > loss_disc_real_0: 0.11506348103284836 (0.11477696845928828) + | > loss_disc_real_1: 0.21857813000679016 (0.20647076103422377) + | > loss_disc_real_2: 0.22050197422504425 (0.22739335232310826) + | > loss_disc_real_3: 0.21932707726955414 (0.23035488492912717) + | > loss_disc_real_4: 0.2531417906284332 (0.22731441391838922) + | > loss_disc_real_5: 0.21021029353141785 (0.22668125761879815) + | > loss_0: 2.4321327209472656 (2.430345778995091) + | > grad_norm_0: tensor(6.7853, device='cuda:0') (tensor(8.1432, device='cuda:0')) + | > loss_gen: 2.3750078678131104 (2.3890540017022026) + | > loss_kl: 1.8231889009475708 (1.805881717469957) + | > loss_feat: 6.831216812133789 (7.341934744517008) + | > loss_mel: 17.732563018798828 (17.723104731241854) + | > loss_duration: 1.4482457637786865 (1.4198600504133434) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.210222244262695 (30.679835171169707) + | > grad_norm_1: tensor(166.5148, device='cuda:0') (tensor(211.2291, device='cuda:0')) + | > current_lr_0: 0.0001892017483846843 + | > current_lr_1: 0.0001892017483846843 + | > step_time: 0.3661 (0.34285930527581115) + | > loader_time: 0.006 (0.005339399973551432) + + + --> TIME: 2025-04-19 06:16:46 -- STEP: 70/142 -- GLOBAL_STEP: 442400 + | > loss_disc: 2.3761613368988037 (2.431095518384662) + | > loss_disc_real_0: 0.1364126354455948 (0.11824431866407395) + | > loss_disc_real_1: 0.21819710731506348 (0.20584520782743182) + | > loss_disc_real_2: 0.22258159518241882 (0.22733958980866842) + | > loss_disc_real_3: 0.23687602579593658 (0.22877448818513324) + | > loss_disc_real_4: 0.23278279602527618 (0.22599945366382598) + | > loss_disc_real_5: 0.2412787228822708 (0.22960723830120905) + | > loss_0: 2.3761613368988037 (2.431095518384662) + | > grad_norm_0: tensor(10.2616, device='cuda:0') (tensor(8.8085, device='cuda:0')) + | > loss_gen: 2.415350914001465 (2.3932460580553325) + | > loss_kl: 1.8839102983474731 (1.8131311655044555) + | > loss_feat: 6.951612949371338 (7.320230770111084) + | > loss_mel: 17.190296173095703 (17.713513919285365) + | > loss_duration: 1.4868204593658447 (1.4312154599598474) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.927989959716797 (30.671337291172573) + | > grad_norm_1: tensor(178.1956, device='cuda:0') (tensor(224.6992, device='cuda:0')) + | > current_lr_0: 0.0001892017483846843 + | > current_lr_1: 0.0001892017483846843 + | > step_time: 0.3946 (0.3560361317225866) + | > loader_time: 0.0065 (0.005604154723031179) + + + --> TIME: 2025-04-19 06:16:57 -- STEP: 95/142 -- GLOBAL_STEP: 442425 + | > loss_disc: 2.392125368118286 (2.4444882493270077) + | > loss_disc_real_0: 0.10483226925134659 (0.12021548724488208) + | > loss_disc_real_1: 0.18525353074073792 (0.20633030361250826) + | > loss_disc_real_2: 0.20302267372608185 (0.22910034421243167) + | > loss_disc_real_3: 0.22814299166202545 (0.22868976357736084) + | > loss_disc_real_4: 0.22160480916500092 (0.229123949220306) + | > loss_disc_real_5: 0.2009928673505783 (0.22986772719182466) + | > loss_0: 2.392125368118286 (2.4444882493270077) + | > grad_norm_0: tensor(8.4825, device='cuda:0') (tensor(9.1033, device='cuda:0')) + | > loss_gen: 2.2748210430145264 (2.3844488344694446) + | > loss_kl: 1.7432252168655396 (1.8245270202034398) + | > loss_feat: 6.524130821228027 (7.2682944247597145) + | > loss_mel: 16.705995559692383 (17.724163376657582) + | > loss_duration: 1.492234230041504 (1.4435146243948684) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.740406036376953 (30.644948176333777) + | > grad_norm_1: tensor(289.0976, device='cuda:0') (tensor(236.6382, device='cuda:0')) + | > current_lr_0: 0.0001892017483846843 + | > current_lr_1: 0.0001892017483846843 + | > step_time: 0.4091 (0.36752614473041745) + | > loader_time: 0.0068 (0.005848169326782224) + + + --> TIME: 2025-04-19 06:17:07 -- STEP: 120/142 -- GLOBAL_STEP: 442450 + | > loss_disc: 2.505809783935547 (2.446313941478729) + | > loss_disc_real_0: 0.09522737562656403 (0.12044324123611053) + | > loss_disc_real_1: 0.17552801966667175 (0.20594731134672958) + | > loss_disc_real_2: 0.2199956327676773 (0.22881131507456304) + | > loss_disc_real_3: 0.2664738893508911 (0.22949123283227285) + | > loss_disc_real_4: 0.25259989500045776 (0.22873346569637457) + | > loss_disc_real_5: 0.2557987570762634 (0.22970803193747996) + | > loss_0: 2.505809783935547 (2.446313941478729) + | > grad_norm_0: tensor(14.0313, device='cuda:0') (tensor(10.5185, device='cuda:0')) + | > loss_gen: 2.3612966537475586 (2.3891369144121817) + | > loss_kl: 1.9063396453857422 (1.8334732433160146) + | > loss_feat: 6.937170028686523 (7.272603718439738) + | > loss_mel: 17.5609130859375 (17.723463265101106) + | > loss_duration: 1.496589183807373 (1.4513409028450648) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.262310028076172 (30.67001797358195) + | > grad_norm_1: tensor(452.0897, device='cuda:0') (tensor(260.6303, device='cuda:0')) + | > current_lr_0: 0.0001892017483846843 + | > current_lr_1: 0.0001892017483846843 + | > step_time: 0.4349 (0.3790119290351868) + | > loader_time: 0.0075 (0.006108017762502032) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.616093397140503 (2.616093397140503) + | > loss_disc_real_0: 0.12283816188573837 (0.12283816188573837) + | > loss_disc_real_1: 0.23517589271068573 (0.23517589271068573) + | > loss_disc_real_2: 0.2686660885810852 (0.2686660885810852) + | > loss_disc_real_3: 0.2844087779521942 (0.2844087779521942) + | > loss_disc_real_4: 0.2630853056907654 (0.2630853056907654) + | > loss_disc_real_5: 0.2799901068210602 (0.2799901068210602) + | > loss_0: 2.616093397140503 (2.616093397140503) + | > loss_gen: 2.268258571624756 (2.268258571624756) + | > loss_kl: 2.376155138015747 (2.376155138015747) + | > loss_feat: 6.635172367095947 (6.635172367095947) + | > loss_mel: 18.1928653717041 (18.1928653717041) + | > loss_duration: 1.7505769729614258 (1.7505769729614258) + | > loss_1: 31.22303009033203 (31.22303009033203) + + --> STEP: 1 + | > loss_disc: 2.423341751098633 (2.423341751098633) + | > loss_disc_real_0: 0.11380746215581894 (0.11380746215581894) + | > loss_disc_real_1: 0.25796952843666077 (0.25796952843666077) + | > loss_disc_real_2: 0.23803511261940002 (0.23803511261940002) + | > loss_disc_real_3: 0.2251506745815277 (0.2251506745815277) + | > loss_disc_real_4: 0.2719044089317322 (0.2719044089317322) + | > loss_disc_real_5: 0.18524549901485443 (0.18524549901485443) + | > loss_0: 2.423341751098633 (2.423341751098633) + | > loss_gen: 2.3225934505462646 (2.3225934505462646) + | > loss_kl: 2.8554024696350098 (2.8554024696350098) + | > loss_feat: 7.115042686462402 (7.115042686462402) + | > loss_mel: 18.185882568359375 (18.185882568359375) + | > loss_duration: 1.8219008445739746 (1.8219008445739746) + | > loss_1: 32.30082321166992 (32.30082321166992) + + --> STEP: 2 + | > loss_disc: 2.452423572540283 (2.437882661819458) + | > loss_disc_real_0: 0.13706284761428833 (0.12543515488505363) + | > loss_disc_real_1: 0.22619973123073578 (0.24208462983369827) + | > loss_disc_real_2: 0.23511648178100586 (0.23657579720020294) + | > loss_disc_real_3: 0.26747843623161316 (0.24631455540657043) + | > loss_disc_real_4: 0.2307245284318924 (0.2513144686818123) + | > loss_disc_real_5: 0.22047506272792816 (0.2028602808713913) + | > loss_0: 2.452423572540283 (2.437882661819458) + | > loss_gen: 2.2517571449279785 (2.2871752977371216) + | > loss_kl: 2.235271453857422 (2.545336961746216) + | > loss_feat: 6.936794281005859 (7.025918483734131) + | > loss_mel: 18.31125259399414 (18.248567581176758) + | > loss_duration: 1.7402307987213135 (1.781065821647644) + | > loss_1: 31.475305557250977 (31.88806438446045) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003871321678161621 (+0.00047576427459716797) + | > avg_loss_disc: 2.437882661819458 (-0.20866644382476807) + | > avg_loss_disc_real_0: 0.12543515488505363 (-0.04791473224759102) + | > avg_loss_disc_real_1: 0.24208462983369827 (-0.013237513601779938) + | > avg_loss_disc_real_2: 0.23657579720020294 (-0.051936984062194824) + | > avg_loss_disc_real_3: 0.24631455540657043 (-0.012301743030548096) + | > avg_loss_disc_real_4: 0.2513144686818123 (-0.032684363424777985) + | > avg_loss_disc_real_5: 0.2028602808713913 (-0.046805694699287415) + | > avg_loss_0: 2.437882661819458 (-0.20866644382476807) + | > avg_loss_gen: 2.2871752977371216 (-0.023554682731628418) + | > avg_loss_kl: 2.545336961746216 (+0.061057448387145996) + | > avg_loss_feat: 7.025918483734131 (+1.430931568145752) + | > avg_loss_mel: 18.248567581176758 (+0.9781150817871094) + | > avg_loss_duration: 1.781065821647644 (+0.0026376843452453613) + | > avg_loss_1: 31.88806438446045 (+2.449188232421875) + + + > EPOCH: 445/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:17:19)  + + --> TIME: 2025-04-19 06:17:21 -- STEP: 3/142 -- GLOBAL_STEP: 442475 + | > loss_disc: 2.4246411323547363 (2.4761904080708823) + | > loss_disc_real_0: 0.13726560771465302 (0.1500340700149536) + | > loss_disc_real_1: 0.16243068873882294 (0.2001326729853948) + | > loss_disc_real_2: 0.2002236396074295 (0.2263007958730062) + | > loss_disc_real_3: 0.2330818772315979 (0.23756038149197897) + | > loss_disc_real_4: 0.2484465390443802 (0.23523413638273874) + | > loss_disc_real_5: 0.22612513601779938 (0.23347486555576324) + | > loss_0: 2.4246411323547363 (2.4761904080708823) + | > grad_norm_0: tensor(17.6346, device='cuda:0') (tensor(26.7404, device='cuda:0')) + | > loss_gen: 2.329132318496704 (2.3664963245391846) + | > loss_kl: 1.862241506576538 (1.7888818979263306) + | > loss_feat: 7.328649520874023 (7.429211457570394) + | > loss_mel: 17.40464973449707 (17.387893040974934) + | > loss_duration: 1.4045453071594238 (1.408611575762431) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.3292179107666 (30.381094614664715) + | > grad_norm_1: tensor(256.4162, device='cuda:0') (tensor(239.6533, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.3395 (0.34378743171691895) + | > loader_time: 0.0046 (0.004595518112182617) + + + --> TIME: 2025-04-19 06:17:29 -- STEP: 28/142 -- GLOBAL_STEP: 442500 + | > loss_disc: 2.4029765129089355 (2.4059483919824873) + | > loss_disc_real_0: 0.10437897592782974 (0.1102826220116445) + | > loss_disc_real_1: 0.1754714399576187 (0.20126082748174667) + | > loss_disc_real_2: 0.19012515246868134 (0.22546744186963355) + | > loss_disc_real_3: 0.19704467058181763 (0.22654541209340096) + | > loss_disc_real_4: 0.1898830533027649 (0.2275397591292858) + | > loss_disc_real_5: 0.19008390605449677 (0.2281679010816983) + | > loss_0: 2.4029765129089355 (2.4059483919824873) + | > grad_norm_0: tensor(8.4901, device='cuda:0') (tensor(13.7471, device='cuda:0')) + | > loss_gen: 2.3217101097106934 (2.456717686993735) + | > loss_kl: 1.8284472227096558 (1.8148581002439772) + | > loss_feat: 7.142427444458008 (7.516599059104919) + | > loss_mel: 18.229385375976562 (17.55224210875375) + | > loss_duration: 1.4329044818878174 (1.4148853974682942) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.954875946044922 (30.75530229296003) + | > grad_norm_1: tensor(221.7395, device='cuda:0') (tensor(314.6981, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.3547 (0.3405060001782009) + | > loader_time: 0.0061 (0.005199415343148367) + + + --> TIME: 2025-04-19 06:17:39 -- STEP: 53/142 -- GLOBAL_STEP: 442525 + | > loss_disc: 2.5685360431671143 (2.409770587705215) + | > loss_disc_real_0: 0.17573590576648712 (0.11192810437027013) + | > loss_disc_real_1: 0.21824543178081512 (0.20313369243774773) + | > loss_disc_real_2: 0.2170543372631073 (0.222072330567072) + | > loss_disc_real_3: 0.1968613713979721 (0.22650917745986074) + | > loss_disc_real_4: 0.22453488409519196 (0.22403667670375896) + | > loss_disc_real_5: 0.24241751432418823 (0.2313366672902737) + | > loss_0: 2.5685360431671143 (2.409770587705215) + | > grad_norm_0: tensor(16.9510, device='cuda:0') (tensor(12.6158, device='cuda:0')) + | > loss_gen: 2.4036903381347656 (2.444220952267917) + | > loss_kl: 1.8050285577774048 (1.8154508257811923) + | > loss_feat: 7.626981735229492 (7.494359907114281) + | > loss_mel: 17.346065521240234 (17.638243099428575) + | > loss_duration: 1.436678171157837 (1.4213613654082675) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.618444442749023 (30.813636167994087) + | > grad_norm_1: tensor(378.6409, device='cuda:0') (tensor(327.6060, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.3764 (0.35395936246188175) + | > loader_time: 0.0059 (0.005559507405982827) + + + --> TIME: 2025-04-19 06:17:49 -- STEP: 78/142 -- GLOBAL_STEP: 442550 + | > loss_disc: 2.4159579277038574 (2.434055420068593) + | > loss_disc_real_0: 0.17272745072841644 (0.11685958800789638) + | > loss_disc_real_1: 0.22299319505691528 (0.2046797457031714) + | > loss_disc_real_2: 0.2183268666267395 (0.22546547173689574) + | > loss_disc_real_3: 0.24164706468582153 (0.22855941473673552) + | > loss_disc_real_4: 0.2304374873638153 (0.22832036285828322) + | > loss_disc_real_5: 0.22897827625274658 (0.2312474489594117) + | > loss_0: 2.4159579277038574 (2.434055420068593) + | > grad_norm_0: tensor(14.5851, device='cuda:0') (tensor(11.8410, device='cuda:0')) + | > loss_gen: 2.4553465843200684 (2.408555819438054) + | > loss_kl: 1.844483733177185 (1.8296380791908655) + | > loss_feat: 7.527076721191406 (7.374618823711689) + | > loss_mel: 18.37771224975586 (17.63448102657612) + | > loss_duration: 1.464261770248413 (1.4330829702890837) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.668880462646484 (30.680376664186134) + | > grad_norm_1: tensor(404.7867, device='cuda:0') (tensor(290.8290, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.4083 (0.3678298944082015) + | > loader_time: 0.0067 (0.005890962405082506) + + + --> TIME: 2025-04-19 06:18:00 -- STEP: 103/142 -- GLOBAL_STEP: 442575 + | > loss_disc: 2.5104188919067383 (2.444884165976812) + | > loss_disc_real_0: 0.11099856346845627 (0.11811249869541057) + | > loss_disc_real_1: 0.21225866675376892 (0.2058474030888196) + | > loss_disc_real_2: 0.1955382227897644 (0.22664753065525906) + | > loss_disc_real_3: 0.20217561721801758 (0.2290637714480891) + | > loss_disc_real_4: 0.2392338663339615 (0.22935409612447313) + | > loss_disc_real_5: 0.21230308711528778 (0.2326026174049933) + | > loss_0: 2.5104188919067383 (2.444884165976812) + | > grad_norm_0: tensor(8.6678, device='cuda:0') (tensor(11.5308, device='cuda:0')) + | > loss_gen: 2.201188325881958 (2.389583052940739) + | > loss_kl: 1.8593037128448486 (1.839986548840421) + | > loss_feat: 6.9484543800354 (7.2655140636036695) + | > loss_mel: 17.20485496520996 (17.589746169673596) + | > loss_duration: 1.4848077297210693 (1.4439835606269467) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.698610305786133 (30.528813445452347) + | > grad_norm_1: tensor(229.3305, device='cuda:0') (tensor(287.6432, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.4146 (0.3792527300640217) + | > loader_time: 0.007 (0.006127007956643706) + + + --> TIME: 2025-04-19 06:18:11 -- STEP: 128/142 -- GLOBAL_STEP: 442600 + | > loss_disc: 2.4277119636535645 (2.442343696951867) + | > loss_disc_real_0: 0.19662503898143768 (0.12040356663055718) + | > loss_disc_real_1: 0.21483956277370453 (0.20572056062519548) + | > loss_disc_real_2: 0.2544141709804535 (0.2257246463559568) + | > loss_disc_real_3: 0.19504328072071075 (0.2270604734076187) + | > loss_disc_real_4: 0.2068662792444229 (0.22829983581323177) + | > loss_disc_real_5: 0.23013408482074738 (0.23107909760437906) + | > loss_0: 2.4277119636535645 (2.442343696951867) + | > grad_norm_0: tensor(12.1358, device='cuda:0') (tensor(11.9457, device='cuda:0')) + | > loss_gen: 2.4936254024505615 (2.386151561513543) + | > loss_kl: 1.800990343093872 (1.8480875538662076) + | > loss_feat: 7.465673446655273 (7.227736592292786) + | > loss_mel: 18.17120361328125 (17.590008482337005) + | > loss_duration: 1.49791419506073 (1.4519654661417007) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.429405212402344 (30.503949657082558) + | > grad_norm_1: tensor(384.2986, device='cuda:0') (tensor(291.8642, device='cuda:0')) + | > current_lr_0: 0.00018917809816613621 + | > current_lr_1: 0.00018917809816613621 + | > step_time: 0.4355 (0.38891489431262016) + | > loader_time: 0.0079 (0.006385535001754761) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5326833724975586 (2.5326833724975586) + | > loss_disc_real_0: 0.13986122608184814 (0.13986122608184814) + | > loss_disc_real_1: 0.18891994655132294 (0.18891994655132294) + | > loss_disc_real_2: 0.294599324464798 (0.294599324464798) + | > loss_disc_real_3: 0.2972572147846222 (0.2972572147846222) + | > loss_disc_real_4: 0.2468867152929306 (0.2468867152929306) + | > loss_disc_real_5: 0.27056264877319336 (0.27056264877319336) + | > loss_0: 2.5326833724975586 (2.5326833724975586) + | > loss_gen: 2.3221933841705322 (2.3221933841705322) + | > loss_kl: 2.7552056312561035 (2.7552056312561035) + | > loss_feat: 6.3181304931640625 (6.3181304931640625) + | > loss_mel: 17.962343215942383 (17.962343215942383) + | > loss_duration: 1.7423770427703857 (1.7423770427703857) + | > loss_1: 31.100248336791992 (31.100248336791992) + + --> STEP: 1 + | > loss_disc: 2.4135324954986572 (2.4135324954986572) + | > loss_disc_real_0: 0.08577920496463776 (0.08577920496463776) + | > loss_disc_real_1: 0.16499480605125427 (0.16499480605125427) + | > loss_disc_real_2: 0.30680403113365173 (0.30680403113365173) + | > loss_disc_real_3: 0.2673163115978241 (0.2673163115978241) + | > loss_disc_real_4: 0.26416388154029846 (0.26416388154029846) + | > loss_disc_real_5: 0.21512947976589203 (0.21512947976589203) + | > loss_0: 2.4135324954986572 (2.4135324954986572) + | > loss_gen: 2.4080810546875 (2.4080810546875) + | > loss_kl: 2.7971816062927246 (2.7971816062927246) + | > loss_feat: 7.816645622253418 (7.816645622253418) + | > loss_mel: 18.695899963378906 (18.695899963378906) + | > loss_duration: 1.8115081787109375 (1.8115081787109375) + | > loss_1: 33.52931594848633 (33.52931594848633) + + --> STEP: 2 + | > loss_disc: 2.5931320190429688 (2.503332257270813) + | > loss_disc_real_0: 0.14518727362155914 (0.11548323929309845) + | > loss_disc_real_1: 0.19054833054542542 (0.17777156829833984) + | > loss_disc_real_2: 0.2560425102710724 (0.28142327070236206) + | > loss_disc_real_3: 0.30500689148902893 (0.2861616015434265) + | > loss_disc_real_4: 0.22327764332294464 (0.24372076243162155) + | > loss_disc_real_5: 0.19525885581970215 (0.2051941677927971) + | > loss_0: 2.5931320190429688 (2.503332257270813) + | > loss_gen: 2.1376309394836426 (2.2728559970855713) + | > loss_kl: 2.4915611743927 (2.6443713903427124) + | > loss_feat: 7.828869342803955 (7.8227574825286865) + | > loss_mel: 17.607460021972656 (18.15167999267578) + | > loss_duration: 1.7212855815887451 (1.7663968801498413) + | > loss_1: 31.786806106567383 (32.658061027526855) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00340425968170166 (-0.00046706199645996094) + | > avg_loss_disc: 2.503332257270813 (+0.06544959545135498) + | > avg_loss_disc_real_0: 0.11548323929309845 (-0.009951915591955185) + | > avg_loss_disc_real_1: 0.17777156829833984 (-0.06431306153535843) + | > avg_loss_disc_real_2: 0.28142327070236206 (+0.04484747350215912) + | > avg_loss_disc_real_3: 0.2861616015434265 (+0.03984704613685608) + | > avg_loss_disc_real_4: 0.24372076243162155 (-0.007593706250190735) + | > avg_loss_disc_real_5: 0.2051941677927971 (+0.0023338869214057922) + | > avg_loss_0: 2.503332257270813 (+0.06544959545135498) + | > avg_loss_gen: 2.2728559970855713 (-0.014319300651550293) + | > avg_loss_kl: 2.6443713903427124 (+0.09903442859649658) + | > avg_loss_feat: 7.8227574825286865 (+0.7968389987945557) + | > avg_loss_mel: 18.15167999267578 (-0.09688758850097656) + | > avg_loss_duration: 1.7663968801498413 (-0.014668941497802734) + | > avg_loss_1: 32.658061027526855 (+0.7699966430664062) + + + > EPOCH: 446/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:18:18)  + + --> TIME: 2025-04-19 06:18:23 -- STEP: 11/142 -- GLOBAL_STEP: 442625 + | > loss_disc: 2.4973714351654053 (2.4092572819102895) + | > loss_disc_real_0: 0.1795414537191391 (0.11487488109957088) + | > loss_disc_real_1: 0.20583371818065643 (0.19910791516304016) + | > loss_disc_real_2: 0.23748356103897095 (0.22429962320761246) + | > loss_disc_real_3: 0.23429618775844574 (0.22823173078623685) + | > loss_disc_real_4: 0.2266116887331009 (0.225648202679374) + | > loss_disc_real_5: 0.27762407064437866 (0.23830516771836716) + | > loss_0: 2.4973714351654053 (2.4092572819102895) + | > grad_norm_0: tensor(9.9572, device='cuda:0') (tensor(7.4885, device='cuda:0')) + | > loss_gen: 2.3937177658081055 (2.401174155148593) + | > loss_kl: 1.7471623420715332 (1.7653251777995715) + | > loss_feat: 7.457585334777832 (7.487109617753462) + | > loss_mel: 18.57957649230957 (17.601002172990277) + | > loss_duration: 1.3878374099731445 (1.3967970392920754) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.565879821777344 (30.65140828219327) + | > grad_norm_1: tensor(153.5916, device='cuda:0') (tensor(204.3741, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.3301 (0.3256100958043879) + | > loader_time: 0.005 (0.004551692442460494) + + + --> TIME: 2025-04-19 06:18:32 -- STEP: 36/142 -- GLOBAL_STEP: 442650 + | > loss_disc: 2.363563060760498 (2.3934280806117587) + | > loss_disc_real_0: 0.09124305099248886 (0.1119500578691562) + | > loss_disc_real_1: 0.22182132303714752 (0.20172238598267236) + | > loss_disc_real_2: 0.2104232907295227 (0.22280476780401337) + | > loss_disc_real_3: 0.20805120468139648 (0.22507618036535051) + | > loss_disc_real_4: 0.2108895182609558 (0.224067695852783) + | > loss_disc_real_5: 0.22436651587486267 (0.22993682821591696) + | > loss_0: 2.363563060760498 (2.3934280806117587) + | > grad_norm_0: tensor(5.2941, device='cuda:0') (tensor(7.1936, device='cuda:0')) + | > loss_gen: 2.4132730960845947 (2.4218711190753512) + | > loss_kl: 1.8420164585113525 (1.8046372135480244) + | > loss_feat: 6.989491939544678 (7.475496914651659) + | > loss_mel: 17.974903106689453 (17.84572458267212) + | > loss_duration: 1.4214842319488525 (1.4139189687040117) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.64116859436035 (30.961648888058132) + | > grad_norm_1: tensor(118.5144, device='cuda:0') (tensor(201.4093, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.3543 (0.3392469022009108) + | > loader_time: 0.0058 (0.00511267450120714) + + + --> TIME: 2025-04-19 06:18:41 -- STEP: 61/142 -- GLOBAL_STEP: 442675 + | > loss_disc: 2.4997849464416504 (2.404761197137051) + | > loss_disc_real_0: 0.1014246940612793 (0.11397696310868029) + | > loss_disc_real_1: 0.24818113446235657 (0.20352371965275437) + | > loss_disc_real_2: 0.2421470582485199 (0.2230810097983626) + | > loss_disc_real_3: 0.22783371806144714 (0.22569696849486867) + | > loss_disc_real_4: 0.2592276930809021 (0.22472237026105163) + | > loss_disc_real_5: 0.24226462841033936 (0.23033480614912313) + | > loss_0: 2.4997849464416504 (2.404761197137051) + | > grad_norm_0: tensor(7.7139, device='cuda:0') (tensor(7.7085, device='cuda:0')) + | > loss_gen: 2.282430648803711 (2.410325953217804) + | > loss_kl: 1.7895569801330566 (1.8190051649437575) + | > loss_feat: 6.238411903381348 (7.40112684593826) + | > loss_mel: 16.475460052490234 (17.904161453247074) + | > loss_duration: 1.4462294578552246 (1.426066117208512) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.23208999633789 (30.960685511104398) + | > grad_norm_1: tensor(147.7108, device='cuda:0') (tensor(205.3330, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.3814 (0.3502249756797416) + | > loader_time: 0.0059 (0.005424300178152617) + + + --> TIME: 2025-04-19 06:18:51 -- STEP: 86/142 -- GLOBAL_STEP: 442700 + | > loss_disc: 2.495708465576172 (2.415607380312542) + | > loss_disc_real_0: 0.11584609001874924 (0.1169419197668863) + | > loss_disc_real_1: 0.23722223937511444 (0.20446199438599652) + | > loss_disc_real_2: 0.21378038823604584 (0.22372805309850116) + | > loss_disc_real_3: 0.2111399918794632 (0.2254445779808732) + | > loss_disc_real_4: 0.22089463472366333 (0.22513954202796138) + | > loss_disc_real_5: 0.2045067548751831 (0.22952579083137734) + | > loss_0: 2.495708465576172 (2.415607380312542) + | > grad_norm_0: tensor(7.8200, device='cuda:0') (tensor(8.3435, device='cuda:0')) + | > loss_gen: 2.1863925457000732 (2.397277244301729) + | > loss_kl: 1.9326330423355103 (1.8402953549872998) + | > loss_feat: 7.2672929763793945 (7.344262588855832) + | > loss_mel: 17.83297348022461 (17.79893306244251) + | > loss_duration: 1.5031697750091553 (1.438774587110032) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.722461700439453 (30.819542818291243) + | > grad_norm_1: tensor(178.6725, device='cuda:0') (tensor(210.9373, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.3965 (0.3624805406082508) + | > loader_time: 0.0065 (0.005725874457248424) + + + --> TIME: 2025-04-19 06:19:01 -- STEP: 111/142 -- GLOBAL_STEP: 442725 + | > loss_disc: 2.482164144515991 (2.425469971991874) + | > loss_disc_real_0: 0.1269908845424652 (0.11890812661196734) + | > loss_disc_real_1: 0.1949208378791809 (0.20454278064740672) + | > loss_disc_real_2: 0.22029347717761993 (0.22419959763148883) + | > loss_disc_real_3: 0.2558569610118866 (0.2267930036461031) + | > loss_disc_real_4: 0.2238398641347885 (0.22577235532236528) + | > loss_disc_real_5: 0.2162722647190094 (0.2294731016631599) + | > loss_0: 2.482164144515991 (2.425469971991874) + | > grad_norm_0: tensor(5.3270, device='cuda:0') (tensor(8.0001, device='cuda:0')) + | > loss_gen: 2.3966708183288574 (2.387987396738551) + | > loss_kl: 1.885596513748169 (1.847801170907579) + | > loss_feat: 6.968654632568359 (7.306843998195888) + | > loss_mel: 18.509313583374023 (17.737639289718484) + | > loss_duration: 1.4890137910842896 (1.4469971871590832) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.249248504638672 (30.727268992243587) + | > grad_norm_1: tensor(129.9128, device='cuda:0') (tensor(196.1094, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.425 (0.37282524882136164) + | > loader_time: 0.0069 (0.005972159875405804) + + + --> TIME: 2025-04-19 06:19:13 -- STEP: 136/142 -- GLOBAL_STEP: 442750 + | > loss_disc: 2.486053943634033 (2.426749685231377) + | > loss_disc_real_0: 0.1226409450173378 (0.11987317643840523) + | > loss_disc_real_1: 0.2113926261663437 (0.20509935499114149) + | > loss_disc_real_2: 0.23361867666244507 (0.22443894483149052) + | > loss_disc_real_3: 0.23906181752681732 (0.22672518737175884) + | > loss_disc_real_4: 0.2250664085149765 (0.225827174151645) + | > loss_disc_real_5: 0.20242342352867126 (0.22973971649566116) + | > loss_0: 2.486053943634033 (2.426749685231377) + | > grad_norm_0: tensor(17.6791, device='cuda:0') (tensor(8.3866, device='cuda:0')) + | > loss_gen: 2.25892972946167 (2.3897872619769163) + | > loss_kl: 1.9457120895385742 (nan) + | > loss_feat: 6.522904396057129 (7.289202223805821) + | > loss_mel: 16.22384262084961 (17.66973437982446) + | > loss_duration: 1.480926275253296 (nan) + | > amp_scaler: 64.0 (121.41176470588235) + | > loss_1: 28.432315826416016 (nan) + | > grad_norm_1: tensor(365.2897, device='cuda:0') (tensor(205.8554, device='cuda:0')) + | > current_lr_0: 0.00018915445090386543 + | > current_lr_1: 0.00018915445090386543 + | > step_time: 0.4487 (0.3846505705048056) + | > loader_time: 0.0073 (0.006207169855342193) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.432095766067505 (2.432095766067505) + | > loss_disc_real_0: 0.11881871521472931 (0.11881871521472931) + | > loss_disc_real_1: 0.2066054493188858 (0.2066054493188858) + | > loss_disc_real_2: 0.2618063986301422 (0.2618063986301422) + | > loss_disc_real_3: 0.2399752289056778 (0.2399752289056778) + | > loss_disc_real_4: 0.22325822710990906 (0.22325822710990906) + | > loss_disc_real_5: 0.24187642335891724 (0.24187642335891724) + | > loss_0: 2.432095766067505 (2.432095766067505) + | > loss_gen: 2.315096855163574 (2.315096855163574) + | > loss_kl: 2.5984930992126465 (2.5984930992126465) + | > loss_feat: 6.725659370422363 (6.725659370422363) + | > loss_mel: 17.777862548828125 (17.777862548828125) + | > loss_duration: 1.748570203781128 (1.748570203781128) + | > loss_1: 31.165679931640625 (31.165679931640625) + + --> STEP: 1 + | > loss_disc: 2.638305902481079 (2.638305902481079) + | > loss_disc_real_0: 0.14985382556915283 (0.14985382556915283) + | > loss_disc_real_1: 0.2587578296661377 (0.2587578296661377) + | > loss_disc_real_2: 0.31240713596343994 (0.31240713596343994) + | > loss_disc_real_3: 0.24979053437709808 (0.24979053437709808) + | > loss_disc_real_4: 0.237802654504776 (0.237802654504776) + | > loss_disc_real_5: 0.23673923313617706 (0.23673923313617706) + | > loss_0: 2.638305902481079 (2.638305902481079) + | > loss_gen: 2.283536911010742 (2.283536911010742) + | > loss_kl: 2.836873769760132 (2.836873769760132) + | > loss_feat: 7.090423583984375 (7.090423583984375) + | > loss_mel: 18.710472106933594 (18.710472106933594) + | > loss_duration: 1.7877545356750488 (1.7877545356750488) + | > loss_1: 32.70906066894531 (32.70906066894531) + + --> STEP: 2 + | > loss_disc: 2.5003089904785156 (2.5693074464797974) + | > loss_disc_real_0: 0.18130892515182495 (0.1655813753604889) + | > loss_disc_real_1: 0.2467995285987854 (0.25277867913246155) + | > loss_disc_real_2: 0.2542862594127655 (0.2833466976881027) + | > loss_disc_real_3: 0.2172316461801529 (0.2335110902786255) + | > loss_disc_real_4: 0.2504301369190216 (0.2441163957118988) + | > loss_disc_real_5: 0.259642094373703 (0.24819066375494003) + | > loss_0: 2.5003089904785156 (2.5693074464797974) + | > loss_gen: 2.3814587593078613 (2.3324978351593018) + | > loss_kl: 2.3169147968292236 (2.5768942832946777) + | > loss_feat: 6.273565769195557 (6.681994676589966) + | > loss_mel: 18.06733512878418 (18.388903617858887) + | > loss_duration: 1.7137093544006348 (1.7507319450378418) + | > loss_1: 30.75298309326172 (31.731021881103516) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003827333450317383 (+0.00042307376861572266) + | > avg_loss_disc: 2.5693074464797974 (+0.06597518920898438) + | > avg_loss_disc_real_0: 0.1655813753604889 (+0.05009813606739044) + | > avg_loss_disc_real_1: 0.25277867913246155 (+0.0750071108341217) + | > avg_loss_disc_real_2: 0.2833466976881027 (+0.0019234269857406616) + | > avg_loss_disc_real_3: 0.2335110902786255 (-0.052650511264801025) + | > avg_loss_disc_real_4: 0.2441163957118988 (+0.0003956332802772522) + | > avg_loss_disc_real_5: 0.24819066375494003 (+0.042996495962142944) + | > avg_loss_0: 2.5693074464797974 (+0.06597518920898438) + | > avg_loss_gen: 2.3324978351593018 (+0.05964183807373047) + | > avg_loss_kl: 2.5768942832946777 (-0.06747710704803467) + | > avg_loss_feat: 6.681994676589966 (-1.1407628059387207) + | > avg_loss_mel: 18.388903617858887 (+0.23722362518310547) + | > avg_loss_duration: 1.7507319450378418 (-0.01566493511199951) + | > avg_loss_1: 31.731021881103516 (-0.9270391464233398) + + + > EPOCH: 447/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:19:17)  + + --> TIME: 2025-04-19 06:19:24 -- STEP: 19/142 -- GLOBAL_STEP: 442775 + | > loss_disc: 2.390035390853882 (2.424788751100239) + | > loss_disc_real_0: 0.1002892255783081 (0.11197521419901597) + | > loss_disc_real_1: 0.1898210197687149 (0.20534760465747431) + | > loss_disc_real_2: 0.24384155869483948 (0.2288459688425064) + | > loss_disc_real_3: 0.2246827334165573 (0.2260140298228515) + | > loss_disc_real_4: 0.21842849254608154 (0.22827801892631933) + | > loss_disc_real_5: 0.19583922624588013 (0.22556801218735545) + | > loss_0: 2.390035390853882 (2.424788751100239) + | > grad_norm_0: tensor(15.6128, device='cuda:0') (tensor(11.7750, device='cuda:0')) + | > loss_gen: 2.3154349327087402 (2.4057246634834693) + | > loss_kl: 1.866381287574768 (1.802439852764732) + | > loss_feat: 7.371067523956299 (7.3433965632790015) + | > loss_mel: 17.97945785522461 (17.796001133165863) + | > loss_duration: 1.4157791137695312 (1.4060112112446834) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.948122024536133 (30.753573267083418) + | > grad_norm_1: tensor(447.1817, device='cuda:0') (tensor(284.8527, device='cuda:0')) + | > current_lr_0: 0.00018913080659750244 + | > current_lr_1: 0.00018913080659750244 + | > step_time: 0.3587 (0.34738876945094055) + | > loader_time: 0.0059 (0.004813683660406815) + + + --> TIME: 2025-04-19 06:19:33 -- STEP: 44/142 -- GLOBAL_STEP: 442800 + | > loss_disc: 2.3691418170928955 (2.429750843481585) + | > loss_disc_real_0: 0.12836304306983948 (0.11561080525544556) + | > loss_disc_real_1: 0.17482371628284454 (0.20309983837333592) + | > loss_disc_real_2: 0.24657216668128967 (0.2267471230165525) + | > loss_disc_real_3: 0.24157214164733887 (0.22791565920818935) + | > loss_disc_real_4: 0.24238227307796478 (0.2315717793323777) + | > loss_disc_real_5: 0.23533236980438232 (0.22827580232511868) + | > loss_0: 2.3691418170928955 (2.429750843481585) + | > grad_norm_0: tensor(7.4393, device='cuda:0') (tensor(10.9760, device='cuda:0')) + | > loss_gen: 2.3738088607788086 (2.390205529603091) + | > loss_kl: 1.8022327423095703 (1.8145520009777762) + | > loss_feat: 6.970681667327881 (7.26054345477711) + | > loss_mel: 17.580591201782227 (17.76962345296687) + | > loss_duration: 1.4503642320632935 (1.4223958822813902) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.17767906188965 (30.65732019597834) + | > grad_norm_1: tensor(293.1035, device='cuda:0') (tensor(284.2649, device='cuda:0')) + | > current_lr_0: 0.00018913080659750244 + | > current_lr_1: 0.00018913080659750244 + | > step_time: 0.363 (0.3553743470798839) + | > loader_time: 0.0056 (0.005252832716161554) + + + --> TIME: 2025-04-19 06:19:43 -- STEP: 69/142 -- GLOBAL_STEP: 442825 + | > loss_disc: 2.4763295650482178 (2.43877617172573) + | > loss_disc_real_0: 0.10440874099731445 (0.11721417772165243) + | > loss_disc_real_1: 0.2420550286769867 (0.2057842482691226) + | > loss_disc_real_2: 0.2178378403186798 (0.22524887388167175) + | > loss_disc_real_3: 0.2589055001735687 (0.22840765831263168) + | > loss_disc_real_4: 0.24489368498325348 (0.22948110427545465) + | > loss_disc_real_5: 0.24782471358776093 (0.22955962194912677) + | > loss_0: 2.4763295650482178 (2.43877617172573) + | > grad_norm_0: tensor(8.0804, device='cuda:0') (tensor(11.8971, device='cuda:0')) + | > loss_gen: 2.4415876865386963 (2.38079912075098) + | > loss_kl: 1.9538722038269043 (1.831118573313174) + | > loss_feat: 6.881257057189941 (7.264245447905167) + | > loss_mel: 17.032814025878906 (17.70790838158649) + | > loss_duration: 1.5075256824493408 (1.4333742891532788) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.81705665588379 (30.61744562784831) + | > grad_norm_1: tensor(272.1944, device='cuda:0') (tensor(294.1471, device='cuda:0')) + | > current_lr_0: 0.00018913080659750244 + | > current_lr_1: 0.00018913080659750244 + | > step_time: 0.3936 (0.3632491913394651) + | > loader_time: 0.0062 (0.005648937778196473) + + + --> TIME: 2025-04-19 06:19:53 -- STEP: 94/142 -- GLOBAL_STEP: 442850 + | > loss_disc: 2.5033481121063232 (2.4412138512793997) + | > loss_disc_real_0: 0.16588518023490906 (0.11831065497182786) + | > loss_disc_real_1: 0.22129179537296295 (0.20609043506865807) + | > loss_disc_real_2: 0.22301945090293884 (0.22683886945881743) + | > loss_disc_real_3: 0.22638258337974548 (0.22832254003337088) + | > loss_disc_real_4: 0.2653747498989105 (0.2286822553327743) + | > loss_disc_real_5: 0.2235347032546997 (0.2299492470127471) + | > loss_0: 2.5033481121063232 (2.4412138512793997) + | > grad_norm_0: tensor(12.9247, device='cuda:0') (tensor(11.9586, device='cuda:0')) + | > loss_gen: 2.3452978134155273 (2.37884129869177) + | > loss_kl: 1.909893274307251 (1.8384197848908446) + | > loss_feat: 6.11459493637085 (7.23438922902371) + | > loss_mel: 16.842384338378906 (17.67371868579946) + | > loss_duration: 1.482253074645996 (1.4440672346886168) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.69442367553711 (30.569436154466995) + | > grad_norm_1: tensor(153.1337, device='cuda:0') (tensor(292.9492, device='cuda:0')) + | > current_lr_0: 0.00018913080659750244 + | > current_lr_1: 0.00018913080659750244 + | > step_time: 0.3963 (0.37238127119997705) + | > loader_time: 0.007 (0.00592573399239398) + + + --> TIME: 2025-04-19 06:20:04 -- STEP: 119/142 -- GLOBAL_STEP: 442875 + | > loss_disc: 2.452789068222046 (2.447848205806828) + | > loss_disc_real_0: 0.10223330557346344 (0.1200591544268512) + | > loss_disc_real_1: 0.21064849197864532 (0.20662669900084743) + | > loss_disc_real_2: 0.2141883224248886 (0.22540751074542517) + | > loss_disc_real_3: 0.20368918776512146 (0.22861325139758967) + | > loss_disc_real_4: 0.2433057725429535 (0.22890326856565074) + | > loss_disc_real_5: 0.24165266752243042 (0.23168488372774684) + | > loss_0: 2.452789068222046 (2.447848205806828) + | > grad_norm_0: tensor(17.9386, device='cuda:0') (tensor(12.0381, device='cuda:0')) + | > loss_gen: 2.266472339630127 (2.37078734606254) + | > loss_kl: 1.9968441724777222 (1.8476452807418438) + | > loss_feat: 6.6349873542785645 (7.2120206977139) + | > loss_mel: 18.14455795288086 (17.617099818061387) + | > loss_duration: 1.5215458869934082 (1.4507500874896009) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.564407348632812 (30.498303164954947) + | > grad_norm_1: tensor(323.6659, device='cuda:0') (tensor(300.3151, device='cuda:0')) + | > current_lr_0: 0.00018913080659750244 + | > current_lr_1: 0.00018913080659750244 + | > step_time: 0.4267 (0.38214167426614193) + | > loader_time: 0.0073 (0.006183079310825893) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3728041648864746 (2.3728041648864746) + | > loss_disc_real_0: 0.12141478061676025 (0.12141478061676025) + | > loss_disc_real_1: 0.25308504700660706 (0.25308504700660706) + | > loss_disc_real_2: 0.2451528012752533 (0.2451528012752533) + | > loss_disc_real_3: 0.22762349247932434 (0.22762349247932434) + | > loss_disc_real_4: 0.2374749332666397 (0.2374749332666397) + | > loss_disc_real_5: 0.2389247864484787 (0.2389247864484787) + | > loss_0: 2.3728041648864746 (2.3728041648864746) + | > loss_gen: 2.585256338119507 (2.585256338119507) + | > loss_kl: 2.540724992752075 (2.540724992752075) + | > loss_feat: 7.143192291259766 (7.143192291259766) + | > loss_mel: 18.13348960876465 (18.13348960876465) + | > loss_duration: 1.754835605621338 (1.754835605621338) + | > loss_1: 32.15749740600586 (32.15749740600586) + + --> STEP: 1 + | > loss_disc: 2.4922866821289062 (2.4922866821289062) + | > loss_disc_real_0: 0.14101815223693848 (0.14101815223693848) + | > loss_disc_real_1: 0.2645938992500305 (0.2645938992500305) + | > loss_disc_real_2: 0.2369571030139923 (0.2369571030139923) + | > loss_disc_real_3: 0.23679713904857635 (0.23679713904857635) + | > loss_disc_real_4: 0.2351997047662735 (0.2351997047662735) + | > loss_disc_real_5: 0.2920602262020111 (0.2920602262020111) + | > loss_0: 2.4922866821289062 (2.4922866821289062) + | > loss_gen: 2.4299862384796143 (2.4299862384796143) + | > loss_kl: 2.737293243408203 (2.737293243408203) + | > loss_feat: 6.423213481903076 (6.423213481903076) + | > loss_mel: 16.778282165527344 (16.778282165527344) + | > loss_duration: 1.8101599216461182 (1.8101599216461182) + | > loss_1: 30.178936004638672 (30.178936004638672) + + --> STEP: 2 + | > loss_disc: 2.436673164367676 (2.464479923248291) + | > loss_disc_real_0: 0.18196386098861694 (0.1614910066127777) + | > loss_disc_real_1: 0.261063814163208 (0.26282885670661926) + | > loss_disc_real_2: 0.22520899772644043 (0.23108305037021637) + | > loss_disc_real_3: 0.23791638016700745 (0.2373567596077919) + | > loss_disc_real_4: 0.24506855010986328 (0.2401341274380684) + | > loss_disc_real_5: 0.2676738202571869 (0.279867023229599) + | > loss_0: 2.436673164367676 (2.464479923248291) + | > loss_gen: 2.6001291275024414 (2.515057682991028) + | > loss_kl: 2.271822929382324 (2.5045580863952637) + | > loss_feat: 6.700069904327393 (6.561641693115234) + | > loss_mel: 17.573579788208008 (17.175930976867676) + | > loss_duration: 1.7169220447540283 (1.7635409832000732) + | > loss_1: 30.862525939941406 (30.52073097229004) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004000544548034668 (+0.00017321109771728516) + | > avg_loss_disc: 2.464479923248291 (-0.10482752323150635) + | > avg_loss_disc_real_0: 0.1614910066127777 (-0.004090368747711182) + | > avg_loss_disc_real_1: 0.26282885670661926 (+0.010050177574157715) + | > avg_loss_disc_real_2: 0.23108305037021637 (-0.05226364731788635) + | > avg_loss_disc_real_3: 0.2373567596077919 (+0.0038456693291664124) + | > avg_loss_disc_real_4: 0.2401341274380684 (-0.003982268273830414) + | > avg_loss_disc_real_5: 0.279867023229599 (+0.031676359474658966) + | > avg_loss_0: 2.464479923248291 (-0.10482752323150635) + | > avg_loss_gen: 2.515057682991028 (+0.18255984783172607) + | > avg_loss_kl: 2.5045580863952637 (-0.07233619689941406) + | > avg_loss_feat: 6.561641693115234 (-0.12035298347473145) + | > avg_loss_mel: 17.175930976867676 (-1.212972640991211) + | > avg_loss_duration: 1.7635409832000732 (+0.012809038162231445) + | > avg_loss_1: 30.52073097229004 (-1.2102909088134766) + + + > EPOCH: 448/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:20:16)  + + --> TIME: 2025-04-19 06:20:17 -- STEP: 2/142 -- GLOBAL_STEP: 442900 + | > loss_disc: 2.4679503440856934 (2.4775967597961426) + | > loss_disc_real_0: 0.13733211159706116 (0.12826482579112053) + | > loss_disc_real_1: 0.21303966641426086 (0.221063069999218) + | > loss_disc_real_2: 0.2413291037082672 (0.23675763607025146) + | > loss_disc_real_3: 0.2638777494430542 (0.24826445430517197) + | > loss_disc_real_4: 0.26995283365249634 (0.257980115711689) + | > loss_disc_real_5: 0.30589067935943604 (0.32815736532211304) + | > loss_0: 2.4679503440856934 (2.4775967597961426) + | > grad_norm_0: tensor(9.7961, device='cuda:0') (tensor(8.7116, device='cuda:0')) + | > loss_gen: 2.456888198852539 (2.486605644226074) + | > loss_kl: 1.8101922273635864 (1.840727150440216) + | > loss_feat: 7.872793674468994 (7.597012281417847) + | > loss_mel: 17.577503204345703 (17.044089317321777) + | > loss_duration: 1.418035864830017 (1.3965967297554016) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.135414123535156 (30.365032196044922) + | > grad_norm_1: tensor(365.3656, device='cuda:0') (tensor(400.7379, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.3224 (0.32422852516174316) + | > loader_time: 0.0041 (0.0040972232818603516) + + + --> TIME: 2025-04-19 06:20:26 -- STEP: 27/142 -- GLOBAL_STEP: 442925 + | > loss_disc: 2.555758476257324 (2.430189653679177) + | > loss_disc_real_0: 0.09222159534692764 (0.11320636255873574) + | > loss_disc_real_1: 0.21038664877414703 (0.20365729376121802) + | > loss_disc_real_2: 0.22304928302764893 (0.22752955059210458) + | > loss_disc_real_3: 0.23976992070674896 (0.22529938596266288) + | > loss_disc_real_4: 0.24391944706439972 (0.22766794761021933) + | > loss_disc_real_5: 0.23139484226703644 (0.23896582645398598) + | > loss_0: 2.555758476257324 (2.430189653679177) + | > grad_norm_0: tensor(10.8476, device='cuda:0') (tensor(9.8455, device='cuda:0')) + | > loss_gen: 2.3526360988616943 (2.4136844829276756) + | > loss_kl: 1.783637523651123 (1.8177177861884788) + | > loss_feat: 8.071805000305176 (7.485211160447863) + | > loss_mel: 18.068506240844727 (17.676597948427553) + | > loss_duration: 1.415905475616455 (1.4104971179255732) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.692489624023438 (30.803708606296116) + | > grad_norm_1: tensor(364.7878, device='cuda:0') (tensor(249.9001, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.3494 (0.3393794430626763) + | > loader_time: 0.0056 (0.004981853343822339) + + + --> TIME: 2025-04-19 06:20:35 -- STEP: 52/142 -- GLOBAL_STEP: 442950 + | > loss_disc: 2.494539260864258 (2.4150506670658407) + | > loss_disc_real_0: 0.18433453142642975 (0.11742838027958687) + | > loss_disc_real_1: 0.23784701526165009 (0.20400485310416955) + | > loss_disc_real_2: 0.20963194966316223 (0.2237842011337097) + | > loss_disc_real_3: 0.21687127649784088 (0.22264020918653563) + | > loss_disc_real_4: 0.2335195243358612 (0.22413358493493155) + | > loss_disc_real_5: 0.22873590886592865 (0.2316746161534236) + | > loss_0: 2.494539260864258 (2.4150506670658407) + | > grad_norm_0: tensor(4.1040, device='cuda:0') (tensor(9.2032, device='cuda:0')) + | > loss_gen: 2.3695266246795654 (2.4128590638820953) + | > loss_kl: 1.8621511459350586 (1.8222961998902834) + | > loss_feat: 6.928830146789551 (7.468872684698838) + | > loss_mel: 16.449678421020508 (17.746880091153663) + | > loss_duration: 1.4704749584197998 (1.4211899752800277) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.080659866333008 (30.87209800573496) + | > grad_norm_1: tensor(45.5564, device='cuda:0') (tensor(219.4565, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.366 (0.34914427537184495) + | > loader_time: 0.0063 (0.0053345377628619866) + + + --> TIME: 2025-04-19 06:20:45 -- STEP: 77/142 -- GLOBAL_STEP: 442975 + | > loss_disc: 2.474865674972534 (2.424887654069182) + | > loss_disc_real_0: 0.08609296381473541 (0.11931708387353203) + | > loss_disc_real_1: 0.22374828159809113 (0.20373196493495593) + | > loss_disc_real_2: 0.21480965614318848 (0.22418444458540385) + | > loss_disc_real_3: 0.19583381712436676 (0.22500885016732403) + | > loss_disc_real_4: 0.1907949596643448 (0.22660811051919863) + | > loss_disc_real_5: 0.23277494311332703 (0.23290412689184214) + | > loss_0: 2.474865674972534 (2.424887654069182) + | > grad_norm_0: tensor(9.1296, device='cuda:0') (tensor(9.2250, device='cuda:0')) + | > loss_gen: 2.2942912578582764 (2.404870150925278) + | > loss_kl: 1.8369636535644531 (1.8414987511449046) + | > loss_feat: 7.173063278198242 (7.416935871173809) + | > loss_mel: 18.493844985961914 (17.74915567621008) + | > loss_duration: 1.461856722831726 (1.4331854148344558) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.260019302368164 (30.84564592931178) + | > grad_norm_1: tensor(61.1437, device='cuda:0') (tensor(214.9841, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.3914 (0.36068165766728383) + | > loader_time: 0.0062 (0.005641497574843371) + + + --> TIME: 2025-04-19 06:20:55 -- STEP: 102/142 -- GLOBAL_STEP: 443000 + | > loss_disc: 2.539027214050293 (2.4337285219454294) + | > loss_disc_real_0: 0.15512096881866455 (0.12184303750594457) + | > loss_disc_real_1: 0.20744158327579498 (0.2054888772613862) + | > loss_disc_real_2: 0.21571427583694458 (0.2251187386173828) + | > loss_disc_real_3: 0.235981747508049 (0.22468212860472062) + | > loss_disc_real_4: 0.23263557255268097 (0.2260832659461919) + | > loss_disc_real_5: 0.22258907556533813 (0.23128335601558872) + | > loss_0: 2.539027214050293 (2.4337285219454294) + | > grad_norm_0: tensor(13.4616, device='cuda:0') (tensor(9.7242, device='cuda:0')) + | > loss_gen: 2.1208035945892334 (2.397615191983244) + | > loss_kl: 1.8531626462936401 (1.8503317318710626) + | > loss_feat: 5.3636698722839355 (7.3380754695219155) + | > loss_mel: 14.451006889343262 (17.735407081304814) + | > loss_duration: 1.4604729413986206 (1.4435585783977134) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 25.249114990234375 (30.76498805775362) + | > grad_norm_1: tensor(222.7708, device='cuda:0') (tensor(232.7545, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.4185 (0.37075307322483436) + | > loader_time: 0.0072 (0.005906186851800657) + + + --> TIME: 2025-04-19 06:21:06 -- STEP: 127/142 -- GLOBAL_STEP: 443025 + | > loss_disc: 2.5474417209625244 (2.4472254336349604) + | > loss_disc_real_0: 0.24400214850902557 (0.12501310680325575) + | > loss_disc_real_1: 0.20616458356380463 (0.20554984284667518) + | > loss_disc_real_2: 0.24617670476436615 (0.22570837611757863) + | > loss_disc_real_3: 0.23675237596035004 (0.22636531750986896) + | > loss_disc_real_4: 0.25706419348716736 (0.22740953867360364) + | > loss_disc_real_5: 0.2361481487751007 (0.23313346059303583) + | > loss_0: 2.5474417209625244 (2.4472254336349604) + | > grad_norm_0: tensor(18.1340, device='cuda:0') (tensor(11.2351, device='cuda:0')) + | > loss_gen: 2.379291534423828 (2.3962931257533295) + | > loss_kl: 1.849168062210083 (1.8538749790567113) + | > loss_feat: 6.869812965393066 (7.2769661400261825) + | > loss_mel: 17.67383575439453 (17.70322305574192) + | > loss_duration: 1.4889570474624634 (1.451811203806419) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.261064529418945 (30.682168464961013) + | > grad_norm_1: tensor(220.2565, device='cuda:0') (tensor(245.8742, device='cuda:0')) + | > current_lr_0: 0.00018910716524667774 + | > current_lr_1: 0.00018910716524667774 + | > step_time: 0.4381 (0.3818424329982968) + | > loader_time: 0.0072 (0.006140230208869994) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4498109817504883 (2.4498109817504883) + | > loss_disc_real_0: 0.17246301472187042 (0.17246301472187042) + | > loss_disc_real_1: 0.3014630675315857 (0.3014630675315857) + | > loss_disc_real_2: 0.2882218658924103 (0.2882218658924103) + | > loss_disc_real_3: 0.24445632100105286 (0.24445632100105286) + | > loss_disc_real_4: 0.25007423758506775 (0.25007423758506775) + | > loss_disc_real_5: 0.2524094581604004 (0.2524094581604004) + | > loss_0: 2.4498109817504883 (2.4498109817504883) + | > loss_gen: 2.6799962520599365 (2.6799962520599365) + | > loss_kl: 2.3634939193725586 (2.3634939193725586) + | > loss_feat: 7.806065559387207 (7.806065559387207) + | > loss_mel: 19.546768188476562 (19.546768188476562) + | > loss_duration: 1.7899646759033203 (1.7899646759033203) + | > loss_1: 34.18628692626953 (34.18628692626953) + + --> STEP: 1 + | > loss_disc: 2.434103488922119 (2.434103488922119) + | > loss_disc_real_0: 0.14817079901695251 (0.14817079901695251) + | > loss_disc_real_1: 0.26889944076538086 (0.26889944076538086) + | > loss_disc_real_2: 0.27373814582824707 (0.27373814582824707) + | > loss_disc_real_3: 0.23086392879486084 (0.23086392879486084) + | > loss_disc_real_4: 0.2454972267150879 (0.2454972267150879) + | > loss_disc_real_5: 0.1995328813791275 (0.1995328813791275) + | > loss_0: 2.434103488922119 (2.434103488922119) + | > loss_gen: 2.5039782524108887 (2.5039782524108887) + | > loss_kl: 2.7153968811035156 (2.7153968811035156) + | > loss_feat: 7.485129356384277 (7.485129356384277) + | > loss_mel: 18.904394149780273 (18.904394149780273) + | > loss_duration: 1.8052678108215332 (1.8052678108215332) + | > loss_1: 33.41416549682617 (33.41416549682617) + + --> STEP: 2 + | > loss_disc: 2.4169697761535645 (2.425536632537842) + | > loss_disc_real_0: 0.18421699106693268 (0.1661938950419426) + | > loss_disc_real_1: 0.23769739270210266 (0.25329841673374176) + | > loss_disc_real_2: 0.2530146539211273 (0.2633763998746872) + | > loss_disc_real_3: 0.2278011441230774 (0.22933253645896912) + | > loss_disc_real_4: 0.21793314814567566 (0.23171518743038177) + | > loss_disc_real_5: 0.2577422261238098 (0.22863755375146866) + | > loss_0: 2.4169697761535645 (2.425536632537842) + | > loss_gen: 2.505690097808838 (2.5048341751098633) + | > loss_kl: 2.2307145595550537 (2.4730557203292847) + | > loss_feat: 7.360193252563477 (7.422661304473877) + | > loss_mel: 18.972286224365234 (18.938340187072754) + | > loss_duration: 1.7145167589187622 (1.7598922848701477) + | > loss_1: 32.78340148925781 (33.09878349304199) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003545403480529785 (-0.0004551410675048828) + | > avg_loss_disc: 2.425536632537842 (-0.03894329071044922) + | > avg_loss_disc_real_0: 0.1661938950419426 (+0.0047028884291648865) + | > avg_loss_disc_real_1: 0.25329841673374176 (-0.009530439972877502) + | > avg_loss_disc_real_2: 0.2633763998746872 (+0.032293349504470825) + | > avg_loss_disc_real_3: 0.22933253645896912 (-0.008024223148822784) + | > avg_loss_disc_real_4: 0.23171518743038177 (-0.008418940007686615) + | > avg_loss_disc_real_5: 0.22863755375146866 (-0.05122946947813034) + | > avg_loss_0: 2.425536632537842 (-0.03894329071044922) + | > avg_loss_gen: 2.5048341751098633 (-0.01022350788116455) + | > avg_loss_kl: 2.4730557203292847 (-0.031502366065979004) + | > avg_loss_feat: 7.422661304473877 (+0.8610196113586426) + | > avg_loss_mel: 18.938340187072754 (+1.7624092102050781) + | > avg_loss_duration: 1.7598922848701477 (-0.003648698329925537) + | > avg_loss_1: 33.09878349304199 (+2.578052520751953) + + + > EPOCH: 449/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:21:14)  + + --> TIME: 2025-04-19 06:21:18 -- STEP: 10/142 -- GLOBAL_STEP: 443050 + | > loss_disc: 2.397416114807129 (2.4086065530776977) + | > loss_disc_real_0: 0.07469429820775986 (0.11628263480961323) + | > loss_disc_real_1: 0.19271881878376007 (0.21474217921495437) + | > loss_disc_real_2: 0.23702609539031982 (0.23230525404214858) + | > loss_disc_real_3: 0.22680675983428955 (0.22772467732429505) + | > loss_disc_real_4: 0.22013624012470245 (0.21985168159008026) + | > loss_disc_real_5: 0.22565381228923798 (0.21979646980762482) + | > loss_0: 2.397416114807129 (2.4086065530776977) + | > grad_norm_0: tensor(7.2974, device='cuda:0') (tensor(9.9136, device='cuda:0')) + | > loss_gen: 2.4120256900787354 (2.4506922006607055) + | > loss_kl: 1.758098840713501 (1.752250647544861) + | > loss_feat: 7.828232765197754 (7.7346090316772464) + | > loss_mel: 19.405384063720703 (18.376675415039063) + | > loss_duration: 1.3854812383651733 (1.4027290344238281) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.789222717285156 (31.716956901550294) + | > grad_norm_1: tensor(129.6004, device='cuda:0') (tensor(163.3762, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.3326 (0.32509918212890626) + | > loader_time: 0.0047 (0.004216241836547852) + + + --> TIME: 2025-04-19 06:21:27 -- STEP: 35/142 -- GLOBAL_STEP: 443075 + | > loss_disc: 2.424132823944092 (2.43483874457223) + | > loss_disc_real_0: 0.11775026470422745 (0.11272663910474096) + | > loss_disc_real_1: 0.1935155987739563 (0.2090127659695489) + | > loss_disc_real_2: 0.19383934140205383 (0.22628392917769297) + | > loss_disc_real_3: 0.2163393646478653 (0.2289892933198384) + | > loss_disc_real_4: 0.21857360005378723 (0.22883031283106123) + | > loss_disc_real_5: 0.2265579253435135 (0.22829270660877227) + | > loss_0: 2.424132823944092 (2.43483874457223) + | > grad_norm_0: tensor(12.3519, device='cuda:0') (tensor(10.4746, device='cuda:0')) + | > loss_gen: 2.3691935539245605 (2.4182989869798934) + | > loss_kl: 1.8179681301116943 (nan) + | > loss_feat: 7.598676681518555 (7.587809031350272) + | > loss_mel: 17.549808502197266 (18.056655120849612) + | > loss_duration: 1.4563543796539307 (nan) + | > amp_scaler: 32.0 (44.8) + | > loss_1: 30.792001724243164 (nan) + | > grad_norm_1: tensor(336.0197, device='cuda:0') (tensor(249.7847, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.3575 (0.33764313289097375) + | > loader_time: 0.0054 (0.0047814369201660155) + + + --> TIME: 2025-04-19 06:21:37 -- STEP: 60/142 -- GLOBAL_STEP: 443100 + | > loss_disc: 2.4500861167907715 (2.4343411207199086) + | > loss_disc_real_0: 0.12148167192935944 (0.11771342984090248) + | > loss_disc_real_1: 0.1979425996541977 (0.20724830304582914) + | > loss_disc_real_2: 0.21841159462928772 (0.2256067919234435) + | > loss_disc_real_3: 0.2314988076686859 (0.22778941467404365) + | > loss_disc_real_4: 0.22224755585193634 (0.22565486282110214) + | > loss_disc_real_5: 0.22077716886997223 (0.23020808895428976) + | > loss_0: 2.4500861167907715 (2.4343411207199086) + | > grad_norm_0: tensor(4.7143, device='cuda:0') (tensor(9.6751, device='cuda:0')) + | > loss_gen: 2.306985855102539 (2.4004113554954527) + | > loss_kl: 1.775334119796753 (nan) + | > loss_feat: 7.010347366333008 (7.500706060727437) + | > loss_mel: 17.32546043395996 (17.995083713531493) + | > loss_duration: 1.451781153678894 (nan) + | > amp_scaler: 32.0 (39.46666666666667) + | > loss_1: 29.86990737915039 (nan) + | > grad_norm_1: tensor(98.3607, device='cuda:0') (tensor(236.0829, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.3754 (0.35256037712097166) + | > loader_time: 0.0063 (0.005168644587198893) + + + --> TIME: 2025-04-19 06:21:47 -- STEP: 85/142 -- GLOBAL_STEP: 443125 + | > loss_disc: 2.471421718597412 (2.445746514376472) + | > loss_disc_real_0: 0.1336067020893097 (0.12037989659344449) + | > loss_disc_real_1: 0.23911017179489136 (0.20762007464380824) + | > loss_disc_real_2: 0.22080440819263458 (0.22663930111071642) + | > loss_disc_real_3: 0.2427302449941635 (0.2284538572325426) + | > loss_disc_real_4: 0.2527158558368683 (0.22617233185207142) + | > loss_disc_real_5: 0.2509910464286804 (0.23184579277739806) + | > loss_0: 2.471421718597412 (2.445746514376472) + | > grad_norm_0: tensor(9.2940, device='cuda:0') (tensor(9.8093, device='cuda:0')) + | > loss_gen: 2.37322998046875 (2.3790861017563767) + | > loss_kl: 1.9813435077667236 (nan) + | > loss_feat: 7.1116943359375 (7.3654977069181555) + | > loss_mel: 18.068687438964844 (17.858675810869997) + | > loss_duration: 1.4732680320739746 (nan) + | > amp_scaler: 32.0 (37.270588235294134) + | > loss_1: 31.008222579956055 (nan) + | > grad_norm_1: tensor(398.7883, device='cuda:0') (tensor(242.2669, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.394 (0.3640742273891672) + | > loader_time: 0.0065 (0.005498630860272577) + + + --> TIME: 2025-04-19 06:21:57 -- STEP: 110/142 -- GLOBAL_STEP: 443150 + | > loss_disc: 2.45621395111084 (2.4482861020348285) + | > loss_disc_real_0: 0.07891702651977539 (0.11958849602802234) + | > loss_disc_real_1: 0.21439264714717865 (0.2069134479219263) + | > loss_disc_real_2: 0.21200652420520782 (0.22727884663776918) + | > loss_disc_real_3: 0.24901901185512543 (0.22877546941692178) + | > loss_disc_real_4: 0.22854967415332794 (0.227902528508143) + | > loss_disc_real_5: 0.21483856439590454 (0.2312250782142986) + | > loss_0: 2.45621395111084 (2.4482861020348285) + | > grad_norm_0: tensor(9.0709, device='cuda:0') (tensor(10.5710, device='cuda:0')) + | > loss_gen: 2.353612184524536 (2.3770446582274007) + | > loss_kl: 1.9885470867156982 (nan) + | > loss_feat: 7.003133773803711 (7.323648821223866) + | > loss_mel: 17.61986541748047 (17.826953437111587) + | > loss_duration: 1.4665071964263916 (nan) + | > amp_scaler: 16.0 (33.309090909090926) + | > loss_1: 30.431665420532227 (nan) + | > grad_norm_1: tensor(252.4991, device='cuda:0') (tensor(257.7744, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.4298 (0.37405608133836216) + | > loader_time: 0.0075 (0.005823538520119407) + + + --> TIME: 2025-04-19 06:22:08 -- STEP: 135/142 -- GLOBAL_STEP: 443175 + | > loss_disc: 2.464923620223999 (2.448020491776642) + | > loss_disc_real_0: 0.16444717347621918 (0.12139643902579943) + | > loss_disc_real_1: 0.2568279802799225 (0.20667064443782523) + | > loss_disc_real_2: 0.26103946566581726 (0.22730436358186934) + | > loss_disc_real_3: 0.23385761678218842 (0.22890117764472961) + | > loss_disc_real_4: 0.22172604501247406 (0.22796118822362688) + | > loss_disc_real_5: 0.22839556634426117 (0.23058490896666492) + | > loss_0: 2.464923620223999 (2.448020491776642) + | > grad_norm_0: tensor(9.1332, device='cuda:0') (tensor(10.3637, device='cuda:0')) + | > loss_gen: 2.496030330657959 (2.374993624510589) + | > loss_kl: 1.79129159450531 (nan) + | > loss_feat: 7.124166965484619 (7.273281849755181) + | > loss_mel: 17.447010040283203 (17.78883111741807) + | > loss_duration: 1.4612255096435547 (nan) + | > amp_scaler: 16.0 (30.10370370370371) + | > loss_1: 30.319725036621094 (nan) + | > grad_norm_1: tensor(215.2834, device='cuda:0') (tensor(253.6944, device='cuda:0')) + | > current_lr_0: 0.0001890835268510219 + | > current_lr_1: 0.0001890835268510219 + | > step_time: 0.4529 (0.38553700623688864) + | > loader_time: 0.0074 (0.006128763269495081) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.45127272605896 (2.45127272605896) + | > loss_disc_real_0: 0.2249559462070465 (0.2249559462070465) + | > loss_disc_real_1: 0.2544575035572052 (0.2544575035572052) + | > loss_disc_real_2: 0.22883912920951843 (0.22883912920951843) + | > loss_disc_real_3: 0.20499415695667267 (0.20499415695667267) + | > loss_disc_real_4: 0.17952512204647064 (0.17952512204647064) + | > loss_disc_real_5: 0.23511537909507751 (0.23511537909507751) + | > loss_0: 2.45127272605896 (2.45127272605896) + | > loss_gen: 2.5759968757629395 (2.5759968757629395) + | > loss_kl: 2.310157299041748 (2.310157299041748) + | > loss_feat: 7.046628952026367 (7.046628952026367) + | > loss_mel: 20.767370223999023 (20.767370223999023) + | > loss_duration: 1.7655854225158691 (1.7655854225158691) + | > loss_1: 34.46574020385742 (34.46574020385742) + + --> STEP: 1 + | > loss_disc: 2.628896474838257 (2.628896474838257) + | > loss_disc_real_0: 0.2187187224626541 (0.2187187224626541) + | > loss_disc_real_1: 0.2576850652694702 (0.2576850652694702) + | > loss_disc_real_2: 0.24514463543891907 (0.24514463543891907) + | > loss_disc_real_3: 0.2148655503988266 (0.2148655503988266) + | > loss_disc_real_4: 0.25181668996810913 (0.25181668996810913) + | > loss_disc_real_5: 0.23341383039951324 (0.23341383039951324) + | > loss_0: 2.628896474838257 (2.628896474838257) + | > loss_gen: 2.3437910079956055 (2.3437910079956055) + | > loss_kl: 2.781264543533325 (2.781264543533325) + | > loss_feat: 5.901984691619873 (5.901984691619873) + | > loss_mel: 15.107046127319336 (15.107046127319336) + | > loss_duration: 1.8100782632827759 (1.8100782632827759) + | > loss_1: 27.944164276123047 (27.944164276123047) + + --> STEP: 2 + | > loss_disc: 2.479149580001831 (2.554023027420044) + | > loss_disc_real_0: 0.20817184448242188 (0.213445283472538) + | > loss_disc_real_1: 0.20662735402584076 (0.2321562096476555) + | > loss_disc_real_2: 0.23752452433109283 (0.24133457988500595) + | > loss_disc_real_3: 0.18156041204929352 (0.19821298122406006) + | > loss_disc_real_4: 0.2546080946922302 (0.2532123923301697) + | > loss_disc_real_5: 0.23099087178707123 (0.23220235109329224) + | > loss_0: 2.479149580001831 (2.554023027420044) + | > loss_gen: 2.475764751434326 (2.409777879714966) + | > loss_kl: 2.5088610649108887 (2.645062804222107) + | > loss_feat: 7.317759990692139 (6.609872341156006) + | > loss_mel: 17.075382232666016 (16.091214179992676) + | > loss_duration: 1.7684392929077148 (1.7892587780952454) + | > loss_1: 31.146209716796875 (29.54518699645996) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003957509994506836 (+0.0004121065139770508) + | > avg_loss_disc: 2.554023027420044 (+0.12848639488220215) + | > avg_loss_disc_real_0: 0.213445283472538 (+0.0472513884305954) + | > avg_loss_disc_real_1: 0.2321562096476555 (-0.021142207086086273) + | > avg_loss_disc_real_2: 0.24133457988500595 (-0.022041819989681244) + | > avg_loss_disc_real_3: 0.19821298122406006 (-0.031119555234909058) + | > avg_loss_disc_real_4: 0.2532123923301697 (+0.021497204899787903) + | > avg_loss_disc_real_5: 0.23220235109329224 (+0.003564797341823578) + | > avg_loss_0: 2.554023027420044 (+0.12848639488220215) + | > avg_loss_gen: 2.409777879714966 (-0.09505629539489746) + | > avg_loss_kl: 2.645062804222107 (+0.17200708389282227) + | > avg_loss_feat: 6.609872341156006 (-0.8127889633178711) + | > avg_loss_mel: 16.091214179992676 (-2.847126007080078) + | > avg_loss_duration: 1.7892587780952454 (+0.029366493225097656) + | > avg_loss_1: 29.54518699645996 (-3.5535964965820312) + + + > EPOCH: 450/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:22:13)  + + --> TIME: 2025-04-19 06:22:20 -- STEP: 18/142 -- GLOBAL_STEP: 443200 + | > loss_disc: 2.378520965576172 (2.4253022803200617) + | > loss_disc_real_0: 0.10846175998449326 (0.10987465787265036) + | > loss_disc_real_1: 0.2291102260351181 (0.20532130698362985) + | > loss_disc_real_2: 0.22108078002929688 (0.22509170985884136) + | > loss_disc_real_3: 0.2271515429019928 (0.22331689298152924) + | > loss_disc_real_4: 0.2097412645816803 (0.2274745363328192) + | > loss_disc_real_5: 0.23230718076229095 (0.22459388193156984) + | > loss_0: 2.378520965576172 (2.4253022803200617) + | > grad_norm_0: tensor(6.2388, device='cuda:0') (tensor(6.5994, device='cuda:0')) + | > loss_gen: 2.4105045795440674 (2.3793187936147056) + | > loss_kl: 1.8570470809936523 (1.8006739616394043) + | > loss_feat: 7.577712535858154 (7.59794118669298) + | > loss_mel: 18.102581024169922 (17.8715697394477) + | > loss_duration: 1.40293550491333 (1.4071123467551336) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.350780487060547 (31.0566160413954) + | > grad_norm_1: tensor(79.2973, device='cuda:0') (tensor(147.5826, device='cuda:0')) + | > current_lr_0: 0.0001890598914101655 + | > current_lr_1: 0.0001890598914101655 + | > step_time: 0.3362 (0.3289279407925076) + | > loader_time: 0.005 (0.004722926351759169) + + + --> TIME: 2025-04-19 06:22:29 -- STEP: 43/142 -- GLOBAL_STEP: 443225 + | > loss_disc: 2.459439992904663 (2.4192787270213283) + | > loss_disc_real_0: 0.10834246128797531 (0.11396783122489619) + | > loss_disc_real_1: 0.23270268738269806 (0.19993988441866498) + | > loss_disc_real_2: 0.19889773428440094 (0.22205006175262984) + | > loss_disc_real_3: 0.24615658819675446 (0.2298648329668267) + | > loss_disc_real_4: 0.22697468101978302 (0.22868234334990037) + | > loss_disc_real_5: 0.27998578548431396 (0.22835612782212192) + | > loss_0: 2.459439992904663 (2.4192787270213283) + | > grad_norm_0: tensor(4.1876, device='cuda:0') (tensor(6.2849, device='cuda:0')) + | > loss_gen: 2.396230697631836 (2.408942904583243) + | > loss_kl: 1.7024362087249756 (1.7957756602486898) + | > loss_feat: 6.413547992706299 (7.563429610673771) + | > loss_mel: 16.33479118347168 (17.850702019624933) + | > loss_duration: 1.4230663776397705 (1.418150818625162) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.27007293701172 (31.037001010983488) + | > grad_norm_1: tensor(55.7202, device='cuda:0') (tensor(141.2352, device='cuda:0')) + | > current_lr_0: 0.0001890598914101655 + | > current_lr_1: 0.0001890598914101655 + | > step_time: 0.3712 (0.34625960505285935) + | > loader_time: 0.0052 (0.0051277016484460175) + + + --> TIME: 2025-04-19 06:22:39 -- STEP: 68/142 -- GLOBAL_STEP: 443250 + | > loss_disc: 2.5535879135131836 (2.4361156014835132) + | > loss_disc_real_0: 0.13853216171264648 (0.1189041099346736) + | > loss_disc_real_1: 0.24335463345050812 (0.203579240862061) + | > loss_disc_real_2: 0.23780938982963562 (0.22426800451734485) + | > loss_disc_real_3: 0.23210133612155914 (0.22854170093641563) + | > loss_disc_real_4: 0.24626976251602173 (0.2296869259546785) + | > loss_disc_real_5: 0.1975787878036499 (0.2289101395975141) + | > loss_0: 2.5535879135131836 (2.4361156014835132) + | > grad_norm_0: tensor(6.0994, device='cuda:0') (tensor(7.6675, device='cuda:0')) + | > loss_gen: 2.361130475997925 (2.3994767210062813) + | > loss_kl: 1.8758772611618042 (1.8181510374826544) + | > loss_feat: 6.884335517883301 (7.388322900323307) + | > loss_mel: 17.174224853515625 (17.730610763325416) + | > loss_duration: 1.4284741878509521 (1.4275965024443233) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.724040985107422 (30.764157884261188) + | > grad_norm_1: tensor(83.5439, device='cuda:0') (tensor(180.3900, device='cuda:0')) + | > current_lr_0: 0.0001890598914101655 + | > current_lr_1: 0.0001890598914101655 + | > step_time: 0.3995 (0.3600302303538603) + | > loader_time: 0.006 (0.005403073394999786) + + + --> TIME: 2025-04-19 06:22:49 -- STEP: 93/142 -- GLOBAL_STEP: 443275 + | > loss_disc: 2.6085119247436523 (2.4514738975032686) + | > loss_disc_real_0: 0.11362898349761963 (0.1217169003781452) + | > loss_disc_real_1: 0.2135886549949646 (0.20397720990642423) + | > loss_disc_real_2: 0.2510368227958679 (0.22542194509378044) + | > loss_disc_real_3: 0.24145129323005676 (0.22952886518611704) + | > loss_disc_real_4: 0.24048326909542084 (0.23065232237180075) + | > loss_disc_real_5: 0.2538687288761139 (0.23246411178060758) + | > loss_0: 2.6085119247436523 (2.4514738975032686) + | > grad_norm_0: tensor(9.7892, device='cuda:0') (tensor(9.0918, device='cuda:0')) + | > loss_gen: 2.1547703742980957 (2.384165325472433) + | > loss_kl: 1.7882225513458252 (1.8228626776767034) + | > loss_feat: 5.983505725860596 (7.2798657160933296) + | > loss_mel: 16.946575164794922 (17.69640492880217) + | > loss_duration: 1.464463233947754 (1.4400371595095562) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.337535858154297 (30.623335705008557) + | > grad_norm_1: tensor(270.1495, device='cuda:0') (tensor(210.2838, device='cuda:0')) + | > current_lr_0: 0.0001890598914101655 + | > current_lr_1: 0.0001890598914101655 + | > step_time: 0.4094 (0.37281237622743) + | > loader_time: 0.0077 (0.005703285176266909) + + + --> TIME: 2025-04-19 06:23:00 -- STEP: 118/142 -- GLOBAL_STEP: 443300 + | > loss_disc: 2.4495797157287598 (2.4576687954239924) + | > loss_disc_real_0: 0.10108935832977295 (0.1242131363537352) + | > loss_disc_real_1: 0.19778357446193695 (0.20516482387053764) + | > loss_disc_real_2: 0.26208215951919556 (0.2255851508954824) + | > loss_disc_real_3: 0.23865264654159546 (0.22921110575987122) + | > loss_disc_real_4: 0.24126973748207092 (0.22973520523410731) + | > loss_disc_real_5: 0.2629103362560272 (0.23113684267815898) + | > loss_0: 2.4495797157287598 (2.4576687954239924) + | > grad_norm_0: tensor(5.4024, device='cuda:0') (tensor(9.4714, device='cuda:0')) + | > loss_gen: 2.3759706020355225 (2.3674741559109456) + | > loss_kl: 1.9581375122070312 (1.8388998548863298) + | > loss_feat: 7.652955055236816 (7.195588346255028) + | > loss_mel: 18.628278732299805 (17.64181275286918) + | > loss_duration: 1.4565621614456177 (1.4480209572840546) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.07190704345703 (30.49179604093907) + | > grad_norm_1: tensor(130.1894, device='cuda:0') (tensor(211.0173, device='cuda:0')) + | > current_lr_0: 0.0001890598914101655 + | > current_lr_1: 0.0001890598914101655 + | > step_time: 0.4492 (0.38499340162438855) + | > loader_time: 0.0075 (0.005991024486089159) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.3718719482421875 (2.3718719482421875) + | > loss_disc_real_0: 0.18738782405853271 (0.18738782405853271) + | > loss_disc_real_1: 0.23079167306423187 (0.23079167306423187) + | > loss_disc_real_2: 0.25784558057785034 (0.25784558057785034) + | > loss_disc_real_3: 0.26750320196151733 (0.26750320196151733) + | > loss_disc_real_4: 0.2130744755268097 (0.2130744755268097) + | > loss_disc_real_5: 0.24404990673065186 (0.24404990673065186) + | > loss_0: 2.3718719482421875 (2.3718719482421875) + | > loss_gen: 2.7677860260009766 (2.7677860260009766) + | > loss_kl: 2.543085813522339 (2.543085813522339) + | > loss_feat: 7.652047157287598 (7.652047157287598) + | > loss_mel: 19.027667999267578 (19.027667999267578) + | > loss_duration: 1.7701201438903809 (1.7701201438903809) + | > loss_1: 33.76070785522461 (33.76070785522461) + + --> STEP: 1 + | > loss_disc: 2.525800943374634 (2.525800943374634) + | > loss_disc_real_0: 0.2133161574602127 (0.2133161574602127) + | > loss_disc_real_1: 0.22220966219902039 (0.22220966219902039) + | > loss_disc_real_2: 0.26345157623291016 (0.26345157623291016) + | > loss_disc_real_3: 0.22476732730865479 (0.22476732730865479) + | > loss_disc_real_4: 0.24253030121326447 (0.24253030121326447) + | > loss_disc_real_5: 0.22854846715927124 (0.22854846715927124) + | > loss_0: 2.525800943374634 (2.525800943374634) + | > loss_gen: 2.43581485748291 (2.43581485748291) + | > loss_kl: 2.8145272731781006 (2.8145272731781006) + | > loss_feat: 6.722233772277832 (6.722233772277832) + | > loss_mel: 16.693941116333008 (16.693941116333008) + | > loss_duration: 1.8067057132720947 (1.8067057132720947) + | > loss_1: 30.473220825195312 (30.473220825195312) + + --> STEP: 2 + | > loss_disc: 2.519436836242676 (2.522618889808655) + | > loss_disc_real_0: 0.22915346920490265 (0.22123481333255768) + | > loss_disc_real_1: 0.20677286386489868 (0.21449126303195953) + | > loss_disc_real_2: 0.24861104786396027 (0.2560313120484352) + | > loss_disc_real_3: 0.21342869102954865 (0.21909800916910172) + | > loss_disc_real_4: 0.20256966352462769 (0.22254998236894608) + | > loss_disc_real_5: 0.21488751471042633 (0.22171799093484879) + | > loss_0: 2.519436836242676 (2.522618889808655) + | > loss_gen: 2.3841004371643066 (2.4099576473236084) + | > loss_kl: 2.317228317260742 (2.5658777952194214) + | > loss_feat: 7.3885369300842285 (7.05538535118103) + | > loss_mel: 16.011247634887695 (16.35259437561035) + | > loss_duration: 1.7438322305679321 (1.7752689719200134) + | > loss_1: 29.84494400024414 (30.159082412719727) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033893585205078125 (-0.0005681514739990234) + | > avg_loss_disc: 2.522618889808655 (-0.03140413761138916) + | > avg_loss_disc_real_0: 0.22123481333255768 (+0.007789529860019684) + | > avg_loss_disc_real_1: 0.21449126303195953 (-0.017664946615695953) + | > avg_loss_disc_real_2: 0.2560313120484352 (+0.01469673216342926) + | > avg_loss_disc_real_3: 0.21909800916910172 (+0.020885027945041656) + | > avg_loss_disc_real_4: 0.22254998236894608 (-0.030662409961223602) + | > avg_loss_disc_real_5: 0.22171799093484879 (-0.010484360158443451) + | > avg_loss_0: 2.522618889808655 (-0.03140413761138916) + | > avg_loss_gen: 2.4099576473236084 (+0.00017976760864257812) + | > avg_loss_kl: 2.5658777952194214 (-0.07918500900268555) + | > avg_loss_feat: 7.05538535118103 (+0.4455130100250244) + | > avg_loss_mel: 16.35259437561035 (+0.2613801956176758) + | > avg_loss_duration: 1.7752689719200134 (-0.013989806175231934) + | > avg_loss_1: 30.159082412719727 (+0.6138954162597656) + + + > EPOCH: 451/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:23:13)  + + --> TIME: 2025-04-19 06:23:14 -- STEP: 1/142 -- GLOBAL_STEP: 443325 + | > loss_disc: 2.4490647315979004 (2.4490647315979004) + | > loss_disc_real_0: 0.15686360001564026 (0.15686360001564026) + | > loss_disc_real_1: 0.21096979081630707 (0.21096979081630707) + | > loss_disc_real_2: 0.22134914994239807 (0.22134914994239807) + | > loss_disc_real_3: 0.2179865688085556 (0.2179865688085556) + | > loss_disc_real_4: 0.2268526405096054 (0.2268526405096054) + | > loss_disc_real_5: 0.2313344031572342 (0.2313344031572342) + | > loss_0: 2.4490647315979004 (2.4490647315979004) + | > grad_norm_0: tensor(6.9143, device='cuda:0') (tensor(6.9143, device='cuda:0')) + | > loss_gen: 2.1565136909484863 (2.1565136909484863) + | > loss_kl: 1.854958415031433 (1.854958415031433) + | > loss_feat: 5.6602935791015625 (5.6602935791015625) + | > loss_mel: 14.918436050415039 (14.918436050415039) + | > loss_duration: 1.3946573734283447 (1.3946573734283447) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 25.984859466552734 (25.984859466552734) + | > grad_norm_1: tensor(223.1436, device='cuda:0') (tensor(223.1436, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.3245 (0.32448577880859375) + | > loader_time: 0.0039 (0.003946542739868164) + + + --> TIME: 2025-04-19 06:23:22 -- STEP: 26/142 -- GLOBAL_STEP: 443350 + | > loss_disc: 2.4385650157928467 (2.3884135576394887) + | > loss_disc_real_0: 0.09274879842996597 (0.11004329587404545) + | > loss_disc_real_1: 0.2346709668636322 (0.19935374477734932) + | > loss_disc_real_2: 0.25311562418937683 (0.21801472111390188) + | > loss_disc_real_3: 0.22652851045131683 (0.22626300843862387) + | > loss_disc_real_4: 0.24536295235157013 (0.22322829468892172) + | > loss_disc_real_5: 0.26223024725914 (0.23619379905553964) + | > loss_0: 2.4385650157928467 (2.3884135576394887) + | > grad_norm_0: tensor(12.9747, device='cuda:0') (tensor(9.2580, device='cuda:0')) + | > loss_gen: 2.4380459785461426 (2.4323723866389346) + | > loss_kl: 1.8387285470962524 (1.8295416511022127) + | > loss_feat: 7.425448417663574 (7.522835713166457) + | > loss_mel: 18.480369567871094 (17.609505873460034) + | > loss_duration: 1.4290785789489746 (1.4151858412302458) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.611671447753906 (30.80944141974816) + | > grad_norm_1: tensor(294.8248, device='cuda:0') (tensor(231.6428, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.3497 (0.33286574253669154) + | > loader_time: 0.0049 (0.004710105749276969) + + + --> TIME: 2025-04-19 06:23:31 -- STEP: 51/142 -- GLOBAL_STEP: 443375 + | > loss_disc: 2.3761403560638428 (2.4100586853775323) + | > loss_disc_real_0: 0.11883469671010971 (0.11280283353784505) + | > loss_disc_real_1: 0.20128485560417175 (0.20081797154510722) + | > loss_disc_real_2: 0.21839675307273865 (0.22486386433535932) + | > loss_disc_real_3: 0.21331529319286346 (0.22485876492425508) + | > loss_disc_real_4: 0.24218688905239105 (0.2287366489569346) + | > loss_disc_real_5: 0.21635691821575165 (0.23059105639364205) + | > loss_0: 2.3761403560638428 (2.4100586853775323) + | > grad_norm_0: tensor(9.8847, device='cuda:0') (tensor(9.9890, device='cuda:0')) + | > loss_gen: 2.4150853157043457 (2.41343043832218) + | > loss_kl: 1.8385640382766724 (1.827816177816952) + | > loss_feat: 7.416642665863037 (7.429529685600131) + | > loss_mel: 16.530471801757812 (17.62745666503906) + | > loss_duration: 1.4150491952896118 (1.422091575229869) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.615812301635742 (30.72032460979387) + | > grad_norm_1: tensor(455.6230, device='cuda:0') (tensor(271.2454, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.3668 (0.3452967054703656) + | > loader_time: 0.0059 (0.005079404980528588) + + + --> TIME: 2025-04-19 06:23:41 -- STEP: 76/142 -- GLOBAL_STEP: 443400 + | > loss_disc: 2.5694313049316406 (2.426932949768869) + | > loss_disc_real_0: 0.13962553441524506 (0.11608853705815579) + | > loss_disc_real_1: 0.2091434895992279 (0.20161085418964686) + | > loss_disc_real_2: 0.24607037007808685 (0.2255227777518724) + | > loss_disc_real_3: 0.19740286469459534 (0.22668761348253802) + | > loss_disc_real_4: 0.22433143854141235 (0.22869673606596494) + | > loss_disc_real_5: 0.21330608427524567 (0.23084804905872597) + | > loss_0: 2.5694313049316406 (2.426932949768869) + | > grad_norm_0: tensor(3.5388, device='cuda:0') (tensor(9.5420, device='cuda:0')) + | > loss_gen: 2.1072678565979004 (2.396231510137256) + | > loss_kl: 1.9558392763137817 (1.8426474646518105) + | > loss_feat: 6.079381465911865 (7.3593262684972665) + | > loss_mel: 17.335268020629883 (17.6775672310277) + | > loss_duration: 1.4687199592590332 (1.4325990441598386) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.946475982666016 (30.70837156396163) + | > grad_norm_1: tensor(44.4457, device='cuda:0') (tensor(250.1351, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.3929 (0.35950864302484614) + | > loader_time: 0.0064 (0.005408826627229388) + + + --> TIME: 2025-04-19 06:23:52 -- STEP: 101/142 -- GLOBAL_STEP: 443425 + | > loss_disc: 2.3168883323669434 (2.433429328521879) + | > loss_disc_real_0: 0.1454477608203888 (0.1187968785588694) + | > loss_disc_real_1: 0.19745947420597076 (0.2026637982909042) + | > loss_disc_real_2: 0.24793235957622528 (0.22474719067611318) + | > loss_disc_real_3: 0.2642279267311096 (0.22766626309050192) + | > loss_disc_real_4: 0.25332751870155334 (0.22855972461771257) + | > loss_disc_real_5: 0.23755307495594025 (0.23014622793929412) + | > loss_0: 2.3168883323669434 (2.433429328521879) + | > grad_norm_0: tensor(14.8165, device='cuda:0') (tensor(10.4354, device='cuda:0')) + | > loss_gen: 2.611579418182373 (2.3997628334725243) + | > loss_kl: 1.9422669410705566 (1.8520516900732966) + | > loss_feat: 6.99284553527832 (7.339298422973935) + | > loss_mel: 17.90769386291504 (17.75128411774588) + | > loss_duration: 1.485142469406128 (1.4434977852471984) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.93952751159668 (30.785894847152257) + | > grad_norm_1: tensor(382.2261, device='cuda:0') (tensor(263.2544, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.416 (0.3699648002586741) + | > loader_time: 0.0065 (0.005732814864356918) + + + --> TIME: 2025-04-19 06:24:02 -- STEP: 126/142 -- GLOBAL_STEP: 443450 + | > loss_disc: 2.4678750038146973 (2.4358304303789886) + | > loss_disc_real_0: 0.1559859961271286 (0.12071837274919427) + | > loss_disc_real_1: 0.19300252199172974 (0.2029585981416324) + | > loss_disc_real_2: 0.21574163436889648 (0.22562918882994426) + | > loss_disc_real_3: 0.23881861567497253 (0.22736059968906736) + | > loss_disc_real_4: 0.21629177033901215 (0.22822808095860103) + | > loss_disc_real_5: 0.21742282807826996 (0.2286757631670861) + | > loss_0: 2.4678750038146973 (2.4358304303789886) + | > grad_norm_0: tensor(14.9621, device='cuda:0') (tensor(11.8845, device='cuda:0')) + | > loss_gen: 2.572333812713623 (2.396315695747495) + | > loss_kl: 2.0099732875823975 (1.857740719167013) + | > loss_feat: 7.383148670196533 (7.264655438680498) + | > loss_mel: 17.839977264404297 (17.649224500807506) + | > loss_duration: 1.4763381481170654 (1.4507945624608842) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.281770706176758 (30.618730938623823) + | > grad_norm_1: tensor(81.1467, device='cuda:0') (tensor(272.3771, device='cuda:0')) + | > current_lr_0: 0.00018903625892373923 + | > current_lr_1: 0.00018903625892373923 + | > step_time: 0.4395 (0.38141413529713936) + | > loader_time: 0.0078 (0.006042643198891291) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.446397304534912 (2.446397304534912) + | > loss_disc_real_0: 0.20684197545051575 (0.20684197545051575) + | > loss_disc_real_1: 0.2081286758184433 (0.2081286758184433) + | > loss_disc_real_2: 0.2446344792842865 (0.2446344792842865) + | > loss_disc_real_3: 0.24329409003257751 (0.24329409003257751) + | > loss_disc_real_4: 0.2605411112308502 (0.2605411112308502) + | > loss_disc_real_5: 0.22078877687454224 (0.22078877687454224) + | > loss_0: 2.446397304534912 (2.446397304534912) + | > loss_gen: 2.6318774223327637 (2.6318774223327637) + | > loss_kl: 2.628976583480835 (2.628976583480835) + | > loss_feat: 7.116163730621338 (7.116163730621338) + | > loss_mel: 18.343753814697266 (18.343753814697266) + | > loss_duration: 1.7481904029846191 (1.7481904029846191) + | > loss_1: 32.468963623046875 (32.468963623046875) + + --> STEP: 1 + | > loss_disc: 2.5124120712280273 (2.5124120712280273) + | > loss_disc_real_0: 0.2035883516073227 (0.2035883516073227) + | > loss_disc_real_1: 0.22509652376174927 (0.22509652376174927) + | > loss_disc_real_2: 0.2225000411272049 (0.2225000411272049) + | > loss_disc_real_3: 0.20947428047657013 (0.20947428047657013) + | > loss_disc_real_4: 0.20853997766971588 (0.20853997766971588) + | > loss_disc_real_5: 0.1820206195116043 (0.1820206195116043) + | > loss_0: 2.5124120712280273 (2.5124120712280273) + | > loss_gen: 2.3737781047821045 (2.3737781047821045) + | > loss_kl: 2.8184664249420166 (2.8184664249420166) + | > loss_feat: 6.716570854187012 (6.716570854187012) + | > loss_mel: 17.951953887939453 (17.951953887939453) + | > loss_duration: 1.7839429378509521 (1.7839429378509521) + | > loss_1: 31.644712448120117 (31.644712448120117) + + --> STEP: 2 + | > loss_disc: 2.574979305267334 (2.5436956882476807) + | > loss_disc_real_0: 0.2544386088848114 (0.22901348024606705) + | > loss_disc_real_1: 0.201510488986969 (0.21330350637435913) + | > loss_disc_real_2: 0.2105289101600647 (0.2165144756436348) + | > loss_disc_real_3: 0.24022570252418518 (0.22484999150037766) + | > loss_disc_real_4: 0.2412320226430893 (0.2248860001564026) + | > loss_disc_real_5: 0.2780512273311615 (0.2300359234213829) + | > loss_0: 2.574979305267334 (2.5436956882476807) + | > loss_gen: 2.7441885471343994 (2.558983325958252) + | > loss_kl: 2.587465524673462 (2.7029659748077393) + | > loss_feat: 8.493884086608887 (7.605227470397949) + | > loss_mel: 19.154272079467773 (18.553112983703613) + | > loss_duration: 1.7397644519805908 (1.7618536949157715) + | > loss_1: 34.71957778930664 (33.18214511871338) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00363922119140625 (+0.0002498626708984375) + | > avg_loss_disc: 2.5436956882476807 (+0.02107679843902588) + | > avg_loss_disc_real_0: 0.22901348024606705 (+0.007778666913509369) + | > avg_loss_disc_real_1: 0.21330350637435913 (-0.0011877566576004028) + | > avg_loss_disc_real_2: 0.2165144756436348 (-0.039516836404800415) + | > avg_loss_disc_real_3: 0.22484999150037766 (+0.00575198233127594) + | > avg_loss_disc_real_4: 0.2248860001564026 (+0.0023360177874565125) + | > avg_loss_disc_real_5: 0.2300359234213829 (+0.008317932486534119) + | > avg_loss_0: 2.5436956882476807 (+0.02107679843902588) + | > avg_loss_gen: 2.558983325958252 (+0.14902567863464355) + | > avg_loss_kl: 2.7029659748077393 (+0.13708817958831787) + | > avg_loss_feat: 7.605227470397949 (+0.549842119216919) + | > avg_loss_mel: 18.553112983703613 (+2.2005186080932617) + | > avg_loss_duration: 1.7618536949157715 (-0.013415277004241943) + | > avg_loss_1: 33.18214511871338 (+3.0230627059936523) + + + > EPOCH: 452/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:24:11)  + + --> TIME: 2025-04-19 06:24:15 -- STEP: 9/142 -- GLOBAL_STEP: 443475 + | > loss_disc: 2.456162452697754 (2.409368779924181) + | > loss_disc_real_0: 0.12014169991016388 (0.11608364681402843) + | > loss_disc_real_1: 0.2182082235813141 (0.2020741601785024) + | > loss_disc_real_2: 0.2728625535964966 (0.22827749285433027) + | > loss_disc_real_3: 0.24232569336891174 (0.2256939642959171) + | > loss_disc_real_4: 0.27477341890335083 (0.22576378451453316) + | > loss_disc_real_5: 0.2647525370121002 (0.2298026465707355) + | > loss_0: 2.456162452697754 (2.409368779924181) + | > grad_norm_0: tensor(10.5341, device='cuda:0') (tensor(9.9195, device='cuda:0')) + | > loss_gen: 2.4768497943878174 (2.3963319990370007) + | > loss_kl: 1.7321996688842773 (1.8057536681493123) + | > loss_feat: 6.448557376861572 (7.343325826856825) + | > loss_mel: 16.649991989135742 (17.375930786132812) + | > loss_duration: 1.4020516872406006 (1.4042950603697035) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.709651947021484 (30.325637605455185) + | > grad_norm_1: tensor(178.0489, device='cuda:0') (tensor(292.0092, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.3242 (0.3239287535349528) + | > loader_time: 0.0047 (0.004257573021782769) + + + --> TIME: 2025-04-19 06:24:23 -- STEP: 34/142 -- GLOBAL_STEP: 443500 + | > loss_disc: 2.5428311824798584 (2.404178373953876) + | > loss_disc_real_0: 0.09515922516584396 (0.11076672024586622) + | > loss_disc_real_1: 0.2139551341533661 (0.20177621964146109) + | > loss_disc_real_2: 0.22179381549358368 (0.22353427857160568) + | > loss_disc_real_3: 0.20529848337173462 (0.22490527962937074) + | > loss_disc_real_4: 0.22397884726524353 (0.2271629405372283) + | > loss_disc_real_5: 0.2423275262117386 (0.23059176390661912) + | > loss_0: 2.5428311824798584 (2.404178373953876) + | > grad_norm_0: tensor(12.5392, device='cuda:0') (tensor(8.1150, device='cuda:0')) + | > loss_gen: 2.208690881729126 (2.406387244953829) + | > loss_kl: 1.800513744354248 (1.8172860671492184) + | > loss_feat: 6.731457710266113 (7.401115010766422) + | > loss_mel: 16.024885177612305 (17.58754006554098) + | > loss_duration: 1.4388556480407715 (1.4145463985555313) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.204402923583984 (30.626875092001523) + | > grad_norm_1: tensor(337.7981, device='cuda:0') (tensor(242.0426, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.3554 (0.337043516776141) + | > loader_time: 0.0055 (0.004980767474454992) + + + --> TIME: 2025-04-19 06:24:33 -- STEP: 59/142 -- GLOBAL_STEP: 443525 + | > loss_disc: 2.5377042293548584 (2.420539528636609) + | > loss_disc_real_0: 0.159173384308815 (0.1162821870234053) + | > loss_disc_real_1: 0.2289511114358902 (0.20505536259230922) + | > loss_disc_real_2: 0.22712481021881104 (0.22450816201961646) + | > loss_disc_real_3: 0.2615889310836792 (0.22472070208040335) + | > loss_disc_real_4: 0.23588864505290985 (0.22582196810487973) + | > loss_disc_real_5: 0.24078340828418732 (0.23267265750189958) + | > loss_0: 2.5377042293548584 (2.420539528636609) + | > grad_norm_0: tensor(8.5065, device='cuda:0') (tensor(8.7962, device='cuda:0')) + | > loss_gen: 2.21146297454834 (2.3992348000154653) + | > loss_kl: 1.7048076391220093 (1.830074150683516) + | > loss_feat: 6.700395107269287 (7.329392594806219) + | > loss_mel: 15.886171340942383 (17.62513332043664) + | > loss_duration: 1.4597675800323486 (1.4275067337488725) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 27.96260643005371 (30.61134189670369) + | > grad_norm_1: tensor(337.5470, device='cuda:0') (tensor(234.0424, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.378 (0.3527012154207391) + | > loader_time: 0.0058 (0.005263740733518439) + + + --> TIME: 2025-04-19 06:24:43 -- STEP: 84/142 -- GLOBAL_STEP: 443550 + | > loss_disc: 2.4074363708496094 (2.429486995651609) + | > loss_disc_real_0: 0.10226929187774658 (0.11796252260960284) + | > loss_disc_real_1: 0.2091463953256607 (0.20490737702874912) + | > loss_disc_real_2: 0.23026317358016968 (0.22538374754644575) + | > loss_disc_real_3: 0.23386512696743011 (0.22530670960744223) + | > loss_disc_real_4: 0.2210400402545929 (0.22699458098837308) + | > loss_disc_real_5: 0.2374088317155838 (0.23252756379189946) + | > loss_0: 2.4074363708496094 (2.429486995651609) + | > grad_norm_0: tensor(4.9136, device='cuda:0') (tensor(8.4452, device='cuda:0')) + | > loss_gen: 2.4160993099212646 (2.3842281528881606) + | > loss_kl: 1.958485722541809 (1.8470052920636677) + | > loss_feat: 7.89521598815918 (7.258105590229943) + | > loss_mel: 18.809736251831055 (17.52827025595166) + | > loss_duration: 1.4934263229370117 (1.4398526889937266) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.57296371459961 (30.457462129138765) + | > grad_norm_1: tensor(80.5597, device='cuda:0') (tensor(221.6676, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.3959 (0.3640177590506417) + | > loader_time: 0.0059 (0.005536661261603947) + + + --> TIME: 2025-04-19 06:24:53 -- STEP: 109/142 -- GLOBAL_STEP: 443575 + | > loss_disc: 2.471554756164551 (2.444355813735122) + | > loss_disc_real_0: 0.1318269819021225 (0.12200067780160029) + | > loss_disc_real_1: 0.18863694369792938 (0.20584529323862233) + | > loss_disc_real_2: 0.25070858001708984 (0.2255399855998678) + | > loss_disc_real_3: 0.2565292418003082 (0.2260448574473005) + | > loss_disc_real_4: 0.2343263030052185 (0.2273971282560891) + | > loss_disc_real_5: 0.2449304759502411 (0.2316153028962809) + | > loss_0: 2.471554756164551 (2.444355813735122) + | > grad_norm_0: tensor(6.6314, device='cuda:0') (tensor(8.7644, device='cuda:0')) + | > loss_gen: 2.258519411087036 (2.366111451332721) + | > loss_kl: 1.9057883024215698 (1.8540380624456143) + | > loss_feat: 6.224283218383789 (7.178256971026779) + | > loss_mel: 16.398515701293945 (17.4855483606321) + | > loss_duration: 1.4988083839416504 (1.447552288344147) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.28591537475586 (30.33150734157737) + | > grad_norm_1: tensor(80.5241, device='cuda:0') (tensor(215.7043, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.4121 (0.37430829739351884) + | > loader_time: 0.0068 (0.005794157675646861) + + + --> TIME: 2025-04-19 06:25:05 -- STEP: 134/142 -- GLOBAL_STEP: 443600 + | > loss_disc: 2.333894968032837 (2.44559521639525) + | > loss_disc_real_0: 0.14166979491710663 (0.12177617280785717) + | > loss_disc_real_1: 0.17915722727775574 (0.20555674785108707) + | > loss_disc_real_2: 0.22586585581302643 (0.2256407064955626) + | > loss_disc_real_3: 0.2666224241256714 (0.2267181980076121) + | > loss_disc_real_4: 0.2053186446428299 (0.22792651971329503) + | > loss_disc_real_5: 0.24177496135234833 (0.23265990892897792) + | > loss_0: 2.333894968032837 (2.44559521639525) + | > grad_norm_0: tensor(15.0319, device='cuda:0') (tensor(9.1179, device='cuda:0')) + | > loss_gen: 2.472195625305176 (2.369544545216346) + | > loss_kl: 1.8909331560134888 (1.8575163732713729) + | > loss_feat: 7.808597564697266 (7.184959553960544) + | > loss_mel: 18.416715621948242 (17.506822529123795) + | > loss_duration: 1.4983100891113281 (1.454722913343515) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.08675003051758 (30.373566100846475) + | > grad_norm_1: tensor(399.9279, device='cuda:0') (tensor(226.5197, device='cuda:0')) + | > current_lr_0: 0.00018901262939137375 + | > current_lr_1: 0.00018901262939137375 + | > step_time: 0.4407 (0.3859081713121329) + | > loader_time: 0.0077 (0.006067010893750547) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4363009929656982 (2.4363009929656982) + | > loss_disc_real_0: 0.14205753803253174 (0.14205753803253174) + | > loss_disc_real_1: 0.24599754810333252 (0.24599754810333252) + | > loss_disc_real_2: 0.2578909695148468 (0.2578909695148468) + | > loss_disc_real_3: 0.27820712327957153 (0.27820712327957153) + | > loss_disc_real_4: 0.2334931194782257 (0.2334931194782257) + | > loss_disc_real_5: 0.2644841969013214 (0.2644841969013214) + | > loss_0: 2.4363009929656982 (2.4363009929656982) + | > loss_gen: 2.549182653427124 (2.549182653427124) + | > loss_kl: 2.758784532546997 (2.758784532546997) + | > loss_feat: 6.366100311279297 (6.366100311279297) + | > loss_mel: 16.756851196289062 (16.756851196289062) + | > loss_duration: 1.775923728942871 (1.775923728942871) + | > loss_1: 30.206844329833984 (30.206844329833984) + + --> STEP: 1 + | > loss_disc: 2.5585551261901855 (2.5585551261901855) + | > loss_disc_real_0: 0.15342621505260468 (0.15342621505260468) + | > loss_disc_real_1: 0.24143216013908386 (0.24143216013908386) + | > loss_disc_real_2: 0.23149891197681427 (0.23149891197681427) + | > loss_disc_real_3: 0.29506075382232666 (0.29506075382232666) + | > loss_disc_real_4: 0.2546479105949402 (0.2546479105949402) + | > loss_disc_real_5: 0.215461865067482 (0.215461865067482) + | > loss_0: 2.5585551261901855 (2.5585551261901855) + | > loss_gen: 2.357297897338867 (2.357297897338867) + | > loss_kl: 2.667135000228882 (2.667135000228882) + | > loss_feat: 7.0882344245910645 (7.0882344245910645) + | > loss_mel: 18.702478408813477 (18.702478408813477) + | > loss_duration: 1.7996021509170532 (1.7996021509170532) + | > loss_1: 32.61474609375 (32.61474609375) + + --> STEP: 2 + | > loss_disc: 2.5648534297943115 (2.5617042779922485) + | > loss_disc_real_0: 0.1857367902994156 (0.16958150267601013) + | > loss_disc_real_1: 0.28356364369392395 (0.2624979019165039) + | > loss_disc_real_2: 0.2388859987258911 (0.2351924553513527) + | > loss_disc_real_3: 0.27082905173301697 (0.2829449027776718) + | > loss_disc_real_4: 0.19027750194072723 (0.2224627062678337) + | > loss_disc_real_5: 0.21359236538410187 (0.21452711522579193) + | > loss_0: 2.5648534297943115 (2.5617042779922485) + | > loss_gen: 2.3670332431793213 (2.3621655702590942) + | > loss_kl: 2.520456075668335 (2.5937955379486084) + | > loss_feat: 7.3592000007629395 (7.223717212677002) + | > loss_mel: 16.955461502075195 (17.828969955444336) + | > loss_duration: 1.7214281558990479 (1.7605151534080505) + | > loss_1: 30.9235782623291 (31.76916217803955) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003300309181213379 (-0.0003389120101928711) + | > avg_loss_disc: 2.5617042779922485 (+0.01800858974456787) + | > avg_loss_disc_real_0: 0.16958150267601013 (-0.059431977570056915) + | > avg_loss_disc_real_1: 0.2624979019165039 (+0.049194395542144775) + | > avg_loss_disc_real_2: 0.2351924553513527 (+0.018677979707717896) + | > avg_loss_disc_real_3: 0.2829449027776718 (+0.05809491127729416) + | > avg_loss_disc_real_4: 0.2224627062678337 (-0.002423293888568878) + | > avg_loss_disc_real_5: 0.21452711522579193 (-0.015508808195590973) + | > avg_loss_0: 2.5617042779922485 (+0.01800858974456787) + | > avg_loss_gen: 2.3621655702590942 (-0.19681775569915771) + | > avg_loss_kl: 2.5937955379486084 (-0.10917043685913086) + | > avg_loss_feat: 7.223717212677002 (-0.38151025772094727) + | > avg_loss_mel: 17.828969955444336 (-0.7241430282592773) + | > avg_loss_duration: 1.7605151534080505 (-0.0013385415077209473) + | > avg_loss_1: 31.76916217803955 (-1.4129829406738281) + + + > EPOCH: 453/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:25:10)  + + --> TIME: 2025-04-19 06:25:16 -- STEP: 17/142 -- GLOBAL_STEP: 443625 + | > loss_disc: 2.406454563140869 (2.4007856004378376) + | > loss_disc_real_0: 0.11942235380411148 (0.11747785789125106) + | > loss_disc_real_1: 0.2589414119720459 (0.19968123821651235) + | > loss_disc_real_2: 0.21612122654914856 (0.2221421546795789) + | > loss_disc_real_3: 0.19461660087108612 (0.22446415178916035) + | > loss_disc_real_4: 0.21957623958587646 (0.22559409720056198) + | > loss_disc_real_5: 0.23319758474826813 (0.2394047142828212) + | > loss_0: 2.406454563140869 (2.4007856004378376) + | > grad_norm_0: tensor(3.6871, device='cuda:0') (tensor(9.9098, device='cuda:0')) + | > loss_gen: 2.3444793224334717 (2.419483507380766) + | > loss_kl: 1.8946362733840942 (1.8289460504756254) + | > loss_feat: 8.221321105957031 (7.377217517179601) + | > loss_mel: 18.614402770996094 (17.69545891705681) + | > loss_duration: 1.3876450061798096 (1.4036512725493486) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.46248245239258 (30.72475736281451) + | > grad_norm_1: tensor(52.5308, device='cuda:0') (tensor(258.4696, device='cuda:0')) + | > current_lr_0: 0.0001889890028126998 + | > current_lr_1: 0.0001889890028126998 + | > step_time: 0.337 (0.3302345556371352) + | > loader_time: 0.0054 (0.004785313325769761) + + + --> TIME: 2025-04-19 06:25:25 -- STEP: 42/142 -- GLOBAL_STEP: 443650 + | > loss_disc: 2.3827013969421387 (2.4277049813951765) + | > loss_disc_real_0: 0.12182580679655075 (0.12046299857043084) + | > loss_disc_real_1: 0.2002059817314148 (0.20504924122776305) + | > loss_disc_real_2: 0.20763102173805237 (0.22456821160657064) + | > loss_disc_real_3: 0.18922914564609528 (0.22299532627775556) + | > loss_disc_real_4: 0.2619662284851074 (0.22641963901973905) + | > loss_disc_real_5: 0.24594177305698395 (0.23351451790048963) + | > loss_0: 2.3827013969421387 (2.4277049813951765) + | > grad_norm_0: tensor(14.6301, device='cuda:0') (tensor(12.8804, device='cuda:0')) + | > loss_gen: 2.511495590209961 (2.3943599462509146) + | > loss_kl: 1.7374128103256226 (1.7949653324626742) + | > loss_feat: 7.719135284423828 (7.235344114757719) + | > loss_mel: 18.037521362304688 (17.498752548581077) + | > loss_duration: 1.442556619644165 (1.4191873101961041) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.4481201171875 (30.342609496343705) + | > grad_norm_1: tensor(307.7477, device='cuda:0') (tensor(289.4641, device='cuda:0')) + | > current_lr_0: 0.0001889890028126998 + | > current_lr_1: 0.0001889890028126998 + | > step_time: 0.3604 (0.342383379027957) + | > loader_time: 0.0058 (0.005115514709835961) + + + --> TIME: 2025-04-19 06:25:35 -- STEP: 67/142 -- GLOBAL_STEP: 443675 + | > loss_disc: 2.334662914276123 (2.4361049310484924) + | > loss_disc_real_0: 0.12793758511543274 (0.12164209938761014) + | > loss_disc_real_1: 0.20564329624176025 (0.20364966766158146) + | > loss_disc_real_2: 0.18913467228412628 (0.22488513811310726) + | > loss_disc_real_3: 0.19651125371456146 (0.22557941168101867) + | > loss_disc_real_4: 0.16182845830917358 (0.22740978647523852) + | > loss_disc_real_5: 0.22850264608860016 (0.2339107705585992) + | > loss_0: 2.334662914276123 (2.4361049310484924) + | > grad_norm_0: tensor(17.0189, device='cuda:0') (tensor(13.1674, device='cuda:0')) + | > loss_gen: 2.4832029342651367 (2.3874070359699755) + | > loss_kl: 1.8906978368759155 (1.8097308304772448) + | > loss_feat: 8.042394638061523 (7.238734828892039) + | > loss_mel: 18.03352165222168 (17.5656099319458) + | > loss_duration: 1.4067273139953613 (1.4304222206571209) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.856544494628906 (30.43190483548748) + | > grad_norm_1: tensor(441.5815, device='cuda:0') (tensor(297.2491, device='cuda:0')) + | > current_lr_0: 0.0001889890028126998 + | > current_lr_1: 0.0001889890028126998 + | > step_time: 0.3826 (0.35414239185959545) + | > loader_time: 0.0059 (0.005446401994619796) + + + --> TIME: 2025-04-19 06:25:45 -- STEP: 92/142 -- GLOBAL_STEP: 443700 + | > loss_disc: 2.5572752952575684 (2.4452019022858655) + | > loss_disc_real_0: 0.1314014047384262 (0.12130486252515213) + | > loss_disc_real_1: 0.2305707186460495 (0.204580705774867) + | > loss_disc_real_2: 0.24361199140548706 (0.22544302969523097) + | > loss_disc_real_3: 0.23941278457641602 (0.2256943146171777) + | > loss_disc_real_4: 0.23480378091335297 (0.22738510078710059) + | > loss_disc_real_5: 0.25731658935546875 (0.23411625753278317) + | > loss_0: 2.5572752952575684 (2.4452019022858655) + | > grad_norm_0: tensor(5.9748, device='cuda:0') (tensor(12.9485, device='cuda:0')) + | > loss_gen: 2.221404790878296 (2.3752920990404873) + | > loss_kl: 1.9442138671875 (1.8309404396492501) + | > loss_feat: 6.69907283782959 (7.237612610277922) + | > loss_mel: 17.252418518066406 (17.57163590970246) + | > loss_duration: 1.439084768295288 (1.441045841445094) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.556196212768555 (30.456526942875076) + | > grad_norm_1: tensor(260.1931, device='cuda:0') (tensor(299.7963, device='cuda:0')) + | > current_lr_0: 0.0001889890028126998 + | > current_lr_1: 0.0001889890028126998 + | > step_time: 0.4105 (0.36621279561001324) + | > loader_time: 0.0063 (0.0057983968568884785) + + + --> TIME: 2025-04-19 06:25:55 -- STEP: 117/142 -- GLOBAL_STEP: 443725 + | > loss_disc: 2.4440364837646484 (2.452182213465372) + | > loss_disc_real_0: 0.12490997463464737 (0.12243558689315096) + | > loss_disc_real_1: 0.2062082141637802 (0.20586209546806467) + | > loss_disc_real_2: 0.23341959714889526 (0.22571859744369474) + | > loss_disc_real_3: 0.2229345142841339 (0.22717845376230714) + | > loss_disc_real_4: 0.2456521987915039 (0.22792007933315048) + | > loss_disc_real_5: 0.22265847027301788 (0.23403124625866228) + | > loss_0: 2.4440364837646484 (2.452182213465372) + | > grad_norm_0: tensor(10.0356, device='cuda:0') (tensor(12.3804, device='cuda:0')) + | > loss_gen: 2.3850860595703125 (2.373748985111203) + | > loss_kl: 1.7439419031143188 (1.8392109707889395) + | > loss_feat: 6.705429553985596 (7.224621271475767) + | > loss_mel: 16.74509048461914 (17.650612749605106) + | > loss_duration: 1.464393973350525 (1.4480933980045159) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.043941497802734 (30.536287389249882) + | > grad_norm_1: tensor(230.5999, device='cuda:0') (tensor(285.5410, device='cuda:0')) + | > current_lr_0: 0.0001889890028126998 + | > current_lr_1: 0.0001889890028126998 + | > step_time: 0.4349 (0.37683745530935436) + | > loader_time: 0.0075 (0.006059754608023878) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4562830924987793 (2.4562830924987793) + | > loss_disc_real_0: 0.20614174008369446 (0.20614174008369446) + | > loss_disc_real_1: 0.18296225368976593 (0.18296225368976593) + | > loss_disc_real_2: 0.21756108105182648 (0.21756108105182648) + | > loss_disc_real_3: 0.23748640716075897 (0.23748640716075897) + | > loss_disc_real_4: 0.2259882688522339 (0.2259882688522339) + | > loss_disc_real_5: 0.27638116478919983 (0.27638116478919983) + | > loss_0: 2.4562830924987793 (2.4562830924987793) + | > loss_gen: 2.468106269836426 (2.468106269836426) + | > loss_kl: 2.798098564147949 (2.798098564147949) + | > loss_feat: 6.830353736877441 (6.830353736877441) + | > loss_mel: 18.00106430053711 (18.00106430053711) + | > loss_duration: 1.7539277076721191 (1.7539277076721191) + | > loss_1: 31.85154914855957 (31.85154914855957) + + --> STEP: 1 + | > loss_disc: 2.5203378200531006 (2.5203378200531006) + | > loss_disc_real_0: 0.20593415200710297 (0.20593415200710297) + | > loss_disc_real_1: 0.19406259059906006 (0.19406259059906006) + | > loss_disc_real_2: 0.2166018933057785 (0.2166018933057785) + | > loss_disc_real_3: 0.24348986148834229 (0.24348986148834229) + | > loss_disc_real_4: 0.23458227515220642 (0.23458227515220642) + | > loss_disc_real_5: 0.23865389823913574 (0.23865389823913574) + | > loss_0: 2.5203378200531006 (2.5203378200531006) + | > loss_gen: 2.421505928039551 (2.421505928039551) + | > loss_kl: 2.9438798427581787 (2.9438798427581787) + | > loss_feat: 7.20890998840332 (7.20890998840332) + | > loss_mel: 18.33800506591797 (18.33800506591797) + | > loss_duration: 1.8019626140594482 (1.8019626140594482) + | > loss_1: 32.714263916015625 (32.714263916015625) + + --> STEP: 2 + | > loss_disc: 2.265806198120117 (2.393072009086609) + | > loss_disc_real_0: 0.12490526586771011 (0.16541970893740654) + | > loss_disc_real_1: 0.13475926220417023 (0.16441092640161514) + | > loss_disc_real_2: 0.18247045576572418 (0.19953617453575134) + | > loss_disc_real_3: 0.17684032022953033 (0.2101650908589363) + | > loss_disc_real_4: 0.17851479351520538 (0.2065485343337059) + | > loss_disc_real_5: 0.1663302630186081 (0.20249208062887192) + | > loss_0: 2.265806198120117 (2.393072009086609) + | > loss_gen: 2.2692337036132812 (2.345369815826416) + | > loss_kl: 2.4057457447052 (2.6748127937316895) + | > loss_feat: 9.289385795593262 (8.249147891998291) + | > loss_mel: 19.171571731567383 (18.754788398742676) + | > loss_duration: 1.7378337383270264 (1.7698981761932373) + | > loss_1: 34.87377166748047 (33.79401779174805) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003612041473388672 (+0.00031173229217529297) + | > avg_loss_disc: 2.393072009086609 (-0.16863226890563965) + | > avg_loss_disc_real_0: 0.16541970893740654 (-0.004161793738603592) + | > avg_loss_disc_real_1: 0.16441092640161514 (-0.09808697551488876) + | > avg_loss_disc_real_2: 0.19953617453575134 (-0.03565628081560135) + | > avg_loss_disc_real_3: 0.2101650908589363 (-0.0727798119187355) + | > avg_loss_disc_real_4: 0.2065485343337059 (-0.015914171934127808) + | > avg_loss_disc_real_5: 0.20249208062887192 (-0.012035034596920013) + | > avg_loss_0: 2.393072009086609 (-0.16863226890563965) + | > avg_loss_gen: 2.345369815826416 (-0.016795754432678223) + | > avg_loss_kl: 2.6748127937316895 (+0.08101725578308105) + | > avg_loss_feat: 8.249147891998291 (+1.025430679321289) + | > avg_loss_mel: 18.754788398742676 (+0.9258184432983398) + | > avg_loss_duration: 1.7698981761932373 (+0.009383022785186768) + | > avg_loss_1: 33.79401779174805 (+2.024855613708496) + + + > EPOCH: 454/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:26:08)  + + --> TIME: 2025-04-19 06:26:09 -- STEP: 0/142 -- GLOBAL_STEP: 443750 + | > loss_disc: 2.3758621215820312 (2.3758621215820312) + | > loss_disc_real_0: 0.15109920501708984 (0.15109920501708984) + | > loss_disc_real_1: 0.16612224280834198 (0.16612224280834198) + | > loss_disc_real_2: 0.20524221658706665 (0.20524221658706665) + | > loss_disc_real_3: 0.2116253823041916 (0.2116253823041916) + | > loss_disc_real_4: 0.20169851183891296 (0.20169851183891296) + | > loss_disc_real_5: 0.2159116566181183 (0.2159116566181183) + | > loss_0: 2.3758621215820312 (2.3758621215820312) + | > grad_norm_0: tensor(19.2099, device='cuda:0') (tensor(19.2099, device='cuda:0')) + | > loss_gen: 2.655285120010376 (2.655285120010376) + | > loss_kl: 1.8187096118927002 (1.8187096118927002) + | > loss_feat: 7.957112789154053 (7.957112789154053) + | > loss_mel: 18.234987258911133 (18.234987258911133) + | > loss_duration: 1.4381868839263916 (1.4381868839263916) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.10428237915039 (32.10428237915039) + | > grad_norm_1: tensor(550.7729, device='cuda:0') (tensor(550.7729, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.3574 (0.3573784828186035) + | > loader_time: 0.3277 (0.32767415046691895) + + + --> TIME: 2025-04-19 06:26:17 -- STEP: 25/142 -- GLOBAL_STEP: 443775 + | > loss_disc: 2.362192153930664 (2.471255512237549) + | > loss_disc_real_0: 0.12505413591861725 (0.12534327536821366) + | > loss_disc_real_1: 0.18553034961223602 (0.2043185019493103) + | > loss_disc_real_2: 0.20234966278076172 (0.22890184581279754) + | > loss_disc_real_3: 0.20820604264736176 (0.23173854172229766) + | > loss_disc_real_4: 0.19812296330928802 (0.23314755856990815) + | > loss_disc_real_5: 0.23204641044139862 (0.2394723790884018) + | > loss_0: 2.362192153930664 (2.471255512237549) + | > grad_norm_0: tensor(18.1273, device='cuda:0') (tensor(12.6483, device='cuda:0')) + | > loss_gen: 2.3172664642333984 (2.368225107192993) + | > loss_kl: 1.8942188024520874 (1.7937128257751465) + | > loss_feat: 6.877529621124268 (7.133908214569092) + | > loss_mel: 16.034061431884766 (17.503307113647463) + | > loss_duration: 1.4715781211853027 (1.4142593336105347) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.594654083251953 (30.213412857055665) + | > grad_norm_1: tensor(255.4333, device='cuda:0') (tensor(306.7824, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.3455 (0.33322041511535644) + | > loader_time: 0.0056 (0.004947576522827148) + + + --> TIME: 2025-04-19 06:26:27 -- STEP: 50/142 -- GLOBAL_STEP: 443800 + | > loss_disc: 2.2933239936828613 (2.4386833906173706) + | > loss_disc_real_0: 0.07947000116109848 (0.11771659925580025) + | > loss_disc_real_1: 0.18630370497703552 (0.2015320685505867) + | > loss_disc_real_2: 0.23116172850131989 (0.22661615580320357) + | > loss_disc_real_3: 0.2012551873922348 (0.22715024620294572) + | > loss_disc_real_4: 0.24601709842681885 (0.22933868318796158) + | > loss_disc_real_5: 0.23135989904403687 (0.23608218163251876) + | > loss_0: 2.2933239936828613 (2.4386833906173706) + | > grad_norm_0: tensor(16.3250, device='cuda:0') (tensor(12.2165, device='cuda:0')) + | > loss_gen: 2.5867605209350586 (2.393275799751282) + | > loss_kl: 1.971919298171997 (1.8070098567008972) + | > loss_feat: 7.381702899932861 (7.293958415985108) + | > loss_mel: 18.019277572631836 (17.649250755310053) + | > loss_duration: 1.4615687131881714 (1.4211250281333923) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.421228408813477 (30.564619941711427) + | > grad_norm_1: tensor(466.8344, device='cuda:0') (tensor(318.7894, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.3692 (0.3459634494781494) + | > loader_time: 0.0067 (0.005337662696838378) + + + --> TIME: 2025-04-19 06:26:36 -- STEP: 75/142 -- GLOBAL_STEP: 443825 + | > loss_disc: 2.4069600105285645 (2.449309345881144) + | > loss_disc_real_0: 0.10067956894636154 (0.12025892635186514) + | > loss_disc_real_1: 0.19924408197402954 (0.20459214369455972) + | > loss_disc_real_2: 0.23122866451740265 (0.22625317017237345) + | > loss_disc_real_3: 0.21632914245128632 (0.22630751272042593) + | > loss_disc_real_4: 0.2210659384727478 (0.22824556608994803) + | > loss_disc_real_5: 0.263823539018631 (0.23757354418436685) + | > loss_0: 2.4069600105285645 (2.449309345881144) + | > grad_norm_0: tensor(4.1182, device='cuda:0') (tensor(11.7716, device='cuda:0')) + | > loss_gen: 2.4605507850646973 (2.3779028828938804) + | > loss_kl: 1.763362169265747 (1.8151266956329346) + | > loss_feat: 7.626300811767578 (7.20970447540283) + | > loss_mel: 17.484521865844727 (17.493747126261393) + | > loss_duration: 1.412118673324585 (1.4319011878967285) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.74685287475586 (30.328382263183595) + | > grad_norm_1: tensor(109.4498, device='cuda:0') (tensor(313.3066, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.3978 (0.358458137512207) + | > loader_time: 0.0061 (0.005597197214762368) + + + --> TIME: 2025-04-19 06:26:47 -- STEP: 100/142 -- GLOBAL_STEP: 443850 + | > loss_disc: 2.4579017162323 (2.4499122500419617) + | > loss_disc_real_0: 0.10060770809650421 (0.12076973430812359) + | > loss_disc_real_1: 0.19965584576129913 (0.20420086950063707) + | > loss_disc_real_2: 0.19166754186153412 (0.22560389578342438) + | > loss_disc_real_3: 0.2153591513633728 (0.22736371532082558) + | > loss_disc_real_4: 0.2345912754535675 (0.22838501140475273) + | > loss_disc_real_5: 0.21612851321697235 (0.23635926753282546) + | > loss_0: 2.4579017162323 (2.4499122500419617) + | > grad_norm_0: tensor(14.0394, device='cuda:0') (tensor(11.9854, device='cuda:0')) + | > loss_gen: 2.2933197021484375 (2.375166375637056) + | > loss_kl: 1.8907753229141235 (1.8336942267417908) + | > loss_feat: 7.320028781890869 (7.2043285751342765) + | > loss_mel: 17.640403747558594 (17.539513225555424) + | > loss_duration: 1.4792696237564087 (1.443496081829071) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.623796463012695 (30.39619846343994) + | > grad_norm_1: tensor(340.8333, device='cuda:0') (tensor(316.0760, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.417 (0.3700634503364563) + | > loader_time: 0.0067 (0.005907652378082273) + + + --> TIME: 2025-04-19 06:26:58 -- STEP: 125/142 -- GLOBAL_STEP: 443875 + | > loss_disc: 2.417490243911743 (2.4591152667999268) + | > loss_disc_real_0: 0.21085743606090546 (0.12481450831890106) + | > loss_disc_real_1: 0.21239671111106873 (0.20532529175281525) + | > loss_disc_real_2: 0.26791954040527344 (0.22604467642307283) + | > loss_disc_real_3: 0.26030558347702026 (0.22805901420116426) + | > loss_disc_real_4: 0.2221827656030655 (0.2285850284099579) + | > loss_disc_real_5: 0.2470729798078537 (0.23688805854320527) + | > loss_0: 2.417490243911743 (2.4591152667999268) + | > grad_norm_0: tensor(36.0990, device='cuda:0') (tensor(13.4071, device='cuda:0')) + | > loss_gen: 2.607558488845825 (2.3772304916381852) + | > loss_kl: 1.9021879434585571 (1.8439023237228394) + | > loss_feat: 7.229796409606934 (7.162936981201171) + | > loss_mel: 18.981679916381836 (17.524302619934087) + | > loss_duration: 1.5117647647857666 (1.4505595541000367) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.23299026489258 (30.358931930541992) + | > grad_norm_1: tensor(31.7064, device='cuda:0') (tensor(324.9089, device='cuda:0')) + | > current_lr_0: 0.00018896537918734822 + | > current_lr_1: 0.00018896537918734822 + | > step_time: 0.4386 (0.3818793716430664) + | > loader_time: 0.0072 (0.006139207839965819) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.554360866546631 (2.554360866546631) + | > loss_disc_real_0: 0.15463629364967346 (0.15463629364967346) + | > loss_disc_real_1: 0.3013560175895691 (0.3013560175895691) + | > loss_disc_real_2: 0.21886764466762543 (0.21886764466762543) + | > loss_disc_real_3: 0.2719433009624481 (0.2719433009624481) + | > loss_disc_real_4: 0.25682976841926575 (0.25682976841926575) + | > loss_disc_real_5: 0.2469041347503662 (0.2469041347503662) + | > loss_0: 2.554360866546631 (2.554360866546631) + | > loss_gen: 2.5321667194366455 (2.5321667194366455) + | > loss_kl: 2.407090187072754 (2.407090187072754) + | > loss_feat: 7.141308784484863 (7.141308784484863) + | > loss_mel: 17.521900177001953 (17.521900177001953) + | > loss_duration: 1.7736687660217285 (1.7736687660217285) + | > loss_1: 31.376134872436523 (31.376134872436523) + + --> STEP: 1 + | > loss_disc: 2.510810613632202 (2.510810613632202) + | > loss_disc_real_0: 0.1654243916273117 (0.1654243916273117) + | > loss_disc_real_1: 0.2584545314311981 (0.2584545314311981) + | > loss_disc_real_2: 0.2345830649137497 (0.2345830649137497) + | > loss_disc_real_3: 0.2636932134628296 (0.2636932134628296) + | > loss_disc_real_4: 0.25314953923225403 (0.25314953923225403) + | > loss_disc_real_5: 0.21729490160942078 (0.21729490160942078) + | > loss_0: 2.510810613632202 (2.510810613632202) + | > loss_gen: 2.342202663421631 (2.342202663421631) + | > loss_kl: 2.768315076828003 (2.768315076828003) + | > loss_feat: 6.456678867340088 (6.456678867340088) + | > loss_mel: 18.05696678161621 (18.05696678161621) + | > loss_duration: 1.7955200672149658 (1.7955200672149658) + | > loss_1: 31.4196834564209 (31.4196834564209) + + --> STEP: 2 + | > loss_disc: 2.579861879348755 (2.5453362464904785) + | > loss_disc_real_0: 0.13315558433532715 (0.14928998798131943) + | > loss_disc_real_1: 0.2941822111606598 (0.27631837129592896) + | > loss_disc_real_2: 0.28797677159309387 (0.2612799182534218) + | > loss_disc_real_3: 0.2677179276943207 (0.26570557057857513) + | > loss_disc_real_4: 0.29226335883140564 (0.27270644903182983) + | > loss_disc_real_5: 0.25794678926467896 (0.23762084543704987) + | > loss_0: 2.579861879348755 (2.5453362464904785) + | > loss_gen: 2.539163112640381 (2.440682888031006) + | > loss_kl: 2.2995824813842773 (2.53394877910614) + | > loss_feat: 7.2664008140563965 (6.861539840698242) + | > loss_mel: 19.24597930908203 (18.65147304534912) + | > loss_duration: 1.7381770610809326 (1.7668485641479492) + | > loss_1: 33.08930206298828 (32.25449275970459) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033854246139526367 (-0.00022661685943603516) + | > avg_loss_disc: 2.5453362464904785 (+0.15226423740386963) + | > avg_loss_disc_real_0: 0.14928998798131943 (-0.016129720956087112) + | > avg_loss_disc_real_1: 0.27631837129592896 (+0.11190744489431381) + | > avg_loss_disc_real_2: 0.2612799182534218 (+0.06174374371767044) + | > avg_loss_disc_real_3: 0.26570557057857513 (+0.055540479719638824) + | > avg_loss_disc_real_4: 0.27270644903182983 (+0.06615791469812393) + | > avg_loss_disc_real_5: 0.23762084543704987 (+0.03512876480817795) + | > avg_loss_0: 2.5453362464904785 (+0.15226423740386963) + | > avg_loss_gen: 2.440682888031006 (+0.09531307220458984) + | > avg_loss_kl: 2.53394877910614 (-0.14086401462554932) + | > avg_loss_feat: 6.861539840698242 (-1.3876080513000488) + | > avg_loss_mel: 18.65147304534912 (-0.10331535339355469) + | > avg_loss_duration: 1.7668485641479492 (-0.003049612045288086) + | > avg_loss_1: 32.25449275970459 (-1.539525032043457) + + + > EPOCH: 455/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:27:07)  + + --> TIME: 2025-04-19 06:27:10 -- STEP: 8/142 -- GLOBAL_STEP: 443900 + | > loss_disc: 2.4478442668914795 (2.4939865171909332) + | > loss_disc_real_0: 0.09014150500297546 (0.1078050471842289) + | > loss_disc_real_1: 0.19095592200756073 (0.21975954994559288) + | > loss_disc_real_2: 0.23041251301765442 (0.2331753373146057) + | > loss_disc_real_3: 0.22044411301612854 (0.23887266963720322) + | > loss_disc_real_4: 0.21862588822841644 (0.24042311497032642) + | > loss_disc_real_5: 0.26405584812164307 (0.2462493572384119) + | > loss_0: 2.4478442668914795 (2.4939865171909332) + | > grad_norm_0: tensor(12.3821, device='cuda:0') (tensor(13.0315, device='cuda:0')) + | > loss_gen: 2.4826087951660156 (2.3572953641414642) + | > loss_kl: 1.8127578496932983 (1.8561936169862747) + | > loss_feat: 8.344246864318848 (7.263184368610382) + | > loss_mel: 18.25705909729004 (17.53316569328308) + | > loss_duration: 1.4250478744506836 (1.4127329885959625) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.321720123291016 (30.422572135925293) + | > grad_norm_1: tensor(303.5219, device='cuda:0') (tensor(366.9101, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.3264 (0.3238038718700409) + | > loader_time: 0.0048 (0.004377037286758423) + + + --> TIME: 2025-04-19 06:27:19 -- STEP: 33/142 -- GLOBAL_STEP: 443925 + | > loss_disc: 2.4741756916046143 (2.439715681654035) + | > loss_disc_real_0: 0.08331577479839325 (0.1083239633025545) + | > loss_disc_real_1: 0.22564193606376648 (0.2096648049173933) + | > loss_disc_real_2: 0.22618822753429413 (0.22783225684454947) + | > loss_disc_real_3: 0.23805956542491913 (0.23149529189774484) + | > loss_disc_real_4: 0.23887646198272705 (0.23408957503058694) + | > loss_disc_real_5: 0.24636194109916687 (0.23585466453523346) + | > loss_0: 2.4741756916046143 (2.439715681654035) + | > grad_norm_0: tensor(12.9531, device='cuda:0') (tensor(11.4671, device='cuda:0')) + | > loss_gen: 2.3538107872009277 (2.3872229330467465) + | > loss_kl: 1.8994255065917969 (1.8626983454733184) + | > loss_feat: 7.192688465118408 (7.350303115266742) + | > loss_mel: 18.198179244995117 (17.570493351329457) + | > loss_duration: 1.4276385307312012 (1.4136312369144324) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.07174301147461 (30.584349083177973) + | > grad_norm_1: tensor(384.1292, device='cuda:0') (tensor(303.9294, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.3542 (0.3376714749769731) + | > loader_time: 0.0055 (0.0050848686333858605) + + + --> TIME: 2025-04-19 06:27:28 -- STEP: 58/142 -- GLOBAL_STEP: 443950 + | > loss_disc: 2.495176315307617 (2.4307037304187653) + | > loss_disc_real_0: 0.16019323468208313 (0.11401237084947784) + | > loss_disc_real_1: 0.1903713494539261 (0.20525515516256465) + | > loss_disc_real_2: 0.20404109358787537 (0.22488946313488073) + | > loss_disc_real_3: 0.19985435903072357 (0.22786076315518083) + | > loss_disc_real_4: 0.24155685305595398 (0.23075594573185362) + | > loss_disc_real_5: 0.23910287022590637 (0.23428479427921362) + | > loss_0: 2.495176315307617 (2.4307037304187653) + | > grad_norm_0: tensor(16.8810, device='cuda:0') (tensor(11.4591, device='cuda:0')) + | > loss_gen: 2.3190948963165283 (2.382913996433389) + | > loss_kl: 1.8072638511657715 (1.872191320205557) + | > loss_feat: 7.394243240356445 (7.3235937891335325) + | > loss_mel: 19.14715003967285 (17.61861603835533) + | > loss_duration: 1.4602543115615845 (1.425717343544138) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.12800598144531 (30.62303240545865) + | > grad_norm_1: tensor(369.0612, device='cuda:0') (tensor(292.5293, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.3738 (0.34948507259631983) + | > loader_time: 0.0061 (0.005460977554321289) + + + --> TIME: 2025-04-19 06:27:38 -- STEP: 83/142 -- GLOBAL_STEP: 443975 + | > loss_disc: 2.368528366088867 (2.4336413705205344) + | > loss_disc_real_0: 0.0934315025806427 (0.11548106112034924) + | > loss_disc_real_1: 0.19153930246829987 (0.20579997064119362) + | > loss_disc_real_2: 0.2220057100057602 (0.22589118688939566) + | > loss_disc_real_3: 0.22238458693027496 (0.2271690289658236) + | > loss_disc_real_4: 0.181907057762146 (0.22958007041948386) + | > loss_disc_real_5: 0.1971997618675232 (0.23222311523305364) + | > loss_0: 2.368528366088867 (2.4336413705205344) + | > grad_norm_0: tensor(9.3252, device='cuda:0') (tensor(11.4156, device='cuda:0')) + | > loss_gen: 2.470341920852661 (2.37326261796147) + | > loss_kl: 1.9427464008331299 (1.8660989813057773) + | > loss_feat: 7.129586696624756 (7.299774049276329) + | > loss_mel: 18.074146270751953 (17.672408149903074) + | > loss_duration: 1.5140645503997803 (1.4364425403526029) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.130887985229492 (30.64798635459808) + | > grad_norm_1: tensor(346.7385, device='cuda:0') (tensor(286.1109, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.3967 (0.361912572240255) + | > loader_time: 0.0067 (0.005718236946197878) + + + --> TIME: 2025-04-19 06:27:49 -- STEP: 108/142 -- GLOBAL_STEP: 444000 + | > loss_disc: 2.5076301097869873 (2.436080594857534) + | > loss_disc_real_0: 0.18687358498573303 (0.117441158534752) + | > loss_disc_real_1: 0.21892264485359192 (0.20477490844549956) + | > loss_disc_real_2: 0.2399493157863617 (0.22495274052575784) + | > loss_disc_real_3: 0.22008858621120453 (0.2275828746733842) + | > loss_disc_real_4: 0.20524263381958008 (0.2294945967970071) + | > loss_disc_real_5: 0.2085258960723877 (0.23260147207313114) + | > loss_0: 2.5076301097869873 (2.436080594857534) + | > grad_norm_0: tensor(5.8148, device='cuda:0') (tensor(10.8754, device='cuda:0')) + | > loss_gen: 2.476982831954956 (2.373896863725449) + | > loss_kl: 1.8764747381210327 (1.86907258188283) + | > loss_feat: 7.29606819152832 (7.284560976205049) + | > loss_mel: 16.691648483276367 (17.690709873482035) + | > loss_duration: 1.472584843635559 (1.4455493644431785) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.813758850097656 (30.66378957253915) + | > grad_norm_1: tensor(187.3078, device='cuda:0') (tensor(272.2582, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.4317 (0.3742475752477293) + | > loader_time: 0.0069 (0.005979105278297707) + + + --> TIME: 2025-04-19 06:28:00 -- STEP: 133/142 -- GLOBAL_STEP: 444025 + | > loss_disc: 2.4295244216918945 (2.4403010687433686) + | > loss_disc_real_0: 0.1558162271976471 (0.12011237711386573) + | > loss_disc_real_1: 0.24256937205791473 (0.2045517073089915) + | > loss_disc_real_2: 0.21984805166721344 (0.22523791337371768) + | > loss_disc_real_3: 0.21639904379844666 (0.2277494329482989) + | > loss_disc_real_4: 0.2231765240430832 (0.22988917101594739) + | > loss_disc_real_5: 0.19622628390789032 (0.23095745666134626) + | > loss_0: 2.4295244216918945 (2.4403010687433686) + | > grad_norm_0: tensor(6.8964, device='cuda:0') (tensor(10.5703, device='cuda:0')) + | > loss_gen: 2.483428955078125 (2.3660374135899356) + | > loss_kl: 1.9828459024429321 (1.875347778313142) + | > loss_feat: 7.713609218597412 (7.210408899120819) + | > loss_mel: 18.24471092224121 (17.6201501215311) + | > loss_duration: 1.5141010284423828 (1.4523754854847615) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.938695907592773 (30.52431964157219) + | > grad_norm_1: tensor(206.4872, device='cuda:0') (tensor(258.5767, device='cuda:0')) + | > current_lr_0: 0.0001889417585149498 + | > current_lr_1: 0.0001889417585149498 + | > step_time: 0.437 (0.38595431370842725) + | > loader_time: 0.0079 (0.00622429883569703) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.614384174346924 (2.614384174346924) + | > loss_disc_real_0: 0.2789863646030426 (0.2789863646030426) + | > loss_disc_real_1: 0.22743140161037445 (0.22743140161037445) + | > loss_disc_real_2: 0.25498491525650024 (0.25498491525650024) + | > loss_disc_real_3: 0.2511787712574005 (0.2511787712574005) + | > loss_disc_real_4: 0.26756832003593445 (0.26756832003593445) + | > loss_disc_real_5: 0.23624029755592346 (0.23624029755592346) + | > loss_0: 2.614384174346924 (2.614384174346924) + | > loss_gen: 2.6622934341430664 (2.6622934341430664) + | > loss_kl: 2.4524617195129395 (2.4524617195129395) + | > loss_feat: 7.522793769836426 (7.522793769836426) + | > loss_mel: 18.81731414794922 (18.81731414794922) + | > loss_duration: 1.7346057891845703 (1.7346057891845703) + | > loss_1: 33.18946838378906 (33.18946838378906) + + --> STEP: 1 + | > loss_disc: 2.518756628036499 (2.518756628036499) + | > loss_disc_real_0: 0.1888788491487503 (0.1888788491487503) + | > loss_disc_real_1: 0.20058943331241608 (0.20058943331241608) + | > loss_disc_real_2: 0.22372892498970032 (0.22372892498970032) + | > loss_disc_real_3: 0.2370815873146057 (0.2370815873146057) + | > loss_disc_real_4: 0.2272929847240448 (0.2272929847240448) + | > loss_disc_real_5: 0.26392510533332825 (0.26392510533332825) + | > loss_0: 2.518756628036499 (2.518756628036499) + | > loss_gen: 2.451108455657959 (2.451108455657959) + | > loss_kl: 2.9791128635406494 (2.9791128635406494) + | > loss_feat: 7.410121440887451 (7.410121440887451) + | > loss_mel: 19.059152603149414 (19.059152603149414) + | > loss_duration: 1.8123528957366943 (1.8123528957366943) + | > loss_1: 33.711849212646484 (33.711849212646484) + + --> STEP: 2 + | > loss_disc: 2.3708765506744385 (2.4448165893554688) + | > loss_disc_real_0: 0.20733478665351868 (0.1981068179011345) + | > loss_disc_real_1: 0.23607265949249268 (0.21833104640245438) + | > loss_disc_real_2: 0.19565549492835999 (0.20969220995903015) + | > loss_disc_real_3: 0.20285920798778534 (0.21997039765119553) + | > loss_disc_real_4: 0.23249207437038422 (0.2298925295472145) + | > loss_disc_real_5: 0.21292869746685028 (0.23842690140008926) + | > loss_0: 2.3708765506744385 (2.4448165893554688) + | > loss_gen: 2.588712215423584 (2.5199103355407715) + | > loss_kl: 2.3636698722839355 (2.6713913679122925) + | > loss_feat: 7.247501850128174 (7.3288116455078125) + | > loss_mel: 18.9344425201416 (18.996797561645508) + | > loss_duration: 1.745896816253662 (1.7791248559951782) + | > loss_1: 32.88022232055664 (33.29603576660156) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003657698631286621 (+0.0002722740173339844) + | > avg_loss_disc: 2.4448165893554688 (-0.10051965713500977) + | > avg_loss_disc_real_0: 0.1981068179011345 (+0.048816829919815063) + | > avg_loss_disc_real_1: 0.21833104640245438 (-0.05798732489347458) + | > avg_loss_disc_real_2: 0.20969220995903015 (-0.05158770829439163) + | > avg_loss_disc_real_3: 0.21997039765119553 (-0.04573517292737961) + | > avg_loss_disc_real_4: 0.2298925295472145 (-0.042813919484615326) + | > avg_loss_disc_real_5: 0.23842690140008926 (+0.0008060559630393982) + | > avg_loss_0: 2.4448165893554688 (-0.10051965713500977) + | > avg_loss_gen: 2.5199103355407715 (+0.07922744750976562) + | > avg_loss_kl: 2.6713913679122925 (+0.13744258880615234) + | > avg_loss_feat: 7.3288116455078125 (+0.4672718048095703) + | > avg_loss_mel: 18.996797561645508 (+0.3453245162963867) + | > avg_loss_duration: 1.7791248559951782 (+0.012276291847229004) + | > avg_loss_1: 33.29603576660156 (+1.0415430068969727) + + + > EPOCH: 456/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:28:06)  + + --> TIME: 2025-04-19 06:28:12 -- STEP: 16/142 -- GLOBAL_STEP: 444050 + | > loss_disc: 2.4506688117980957 (2.428747370839119) + | > loss_disc_real_0: 0.0773807168006897 (0.12482493394054472) + | > loss_disc_real_1: 0.20577393472194672 (0.20150151755660772) + | > loss_disc_real_2: 0.20677468180656433 (0.22419897839426994) + | > loss_disc_real_3: 0.21173816919326782 (0.22558680549263954) + | > loss_disc_real_4: 0.1953376680612564 (0.22556675877422094) + | > loss_disc_real_5: 0.1926821768283844 (0.2213637251406908) + | > loss_0: 2.4506688117980957 (2.428747370839119) + | > grad_norm_0: tensor(12.5551, device='cuda:0') (tensor(8.4817, device='cuda:0')) + | > loss_gen: 2.352234363555908 (2.385613590478897) + | > loss_kl: 1.7688714265823364 (1.7898810282349586) + | > loss_feat: 7.317022323608398 (7.344738125801086) + | > loss_mel: 17.393726348876953 (17.332392930984494) + | > loss_duration: 1.3798320293426514 (1.4066836833953857) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.211685180664062 (30.25930905342102) + | > grad_norm_1: tensor(182.5278, device='cuda:0') (tensor(189.4444, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.3408 (0.32765132188796997) + | > loader_time: 0.0051 (0.0048150718212127686) + + + --> TIME: 2025-04-19 06:28:21 -- STEP: 41/142 -- GLOBAL_STEP: 444075 + | > loss_disc: 2.441021680831909 (2.4649125890034) + | > loss_disc_real_0: 0.1463395357131958 (0.1265562310269693) + | > loss_disc_real_1: 0.21958450973033905 (0.204914852976799) + | > loss_disc_real_2: 0.19835591316223145 (0.22945242502340457) + | > loss_disc_real_3: 0.23214378952980042 (0.22940110397048113) + | > loss_disc_real_4: 0.20432305335998535 (0.22919738765169934) + | > loss_disc_real_5: 0.2145659625530243 (0.2308313821147128) + | > loss_0: 2.441021680831909 (2.4649125890034) + | > grad_norm_0: tensor(5.7259, device='cuda:0') (tensor(7.1156, device='cuda:0')) + | > loss_gen: 2.3602442741394043 (2.355885860396595) + | > loss_kl: 1.8591283559799194 (1.782908244830806) + | > loss_feat: 7.232968330383301 (7.163467162992896) + | > loss_mel: 17.74516487121582 (17.52013306501435) + | > loss_duration: 1.403229832649231 (1.4164925639222308) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.600736618041992 (30.238886949492663) + | > grad_norm_1: tensor(151.1800, device='cuda:0') (tensor(144.5726, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.367 (0.34310494399652247) + | > loader_time: 0.0057 (0.005272027922839653) + + + --> TIME: 2025-04-19 06:28:30 -- STEP: 66/142 -- GLOBAL_STEP: 444100 + | > loss_disc: 2.5079357624053955 (2.454227899060105) + | > loss_disc_real_0: 0.12084567546844482 (0.12489514148822334) + | > loss_disc_real_1: 0.2200697660446167 (0.2033062647237922) + | > loss_disc_real_2: 0.23682095110416412 (0.22664101670185724) + | > loss_disc_real_3: 0.1957450956106186 (0.22877175135142874) + | > loss_disc_real_4: 0.21014587581157684 (0.22832535658821915) + | > loss_disc_real_5: 0.195413738489151 (0.2323435949105205) + | > loss_0: 2.5079357624053955 (2.454227899060105) + | > grad_norm_0: tensor(8.1166, device='cuda:0') (tensor(6.9547, device='cuda:0')) + | > loss_gen: 2.317049503326416 (2.3487587155717793) + | > loss_kl: 1.7872360944747925 (1.8006007562984119) + | > loss_feat: 7.849650859832764 (7.2001706036654385) + | > loss_mel: 18.301897048950195 (17.639531858039625) + | > loss_duration: 1.4162356853485107 (1.427775912212603) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.672067642211914 (30.416837778958406) + | > grad_norm_1: tensor(131.6342, device='cuda:0') (tensor(132.7096, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.3846 (0.3539343602729566) + | > loader_time: 0.0057 (0.005537210088787657) + + + --> TIME: 2025-04-19 06:28:40 -- STEP: 91/142 -- GLOBAL_STEP: 444125 + | > loss_disc: 2.5298354625701904 (2.4611404806703003) + | > loss_disc_real_0: 0.1242881789803505 (0.12610016619929898) + | > loss_disc_real_1: 0.23963774740695953 (0.20526418826737247) + | > loss_disc_real_2: 0.23293527960777283 (0.22640868906791395) + | > loss_disc_real_3: 0.23815889656543732 (0.22826749512127467) + | > loss_disc_real_4: 0.22575439512729645 (0.22860331947986895) + | > loss_disc_real_5: 0.23084911704063416 (0.23251712060236668) + | > loss_0: 2.5298354625701904 (2.4611404806703003) + | > grad_norm_0: tensor(8.0766, device='cuda:0') (tensor(7.1496, device='cuda:0')) + | > loss_gen: 2.4080698490142822 (2.3439198928874934) + | > loss_kl: 1.8431098461151123 (1.8152369601385934) + | > loss_feat: 7.318406105041504 (7.14009939969241) + | > loss_mel: 18.389320373535156 (17.70200773385854) + | > loss_duration: 1.4703868627548218 (1.4395589684392067) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.429292678833008 (30.440822999555987) + | > grad_norm_1: tensor(75.1726, device='cuda:0') (tensor(140.5850, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.401 (0.3657176730396983) + | > loader_time: 0.0065 (0.005768175963517071) + + + --> TIME: 2025-04-19 06:28:51 -- STEP: 116/142 -- GLOBAL_STEP: 444150 + | > loss_disc: 2.418952703475952 (2.4639442501396966) + | > loss_disc_real_0: 0.10185765475034714 (0.1287947233830545) + | > loss_disc_real_1: 0.17652414739131927 (0.2056531922827507) + | > loss_disc_real_2: 0.21773387491703033 (0.225513547530462) + | > loss_disc_real_3: 0.25126540660858154 (0.22820168309684458) + | > loss_disc_real_4: 0.19874714314937592 (0.22791916581577268) + | > loss_disc_real_5: 0.2410147339105606 (0.23333141901369753) + | > loss_0: 2.418952703475952 (2.4639442501396966) + | > grad_norm_0: tensor(18.4791, device='cuda:0') (tensor(8.0587, device='cuda:0')) + | > loss_gen: 2.38950252532959 (2.341120197855193) + | > loss_kl: 1.8225027322769165 (1.8354451163061734) + | > loss_feat: 7.010702133178711 (7.109804367196971) + | > loss_mel: 16.396753311157227 (17.65419412481373) + | > loss_duration: 1.4453811645507812 (1.447118040816537) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.064842224121094 (30.387681878846266) + | > grad_norm_1: tensor(483.3342, device='cuda:0') (tensor(156.8311, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.429 (0.37626334099934017) + | > loader_time: 0.0073 (0.006005067249824259) + + + --> TIME: 2025-04-19 06:29:02 -- STEP: 141/142 -- GLOBAL_STEP: 444175 + | > loss_disc: 2.3182661533355713 (2.470051998787738) + | > loss_disc_real_0: 0.043612536042928696 (0.12855428134930044) + | > loss_disc_real_1: 0.14390790462493896 (0.20668055138266686) + | > loss_disc_real_2: 0.22139139473438263 (0.22642344183532903) + | > loss_disc_real_3: 0.22291916608810425 (0.2285365360848447) + | > loss_disc_real_4: 0.21060273051261902 (0.22833360668192518) + | > loss_disc_real_5: 0.22213612496852875 (0.23280666493777688) + | > loss_0: 2.3182661533355713 (2.470051998787738) + | > grad_norm_0: tensor(24.5066, device='cuda:0') (tensor(9.6784, device='cuda:0')) + | > loss_gen: 2.6161105632781982 (2.343464560542546) + | > loss_kl: 1.6430095434188843 (1.845664453844652) + | > loss_feat: 9.178000450134277 (7.106507098421138) + | > loss_mel: 19.13111114501953 (17.627851601188066) + | > loss_duration: 1.3187243938446045 (1.4532075807557883) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 33.886959075927734 (30.376695362388666) + | > grad_norm_1: tensor(708.2303, device='cuda:0') (tensor(185.0447, device='cuda:0')) + | > current_lr_0: 0.00018891814079513542 + | > current_lr_1: 0.00018891814079513542 + | > step_time: 0.2004 (0.3864924417319873) + | > loader_time: 0.0029 (0.006234498734169817) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6131162643432617 (2.6131162643432617) + | > loss_disc_real_0: 0.11675053089857101 (0.11675053089857101) + | > loss_disc_real_1: 0.22933730483055115 (0.22933730483055115) + | > loss_disc_real_2: 0.23769935965538025 (0.23769935965538025) + | > loss_disc_real_3: 0.24106813967227936 (0.24106813967227936) + | > loss_disc_real_4: 0.23666910827159882 (0.23666910827159882) + | > loss_disc_real_5: 0.21459558606147766 (0.21459558606147766) + | > loss_0: 2.6131162643432617 (2.6131162643432617) + | > loss_gen: 2.104060173034668 (2.104060173034668) + | > loss_kl: 2.361623525619507 (2.361623525619507) + | > loss_feat: 6.580374717712402 (6.580374717712402) + | > loss_mel: 17.94260597229004 (17.94260597229004) + | > loss_duration: 1.7580962181091309 (1.7580962181091309) + | > loss_1: 30.746763229370117 (30.746763229370117) + + --> STEP: 1 + | > loss_disc: 2.537386178970337 (2.537386178970337) + | > loss_disc_real_0: 0.0862867459654808 (0.0862867459654808) + | > loss_disc_real_1: 0.26851674914360046 (0.26851674914360046) + | > loss_disc_real_2: 0.237029030919075 (0.237029030919075) + | > loss_disc_real_3: 0.24393892288208008 (0.24393892288208008) + | > loss_disc_real_4: 0.25590139627456665 (0.25590139627456665) + | > loss_disc_real_5: 0.2057693898677826 (0.2057693898677826) + | > loss_0: 2.537386178970337 (2.537386178970337) + | > loss_gen: 2.1611671447753906 (2.1611671447753906) + | > loss_kl: 2.7695674896240234 (2.7695674896240234) + | > loss_feat: 7.358769416809082 (7.358769416809082) + | > loss_mel: 17.955989837646484 (17.955989837646484) + | > loss_duration: 1.7987313270568848 (1.7987313270568848) + | > loss_1: 32.044227600097656 (32.044227600097656) + + --> STEP: 2 + | > loss_disc: 2.6568713188171387 (2.597128748893738) + | > loss_disc_real_0: 0.11679068207740784 (0.10153871402144432) + | > loss_disc_real_1: 0.2693994343280792 (0.26895809173583984) + | > loss_disc_real_2: 0.23633542656898499 (0.23668222874403) + | > loss_disc_real_3: 0.208149254322052 (0.22604408860206604) + | > loss_disc_real_4: 0.24325358867645264 (0.24957749247550964) + | > loss_disc_real_5: 0.21895119547843933 (0.21236029267311096) + | > loss_0: 2.6568713188171387 (2.597128748893738) + | > loss_gen: 1.9921939373016357 (2.076680541038513) + | > loss_kl: 2.4091148376464844 (2.589341163635254) + | > loss_feat: 6.063104152679443 (6.710936784744263) + | > loss_mel: 17.42225456237793 (17.689122200012207) + | > loss_duration: 1.7157258987426758 (1.7572286128997803) + | > loss_1: 29.602394104003906 (30.82331085205078) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003819704055786133 (+0.00016200542449951172) + | > avg_loss_disc: 2.597128748893738 (+0.15231215953826904) + | > avg_loss_disc_real_0: 0.10153871402144432 (-0.09656810387969017) + | > avg_loss_disc_real_1: 0.26895809173583984 (+0.05062704533338547) + | > avg_loss_disc_real_2: 0.23668222874403 (+0.026990018784999847) + | > avg_loss_disc_real_3: 0.22604408860206604 (+0.006073690950870514) + | > avg_loss_disc_real_4: 0.24957749247550964 (+0.019684962928295135) + | > avg_loss_disc_real_5: 0.21236029267311096 (-0.026066608726978302) + | > avg_loss_0: 2.597128748893738 (+0.15231215953826904) + | > avg_loss_gen: 2.076680541038513 (-0.4432297945022583) + | > avg_loss_kl: 2.589341163635254 (-0.08205020427703857) + | > avg_loss_feat: 6.710936784744263 (-0.6178748607635498) + | > avg_loss_mel: 17.689122200012207 (-1.3076753616333008) + | > avg_loss_duration: 1.7572286128997803 (-0.02189624309539795) + | > avg_loss_1: 30.82331085205078 (-2.4727249145507812) + + + > EPOCH: 457/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:29:04)  + + --> TIME: 2025-04-19 06:29:13 -- STEP: 24/142 -- GLOBAL_STEP: 444200 + | > loss_disc: 2.6232104301452637 (2.4342599709828696) + | > loss_disc_real_0: 0.11835141479969025 (0.11743905302137136) + | > loss_disc_real_1: 0.22870460152626038 (0.20660331721107164) + | > loss_disc_real_2: 0.240560844540596 (0.22678261312345663) + | > loss_disc_real_3: 0.23348356783390045 (0.22766360640525818) + | > loss_disc_real_4: 0.23580341041088104 (0.22544641916950545) + | > loss_disc_real_5: 0.25142523646354675 (0.23177092149853706) + | > loss_0: 2.6232104301452637 (2.4342599709828696) + | > grad_norm_0: tensor(4.6475, device='cuda:0') (tensor(9.8196, device='cuda:0')) + | > loss_gen: 2.2623276710510254 (2.375791847705841) + | > loss_kl: 1.8263778686523438 (1.8065549284219742) + | > loss_feat: 7.131890296936035 (7.2486756046613054) + | > loss_mel: 18.313312530517578 (17.553269863128662) + | > loss_duration: 1.4442024230957031 (1.4106407910585403) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.97810935974121 (30.394932746887207) + | > grad_norm_1: tensor(123.6988, device='cuda:0') (tensor(248.2234, device='cuda:0')) + | > current_lr_0: 0.000188894526027536 + | > current_lr_1: 0.000188894526027536 + | > step_time: 0.3454 (0.3330598970254262) + | > loader_time: 0.0054 (0.0049088001251220694) + + + --> TIME: 2025-04-19 06:29:22 -- STEP: 49/142 -- GLOBAL_STEP: 444225 + | > loss_disc: 2.510589122772217 (2.442713829935813) + | > loss_disc_real_0: 0.11610247939825058 (0.11735206447085555) + | > loss_disc_real_1: 0.23824548721313477 (0.20520724021658607) + | > loss_disc_real_2: 0.2590026557445526 (0.22482419287671848) + | > loss_disc_real_3: 0.23196548223495483 (0.227828178478747) + | > loss_disc_real_4: 0.2429361194372177 (0.22853601769525178) + | > loss_disc_real_5: 0.2643563151359558 (0.23090049928548384) + | > loss_0: 2.510589122772217 (2.442713829935813) + | > grad_norm_0: tensor(9.0610, device='cuda:0') (tensor(8.6283, device='cuda:0')) + | > loss_gen: 2.336723804473877 (2.3659104376423117) + | > loss_kl: 1.9121570587158203 (1.8228455368353396) + | > loss_feat: 7.085967063903809 (7.22333101350434) + | > loss_mel: 18.40625762939453 (17.6142848657102) + | > loss_duration: 1.440617561340332 (1.4202503111897682) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.181720733642578 (30.446622070001098) + | > grad_norm_1: tensor(225.1858, device='cuda:0') (tensor(234.9190, device='cuda:0')) + | > current_lr_0: 0.000188894526027536 + | > current_lr_1: 0.000188894526027536 + | > step_time: 0.3715 (0.3458335691568803) + | > loader_time: 0.0054 (0.005214014831854373) + + + --> TIME: 2025-04-19 06:29:32 -- STEP: 74/142 -- GLOBAL_STEP: 444250 + | > loss_disc: 2.5391814708709717 (2.4497781540896435) + | > loss_disc_real_0: 0.15719188749790192 (0.12113234964576927) + | > loss_disc_real_1: 0.21704287827014923 (0.2053918304878312) + | > loss_disc_real_2: 0.20570886135101318 (0.22425647804865967) + | > loss_disc_real_3: 0.2613086998462677 (0.22789819679550222) + | > loss_disc_real_4: 0.21403180062770844 (0.2285735043319496) + | > loss_disc_real_5: 0.2435937523841858 (0.23400668397143082) + | > loss_0: 2.5391814708709717 (2.4497781540896435) + | > grad_norm_0: tensor(10.5410, device='cuda:0') (tensor(8.3531, device='cuda:0')) + | > loss_gen: 2.269362449645996 (2.3631356084668953) + | > loss_kl: 1.879790186882019 (1.8353173072273667) + | > loss_feat: 6.352825164794922 (7.1785816695239095) + | > loss_mel: 16.81485366821289 (17.517048397579714) + | > loss_duration: 1.4606623649597168 (1.4326986283869358) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.777496337890625 (30.326781556412982) + | > grad_norm_1: tensor(340.4729, device='cuda:0') (tensor(220.3841, device='cuda:0')) + | > current_lr_0: 0.000188894526027536 + | > current_lr_1: 0.000188894526027536 + | > step_time: 0.3885 (0.35815289213850693) + | > loader_time: 0.0068 (0.005534162392487397) + + + --> TIME: 2025-04-19 06:29:42 -- STEP: 99/142 -- GLOBAL_STEP: 444275 + | > loss_disc: 2.544279098510742 (2.463661304628005) + | > loss_disc_real_0: 0.16343867778778076 (0.12179313268926409) + | > loss_disc_real_1: 0.21457378566265106 (0.20662595677857448) + | > loss_disc_real_2: 0.23607665300369263 (0.2261531057983938) + | > loss_disc_real_3: 0.21642519533634186 (0.22826816879137599) + | > loss_disc_real_4: 0.2167525738477707 (0.2296408459995732) + | > loss_disc_real_5: 0.22417080402374268 (0.23514374970185636) + | > loss_0: 2.544279098510742 (2.463661304628005) + | > grad_norm_0: tensor(10.9487, device='cuda:0') (tensor(9.2548, device='cuda:0')) + | > loss_gen: 2.1241538524627686 (2.3490851840587577) + | > loss_kl: 1.8981647491455078 (1.849154780609439) + | > loss_feat: 5.305966854095459 (7.070740333711258) + | > loss_mel: 15.081286430358887 (17.461446270798195) + | > loss_duration: 1.4831199645996094 (1.4427028485018798) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 25.892690658569336 (30.173129361085216) + | > grad_norm_1: tensor(303.2115, device='cuda:0') (tensor(233.9211, device='cuda:0')) + | > current_lr_0: 0.000188894526027536 + | > current_lr_1: 0.000188894526027536 + | > step_time: 0.4039 (0.36847905438355727) + | > loader_time: 0.0066 (0.0057683159606625335) + + + --> TIME: 2025-04-19 06:29:53 -- STEP: 124/142 -- GLOBAL_STEP: 444300 + | > loss_disc: 2.518578052520752 (2.4620128658510017) + | > loss_disc_real_0: 0.12767115235328674 (0.1219040002794035) + | > loss_disc_real_1: 0.21131519973278046 (0.2068318042062944) + | > loss_disc_real_2: 0.2246248871088028 (0.2255419677784366) + | > loss_disc_real_3: 0.23158666491508484 (0.22890993195676035) + | > loss_disc_real_4: 0.22478486597537994 (0.22847307269130984) + | > loss_disc_real_5: 0.27264025807380676 (0.23452199550886307) + | > loss_0: 2.518578052520752 (2.4620128658510017) + | > grad_norm_0: tensor(5.6058, device='cuda:0') (tensor(9.2331, device='cuda:0')) + | > loss_gen: 2.3920516967773438 (2.3451477673745926) + | > loss_kl: 1.98077392578125 (1.856208429221184) + | > loss_feat: 7.036041736602783 (7.039852123106679) + | > loss_mel: 17.592864990234375 (17.408649121561368) + | > loss_duration: 1.5052623748779297 (1.4502994206643873) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.506994247436523 (30.100156860966838) + | > grad_norm_1: tensor(323.8721, device='cuda:0') (tensor(229.3615, device='cuda:0')) + | > current_lr_0: 0.000188894526027536 + | > current_lr_1: 0.000188894526027536 + | > step_time: 0.4404 (0.38022174758295857) + | > loader_time: 0.0072 (0.005990649423291606) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4617598056793213 (2.4617598056793213) + | > loss_disc_real_0: 0.2115529328584671 (0.2115529328584671) + | > loss_disc_real_1: 0.216237872838974 (0.216237872838974) + | > loss_disc_real_2: 0.2355874478816986 (0.2355874478816986) + | > loss_disc_real_3: 0.2123109996318817 (0.2123109996318817) + | > loss_disc_real_4: 0.25290974974632263 (0.25290974974632263) + | > loss_disc_real_5: 0.22112129628658295 (0.22112129628658295) + | > loss_0: 2.4617598056793213 (2.4617598056793213) + | > loss_gen: 2.376990795135498 (2.376990795135498) + | > loss_kl: 2.4881129264831543 (2.4881129264831543) + | > loss_feat: 6.490076065063477 (6.490076065063477) + | > loss_mel: 17.507238388061523 (17.507238388061523) + | > loss_duration: 1.7487295866012573 (1.7487295866012573) + | > loss_1: 30.611148834228516 (30.611148834228516) + + --> STEP: 1 + | > loss_disc: 2.5849037170410156 (2.5849037170410156) + | > loss_disc_real_0: 0.22713878750801086 (0.22713878750801086) + | > loss_disc_real_1: 0.2227548211812973 (0.2227548211812973) + | > loss_disc_real_2: 0.24005524814128876 (0.24005524814128876) + | > loss_disc_real_3: 0.2427458018064499 (0.2427458018064499) + | > loss_disc_real_4: 0.25480175018310547 (0.25480175018310547) + | > loss_disc_real_5: 0.25334909558296204 (0.25334909558296204) + | > loss_0: 2.5849037170410156 (2.5849037170410156) + | > loss_gen: 2.366755485534668 (2.366755485534668) + | > loss_kl: 2.8372139930725098 (2.8372139930725098) + | > loss_feat: 6.678465843200684 (6.678465843200684) + | > loss_mel: 18.077146530151367 (18.077146530151367) + | > loss_duration: 1.816875696182251 (1.816875696182251) + | > loss_1: 31.776458740234375 (31.776458740234375) + + --> STEP: 2 + | > loss_disc: 2.6330339908599854 (2.6089688539505005) + | > loss_disc_real_0: 0.24075078964233398 (0.23394478857517242) + | > loss_disc_real_1: 0.21684004366397858 (0.21979743242263794) + | > loss_disc_real_2: 0.24227622151374817 (0.24116573482751846) + | > loss_disc_real_3: 0.19855239987373352 (0.2206491008400917) + | > loss_disc_real_4: 0.2474043071269989 (0.2511030286550522) + | > loss_disc_real_5: 0.21427607536315918 (0.2338125854730606) + | > loss_0: 2.6330339908599854 (2.6089688539505005) + | > loss_gen: 2.252471685409546 (2.309613585472107) + | > loss_kl: 2.4064691066741943 (2.621841549873352) + | > loss_feat: 6.362329006195068 (6.520397424697876) + | > loss_mel: 17.059606552124023 (17.568376541137695) + | > loss_duration: 1.7397830486297607 (1.7783293724060059) + | > loss_1: 29.82065773010254 (30.798558235168457) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035561323165893555 (-0.00026357173919677734) + | > avg_loss_disc: 2.6089688539505005 (+0.011840105056762695) + | > avg_loss_disc_real_0: 0.23394478857517242 (+0.1324060745537281) + | > avg_loss_disc_real_1: 0.21979743242263794 (-0.049160659313201904) + | > avg_loss_disc_real_2: 0.24116573482751846 (+0.004483506083488464) + | > avg_loss_disc_real_3: 0.2206491008400917 (-0.005394987761974335) + | > avg_loss_disc_real_4: 0.2511030286550522 (+0.0015255361795425415) + | > avg_loss_disc_real_5: 0.2338125854730606 (+0.021452292799949646) + | > avg_loss_0: 2.6089688539505005 (+0.011840105056762695) + | > avg_loss_gen: 2.309613585472107 (+0.23293304443359375) + | > avg_loss_kl: 2.621841549873352 (+0.032500386238098145) + | > avg_loss_feat: 6.520397424697876 (-0.19053936004638672) + | > avg_loss_mel: 17.568376541137695 (-0.12074565887451172) + | > avg_loss_duration: 1.7783293724060059 (+0.021100759506225586) + | > avg_loss_1: 30.798558235168457 (-0.02475261688232422) + + + > EPOCH: 458/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:30:03)  + + --> TIME: 2025-04-19 06:30:06 -- STEP: 7/142 -- GLOBAL_STEP: 444325 + | > loss_disc: 2.4028120040893555 (2.4119743279048373) + | > loss_disc_real_0: 0.12141280621290207 (0.11067934760025569) + | > loss_disc_real_1: 0.181880921125412 (0.19512410249028886) + | > loss_disc_real_2: 0.20468054711818695 (0.22881781416279928) + | > loss_disc_real_3: 0.21312899887561798 (0.2275617846420833) + | > loss_disc_real_4: 0.21602210402488708 (0.23566901683807373) + | > loss_disc_real_5: 0.20992013812065125 (0.23105338215827942) + | > loss_0: 2.4028120040893555 (2.4119743279048373) + | > grad_norm_0: tensor(14.9118, device='cuda:0') (tensor(9.1491, device='cuda:0')) + | > loss_gen: 2.40850830078125 (2.4103144236973355) + | > loss_kl: 1.7064324617385864 (1.8071040255682809) + | > loss_feat: 7.3864336013793945 (7.617485931941441) + | > loss_mel: 18.113231658935547 (18.28274236406599) + | > loss_duration: 1.3964735269546509 (1.4032497576304845) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.011077880859375 (31.520896094185964) + | > grad_norm_1: tensor(252.2413, device='cuda:0') (tensor(264.4873, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.3328 (0.3251769883292062) + | > loader_time: 0.004 (0.004255737577165876) + + + --> TIME: 2025-04-19 06:30:15 -- STEP: 32/142 -- GLOBAL_STEP: 444350 + | > loss_disc: 2.4674887657165527 (2.428431950509548) + | > loss_disc_real_0: 0.12943832576274872 (0.11269851226825267) + | > loss_disc_real_1: 0.22149214148521423 (0.20147470198571682) + | > loss_disc_real_2: 0.22797095775604248 (0.22748289536684752) + | > loss_disc_real_3: 0.23186732828617096 (0.22958297515287995) + | > loss_disc_real_4: 0.21624860167503357 (0.227860014885664) + | > loss_disc_real_5: 0.21398207545280457 (0.2308394694700837) + | > loss_0: 2.4674887657165527 (2.428431950509548) + | > grad_norm_0: tensor(8.5643, device='cuda:0') (tensor(9.6586, device='cuda:0')) + | > loss_gen: 2.409457206726074 (2.4015861302614208) + | > loss_kl: 1.9530465602874756 (1.8159030117094517) + | > loss_feat: 6.923289775848389 (7.463812857866287) + | > loss_mel: 17.230215072631836 (17.90500554442406) + | > loss_duration: 1.420868992805481 (1.415667086839676) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.936880111694336 (31.00197422504425) + | > grad_norm_1: tensor(380.5096, device='cuda:0') (tensor(269.1532, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.3556 (0.34089940041303635) + | > loader_time: 0.0063 (0.0052067190408706665) + + + --> TIME: 2025-04-19 06:30:24 -- STEP: 57/142 -- GLOBAL_STEP: 444375 + | > loss_disc: 2.3699347972869873 (2.4318700840598657) + | > loss_disc_real_0: 0.06928824633359909 (0.1165964678583438) + | > loss_disc_real_1: 0.1895119547843933 (0.2027765235357117) + | > loss_disc_real_2: 0.22272968292236328 (0.2247578648098728) + | > loss_disc_real_3: 0.23272523283958435 (0.22776236633459726) + | > loss_disc_real_4: 0.23541823029518127 (0.22547996018016547) + | > loss_disc_real_5: 0.23909002542495728 (0.23171850019379667) + | > loss_0: 2.3699347972869873 (2.4318700840598657) + | > grad_norm_0: tensor(16.6540, device='cuda:0') (tensor(11.0762, device='cuda:0')) + | > loss_gen: 2.626659393310547 (2.404973724432158) + | > loss_kl: 1.8276649713516235 (1.8212700810348779) + | > loss_feat: 6.572179794311523 (7.363501674250553) + | > loss_mel: 15.993316650390625 (17.768593754684716) + | > loss_duration: 1.477158546447754 (1.4262638698544416) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.496978759765625 (30.784602918122943) + | > grad_norm_1: tensor(428.2006, device='cuda:0') (tensor(303.6861, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.379 (0.350599581735176) + | > loader_time: 0.0055 (0.005493063675729853) + + + --> TIME: 2025-04-19 06:30:34 -- STEP: 82/142 -- GLOBAL_STEP: 444400 + | > loss_disc: 2.4948887825012207 (2.453562285841965) + | > loss_disc_real_0: 0.1732662171125412 (0.12032781436857654) + | > loss_disc_real_1: 0.20289023220539093 (0.20604460177625097) + | > loss_disc_real_2: 0.21237491071224213 (0.22585863729075686) + | > loss_disc_real_3: 0.266537606716156 (0.2300886439114082) + | > loss_disc_real_4: 0.22790291905403137 (0.22719067316956637) + | > loss_disc_real_5: 0.2505125403404236 (0.23353596522313794) + | > loss_0: 2.4948887825012207 (2.453562285841965) + | > grad_norm_0: tensor(21.6504, device='cuda:0') (tensor(13.4099, device='cuda:0')) + | > loss_gen: 2.470651865005493 (2.3921261996757686) + | > loss_kl: 2.151113271713257 (1.8345263832952918) + | > loss_feat: 6.89472770690918 (7.2610724902734525) + | > loss_mel: 17.728147506713867 (17.720334320533567) + | > loss_duration: 1.5180940628051758 (1.4360444487594972) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.762733459472656 (30.64410365500101) + | > grad_norm_1: tensor(296.6763, device='cuda:0') (tensor(304.7717, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.3961 (0.36207303768251003) + | > loader_time: 0.0063 (0.005772802887893306) + + + --> TIME: 2025-04-19 06:30:44 -- STEP: 107/142 -- GLOBAL_STEP: 444425 + | > loss_disc: 2.495577573776245 (2.456262893765885) + | > loss_disc_real_0: 0.19018633663654327 (0.1225567320100615) + | > loss_disc_real_1: 0.22610190510749817 (0.2067170308850636) + | > loss_disc_real_2: 0.2065127044916153 (0.22650122085464336) + | > loss_disc_real_3: 0.2184472233057022 (0.2295969939120462) + | > loss_disc_real_4: 0.22557666897773743 (0.22666168672459147) + | > loss_disc_real_5: 0.25653088092803955 (0.23305301911362977) + | > loss_0: 2.495577573776245 (2.456262893765885) + | > grad_norm_0: tensor(25.1390, device='cuda:0') (tensor(14.8916, device='cuda:0')) + | > loss_gen: 2.1777708530426025 (2.383937577220881) + | > loss_kl: 1.8546085357666016 (1.84683380505749) + | > loss_feat: 6.125433444976807 (7.194671247607079) + | > loss_mel: 16.335845947265625 (17.681570177880403) + | > loss_duration: 1.460715413093567 (1.446047361766066) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 27.954374313354492 (30.55306005032263) + | > grad_norm_1: tensor(403.8047, device='cuda:0') (tensor(316.6017, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.4147 (0.3722333239617749) + | > loader_time: 0.0072 (0.006028701211804543) + + + --> TIME: 2025-04-19 06:30:55 -- STEP: 132/142 -- GLOBAL_STEP: 444450 + | > loss_disc: 2.5616517066955566 (2.453393098079796) + | > loss_disc_real_0: 0.11104714870452881 (0.12227725728668949) + | > loss_disc_real_1: 0.2251376509666443 (0.2068650900866046) + | > loss_disc_real_2: 0.27081358432769775 (0.22562558192646864) + | > loss_disc_real_3: 0.23179100453853607 (0.22858402302319353) + | > loss_disc_real_4: 0.21448072791099548 (0.22633109930338283) + | > loss_disc_real_5: 0.25055092573165894 (0.2329960964394338) + | > loss_0: 2.5616517066955566 (2.453393098079796) + | > grad_norm_0: tensor(6.4454, device='cuda:0') (tensor(13.9454, device='cuda:0')) + | > loss_gen: 2.1601948738098145 (2.375709952730121) + | > loss_kl: 1.8652687072753906 (1.8499822625608155) + | > loss_feat: 6.095973491668701 (7.145702921982967) + | > loss_mel: 15.805832862854004 (17.593926321376454) + | > loss_duration: 1.4569900035858154 (1.4521435011516914) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 27.384258270263672 (30.4174648920695) + | > grad_norm_1: tensor(72.2832, device='cuda:0') (tensor(304.6802, device='cuda:0')) + | > current_lr_0: 0.00018887091421178256 + | > current_lr_1: 0.00018887091421178256 + | > step_time: 0.4476 (0.3839545051256816) + | > loader_time: 0.0076 (0.006282049598115864) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.582932472229004 (2.582932472229004) + | > loss_disc_real_0: 0.1265273094177246 (0.1265273094177246) + | > loss_disc_real_1: 0.2967902719974518 (0.2967902719974518) + | > loss_disc_real_2: 0.26601195335388184 (0.26601195335388184) + | > loss_disc_real_3: 0.30508533120155334 (0.30508533120155334) + | > loss_disc_real_4: 0.24731531739234924 (0.24731531739234924) + | > loss_disc_real_5: 0.20262815058231354 (0.20262815058231354) + | > loss_0: 2.582932472229004 (2.582932472229004) + | > loss_gen: 2.4016270637512207 (2.4016270637512207) + | > loss_kl: 2.6270551681518555 (2.6270551681518555) + | > loss_feat: 7.212264537811279 (7.212264537811279) + | > loss_mel: 18.559900283813477 (18.559900283813477) + | > loss_duration: 1.7977339029312134 (1.7977339029312134) + | > loss_1: 32.59857940673828 (32.59857940673828) + + --> STEP: 1 + | > loss_disc: 2.5992870330810547 (2.5992870330810547) + | > loss_disc_real_0: 0.21073012053966522 (0.21073012053966522) + | > loss_disc_real_1: 0.25807589292526245 (0.25807589292526245) + | > loss_disc_real_2: 0.26666882634162903 (0.26666882634162903) + | > loss_disc_real_3: 0.26932841539382935 (0.26932841539382935) + | > loss_disc_real_4: 0.23217786848545074 (0.23217786848545074) + | > loss_disc_real_5: 0.25232887268066406 (0.25232887268066406) + | > loss_0: 2.5992870330810547 (2.5992870330810547) + | > loss_gen: 2.424232244491577 (2.424232244491577) + | > loss_kl: 2.8546411991119385 (2.8546411991119385) + | > loss_feat: 6.579986572265625 (6.579986572265625) + | > loss_mel: 17.278249740600586 (17.278249740600586) + | > loss_duration: 1.802856683731079 (1.802856683731079) + | > loss_1: 30.939966201782227 (30.939966201782227) + + --> STEP: 2 + | > loss_disc: 2.5970711708068848 (2.5981791019439697) + | > loss_disc_real_0: 0.23617860674858093 (0.22345436364412308) + | > loss_disc_real_1: 0.24680642783641815 (0.2524411603808403) + | > loss_disc_real_2: 0.2541360557079315 (0.2604024410247803) + | > loss_disc_real_3: 0.2901875972747803 (0.2797580063343048) + | > loss_disc_real_4: 0.2480950802564621 (0.24013647437095642) + | > loss_disc_real_5: 0.22978927195072174 (0.2410590723156929) + | > loss_0: 2.5970711708068848 (2.5981791019439697) + | > loss_gen: 2.553675889968872 (2.4889540672302246) + | > loss_kl: 2.4112517833709717 (2.632946491241455) + | > loss_feat: 7.003848552703857 (6.791917562484741) + | > loss_mel: 18.56678009033203 (17.92251491546631) + | > loss_duration: 1.712799310684204 (1.7578279972076416) + | > loss_1: 32.248355865478516 (31.59416103363037) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003393411636352539 (-0.0001627206802368164) + | > avg_loss_disc: 2.5981791019439697 (-0.010789752006530762) + | > avg_loss_disc_real_0: 0.22345436364412308 (-0.010490424931049347) + | > avg_loss_disc_real_1: 0.2524411603808403 (+0.03264372795820236) + | > avg_loss_disc_real_2: 0.2604024410247803 (+0.01923670619726181) + | > avg_loss_disc_real_3: 0.2797580063343048 (+0.059108905494213104) + | > avg_loss_disc_real_4: 0.24013647437095642 (-0.010966554284095764) + | > avg_loss_disc_real_5: 0.2410590723156929 (+0.007246486842632294) + | > avg_loss_0: 2.5981791019439697 (-0.010789752006530762) + | > avg_loss_gen: 2.4889540672302246 (+0.17934048175811768) + | > avg_loss_kl: 2.632946491241455 (+0.011104941368103027) + | > avg_loss_feat: 6.791917562484741 (+0.27152013778686523) + | > avg_loss_mel: 17.92251491546631 (+0.3541383743286133) + | > avg_loss_duration: 1.7578279972076416 (-0.020501375198364258) + | > avg_loss_1: 31.59416103363037 (+0.7956027984619141) + + + > EPOCH: 459/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:31:02)  + + --> TIME: 2025-04-19 06:31:07 -- STEP: 15/142 -- GLOBAL_STEP: 444475 + | > loss_disc: 2.3723559379577637 (2.397155253092448) + | > loss_disc_real_0: 0.11774881184101105 (0.11824581027030945) + | > loss_disc_real_1: 0.19017891585826874 (0.19674443900585176) + | > loss_disc_real_2: 0.2190827876329422 (0.22498512168725332) + | > loss_disc_real_3: 0.19835102558135986 (0.23092130621274312) + | > loss_disc_real_4: 0.199661985039711 (0.22056512931982677) + | > loss_disc_real_5: 0.20865824818611145 (0.23327811459700268) + | > loss_0: 2.3723559379577637 (2.397155253092448) + | > grad_norm_0: tensor(6.0938, device='cuda:0') (tensor(11.9266, device='cuda:0')) + | > loss_gen: 2.358550548553467 (2.408074140548706) + | > loss_kl: 1.790099024772644 (1.7364586194356282) + | > loss_feat: 7.175683975219727 (7.42528657913208) + | > loss_mel: 16.610097885131836 (17.562424405415854) + | > loss_duration: 1.4075435400009155 (1.4035426775614421) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.341976165771484 (30.535786692301432) + | > grad_norm_1: tensor(46.4279, device='cuda:0') (tensor(282.2587, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.3332 (0.32654825846354163) + | > loader_time: 0.0052 (0.004606485366821289) + + + --> TIME: 2025-04-19 06:31:16 -- STEP: 40/142 -- GLOBAL_STEP: 444500 + | > loss_disc: 2.4734952449798584 (2.435052335262298) + | > loss_disc_real_0: 0.10920973867177963 (0.1165379973128438) + | > loss_disc_real_1: 0.26530203223228455 (0.20339900068938732) + | > loss_disc_real_2: 0.1921692043542862 (0.22726592160761355) + | > loss_disc_real_3: 0.23380976915359497 (0.23300050012767315) + | > loss_disc_real_4: 0.2302815169095993 (0.22832869328558444) + | > loss_disc_real_5: 0.23312059044837952 (0.23334326669573785) + | > loss_0: 2.4734952449798584 (2.435052335262298) + | > grad_norm_0: tensor(12.7514, device='cuda:0') (tensor(11.3321, device='cuda:0')) + | > loss_gen: 2.153573989868164 (2.3806606113910678) + | > loss_kl: 1.901059865951538 (1.7825115263462066) + | > loss_feat: 7.043773174285889 (7.31518429517746) + | > loss_mel: 16.993074417114258 (17.513838315010066) + | > loss_duration: 1.394399642944336 (1.417333859205246) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.485881805419922 (30.40952863693237) + | > grad_norm_1: tensor(309.5489, device='cuda:0') (tensor(295.0608, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.3617 (0.3400975763797759) + | > loader_time: 0.0059 (0.005077075958251952) + + + --> TIME: 2025-04-19 06:31:26 -- STEP: 65/142 -- GLOBAL_STEP: 444525 + | > loss_disc: 2.4708919525146484 (2.4427083015441893) + | > loss_disc_real_0: 0.12584786117076874 (0.12087475657463073) + | > loss_disc_real_1: 0.1944470852613449 (0.20374316985790547) + | > loss_disc_real_2: 0.23420751094818115 (0.22533410237385676) + | > loss_disc_real_3: 0.23135298490524292 (0.2305887630352607) + | > loss_disc_real_4: 0.2325555980205536 (0.22724219950345848) + | > loss_disc_real_5: 0.24883601069450378 (0.23438631066909202) + | > loss_0: 2.4708919525146484 (2.4427083015441893) + | > grad_norm_0: tensor(10.7984, device='cuda:0') (tensor(10.4525, device='cuda:0')) + | > loss_gen: 2.250458240509033 (2.3697538632612956) + | > loss_kl: 1.8725069761276245 (1.7970802178749672) + | > loss_feat: 7.346611499786377 (7.271489693568303) + | > loss_mel: 18.173324584960938 (17.53221611609825) + | > loss_duration: 1.4376204013824463 (1.4300897451547476) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.080520629882812 (30.400629571767954) + | > grad_norm_1: tensor(413.9485, device='cuda:0') (tensor(261.8612, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.392 (0.3522197870107798) + | > loader_time: 0.0067 (0.005402187200692983) + + + --> TIME: 2025-04-19 06:31:36 -- STEP: 90/142 -- GLOBAL_STEP: 444550 + | > loss_disc: 2.5247557163238525 (2.4497593084971103) + | > loss_disc_real_0: 0.11997417360544205 (0.1228177263504929) + | > loss_disc_real_1: 0.2543104588985443 (0.2049338448378775) + | > loss_disc_real_2: 0.2333759218454361 (0.22461362497674095) + | > loss_disc_real_3: 0.22688858211040497 (0.22892407361004088) + | > loss_disc_real_4: 0.22388474643230438 (0.2274305800596873) + | > loss_disc_real_5: 0.2263769656419754 (0.23400028463866976) + | > loss_0: 2.5247557163238525 (2.4497593084971103) + | > grad_norm_0: tensor(8.2800, device='cuda:0') (tensor(9.7243, device='cuda:0')) + | > loss_gen: 2.361309051513672 (2.355167158444722) + | > loss_kl: 1.962371826171875 (1.8181565059555902) + | > loss_feat: 6.054224967956543 (7.175967889361913) + | > loss_mel: 16.173641204833984 (17.45931754642062) + | > loss_duration: 1.4764429330825806 (1.4407276762856378) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.027990341186523 (30.24933673010932) + | > grad_norm_1: tensor(93.7344, device='cuda:0') (tensor(243.5940, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.4007 (0.36403265794118245) + | > loader_time: 0.0068 (0.005686577161153155) + + + --> TIME: 2025-04-19 06:31:46 -- STEP: 115/142 -- GLOBAL_STEP: 444575 + | > loss_disc: 2.3611793518066406 (2.4563661575317375) + | > loss_disc_real_0: 0.11933298408985138 (0.12411374488602513) + | > loss_disc_real_1: 0.1896209865808487 (0.2052099778600361) + | > loss_disc_real_2: 0.22481505572795868 (0.22462080509766288) + | > loss_disc_real_3: 0.20455718040466309 (0.22951457617075546) + | > loss_disc_real_4: 0.2225818932056427 (0.22751259738984314) + | > loss_disc_real_5: 0.2114890068769455 (0.23407346036123192) + | > loss_0: 2.3611793518066406 (2.4563661575317375) + | > grad_norm_0: tensor(5.5082, device='cuda:0') (tensor(9.3001, device='cuda:0')) + | > loss_gen: 2.5793616771698 (2.350133750749671) + | > loss_kl: 1.8801844120025635 (1.8293848535288935) + | > loss_feat: 7.388198375701904 (7.1549065050871485) + | > loss_mel: 17.357044219970703 (17.502396625021223) + | > loss_duration: 1.484889030456543 (1.4489965013835742) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.689678192138672 (30.28581823266071) + | > grad_norm_1: tensor(196.2861, device='cuda:0') (tensor(221.3984, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.4228 (0.37472295553787893) + | > loader_time: 0.007 (0.005963329646898351) + + + --> TIME: 2025-04-19 06:31:58 -- STEP: 140/142 -- GLOBAL_STEP: 444600 + | > loss_disc: 2.447352647781372 (2.4685103910309922) + | > loss_disc_real_0: 0.1008773222565651 (0.12539950396333419) + | > loss_disc_real_1: 0.18281061947345734 (0.20590596177748272) + | > loss_disc_real_2: 0.19875937700271606 (0.225728326184409) + | > loss_disc_real_3: 0.2067464143037796 (0.2298732089144843) + | > loss_disc_real_4: 0.1817113161087036 (0.22927210352250507) + | > loss_disc_real_5: 0.2120676189661026 (0.23480296454259328) + | > loss_0: 2.447352647781372 (2.4685103910309922) + | > grad_norm_0: tensor(11.1126, device='cuda:0') (tensor(10.1818, device='cuda:0')) + | > loss_gen: 2.273754358291626 (2.3448591589927674) + | > loss_kl: 1.811394453048706 (1.8393941487584795) + | > loss_feat: 6.878362655639648 (7.116369387081692) + | > loss_mel: 17.501211166381836 (17.501058108466005) + | > loss_duration: 1.5106703042984009 (1.455456804377692) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.975391387939453 (30.257137639181956) + | > grad_norm_1: tensor(333.5891, device='cuda:0') (tensor(232.4085, device='cuda:0')) + | > current_lr_0: 0.00018884730534750609 + | > current_lr_1: 0.00018884730534750609 + | > step_time: 0.4538 (0.38675096205302645) + | > loader_time: 0.0074 (0.006244499342782155) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.5236663818359375 (2.5236663818359375) + | > loss_disc_real_0: 0.19731824100017548 (0.19731824100017548) + | > loss_disc_real_1: 0.2685859799385071 (0.2685859799385071) + | > loss_disc_real_2: 0.21742188930511475 (0.21742188930511475) + | > loss_disc_real_3: 0.2550094723701477 (0.2550094723701477) + | > loss_disc_real_4: 0.2026381939649582 (0.2026381939649582) + | > loss_disc_real_5: 0.232225239276886 (0.232225239276886) + | > loss_0: 2.5236663818359375 (2.5236663818359375) + | > loss_gen: 2.505960702896118 (2.505960702896118) + | > loss_kl: 2.557143211364746 (2.557143211364746) + | > loss_feat: 7.180980205535889 (7.180980205535889) + | > loss_mel: 18.720849990844727 (18.720849990844727) + | > loss_duration: 1.7716436386108398 (1.7716436386108398) + | > loss_1: 32.736576080322266 (32.736576080322266) + + --> STEP: 1 + | > loss_disc: 2.633073568344116 (2.633073568344116) + | > loss_disc_real_0: 0.21800532937049866 (0.21800532937049866) + | > loss_disc_real_1: 0.2794140577316284 (0.2794140577316284) + | > loss_disc_real_2: 0.23971018195152283 (0.23971018195152283) + | > loss_disc_real_3: 0.2428332269191742 (0.2428332269191742) + | > loss_disc_real_4: 0.23686692118644714 (0.23686692118644714) + | > loss_disc_real_5: 0.2398785650730133 (0.2398785650730133) + | > loss_0: 2.633073568344116 (2.633073568344116) + | > loss_gen: 2.392777919769287 (2.392777919769287) + | > loss_kl: 2.8349220752716064 (2.8349220752716064) + | > loss_feat: 5.334852695465088 (5.334852695465088) + | > loss_mel: 14.967296600341797 (14.967296600341797) + | > loss_duration: 1.8087387084960938 (1.8087387084960938) + | > loss_1: 27.33858871459961 (27.33858871459961) + + --> STEP: 2 + | > loss_disc: 2.4989173412323 (2.565995454788208) + | > loss_disc_real_0: 0.2958698868751526 (0.2569376081228256) + | > loss_disc_real_1: 0.2597113847732544 (0.2695627212524414) + | > loss_disc_real_2: 0.23543444275856018 (0.2375723123550415) + | > loss_disc_real_3: 0.21814598143100739 (0.2304896041750908) + | > loss_disc_real_4: 0.21557272970676422 (0.22621982544660568) + | > loss_disc_real_5: 0.21166841685771942 (0.22577349096536636) + | > loss_0: 2.4989173412323 (2.565995454788208) + | > loss_gen: 2.4380245208740234 (2.4154012203216553) + | > loss_kl: 2.5760152339935303 (2.7054686546325684) + | > loss_feat: 5.255363464355469 (5.295108079910278) + | > loss_mel: 15.149054527282715 (15.058175563812256) + | > loss_duration: 1.7420823574066162 (1.775410532951355) + | > loss_1: 27.160541534423828 (27.24956512451172) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034766197204589844 (+8.320808410644531e-05) + | > avg_loss_disc: 2.565995454788208 (-0.03218364715576172) + | > avg_loss_disc_real_0: 0.2569376081228256 (+0.033483244478702545) + | > avg_loss_disc_real_1: 0.2695627212524414 (+0.017121560871601105) + | > avg_loss_disc_real_2: 0.2375723123550415 (-0.02283012866973877) + | > avg_loss_disc_real_3: 0.2304896041750908 (-0.04926840215921402) + | > avg_loss_disc_real_4: 0.22621982544660568 (-0.013916648924350739) + | > avg_loss_disc_real_5: 0.22577349096536636 (-0.015285581350326538) + | > avg_loss_0: 2.565995454788208 (-0.03218364715576172) + | > avg_loss_gen: 2.4154012203216553 (-0.07355284690856934) + | > avg_loss_kl: 2.7054686546325684 (+0.07252216339111328) + | > avg_loss_feat: 5.295108079910278 (-1.496809482574463) + | > avg_loss_mel: 15.058175563812256 (-2.8643393516540527) + | > avg_loss_duration: 1.775410532951355 (+0.01758253574371338) + | > avg_loss_1: 27.24956512451172 (-4.344595909118652) + + + > EPOCH: 460/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:32:00)  + + --> TIME: 2025-04-19 06:32:08 -- STEP: 23/142 -- GLOBAL_STEP: 444625 + | > loss_disc: 2.437699556350708 (2.4115084979845127) + | > loss_disc_real_0: 0.1190195083618164 (0.11186631315428278) + | > loss_disc_real_1: 0.19820059835910797 (0.20540259065835373) + | > loss_disc_real_2: 0.2137877345085144 (0.2228244115477023) + | > loss_disc_real_3: 0.20245936512947083 (0.22312747849070508) + | > loss_disc_real_4: 0.21464337408542633 (0.2287518550520358) + | > loss_disc_real_5: 0.20948053896427155 (0.2282289992208066) + | > loss_0: 2.437699556350708 (2.4115084979845127) + | > grad_norm_0: tensor(3.1363, device='cuda:0') (tensor(9.4400, device='cuda:0')) + | > loss_gen: 2.4032840728759766 (2.393035131952037) + | > loss_kl: 1.7319741249084473 (1.7787761636402295) + | > loss_feat: 7.851325035095215 (7.383198530777641) + | > loss_mel: 19.498098373413086 (17.578732449075446) + | > loss_duration: 1.4286484718322754 (1.411605197450389) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.913330078125 (30.545347047888715) + | > grad_norm_1: tensor(93.4195, device='cuda:0') (tensor(190.3395, device='cuda:0')) + | > current_lr_0: 0.00018882369943433764 + | > current_lr_1: 0.00018882369943433764 + | > step_time: 0.3509 (0.3330527699511984) + | > loader_time: 0.0054 (0.004933657853499703) + + + --> TIME: 2025-04-19 06:32:18 -- STEP: 48/142 -- GLOBAL_STEP: 444650 + | > loss_disc: 2.4443717002868652 (2.438296566406885) + | > loss_disc_real_0: 0.11833536624908447 (0.11433628962064783) + | > loss_disc_real_1: 0.23761288821697235 (0.2080515017732978) + | > loss_disc_real_2: 0.21171514689922333 (0.22588716385265192) + | > loss_disc_real_3: 0.21189433336257935 (0.225148964052399) + | > loss_disc_real_4: 0.2517399787902832 (0.23052965415020785) + | > loss_disc_real_5: 0.24159806966781616 (0.2296559720610579) + | > loss_0: 2.4443717002868652 (2.438296566406885) + | > grad_norm_0: tensor(11.5653, device='cuda:0') (tensor(10.2232, device='cuda:0')) + | > loss_gen: 2.4996392726898193 (2.372813865542412) + | > loss_kl: 1.9851487874984741 (1.799501046538353) + | > loss_feat: 7.055319786071777 (7.300773620605469) + | > loss_mel: 17.4339599609375 (17.642118811607364) + | > loss_duration: 1.432279348373413 (1.4222960645953815) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.406347274780273 (30.537503282229107) + | > grad_norm_1: tensor(293.2274, device='cuda:0') (tensor(248.2752, device='cuda:0')) + | > current_lr_0: 0.00018882369943433764 + | > current_lr_1: 0.00018882369943433764 + | > step_time: 0.3687 (0.3537112971146901) + | > loader_time: 0.0056 (0.0054032256205876665) + + + --> TIME: 2025-04-19 06:32:28 -- STEP: 73/142 -- GLOBAL_STEP: 444675 + | > loss_disc: 2.601473808288574 (2.432925563969024) + | > loss_disc_real_0: 0.13314269483089447 (0.11370711691983759) + | > loss_disc_real_1: 0.23179864883422852 (0.2060822405635494) + | > loss_disc_real_2: 0.22327178716659546 (0.22553681939432066) + | > loss_disc_real_3: 0.21011286973953247 (0.22619407307611752) + | > loss_disc_real_4: 0.2225029617547989 (0.23088073485518154) + | > loss_disc_real_5: 0.22213895618915558 (0.2296249664401355) + | > loss_0: 2.601473808288574 (2.432925563969024) + | > grad_norm_0: tensor(14.6682, device='cuda:0') (tensor(10.3749, device='cuda:0')) + | > loss_gen: 2.1684179306030273 (2.3831957235728223) + | > loss_kl: 1.7649102210998535 (1.816520777467179) + | > loss_feat: 6.765468597412109 (7.325337390377097) + | > loss_mel: 17.14634895324707 (17.71272122370054) + | > loss_duration: 1.484527587890625 (1.4342114631443807) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.329673767089844 (30.67198651457486) + | > grad_norm_1: tensor(473.6023, device='cuda:0') (tensor(268.2475, device='cuda:0')) + | > current_lr_0: 0.00018882369943433764 + | > current_lr_1: 0.00018882369943433764 + | > step_time: 0.4058 (0.3663379133564152) + | > loader_time: 0.0066 (0.005708335197135193) + + + --> TIME: 2025-04-19 06:32:38 -- STEP: 98/142 -- GLOBAL_STEP: 444700 + | > loss_disc: 2.474114418029785 (2.452316094417962) + | > loss_disc_real_0: 0.11242508143186569 (0.11722102016210556) + | > loss_disc_real_1: 0.1916351169347763 (0.20793285965919495) + | > loss_disc_real_2: 0.23288261890411377 (0.2273740637667325) + | > loss_disc_real_3: 0.23218801617622375 (0.22701781775270188) + | > loss_disc_real_4: 0.23334352672100067 (0.23095874153837867) + | > loss_disc_real_5: 0.185249462723732 (0.23070908702757895) + | > loss_0: 2.474114418029785 (2.452316094417962) + | > grad_norm_0: tensor(7.4894, device='cuda:0') (tensor(11.5217, device='cuda:0')) + | > loss_gen: 2.3335013389587402 (2.377445712381482) + | > loss_kl: 1.9846702814102173 (1.829660826799821) + | > loss_feat: 7.452983379364014 (7.258340149509664) + | > loss_mel: 16.78694725036621 (17.64608824009798) + | > loss_duration: 1.4458379745483398 (1.4437420453344072) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.00394058227539 (30.555276831802058) + | > grad_norm_1: tensor(317.8807, device='cuda:0') (tensor(283.9472, device='cuda:0')) + | > current_lr_0: 0.00018882369943433764 + | > current_lr_1: 0.00018882369943433764 + | > step_time: 0.4232 (0.3776993751525879) + | > loader_time: 0.007 (0.00599838519582943) + + + --> TIME: 2025-04-19 06:32:50 -- STEP: 123/142 -- GLOBAL_STEP: 444725 + | > loss_disc: 2.3549468517303467 (2.4581421817221303) + | > loss_disc_real_0: 0.07175227254629135 (0.11892014790356643) + | > loss_disc_real_1: 0.20493006706237793 (0.20721224737845786) + | > loss_disc_real_2: 0.24924860894680023 (0.22811813240613393) + | > loss_disc_real_3: 0.2122829109430313 (0.22774308423201242) + | > loss_disc_real_4: 0.19578616321086884 (0.23055157780162688) + | > loss_disc_real_5: 0.21624526381492615 (0.23285534081420278) + | > loss_0: 2.3549468517303467 (2.4581421817221303) + | > grad_norm_0: tensor(19.8576, device='cuda:0') (tensor(13.0441, device='cuda:0')) + | > loss_gen: 2.365360736846924 (2.3771246758902964) + | > loss_kl: 1.9243489503860474 (1.8398301659560785) + | > loss_feat: 8.002846717834473 (7.22993249815654) + | > loss_mel: 17.665708541870117 (17.581653656998295) + | > loss_duration: 1.4673923254013062 (1.450542567222099) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.425655364990234 (30.4790834101235) + | > grad_norm_1: tensor(366.8024, device='cuda:0') (tensor(297.4189, device='cuda:0')) + | > current_lr_0: 0.00018882369943433764 + | > current_lr_1: 0.00018882369943433764 + | > step_time: 0.4501 (0.3904096847627221) + | > loader_time: 0.0076 (0.006269410373718758) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.554720878601074 (2.554720878601074) + | > loss_disc_real_0: 0.17090509831905365 (0.17090509831905365) + | > loss_disc_real_1: 0.27814286947250366 (0.27814286947250366) + | > loss_disc_real_2: 0.2426229864358902 (0.2426229864358902) + | > loss_disc_real_3: 0.24528570473194122 (0.24528570473194122) + | > loss_disc_real_4: 0.24845296144485474 (0.24845296144485474) + | > loss_disc_real_5: 0.23454473912715912 (0.23454473912715912) + | > loss_0: 2.554720878601074 (2.554720878601074) + | > loss_gen: 2.4063901901245117 (2.4063901901245117) + | > loss_kl: 2.6646625995635986 (2.6646625995635986) + | > loss_feat: 6.3467535972595215 (6.3467535972595215) + | > loss_mel: 19.14943504333496 (19.14943504333496) + | > loss_duration: 1.759998083114624 (1.759998083114624) + | > loss_1: 32.327239990234375 (32.327239990234375) + + --> STEP: 1 + | > loss_disc: 2.597578525543213 (2.597578525543213) + | > loss_disc_real_0: 0.17165663838386536 (0.17165663838386536) + | > loss_disc_real_1: 0.30540964007377625 (0.30540964007377625) + | > loss_disc_real_2: 0.2609264552593231 (0.2609264552593231) + | > loss_disc_real_3: 0.2868199348449707 (0.2868199348449707) + | > loss_disc_real_4: 0.24127179384231567 (0.24127179384231567) + | > loss_disc_real_5: 0.22770090401172638 (0.22770090401172638) + | > loss_0: 2.597578525543213 (2.597578525543213) + | > loss_gen: 2.39355731010437 (2.39355731010437) + | > loss_kl: 2.712317705154419 (2.712317705154419) + | > loss_feat: 7.016890048980713 (7.016890048980713) + | > loss_mel: 18.775894165039062 (18.775894165039062) + | > loss_duration: 1.8167414665222168 (1.8167414665222168) + | > loss_1: 32.71540069580078 (32.71540069580078) + + --> STEP: 2 + | > loss_disc: 2.417880058288574 (2.5077292919158936) + | > loss_disc_real_0: 0.1307705044746399 (0.15121357142925262) + | > loss_disc_real_1: 0.26077842712402344 (0.28309403359889984) + | > loss_disc_real_2: 0.2471240609884262 (0.25402525812387466) + | > loss_disc_real_3: 0.2598787248134613 (0.273349329829216) + | > loss_disc_real_4: 0.22518999874591827 (0.23323089629411697) + | > loss_disc_real_5: 0.2519378066062927 (0.23981935530900955) + | > loss_0: 2.417880058288574 (2.5077292919158936) + | > loss_gen: 2.496173620223999 (2.4448654651641846) + | > loss_kl: 2.312204599380493 (2.512261152267456) + | > loss_feat: 7.69003438949585 (7.353462219238281) + | > loss_mel: 19.39032554626465 (19.083109855651855) + | > loss_duration: 1.7463126182556152 (1.781527042388916) + | > loss_1: 33.63505172729492 (33.17522621154785) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033712387084960938 (-0.00010538101196289062) + | > avg_loss_disc: 2.5077292919158936 (-0.05826616287231445) + | > avg_loss_disc_real_0: 0.15121357142925262 (-0.105724036693573) + | > avg_loss_disc_real_1: 0.28309403359889984 (+0.013531312346458435) + | > avg_loss_disc_real_2: 0.25402525812387466 (+0.01645294576883316) + | > avg_loss_disc_real_3: 0.273349329829216 (+0.042859725654125214) + | > avg_loss_disc_real_4: 0.23323089629411697 (+0.0070110708475112915) + | > avg_loss_disc_real_5: 0.23981935530900955 (+0.014045864343643188) + | > avg_loss_0: 2.5077292919158936 (-0.05826616287231445) + | > avg_loss_gen: 2.4448654651641846 (+0.029464244842529297) + | > avg_loss_kl: 2.512261152267456 (-0.1932075023651123) + | > avg_loss_feat: 7.353462219238281 (+2.058354139328003) + | > avg_loss_mel: 19.083109855651855 (+4.0249342918396) + | > avg_loss_duration: 1.781527042388916 (+0.006116509437561035) + | > avg_loss_1: 33.17522621154785 (+5.925661087036133) + + + > EPOCH: 461/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:33:00)  + + --> TIME: 2025-04-19 06:33:03 -- STEP: 6/142 -- GLOBAL_STEP: 444750 + | > loss_disc: 2.335442066192627 (2.337197780609131) + | > loss_disc_real_0: 0.13125741481781006 (0.11532567565639813) + | > loss_disc_real_1: 0.23824971914291382 (0.19936427970727286) + | > loss_disc_real_2: 0.22354795038700104 (0.2269264260927836) + | > loss_disc_real_3: 0.21739967167377472 (0.21948182582855225) + | > loss_disc_real_4: 0.21212077140808105 (0.23220698038736978) + | > loss_disc_real_5: 0.21438677608966827 (0.20855358491341272) + | > loss_0: 2.335442066192627 (2.337197780609131) + | > grad_norm_0: tensor(7.9490, device='cuda:0') (tensor(8.6579, device='cuda:0')) + | > loss_gen: 2.4841527938842773 (2.451281944910685) + | > loss_kl: 1.876753330230713 (1.8173386653264363) + | > loss_feat: 8.470405578613281 (7.82361634572347) + | > loss_mel: 18.436952590942383 (17.927653948465984) + | > loss_duration: 1.4056956768035889 (1.4037079612414043) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.6739616394043 (31.423598607381184) + | > grad_norm_1: tensor(287.3042, device='cuda:0') (tensor(294.1420, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.3292 (0.3229513963063558) + | > loader_time: 0.0041 (0.004437446594238281) + + + --> TIME: 2025-04-19 06:33:11 -- STEP: 31/142 -- GLOBAL_STEP: 444775 + | > loss_disc: 2.3771796226501465 (2.403398136938772) + | > loss_disc_real_0: 0.07027218490839005 (0.11415014740440153) + | > loss_disc_real_1: 0.17377012968063354 (0.19989769497225363) + | > loss_disc_real_2: 0.23143962025642395 (0.22655297431253618) + | > loss_disc_real_3: 0.21368488669395447 (0.22385272960509023) + | > loss_disc_real_4: 0.24161988496780396 (0.22728455066680908) + | > loss_disc_real_5: 0.24627816677093506 (0.2289239033575981) + | > loss_0: 2.3771796226501465 (2.403398136938772) + | > grad_norm_0: tensor(6.6157, device='cuda:0') (tensor(8.3535, device='cuda:0')) + | > loss_gen: 2.369795322418213 (2.394567035859631) + | > loss_kl: 1.8958145380020142 (1.8319869272170528) + | > loss_feat: 7.840676784515381 (7.48460760424214) + | > loss_mel: 18.02285385131836 (17.566406280763687) + | > loss_duration: 1.418095350265503 (1.4126330844817627) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.5472354888916 (30.690200744136686) + | > grad_norm_1: tensor(191.7240, device='cuda:0') (tensor(208.6048, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.3527 (0.33559142389605123) + | > loader_time: 0.0053 (0.0049612675943682275) + + + --> TIME: 2025-04-19 06:33:20 -- STEP: 56/142 -- GLOBAL_STEP: 444800 + | > loss_disc: 2.418036699295044 (2.4082243910857613) + | > loss_disc_real_0: 0.11159393936395645 (0.11501730346520032) + | > loss_disc_real_1: 0.21552346646785736 (0.20218170806765556) + | > loss_disc_real_2: 0.2208261638879776 (0.2253933088587863) + | > loss_disc_real_3: 0.23236173391342163 (0.22286142435457026) + | > loss_disc_real_4: 0.2542823255062103 (0.22787418136639254) + | > loss_disc_real_5: 0.23706971108913422 (0.22708748893014022) + | > loss_0: 2.418036699295044 (2.4082243910857613) + | > grad_norm_0: tensor(10.8846, device='cuda:0') (tensor(8.3723, device='cuda:0')) + | > loss_gen: 2.4116623401641846 (2.3994995398180836) + | > loss_kl: 1.931849718093872 (1.8336007680211748) + | > loss_feat: 7.3956708908081055 (7.449902858052935) + | > loss_mel: 17.680110931396484 (17.652944479669852) + | > loss_duration: 1.4533357620239258 (1.4235202350786758) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.872631072998047 (30.7594678742545) + | > grad_norm_1: tensor(335.6577, device='cuda:0') (tensor(227.3456, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.3724 (0.3473102109772818) + | > loader_time: 0.0059 (0.005292053733553204) + + + --> TIME: 2025-04-19 06:33:30 -- STEP: 81/142 -- GLOBAL_STEP: 444825 + | > loss_disc: 2.555966854095459 (2.4207423027650816) + | > loss_disc_real_0: 0.09989022463560104 (0.1159847721365499) + | > loss_disc_real_1: 0.24387414753437042 (0.20329179991910487) + | > loss_disc_real_2: 0.2078917920589447 (0.22514175761629035) + | > loss_disc_real_3: 0.23139441013336182 (0.2248698185991358) + | > loss_disc_real_4: 0.25903668999671936 (0.22700867627137972) + | > loss_disc_real_5: 0.2567459046840668 (0.22980266275965136) + | > loss_0: 2.555966854095459 (2.4207423027650816) + | > grad_norm_0: tensor(20.8745, device='cuda:0') (tensor(9.5738, device='cuda:0')) + | > loss_gen: 2.3422818183898926 (2.395971604335456) + | > loss_kl: 1.9217984676361084 (1.8408650277573386) + | > loss_feat: 7.119176864624023 (7.394823798426875) + | > loss_mel: 17.152069091796875 (17.625448827390322) + | > loss_duration: 1.4435350894927979 (1.4344990945156704) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.97886085510254 (30.69160826412248) + | > grad_norm_1: tensor(524.3413, device='cuda:0') (tensor(251.9756, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.3944 (0.3596407160346891) + | > loader_time: 0.0061 (0.0055835924030822) + + + --> TIME: 2025-04-19 06:33:41 -- STEP: 106/142 -- GLOBAL_STEP: 444850 + | > loss_disc: 2.3814239501953125 (2.426029997051888) + | > loss_disc_real_0: 0.12485959380865097 (0.1187114647747773) + | > loss_disc_real_1: 0.21367765963077545 (0.20360898577942038) + | > loss_disc_real_2: 0.20195847749710083 (0.22464659191527456) + | > loss_disc_real_3: 0.2203523963689804 (0.2248400310581585) + | > loss_disc_real_4: 0.17373895645141602 (0.22517809707601116) + | > loss_disc_real_5: 0.24172529578208923 (0.2307061588145652) + | > loss_0: 2.3814239501953125 (2.426029997051888) + | > grad_norm_0: tensor(5.2473, device='cuda:0') (tensor(11.2227, device='cuda:0')) + | > loss_gen: 2.595898389816284 (2.399500450997984) + | > loss_kl: 1.6501721143722534 (1.8530433785240605) + | > loss_feat: 6.984121322631836 (7.32723671535276) + | > loss_mel: 18.051748275756836 (17.62202268276575) + | > loss_duration: 1.4851008653640747 (1.4446973047166498) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.76704216003418 (30.64650055147567) + | > grad_norm_1: tensor(455.9090, device='cuda:0') (tensor(271.1834, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.4108 (0.370314730788177) + | > loader_time: 0.0069 (0.005796023134915334) + + + --> TIME: 2025-04-19 06:33:52 -- STEP: 131/142 -- GLOBAL_STEP: 444875 + | > loss_disc: 2.4377143383026123 (2.4331461109277863) + | > loss_disc_real_0: 0.1345984786748886 (0.12149546468872151) + | > loss_disc_real_1: 0.22116117179393768 (0.2051210378417532) + | > loss_disc_real_2: 0.26559311151504517 (0.2256301386665752) + | > loss_disc_real_3: 0.2290913164615631 (0.22499583559181854) + | > loss_disc_real_4: 0.25884726643562317 (0.2261555373441172) + | > loss_disc_real_5: 0.24211114645004272 (0.22989425245132156) + | > loss_0: 2.4377143383026123 (2.4331461109277863) + | > grad_norm_0: tensor(9.3036, device='cuda:0') (tensor(12.6790, device='cuda:0')) + | > loss_gen: 2.588538646697998 (2.4023777837971707) + | > loss_kl: 1.809099555015564 (1.857960734658569) + | > loss_feat: 7.508921146392822 (7.268940594360118) + | > loss_mel: 17.60247230529785 (17.564458344728898) + | > loss_duration: 1.4715485572814941 (1.4506229071216727) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.980581283569336 (30.544360328266638) + | > grad_norm_1: tensor(407.2939, device='cuda:0') (tensor(280.1754, device='cuda:0')) + | > current_lr_0: 0.00018880009647190835 + | > current_lr_1: 0.00018880009647190835 + | > step_time: 0.4479 (0.3825213818149713) + | > loader_time: 0.007 (0.006047911316383886) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.556790590286255 (2.556790590286255) + | > loss_disc_real_0: 0.19058871269226074 (0.19058871269226074) + | > loss_disc_real_1: 0.28007662296295166 (0.28007662296295166) + | > loss_disc_real_2: 0.23473337292671204 (0.23473337292671204) + | > loss_disc_real_3: 0.25345301628112793 (0.25345301628112793) + | > loss_disc_real_4: 0.2764439582824707 (0.2764439582824707) + | > loss_disc_real_5: 0.24811337888240814 (0.24811337888240814) + | > loss_0: 2.556790590286255 (2.556790590286255) + | > loss_gen: 2.4680027961730957 (2.4680027961730957) + | > loss_kl: 2.476473808288574 (2.476473808288574) + | > loss_feat: 6.703789710998535 (6.703789710998535) + | > loss_mel: 17.19563865661621 (17.19563865661621) + | > loss_duration: 1.749873161315918 (1.749873161315918) + | > loss_1: 30.59377670288086 (30.59377670288086) + + --> STEP: 1 + | > loss_disc: 2.519573450088501 (2.519573450088501) + | > loss_disc_real_0: 0.13694779574871063 (0.13694779574871063) + | > loss_disc_real_1: 0.29129791259765625 (0.29129791259765625) + | > loss_disc_real_2: 0.2429330199956894 (0.2429330199956894) + | > loss_disc_real_3: 0.22739121317863464 (0.22739121317863464) + | > loss_disc_real_4: 0.2211209535598755 (0.2211209535598755) + | > loss_disc_real_5: 0.22817298769950867 (0.22817298769950867) + | > loss_0: 2.519573450088501 (2.519573450088501) + | > loss_gen: 2.4107601642608643 (2.4107601642608643) + | > loss_kl: 2.795656681060791 (2.795656681060791) + | > loss_feat: 6.143736839294434 (6.143736839294434) + | > loss_mel: 17.465850830078125 (17.465850830078125) + | > loss_duration: 1.8280404806137085 (1.8280404806137085) + | > loss_1: 30.644046783447266 (30.644046783447266) + + --> STEP: 2 + | > loss_disc: 2.4736483097076416 (2.4966108798980713) + | > loss_disc_real_0: 0.16745170950889587 (0.15219975262880325) + | > loss_disc_real_1: 0.24927499890327454 (0.2702864557504654) + | > loss_disc_real_2: 0.21808041632175446 (0.23050671815872192) + | > loss_disc_real_3: 0.3082462549209595 (0.26781873404979706) + | > loss_disc_real_4: 0.24551232159137726 (0.23331663757562637) + | > loss_disc_real_5: 0.28308531641960144 (0.25562915205955505) + | > loss_0: 2.4736483097076416 (2.4966108798980713) + | > loss_gen: 2.604487895965576 (2.50762403011322) + | > loss_kl: 2.244387626647949 (2.52002215385437) + | > loss_feat: 6.8119940757751465 (6.47786545753479) + | > loss_mel: 16.57297706604004 (17.019413948059082) + | > loss_duration: 1.7473795413970947 (1.7877100110054016) + | > loss_1: 29.981225967407227 (30.312636375427246) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035774707794189453 (+0.00020623207092285156) + | > avg_loss_disc: 2.4966108798980713 (-0.011118412017822266) + | > avg_loss_disc_real_0: 0.15219975262880325 (+0.0009861811995506287) + | > avg_loss_disc_real_1: 0.2702864557504654 (-0.012807577848434448) + | > avg_loss_disc_real_2: 0.23050671815872192 (-0.02351853996515274) + | > avg_loss_disc_real_3: 0.26781873404979706 (-0.005530595779418945) + | > avg_loss_disc_real_4: 0.23331663757562637 (+8.574128150939941e-05) + | > avg_loss_disc_real_5: 0.25562915205955505 (+0.0158097967505455) + | > avg_loss_0: 2.4966108798980713 (-0.011118412017822266) + | > avg_loss_gen: 2.50762403011322 (+0.06275856494903564) + | > avg_loss_kl: 2.52002215385437 (+0.0077610015869140625) + | > avg_loss_feat: 6.47786545753479 (-0.8755967617034912) + | > avg_loss_mel: 17.019413948059082 (-2.0636959075927734) + | > avg_loss_duration: 1.7877100110054016 (+0.006182968616485596) + | > avg_loss_1: 30.312636375427246 (-2.8625898361206055) + + + > EPOCH: 462/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:33:58)  + + --> TIME: 2025-04-19 06:34:04 -- STEP: 14/142 -- GLOBAL_STEP: 444900 + | > loss_disc: 2.421748638153076 (2.36781188419887) + | > loss_disc_real_0: 0.06899420917034149 (0.10067369416356087) + | > loss_disc_real_1: 0.1882711499929428 (0.19174066611698695) + | > loss_disc_real_2: 0.19836057722568512 (0.22760358772107533) + | > loss_disc_real_3: 0.1992088407278061 (0.22256276117903845) + | > loss_disc_real_4: 0.20430175960063934 (0.22938986867666245) + | > loss_disc_real_5: 0.22468680143356323 (0.2244345439331872) + | > loss_0: 2.421748638153076 (2.36781188419887) + | > grad_norm_0: tensor(9.8691, device='cuda:0') (tensor(7.8411, device='cuda:0')) + | > loss_gen: 2.2621583938598633 (2.3892698117664883) + | > loss_kl: 1.8870905637741089 (1.7949822545051575) + | > loss_feat: 7.829888820648193 (7.529984269823347) + | > loss_mel: 17.79912757873535 (17.846328462873185) + | > loss_duration: 1.4085402488708496 (1.4056349056107658) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.186803817749023 (30.966199329921178) + | > grad_norm_1: tensor(209.6178, device='cuda:0') (tensor(212.4414, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.3423 (0.33969082151140484) + | > loader_time: 0.0046 (0.004483767918178013) + + + --> TIME: 2025-04-19 06:34:13 -- STEP: 39/142 -- GLOBAL_STEP: 444925 + | > loss_disc: 2.4385757446289062 (2.3834560712178554) + | > loss_disc_real_0: 0.1384129524230957 (0.10962813194745626) + | > loss_disc_real_1: 0.19255098700523376 (0.19862859562421456) + | > loss_disc_real_2: 0.23360365629196167 (0.22336874367334905) + | > loss_disc_real_3: 0.24205809831619263 (0.2251369024698551) + | > loss_disc_real_4: 0.2515985369682312 (0.22600240585131523) + | > loss_disc_real_5: 0.22200575470924377 (0.225424152918351) + | > loss_0: 2.4385757446289062 (2.3834560712178554) + | > grad_norm_0: tensor(4.6569, device='cuda:0') (tensor(8.2183, device='cuda:0')) + | > loss_gen: 2.2976577281951904 (2.406831117776724) + | > loss_kl: 1.8831443786621094 (1.811546573272118) + | > loss_feat: 7.830719470977783 (7.5909205338893795) + | > loss_mel: 17.878400802612305 (17.94758933629745) + | > loss_duration: 1.431113362312317 (1.4169702010276986) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.321035385131836 (31.173857713356995) + | > grad_norm_1: tensor(147.3012, device='cuda:0') (tensor(215.5691, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.3564 (0.3447351516821446) + | > loader_time: 0.0053 (0.004976327602679913) + + + --> TIME: 2025-04-19 06:34:22 -- STEP: 64/142 -- GLOBAL_STEP: 444950 + | > loss_disc: 2.3071842193603516 (2.4081294313073167) + | > loss_disc_real_0: 0.11031537503004074 (0.1149052280234173) + | > loss_disc_real_1: 0.17525526881217957 (0.20029528252780437) + | > loss_disc_real_2: 0.20403560996055603 (0.22414597705937922) + | > loss_disc_real_3: 0.18265192210674286 (0.22479315800592303) + | > loss_disc_real_4: 0.21121828258037567 (0.2269105522427708) + | > loss_disc_real_5: 0.24632035195827484 (0.228654314763844) + | > loss_0: 2.3071842193603516 (2.4081294313073167) + | > grad_norm_0: tensor(13.6287, device='cuda:0') (tensor(7.7394, device='cuda:0')) + | > loss_gen: 2.5808908939361572 (2.385542217642068) + | > loss_kl: 1.9448168277740479 (1.8330445867031813) + | > loss_feat: 7.888429641723633 (7.454300291836262) + | > loss_mel: 18.63317108154297 (17.72654712200164) + | > loss_duration: 1.4789648056030273 (1.4276708550751205) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.52627182006836 (30.827104955911636) + | > grad_norm_1: tensor(363.6636, device='cuda:0') (tensor(186.8769, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.381 (0.3560759425163269) + | > loader_time: 0.0061 (0.005328919738531113) + + + --> TIME: 2025-04-19 06:34:32 -- STEP: 89/142 -- GLOBAL_STEP: 444975 + | > loss_disc: 2.5223515033721924 (2.43548290649157) + | > loss_disc_real_0: 0.11541853100061417 (0.11813489138410332) + | > loss_disc_real_1: 0.2510119080543518 (0.20227977616733375) + | > loss_disc_real_2: 0.22260820865631104 (0.22516712278462528) + | > loss_disc_real_3: 0.21190418303012848 (0.22939378358005139) + | > loss_disc_real_4: 0.2279108464717865 (0.2284671464997731) + | > loss_disc_real_5: 0.23862770199775696 (0.2314914113350129) + | > loss_0: 2.5223515033721924 (2.43548290649157) + | > grad_norm_0: tensor(14.0783, device='cuda:0') (tensor(8.1562, device='cuda:0')) + | > loss_gen: 2.1936287879943848 (2.3675713619489356) + | > loss_kl: 1.9554308652877808 (1.853744796152865) + | > loss_feat: 6.657320022583008 (7.324386103769367) + | > loss_mel: 16.691734313964844 (17.608796966209834) + | > loss_duration: 1.4488166570663452 (1.4379615314890826) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.946929931640625 (30.592460696616868) + | > grad_norm_1: tensor(181.2120, device='cuda:0') (tensor(190.8197, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.4023 (0.3668701220094488) + | > loader_time: 0.0062 (0.0055979075056783265) + + + --> TIME: 2025-04-19 06:34:43 -- STEP: 114/142 -- GLOBAL_STEP: 445000 + | > loss_disc: 2.4831042289733887 (2.4444975957535866) + | > loss_disc_real_0: 0.19444826245307922 (0.12084813912709554) + | > loss_disc_real_1: 0.22275041043758392 (0.20328145042846074) + | > loss_disc_real_2: 0.20047935843467712 (0.2252992412500214) + | > loss_disc_real_3: 0.24295556545257568 (0.2287295451551153) + | > loss_disc_real_4: 0.2586493194103241 (0.2281232463163242) + | > loss_disc_real_5: 0.2572956383228302 (0.23215263467608838) + | > loss_0: 2.4831042289733887 (2.4444975957535866) + | > grad_norm_0: tensor(11.8437, device='cuda:0') (tensor(8.8083, device='cuda:0')) + | > loss_gen: 2.4397685527801514 (2.3526254955090975) + | > loss_kl: 1.9727754592895508 (1.8629097352948105) + | > loss_feat: 7.536687850952148 (7.219378245504279) + | > loss_mel: 18.141775131225586 (17.557170600221863) + | > loss_duration: 1.4737300872802734 (1.4459958181046602) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.564735412597656 (30.438079716866476) + | > grad_norm_1: tensor(303.7229, device='cuda:0') (tensor(202.9519, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.4262 (0.37765407143977653) + | > loader_time: 0.0067 (0.005844168495713619) + + + --> TIME: 2025-04-19 06:34:54 -- STEP: 139/142 -- GLOBAL_STEP: 445025 + | > loss_disc: 2.486760139465332 (2.4493897081279075) + | > loss_disc_real_0: 0.12397513538599014 (0.12226839288533163) + | > loss_disc_real_1: 0.2005521059036255 (0.20401348580988188) + | > loss_disc_real_2: 0.21413904428482056 (0.22524447910648457) + | > loss_disc_real_3: 0.20179979503154755 (0.2288860918806611) + | > loss_disc_real_4: 0.20342861115932465 (0.22783181907461703) + | > loss_disc_real_5: 0.22604607045650482 (0.23211849668471934) + | > loss_0: 2.486760139465332 (2.4493897081279075) + | > grad_norm_0: tensor(6.1291, device='cuda:0') (tensor(8.8520, device='cuda:0')) + | > loss_gen: 2.1333470344543457 (2.345795355254798) + | > loss_kl: 1.97516930103302 (1.869536502755803) + | > loss_feat: 7.1125969886779785 (7.177433796066175) + | > loss_mel: 16.903715133666992 (17.525917307078405) + | > loss_duration: 1.4801867008209229 (1.4525323240019432) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.60501480102539 (30.371215147937804) + | > grad_norm_1: tensor(122.6399, device='cuda:0') (tensor(194.9747, device='cuda:0')) + | > current_lr_0: 0.00018877649645984934 + | > current_lr_1: 0.00018877649645984934 + | > step_time: 0.4579 (0.3894222588847867) + | > loader_time: 0.0076 (0.0061131484216923335) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.458225965499878 (2.458225965499878) + | > loss_disc_real_0: 0.2589958608150482 (0.2589958608150482) + | > loss_disc_real_1: 0.2138270139694214 (0.2138270139694214) + | > loss_disc_real_2: 0.25678199529647827 (0.25678199529647827) + | > loss_disc_real_3: 0.234212264418602 (0.234212264418602) + | > loss_disc_real_4: 0.1653272658586502 (0.1653272658586502) + | > loss_disc_real_5: 0.19392849504947662 (0.19392849504947662) + | > loss_0: 2.458225965499878 (2.458225965499878) + | > loss_gen: 2.658860445022583 (2.658860445022583) + | > loss_kl: 2.6052088737487793 (2.6052088737487793) + | > loss_feat: 6.691433906555176 (6.691433906555176) + | > loss_mel: 17.231008529663086 (17.231008529663086) + | > loss_duration: 1.7545881271362305 (1.7545881271362305) + | > loss_1: 30.94110107421875 (30.94110107421875) + + --> STEP: 1 + | > loss_disc: 2.632723331451416 (2.632723331451416) + | > loss_disc_real_0: 0.30875474214553833 (0.30875474214553833) + | > loss_disc_real_1: 0.20741066336631775 (0.20741066336631775) + | > loss_disc_real_2: 0.22328516840934753 (0.22328516840934753) + | > loss_disc_real_3: 0.2483004629611969 (0.2483004629611969) + | > loss_disc_real_4: 0.1985076367855072 (0.1985076367855072) + | > loss_disc_real_5: 0.24959757924079895 (0.24959757924079895) + | > loss_0: 2.632723331451416 (2.632723331451416) + | > loss_gen: 2.505260705947876 (2.505260705947876) + | > loss_kl: 2.914750337600708 (2.914750337600708) + | > loss_feat: 7.305395126342773 (7.305395126342773) + | > loss_mel: 19.02395248413086 (19.02395248413086) + | > loss_duration: 1.8158953189849854 (1.8158953189849854) + | > loss_1: 33.56525421142578 (33.56525421142578) + + --> STEP: 2 + | > loss_disc: 2.480792999267578 (2.556758165359497) + | > loss_disc_real_0: 0.2632766366004944 (0.28601568937301636) + | > loss_disc_real_1: 0.2090287208557129 (0.20821969211101532) + | > loss_disc_real_2: 0.258380264043808 (0.24083271622657776) + | > loss_disc_real_3: 0.23008565604686737 (0.23919305950403214) + | > loss_disc_real_4: 0.20118604600429535 (0.19984684139490128) + | > loss_disc_real_5: 0.23204724490642548 (0.2408224120736122) + | > loss_0: 2.480792999267578 (2.556758165359497) + | > loss_gen: 2.6221842765808105 (2.5637224912643433) + | > loss_kl: 2.3298699855804443 (2.622310161590576) + | > loss_feat: 7.020331859588623 (7.162863492965698) + | > loss_mel: 18.461580276489258 (18.74276638031006) + | > loss_duration: 1.7413079738616943 (1.7786016464233398) + | > loss_1: 32.17527389526367 (32.87026405334473) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037741661071777344 (+0.00019669532775878906) + | > avg_loss_disc: 2.556758165359497 (+0.06014728546142578) + | > avg_loss_disc_real_0: 0.28601568937301636 (+0.1338159367442131) + | > avg_loss_disc_real_1: 0.20821969211101532 (-0.06206676363945007) + | > avg_loss_disc_real_2: 0.24083271622657776 (+0.010325998067855835) + | > avg_loss_disc_real_3: 0.23919305950403214 (-0.028625674545764923) + | > avg_loss_disc_real_4: 0.19984684139490128 (-0.0334697961807251) + | > avg_loss_disc_real_5: 0.2408224120736122 (-0.01480673998594284) + | > avg_loss_0: 2.556758165359497 (+0.06014728546142578) + | > avg_loss_gen: 2.5637224912643433 (+0.05609846115112305) + | > avg_loss_kl: 2.622310161590576 (+0.10228800773620605) + | > avg_loss_feat: 7.162863492965698 (+0.6849980354309082) + | > avg_loss_mel: 18.74276638031006 (+1.7233524322509766) + | > avg_loss_duration: 1.7786016464233398 (-0.009108364582061768) + | > avg_loss_1: 32.87026405334473 (+2.5576276779174805) + + + > EPOCH: 463/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:34:57)  + + --> TIME: 2025-04-19 06:35:05 -- STEP: 22/142 -- GLOBAL_STEP: 445050 + | > loss_disc: 2.4458117485046387 (2.421010039069436) + | > loss_disc_real_0: 0.1393308937549591 (0.11879371529953046) + | > loss_disc_real_1: 0.18099986016750336 (0.2054794031110677) + | > loss_disc_real_2: 0.24167798459529877 (0.2296526073054834) + | > loss_disc_real_3: 0.23320001363754272 (0.2284737012603066) + | > loss_disc_real_4: 0.23521019518375397 (0.22693950615145944) + | > loss_disc_real_5: 0.2661612927913666 (0.22663557732647116) + | > loss_0: 2.4458117485046387 (2.421010039069436) + | > grad_norm_0: tensor(9.2701, device='cuda:0') (tensor(7.2601, device='cuda:0')) + | > loss_gen: 2.3936104774475098 (2.413303115151145) + | > loss_kl: 1.8994325399398804 (1.812887435609644) + | > loss_feat: 7.002212047576904 (7.5988194508986036) + | > loss_mel: 18.39192008972168 (17.927085746418346) + | > loss_duration: 1.3660635948181152 (1.408930859782479) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 31.053237915039062 (31.16102660786022) + | > grad_norm_1: tensor(253.4889, device='cuda:0') (tensor(122.0951, device='cuda:0')) + | > current_lr_0: 0.00018875289939779185 + | > current_lr_1: 0.00018875289939779185 + | > step_time: 0.3474 (0.3314203240654685) + | > loader_time: 0.0059 (0.004799983718178489) + + + --> TIME: 2025-04-19 06:35:14 -- STEP: 47/142 -- GLOBAL_STEP: 445075 + | > loss_disc: 2.3493685722351074 (2.4224006926759762) + | > loss_disc_real_0: 0.11232632398605347 (0.11852036678093544) + | > loss_disc_real_1: 0.20307882130146027 (0.20462149032886992) + | > loss_disc_real_2: 0.2089206725358963 (0.22576058354783565) + | > loss_disc_real_3: 0.21237964928150177 (0.22795861261956235) + | > loss_disc_real_4: 0.15402041375637054 (0.2247468150676565) + | > loss_disc_real_5: 0.19398611783981323 (0.2292969318780493) + | > loss_0: 2.3493685722351074 (2.4224006926759762) + | > grad_norm_0: tensor(5.7726, device='cuda:0') (tensor(6.3433, device='cuda:0')) + | > loss_gen: 2.2677788734436035 (2.3944072774116023) + | > loss_kl: 1.8145394325256348 (1.8418574941919206) + | > loss_feat: 8.018394470214844 (7.52354852189409) + | > loss_mel: 17.49126434326172 (17.92525411159435) + | > loss_duration: 1.3982096910476685 (1.420727217451055) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 30.99018669128418 (31.105794784870554) + | > grad_norm_1: tensor(44.9133, device='cuda:0') (tensor(116.3942, device='cuda:0')) + | > current_lr_0: 0.00018875289939779185 + | > current_lr_1: 0.00018875289939779185 + | > step_time: 0.3627 (0.343880349017204) + | > loader_time: 0.0058 (0.005276923483990609) + + + --> TIME: 2025-04-19 06:35:24 -- STEP: 72/142 -- GLOBAL_STEP: 445100 + | > loss_disc: 2.517815351486206 (2.437727097007964) + | > loss_disc_real_0: 0.1552944928407669 (0.12242722640641862) + | > loss_disc_real_1: 0.21134451031684875 (0.20514444965455267) + | > loss_disc_real_2: 0.22008159756660461 (0.22541187889873981) + | > loss_disc_real_3: 0.229237899184227 (0.22915906831622124) + | > loss_disc_real_4: 0.25287312269210815 (0.22783036074704593) + | > loss_disc_real_5: 0.2540556788444519 (0.23087425219515958) + | > loss_0: 2.517815351486206 (2.437727097007964) + | > grad_norm_0: tensor(5.8849, device='cuda:0') (tensor(7.3639, device='cuda:0')) + | > loss_gen: 2.207261562347412 (2.384418553776211) + | > loss_kl: 1.7598530054092407 (1.8495378212796316) + | > loss_feat: 6.563895225524902 (7.36692695485221) + | > loss_mel: 17.564088821411133 (17.769612166616657) + | > loss_duration: 1.435220718383789 (1.4316154685285358) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.530319213867188 (30.802111122343277) + | > grad_norm_1: tensor(139.8560, device='cuda:0') (tensor(148.8585, device='cuda:0')) + | > current_lr_0: 0.00018875289939779185 + | > current_lr_1: 0.00018875289939779185 + | > step_time: 0.3941 (0.3558578458097245) + | > loader_time: 0.0062 (0.005549086464775933) + + + --> TIME: 2025-04-19 06:35:34 -- STEP: 97/142 -- GLOBAL_STEP: 445125 + | > loss_disc: 2.5579166412353516 (2.449509989355029) + | > loss_disc_real_0: 0.1616574376821518 (0.12564533399706035) + | > loss_disc_real_1: 0.2043169140815735 (0.20602927180295139) + | > loss_disc_real_2: 0.2848650813102722 (0.22635062346138904) + | > loss_disc_real_3: 0.23535379767417908 (0.22821137447332598) + | > loss_disc_real_4: 0.2626182734966278 (0.22769770241275275) + | > loss_disc_real_5: 0.26177558302879333 (0.22977301463023903) + | > loss_0: 2.5579166412353516 (2.449509989355029) + | > grad_norm_0: tensor(10.2433, device='cuda:0') (tensor(9.1107, device='cuda:0')) + | > loss_gen: 2.4376769065856934 (2.3716006672259446) + | > loss_kl: 1.9306434392929077 (1.8527446198709232) + | > loss_feat: 6.694464206695557 (7.249168710610301) + | > loss_mel: 16.733489990234375 (17.691937751376756) + | > loss_duration: 1.4528181552886963 (1.441844448600848) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 29.24909210205078 (30.60729622595089) + | > grad_norm_1: tensor(280.5786, device='cuda:0') (tensor(185.5199, device='cuda:0')) + | > current_lr_0: 0.00018875289939779185 + | > current_lr_1: 0.00018875289939779185 + | > step_time: 0.4082 (0.3665192323861661) + | > loader_time: 0.0067 (0.005783729946490414) + + + --> TIME: 2025-04-19 06:35:45 -- STEP: 122/142 -- GLOBAL_STEP: 445150 + | > loss_disc: 2.3688244819641113 (2.4512179484132868) + | > loss_disc_real_0: 0.11308588832616806 (0.12585593353895871) + | > loss_disc_real_1: 0.22318598628044128 (0.2064230605715611) + | > loss_disc_real_2: 0.19990573823451996 (0.2263009947587232) + | > loss_disc_real_3: 0.22394448518753052 (0.22814867633288025) + | > loss_disc_real_4: 0.2220766842365265 (0.22849120529460126) + | > loss_disc_real_5: 0.23238429427146912 (0.23059220318911505) + | > loss_0: 2.3688244819641113 (2.4512179484132868) + | > grad_norm_0: tensor(16.0993, device='cuda:0') (tensor(9.9835, device='cuda:0')) + | > loss_gen: 2.432826280593872 (2.3700409717247113) + | > loss_kl: 1.825774073600769 (1.8676672607171731) + | > loss_feat: 7.696295738220215 (7.250650644302368) + | > loss_mel: 17.326997756958008 (17.708632656785316) + | > loss_duration: 1.463169813156128 (1.4490455437879095) + | > amp_scaler: 32.0 (18.491803278688526) + | > loss_1: 30.74506378173828 (30.64603705484359) + | > grad_norm_1: tensor(343.8568, device='cuda:0') (tensor(211.7457, device='cuda:0')) + | > current_lr_0: 0.00018875289939779185 + | > current_lr_1: 0.00018875289939779185 + | > step_time: 0.4527 (0.3778387855310908) + | > loader_time: 0.0075 (0.006034196400251544) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.376127243041992 (2.376127243041992) + | > loss_disc_real_0: 0.15396906435489655 (0.15396906435489655) + | > loss_disc_real_1: 0.22137515246868134 (0.22137515246868134) + | > loss_disc_real_2: 0.23442551493644714 (0.23442551493644714) + | > loss_disc_real_3: 0.23933248221874237 (0.23933248221874237) + | > loss_disc_real_4: 0.22519497573375702 (0.22519497573375702) + | > loss_disc_real_5: 0.2192179411649704 (0.2192179411649704) + | > loss_0: 2.376127243041992 (2.376127243041992) + | > loss_gen: 2.4940316677093506 (2.4940316677093506) + | > loss_kl: 2.7446043491363525 (2.7446043491363525) + | > loss_feat: 6.149409294128418 (6.149409294128418) + | > loss_mel: 16.922178268432617 (16.922178268432617) + | > loss_duration: 1.767235279083252 (1.767235279083252) + | > loss_1: 30.07745933532715 (30.07745933532715) + + --> STEP: 1 + | > loss_disc: 2.466951608657837 (2.466951608657837) + | > loss_disc_real_0: 0.175866037607193 (0.175866037607193) + | > loss_disc_real_1: 0.27065709233283997 (0.27065709233283997) + | > loss_disc_real_2: 0.23496729135513306 (0.23496729135513306) + | > loss_disc_real_3: 0.2339988797903061 (0.2339988797903061) + | > loss_disc_real_4: 0.21812401711940765 (0.21812401711940765) + | > loss_disc_real_5: 0.21160168945789337 (0.21160168945789337) + | > loss_0: 2.466951608657837 (2.466951608657837) + | > loss_gen: 2.568873405456543 (2.568873405456543) + | > loss_kl: 2.7028591632843018 (2.7028591632843018) + | > loss_feat: 6.3135223388671875 (6.3135223388671875) + | > loss_mel: 17.661088943481445 (17.661088943481445) + | > loss_duration: 1.8096204996109009 (1.8096204996109009) + | > loss_1: 31.055965423583984 (31.055965423583984) + + --> STEP: 2 + | > loss_disc: 2.64622163772583 (2.5565866231918335) + | > loss_disc_real_0: 0.21764305233955383 (0.1967545449733734) + | > loss_disc_real_1: 0.22903753817081451 (0.24984731525182724) + | > loss_disc_real_2: 0.2512543201446533 (0.2431108057498932) + | > loss_disc_real_3: 0.24544362723827362 (0.23972125351428986) + | > loss_disc_real_4: 0.2560098171234131 (0.23706691712141037) + | > loss_disc_real_5: 0.2376672774553299 (0.22463448345661163) + | > loss_0: 2.64622163772583 (2.5565866231918335) + | > loss_gen: 2.184530019760132 (2.3767017126083374) + | > loss_kl: 2.5280487537384033 (2.6154539585113525) + | > loss_feat: 5.939235210418701 (6.126378774642944) + | > loss_mel: 14.658238410949707 (16.159663677215576) + | > loss_duration: 1.7165703773498535 (1.7630954384803772) + | > loss_1: 27.026622772216797 (29.04129409790039) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038809776306152344 (+0.0001068115234375) + | > avg_loss_disc: 2.5565866231918335 (-0.00017154216766357422) + | > avg_loss_disc_real_0: 0.1967545449733734 (-0.08926114439964294) + | > avg_loss_disc_real_1: 0.24984731525182724 (+0.04162762314081192) + | > avg_loss_disc_real_2: 0.2431108057498932 (+0.0022780895233154297) + | > avg_loss_disc_real_3: 0.23972125351428986 (+0.000528194010257721) + | > avg_loss_disc_real_4: 0.23706691712141037 (+0.037220075726509094) + | > avg_loss_disc_real_5: 0.22463448345661163 (-0.01618792861700058) + | > avg_loss_0: 2.5565866231918335 (-0.00017154216766357422) + | > avg_loss_gen: 2.3767017126083374 (-0.18702077865600586) + | > avg_loss_kl: 2.6154539585113525 (-0.006856203079223633) + | > avg_loss_feat: 6.126378774642944 (-1.036484718322754) + | > avg_loss_mel: 16.159663677215576 (-2.5831027030944824) + | > avg_loss_duration: 1.7630954384803772 (-0.015506207942962646) + | > avg_loss_1: 29.04129409790039 (-3.828969955444336) + + + > EPOCH: 464/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:35:55)  + + --> TIME: 2025-04-19 06:35:58 -- STEP: 5/142 -- GLOBAL_STEP: 445175 + | > loss_disc: 2.508105754852295 (2.377639389038086) + | > loss_disc_real_0: 0.14317844808101654 (0.10765777230262756) + | > loss_disc_real_1: 0.22307363152503967 (0.19731132686138153) + | > loss_disc_real_2: 0.2258836030960083 (0.2212759494781494) + | > loss_disc_real_3: 0.2442757785320282 (0.2265470415353775) + | > loss_disc_real_4: 0.21516017615795135 (0.22655433118343354) + | > loss_disc_real_5: 0.2595196068286896 (0.2434719532728195) + | > loss_0: 2.508105754852295 (2.377639389038086) + | > grad_norm_0: tensor(9.4281, device='cuda:0') (tensor(9.7616, device='cuda:0')) + | > loss_gen: 2.294100522994995 (2.39650239944458) + | > loss_kl: 1.8061611652374268 (1.7804328441619872) + | > loss_feat: 6.1513166427612305 (7.082057952880859) + | > loss_mel: 15.694230079650879 (17.1974214553833) + | > loss_duration: 1.4252337217330933 (1.4133960485458374) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 27.37104034423828 (29.869810104370117) + | > grad_norm_1: tensor(238.7866, device='cuda:0') (tensor(238.7404, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.325 (0.32332277297973633) + | > loader_time: 0.0043 (0.004349517822265625) + + + --> TIME: 2025-04-19 06:36:06 -- STEP: 30/142 -- GLOBAL_STEP: 445200 + | > loss_disc: 2.4360480308532715 (2.413621465365092) + | > loss_disc_real_0: 0.12219280004501343 (0.1133311629295349) + | > loss_disc_real_1: 0.24139593541622162 (0.20591702411572138) + | > loss_disc_real_2: 0.2665901482105255 (0.22777727395296096) + | > loss_disc_real_3: 0.21383976936340332 (0.22377586712439854) + | > loss_disc_real_4: 0.23221705853939056 (0.2279170314470927) + | > loss_disc_real_5: 0.22093170881271362 (0.22610210329294206) + | > loss_0: 2.4360480308532715 (2.413621465365092) + | > grad_norm_0: tensor(9.7204, device='cuda:0') (tensor(9.2850, device='cuda:0')) + | > loss_gen: 2.395291805267334 (2.4033793290456136) + | > loss_kl: 1.9479135274887085 (1.8257543524106343) + | > loss_feat: 7.376598358154297 (7.336883497238159) + | > loss_mel: 17.312719345092773 (17.435502115885416) + | > loss_duration: 1.4373962879180908 (1.416256888707479) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.469919204711914 (30.41777572631836) + | > grad_norm_1: tensor(131.3233, device='cuda:0') (tensor(271.5034, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.3522 (0.33688870271046956) + | > loader_time: 0.0054 (0.004902648925781249) + + + --> TIME: 2025-04-19 06:36:16 -- STEP: 55/142 -- GLOBAL_STEP: 445225 + | > loss_disc: 2.4307496547698975 (2.42249047539451) + | > loss_disc_real_0: 0.08068715780973434 (0.11371895467693155) + | > loss_disc_real_1: 0.2305988222360611 (0.20442143461920997) + | > loss_disc_real_2: 0.23270972073078156 (0.2257997607642954) + | > loss_disc_real_3: 0.2442055493593216 (0.22741665325381538) + | > loss_disc_real_4: 0.2255304753780365 (0.22820162231271918) + | > loss_disc_real_5: 0.2693805396556854 (0.22831469476222993) + | > loss_0: 2.4307496547698975 (2.42249047539451) + | > grad_norm_0: tensor(8.8427, device='cuda:0') (tensor(8.8907, device='cuda:0')) + | > loss_gen: 2.437183380126953 (2.39381588155573) + | > loss_kl: 1.8157203197479248 (1.8387510884891858) + | > loss_feat: 7.720333099365234 (7.34482178254561) + | > loss_mel: 18.346965789794922 (17.539564860950826) + | > loss_duration: 1.4938455820083618 (1.4250784332102) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.81404685974121 (30.542031756314366) + | > grad_norm_1: tensor(186.8889, device='cuda:0') (tensor(249.8238, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.3722 (0.34884793541648157) + | > loader_time: 0.0061 (0.005270203677090732) + + + --> TIME: 2025-04-19 06:36:26 -- STEP: 80/142 -- GLOBAL_STEP: 445250 + | > loss_disc: 2.410700559616089 (2.4388828665018085) + | > loss_disc_real_0: 0.11139533668756485 (0.11675085797905922) + | > loss_disc_real_1: 0.19391357898712158 (0.20459585767239333) + | > loss_disc_real_2: 0.21452173590660095 (0.22590785808861255) + | > loss_disc_real_3: 0.18997101485729218 (0.22859196029603482) + | > loss_disc_real_4: 0.18929457664489746 (0.22904774397611619) + | > loss_disc_real_5: 0.21148887276649475 (0.23130794391036033) + | > loss_0: 2.410700559616089 (2.4388828665018085) + | > grad_norm_0: tensor(10.3841, device='cuda:0') (tensor(8.2579, device='cuda:0')) + | > loss_gen: 2.3679769039154053 (2.377557441592215) + | > loss_kl: 1.9998153448104858 (1.8586341932415962) + | > loss_feat: 8.042984008789062 (7.2674600839614865) + | > loss_mel: 18.735933303833008 (17.52412415742875) + | > loss_duration: 1.4579296112060547 (1.436231565475464) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 32.604637145996094 (30.464007258415222) + | > grad_norm_1: tensor(90.4107, device='cuda:0') (tensor(216.3470, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.4081 (0.36110161244869216) + | > loader_time: 0.0066 (0.005554693937301637) + + + --> TIME: 2025-04-19 06:36:36 -- STEP: 105/142 -- GLOBAL_STEP: 445275 + | > loss_disc: 2.568439245223999 (2.443016088576544) + | > loss_disc_real_0: 0.1471206098794937 (0.11743349376178923) + | > loss_disc_real_1: 0.18378375470638275 (0.20401150711945126) + | > loss_disc_real_2: 0.23241327702999115 (0.22592571959609076) + | > loss_disc_real_3: 0.2405788153409958 (0.22824931882676625) + | > loss_disc_real_4: 0.2633986473083496 (0.22940384433383032) + | > loss_disc_real_5: 0.25171467661857605 (0.23191882343519302) + | > loss_0: 2.568439245223999 (2.443016088576544) + | > grad_norm_0: tensor(8.5879, device='cuda:0') (tensor(8.9179, device='cuda:0')) + | > loss_gen: 2.3012919425964355 (2.3804382687523242) + | > loss_kl: 1.9944425821304321 (1.8619890689849854) + | > loss_feat: 6.491390705108643 (7.274817616598947) + | > loss_mel: 17.47755241394043 (17.691700426737476) + | > loss_duration: 1.4852168560028076 (1.4461686009452457) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.749895095825195 (30.655113855997723) + | > grad_norm_1: tensor(207.4524, device='cuda:0') (tensor(231.6200, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.4208 (0.37236449150812045) + | > loader_time: 0.0071 (0.005818764368693034) + + + --> TIME: 2025-04-19 06:36:47 -- STEP: 130/142 -- GLOBAL_STEP: 445300 + | > loss_disc: 2.2866220474243164 (2.4402635977818417) + | > loss_disc_real_0: 0.11514327675104141 (0.11810123659670353) + | > loss_disc_real_1: 0.18684956431388855 (0.20445105467851346) + | > loss_disc_real_2: 0.185969740152359 (0.22454223002378756) + | > loss_disc_real_3: 0.18823257088661194 (0.22788283251799069) + | > loss_disc_real_4: 0.2049928456544876 (0.22852812019678262) + | > loss_disc_real_5: 0.19289791584014893 (0.2317765936255455) + | > loss_0: 2.2866220474243164 (2.4402635977818417) + | > grad_norm_0: tensor(14.3901, device='cuda:0') (tensor(10.1583, device='cuda:0')) + | > loss_gen: 2.4890964031219482 (2.384317854734567) + | > loss_kl: 1.9924782514572144 (1.8708176044317393) + | > loss_feat: 7.403540134429932 (7.267644177950346) + | > loss_mel: 18.14301109313965 (17.699048269712016) + | > loss_duration: 1.4764211177825928 (1.4528221304600055) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.504545211791992 (30.674649869478667) + | > grad_norm_1: tensor(332.4775, device='cuda:0') (tensor(263.5383, device='cuda:0')) + | > current_lr_0: 0.0001887293052853671 + | > current_lr_1: 0.0001887293052853671 + | > step_time: 0.4485 (0.3848432595913226) + | > loader_time: 0.0078 (0.0061219527171208305) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.505728006362915 (2.505728006362915) + | > loss_disc_real_0: 0.1482667475938797 (0.1482667475938797) + | > loss_disc_real_1: 0.227908194065094 (0.227908194065094) + | > loss_disc_real_2: 0.24079947173595428 (0.24079947173595428) + | > loss_disc_real_3: 0.2799903452396393 (0.2799903452396393) + | > loss_disc_real_4: 0.2830091118812561 (0.2830091118812561) + | > loss_disc_real_5: 0.2636941969394684 (0.2636941969394684) + | > loss_0: 2.505728006362915 (2.505728006362915) + | > loss_gen: 2.5381195545196533 (2.5381195545196533) + | > loss_kl: 2.589132308959961 (2.589132308959961) + | > loss_feat: 7.509561538696289 (7.509561538696289) + | > loss_mel: 19.745162963867188 (19.745162963867188) + | > loss_duration: 1.7617961168289185 (1.7617961168289185) + | > loss_1: 34.14377212524414 (34.14377212524414) + + --> STEP: 1 + | > loss_disc: 2.5303456783294678 (2.5303456783294678) + | > loss_disc_real_0: 0.13812042772769928 (0.13812042772769928) + | > loss_disc_real_1: 0.23731453716754913 (0.23731453716754913) + | > loss_disc_real_2: 0.23718616366386414 (0.23718616366386414) + | > loss_disc_real_3: 0.2546643614768982 (0.2546643614768982) + | > loss_disc_real_4: 0.25181642174720764 (0.25181642174720764) + | > loss_disc_real_5: 0.19875307381153107 (0.19875307381153107) + | > loss_0: 2.5303456783294678 (2.5303456783294678) + | > loss_gen: 2.3906378746032715 (2.3906378746032715) + | > loss_kl: 2.903710126876831 (2.903710126876831) + | > loss_feat: 6.621538162231445 (6.621538162231445) + | > loss_mel: 19.037046432495117 (19.037046432495117) + | > loss_duration: 1.8043540716171265 (1.8043540716171265) + | > loss_1: 32.757286071777344 (32.757286071777344) + + --> STEP: 2 + | > loss_disc: 2.5900826454162598 (2.5602141618728638) + | > loss_disc_real_0: 0.1458534598350525 (0.14198694378137589) + | > loss_disc_real_1: 0.2620469033718109 (0.24968072026968002) + | > loss_disc_real_2: 0.29611918330192566 (0.2666526734828949) + | > loss_disc_real_3: 0.2680187225341797 (0.26134154200553894) + | > loss_disc_real_4: 0.2984502613544464 (0.275133341550827) + | > loss_disc_real_5: 0.27110761404037476 (0.2349303439259529) + | > loss_0: 2.5900826454162598 (2.5602141618728638) + | > loss_gen: 2.428368330001831 (2.4095031023025513) + | > loss_kl: 2.3546714782714844 (2.6291908025741577) + | > loss_feat: 7.069879055023193 (6.845708608627319) + | > loss_mel: 17.183006286621094 (18.110026359558105) + | > loss_duration: 1.7621042728424072 (1.7832291722297668) + | > loss_1: 30.79802894592285 (31.777657508850098) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033998489379882812 (-0.0004811286926269531) + | > avg_loss_disc: 2.5602141618728638 (+0.0036275386810302734) + | > avg_loss_disc_real_0: 0.14198694378137589 (-0.05476760119199753) + | > avg_loss_disc_real_1: 0.24968072026968002 (-0.0001665949821472168) + | > avg_loss_disc_real_2: 0.2666526734828949 (+0.02354186773300171) + | > avg_loss_disc_real_3: 0.26134154200553894 (+0.021620288491249084) + | > avg_loss_disc_real_4: 0.275133341550827 (+0.038066424429416656) + | > avg_loss_disc_real_5: 0.2349303439259529 (+0.010295860469341278) + | > avg_loss_0: 2.5602141618728638 (+0.0036275386810302734) + | > avg_loss_gen: 2.4095031023025513 (+0.03280138969421387) + | > avg_loss_kl: 2.6291908025741577 (+0.013736844062805176) + | > avg_loss_feat: 6.845708608627319 (+0.719329833984375) + | > avg_loss_mel: 18.110026359558105 (+1.9503626823425293) + | > avg_loss_duration: 1.7832291722297668 (+0.02013373374938965) + | > avg_loss_1: 31.777657508850098 (+2.736363410949707) + + + > EPOCH: 465/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:36:54)  + + --> TIME: 2025-04-19 06:36:59 -- STEP: 13/142 -- GLOBAL_STEP: 445325 + | > loss_disc: 2.487247943878174 (2.479920038810143) + | > loss_disc_real_0: 0.12004675716161728 (0.11057327945645039) + | > loss_disc_real_1: 0.23259596526622772 (0.2108596689425982) + | > loss_disc_real_2: 0.2217153012752533 (0.234582584637862) + | > loss_disc_real_3: 0.21871066093444824 (0.2386372284247325) + | > loss_disc_real_4: 0.23844397068023682 (0.2446446751172726) + | > loss_disc_real_5: 0.22203366458415985 (0.25052736699581146) + | > loss_0: 2.487247943878174 (2.479920038810143) + | > grad_norm_0: tensor(17.9867, device='cuda:0') (tensor(17.1614, device='cuda:0')) + | > loss_gen: 2.243530035018921 (2.436632651549119) + | > loss_kl: 1.9149283170700073 (1.825777924977816) + | > loss_feat: 7.196478366851807 (7.645737611330473) + | > loss_mel: 16.306514739990234 (18.01556689922626) + | > loss_duration: 1.4175419807434082 (1.4070144158143263) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.07899284362793 (31.330729264479416) + | > grad_norm_1: tensor(449.9187, device='cuda:0') (tensor(375.6052, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.332 (0.32829191134526187) + | > loader_time: 0.0048 (0.004570245742797852) + + + --> TIME: 2025-04-19 06:37:08 -- STEP: 38/142 -- GLOBAL_STEP: 445350 + | > loss_disc: 2.367521286010742 (2.4459699455060457) + | > loss_disc_real_0: 0.11482422798871994 (0.11037941326044108) + | > loss_disc_real_1: 0.20015424489974976 (0.2064831762721664) + | > loss_disc_real_2: 0.22450536489486694 (0.2292369909976658) + | > loss_disc_real_3: 0.23330533504486084 (0.22961414997514926) + | > loss_disc_real_4: 0.2517020106315613 (0.22876054323033282) + | > loss_disc_real_5: 0.22250591218471527 (0.2373973048831287) + | > loss_0: 2.367521286010742 (2.4459699455060457) + | > grad_norm_0: tensor(11.0335, device='cuda:0') (tensor(14.6529, device='cuda:0')) + | > loss_gen: 2.408989667892456 (2.4109906146400855) + | > loss_kl: 1.767243504524231 (1.827417608938719) + | > loss_feat: 7.434375286102295 (7.467911431663914) + | > loss_mel: 17.547311782836914 (17.849727931775547) + | > loss_duration: 1.4463231563568115 (1.4185797634877657) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.604244232177734 (30.97462729403847) + | > grad_norm_1: tensor(241.4743, device='cuda:0') (tensor(350.3720, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.3516 (0.3399758903603805) + | > loader_time: 0.0055 (0.005126219046743293) + + + --> TIME: 2025-04-19 06:37:17 -- STEP: 63/142 -- GLOBAL_STEP: 445375 + | > loss_disc: 2.36076283454895 (2.437566848028274) + | > loss_disc_real_0: 0.09625295549631119 (0.11251517119152206) + | > loss_disc_real_1: 0.18965817987918854 (0.20459395386869944) + | > loss_disc_real_2: 0.239430770277977 (0.22610785776660555) + | > loss_disc_real_3: 0.20954717695713043 (0.22956932939234234) + | > loss_disc_real_4: 0.25177523493766785 (0.2276426660163062) + | > loss_disc_real_5: 0.24558144807815552 (0.2367860712702312) + | > loss_0: 2.36076283454895 (2.437566848028274) + | > grad_norm_0: tensor(12.5624, device='cuda:0') (tensor(12.2260, device='cuda:0')) + | > loss_gen: 2.4378936290740967 (2.3971688936627107) + | > loss_kl: 1.82948899269104 (1.834882823247758) + | > loss_feat: 7.037220001220703 (7.394519230676075) + | > loss_mel: 18.698333740234375 (17.79527379596044) + | > loss_duration: 1.483522653579712 (1.4295281247487144) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.48645782470703 (30.8513728853256) + | > grad_norm_1: tensor(215.6947, device='cuda:0') (tensor(299.6676, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.3921 (0.35163961516486275) + | > loader_time: 0.0062 (0.005452727514599997) + + + --> TIME: 2025-04-19 06:37:28 -- STEP: 88/142 -- GLOBAL_STEP: 445400 + | > loss_disc: 2.385216236114502 (2.4428458322178233) + | > loss_disc_real_0: 0.12519456446170807 (0.1156527164222842) + | > loss_disc_real_1: 0.178375706076622 (0.2048697622323578) + | > loss_disc_real_2: 0.1922490894794464 (0.2262198921631683) + | > loss_disc_real_3: 0.23572012782096863 (0.22953229414468462) + | > loss_disc_real_4: 0.19629555940628052 (0.22705457562750037) + | > loss_disc_real_5: 0.232158362865448 (0.23640804179012775) + | > loss_0: 2.385216236114502 (2.4428458322178233) + | > grad_norm_0: tensor(12.8318, device='cuda:0') (tensor(11.2543, device='cuda:0')) + | > loss_gen: 2.4005072116851807 (2.387013083154505) + | > loss_kl: 1.8853952884674072 (1.849487991495566) + | > loss_feat: 7.379310131072998 (7.296414890072563) + | > loss_mel: 18.09606170654297 (17.67641001397913) + | > loss_duration: 1.4994854927062988 (1.439866668798707) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.260759353637695 (30.649192680012096) + | > grad_norm_1: tensor(184.5822, device='cuda:0') (tensor(274.5191, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.4012 (0.364506342194297) + | > loader_time: 0.007 (0.005757134069095959) + + + --> TIME: 2025-04-19 06:37:38 -- STEP: 113/142 -- GLOBAL_STEP: 445425 + | > loss_disc: 2.502129554748535 (2.447489628749611) + | > loss_disc_real_0: 0.14626459777355194 (0.11736105069254352) + | > loss_disc_real_1: 0.20028412342071533 (0.20550900684521262) + | > loss_disc_real_2: 0.23335182666778564 (0.22747889635837185) + | > loss_disc_real_3: 0.24951171875 (0.22911552056274584) + | > loss_disc_real_4: 0.21621792018413544 (0.22773200471316818) + | > loss_disc_real_5: 0.2407822459936142 (0.23586534979069126) + | > loss_0: 2.502129554748535 (2.447489628749611) + | > grad_norm_0: tensor(9.7811, device='cuda:0') (tensor(10.7586, device='cuda:0')) + | > loss_gen: 2.2879066467285156 (2.378576479126923) + | > loss_kl: 1.7871415615081787 (1.8586554896515028) + | > loss_feat: 7.028090476989746 (7.240503041090164) + | > loss_mel: 17.355451583862305 (17.604213807435148) + | > loss_duration: 1.477053165435791 (1.446997575000324) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.935644149780273 (30.52894636171054) + | > grad_norm_1: tensor(240.0079, device='cuda:0') (tensor(262.6069, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.4286 (0.37577811806602807) + | > loader_time: 0.0071 (0.006012007198502532) + + + --> TIME: 2025-04-19 06:37:50 -- STEP: 138/142 -- GLOBAL_STEP: 445450 + | > loss_disc: 2.5038113594055176 (2.452817706094272) + | > loss_disc_real_0: 0.13340000808238983 (0.120226063941052) + | > loss_disc_real_1: 0.19648052752017975 (0.20576687966999802) + | > loss_disc_real_2: 0.24647675454616547 (0.22668474448331888) + | > loss_disc_real_3: 0.2221444845199585 (0.22878062800652738) + | > loss_disc_real_4: 0.2155018299818039 (0.2271810926604962) + | > loss_disc_real_5: 0.25335872173309326 (0.23618863782157068) + | > loss_0: 2.5038113594055176 (2.452817706094272) + | > grad_norm_0: tensor(18.0460, device='cuda:0') (tensor(11.0549, device='cuda:0')) + | > loss_gen: 2.224374771118164 (2.368379606716876) + | > loss_kl: 2.001739978790283 (1.868130332317905) + | > loss_feat: 6.11085844039917 (7.181106712507165) + | > loss_mel: 15.695026397705078 (17.548901882724472) + | > loss_duration: 1.5156582593917847 (1.4534158983092378) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 27.547657012939453 (30.419934452443883) + | > grad_norm_1: tensor(348.6081, device='cuda:0') (tensor(263.7163, device='cuda:0')) + | > current_lr_0: 0.00018870571412220644 + | > current_lr_1: 0.00018870571412220644 + | > step_time: 0.4491 (0.38768071713654884) + | > loader_time: 0.0075 (0.006287072015845258) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.464127779006958 (2.464127779006958) + | > loss_disc_real_0: 0.1831502616405487 (0.1831502616405487) + | > loss_disc_real_1: 0.2381313145160675 (0.2381313145160675) + | > loss_disc_real_2: 0.22119766473770142 (0.22119766473770142) + | > loss_disc_real_3: 0.24466511607170105 (0.24466511607170105) + | > loss_disc_real_4: 0.21027421951293945 (0.21027421951293945) + | > loss_disc_real_5: 0.24580253660678864 (0.24580253660678864) + | > loss_0: 2.464127779006958 (2.464127779006958) + | > loss_gen: 2.461956024169922 (2.461956024169922) + | > loss_kl: 2.4676976203918457 (2.4676976203918457) + | > loss_feat: 6.494676113128662 (6.494676113128662) + | > loss_mel: 17.022781372070312 (17.022781372070312) + | > loss_duration: 1.774916648864746 (1.774916648864746) + | > loss_1: 30.222026824951172 (30.222026824951172) + + --> STEP: 1 + | > loss_disc: 2.522590160369873 (2.522590160369873) + | > loss_disc_real_0: 0.1505114883184433 (0.1505114883184433) + | > loss_disc_real_1: 0.2713695764541626 (0.2713695764541626) + | > loss_disc_real_2: 0.24380631744861603 (0.24380631744861603) + | > loss_disc_real_3: 0.21674956381320953 (0.21674956381320953) + | > loss_disc_real_4: 0.25768646597862244 (0.25768646597862244) + | > loss_disc_real_5: 0.2190430611371994 (0.2190430611371994) + | > loss_0: 2.522590160369873 (2.522590160369873) + | > loss_gen: 2.2727391719818115 (2.2727391719818115) + | > loss_kl: 3.0009448528289795 (3.0009448528289795) + | > loss_feat: 6.188425540924072 (6.188425540924072) + | > loss_mel: 16.511157989501953 (16.511157989501953) + | > loss_duration: 1.8180698156356812 (1.8180698156356812) + | > loss_1: 29.791336059570312 (29.791336059570312) + + --> STEP: 2 + | > loss_disc: 2.477665901184082 (2.5001280307769775) + | > loss_disc_real_0: 0.20357710123062134 (0.17704429477453232) + | > loss_disc_real_1: 0.24649879336357117 (0.2589341849088669) + | > loss_disc_real_2: 0.24093104898929596 (0.242368683218956) + | > loss_disc_real_3: 0.22120390832424164 (0.21897673606872559) + | > loss_disc_real_4: 0.21168456971645355 (0.234685517847538) + | > loss_disc_real_5: 0.233793705701828 (0.2264183834195137) + | > loss_0: 2.477665901184082 (2.5001280307769775) + | > loss_gen: 2.579906702041626 (2.4263229370117188) + | > loss_kl: 2.3409664630889893 (2.6709556579589844) + | > loss_feat: 7.635927200317383 (6.9121763706207275) + | > loss_mel: 18.856658935546875 (17.683908462524414) + | > loss_duration: 1.7410587072372437 (1.7795642614364624) + | > loss_1: 33.154518127441406 (31.47292709350586) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003459334373474121 (+5.9485435485839844e-05) + | > avg_loss_disc: 2.5001280307769775 (-0.06008613109588623) + | > avg_loss_disc_real_0: 0.17704429477453232 (+0.03505735099315643) + | > avg_loss_disc_real_1: 0.2589341849088669 (+0.009253464639186859) + | > avg_loss_disc_real_2: 0.242368683218956 (-0.024283990263938904) + | > avg_loss_disc_real_3: 0.21897673606872559 (-0.042364805936813354) + | > avg_loss_disc_real_4: 0.234685517847538 (-0.04044782370328903) + | > avg_loss_disc_real_5: 0.2264183834195137 (-0.008511960506439209) + | > avg_loss_0: 2.5001280307769775 (-0.06008613109588623) + | > avg_loss_gen: 2.4263229370117188 (+0.01681983470916748) + | > avg_loss_kl: 2.6709556579589844 (+0.04176485538482666) + | > avg_loss_feat: 6.9121763706207275 (+0.0664677619934082) + | > avg_loss_mel: 17.683908462524414 (-0.4261178970336914) + | > avg_loss_duration: 1.7795642614364624 (-0.0036649107933044434) + | > avg_loss_1: 31.47292709350586 (-0.3047304153442383) + + + > EPOCH: 466/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:37:53)  + + --> TIME: 2025-04-19 06:38:01 -- STEP: 21/142 -- GLOBAL_STEP: 445475 + | > loss_disc: 2.6246142387390137 (2.4422458580562045) + | > loss_disc_real_0: 0.17568610608577728 (0.11588892234223229) + | > loss_disc_real_1: 0.2447870820760727 (0.20419123484974816) + | > loss_disc_real_2: 0.23593616485595703 (0.23085890071732657) + | > loss_disc_real_3: 0.249541774392128 (0.22536835784003847) + | > loss_disc_real_4: 0.24113981425762177 (0.22675573471046628) + | > loss_disc_real_5: 0.2877790927886963 (0.22847797118482135) + | > loss_0: 2.6246142387390137 (2.4422458580562045) + | > grad_norm_0: tensor(11.3980, device='cuda:0') (tensor(10.9890, device='cuda:0')) + | > loss_gen: 2.331352710723877 (2.403050252369472) + | > loss_kl: 1.8682204484939575 (1.8182452803566342) + | > loss_feat: 7.366878986358643 (7.442820344652448) + | > loss_mel: 18.108604431152344 (17.829592795599076) + | > loss_duration: 1.4101982116699219 (1.409480373064677) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.085254669189453 (30.903189159574964) + | > grad_norm_1: tensor(86.9933, device='cuda:0') (tensor(285.2982, device='cuda:0')) + | > current_lr_0: 0.00018868212590794116 + | > current_lr_1: 0.00018868212590794116 + | > step_time: 0.3425 (0.33183717727661133) + | > loader_time: 0.0051 (0.004884504136585054) + + + --> TIME: 2025-04-19 06:38:10 -- STEP: 46/142 -- GLOBAL_STEP: 445500 + | > loss_disc: 2.461719512939453 (2.4466406003288603) + | > loss_disc_real_0: 0.11664988845586777 (0.11637996736428012) + | > loss_disc_real_1: 0.18068060278892517 (0.20416044800177865) + | > loss_disc_real_2: 0.25441116094589233 (0.22774657380321753) + | > loss_disc_real_3: 0.21406616270542145 (0.22795316544563873) + | > loss_disc_real_4: 0.2006944864988327 (0.2270418233845545) + | > loss_disc_real_5: 0.22908516228199005 (0.2333641956033914) + | > loss_0: 2.461719512939453 (2.4466406003288603) + | > grad_norm_0: tensor(25.3872, device='cuda:0') (tensor(11.6314, device='cuda:0')) + | > loss_gen: 2.414508581161499 (2.3767132966414746) + | > loss_kl: 1.8853719234466553 (1.8480962385302004) + | > loss_feat: 8.120285034179688 (7.33568955504376) + | > loss_mel: 17.907482147216797 (17.74905492948449) + | > loss_duration: 1.4094101190567017 (1.418738264104595) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.737058639526367 (30.728292257889457) + | > grad_norm_1: tensor(697.5960, device='cuda:0') (tensor(305.7135, device='cuda:0')) + | > current_lr_0: 0.00018868212590794116 + | > current_lr_1: 0.00018868212590794116 + | > step_time: 0.3652 (0.3437435056852258) + | > loader_time: 0.006 (0.005299319391665252) + + + --> TIME: 2025-04-19 06:38:19 -- STEP: 71/142 -- GLOBAL_STEP: 445525 + | > loss_disc: 2.4066691398620605 (2.44897180543819) + | > loss_disc_real_0: 0.1027512326836586 (0.11803113438293968) + | > loss_disc_real_1: 0.19389931857585907 (0.20531430777529597) + | > loss_disc_real_2: 0.2322292923927307 (0.2269033434945093) + | > loss_disc_real_3: 0.22204703092575073 (0.22832745292656859) + | > loss_disc_real_4: 0.24634957313537598 (0.22865875212239548) + | > loss_disc_real_5: 0.23781192302703857 (0.23393649681353232) + | > loss_0: 2.4066691398620605 (2.44897180543819) + | > grad_norm_0: tensor(4.4514, device='cuda:0') (tensor(11.9742, device='cuda:0')) + | > loss_gen: 2.551250696182251 (2.374736339273587) + | > loss_kl: 1.9500986337661743 (1.8576829064060265) + | > loss_feat: 7.134321689605713 (7.273039367836966) + | > loss_mel: 18.052860260009766 (17.702664979746643) + | > loss_duration: 1.4672036170959473 (1.430457346875903) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.155733108520508 (30.63858091327506) + | > grad_norm_1: tensor(207.0230, device='cuda:0') (tensor(297.2605, device='cuda:0')) + | > current_lr_0: 0.00018868212590794116 + | > current_lr_1: 0.00018868212590794116 + | > step_time: 0.3835 (0.3561778236442888) + | > loader_time: 0.0059 (0.005574733438626142) + + + --> TIME: 2025-04-19 06:38:30 -- STEP: 96/142 -- GLOBAL_STEP: 445550 + | > loss_disc: 2.4857654571533203 (2.4587563027938217) + | > loss_disc_real_0: 0.16473785042762756 (0.12143823861454923) + | > loss_disc_real_1: 0.19459909200668335 (0.2056516231968999) + | > loss_disc_real_2: 0.22353510558605194 (0.22561459429562092) + | > loss_disc_real_3: 0.2191365510225296 (0.22848724201321602) + | > loss_disc_real_4: 0.23452278971672058 (0.22822565600896874) + | > loss_disc_real_5: 0.2159767746925354 (0.23457091317201653) + | > loss_0: 2.4857654571533203 (2.4587563027938217) + | > grad_norm_0: tensor(10.7002, device='cuda:0') (tensor(12.3544, device='cuda:0')) + | > loss_gen: 2.457876682281494 (2.366567182044188) + | > loss_kl: 1.9197814464569092 (1.8607990878323715) + | > loss_feat: 6.197324752807617 (7.200405240058899) + | > loss_mel: 14.817588806152344 (17.624730020761486) + | > loss_duration: 1.4656789302825928 (1.4407497396071751) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 26.858251571655273 (30.493251125017803) + | > grad_norm_1: tensor(344.8466, device='cuda:0') (tensor(294.6636, device='cuda:0')) + | > current_lr_0: 0.00018868212590794116 + | > current_lr_1: 0.00018868212590794116 + | > step_time: 0.4016 (0.3665546004970868) + | > loader_time: 0.0069 (0.005795679986476896) + + + --> TIME: 2025-04-19 06:38:40 -- STEP: 121/142 -- GLOBAL_STEP: 445575 + | > loss_disc: 2.449164390563965 (2.4584030316880923) + | > loss_disc_real_0: 0.1248302012681961 (0.12351200587985929) + | > loss_disc_real_1: 0.2087494134902954 (0.2069269889149784) + | > loss_disc_real_2: 0.2026778608560562 (0.22571460286940426) + | > loss_disc_real_3: 0.21618901193141937 (0.2280723372520494) + | > loss_disc_real_4: 0.24438633024692535 (0.22792899042121634) + | > loss_disc_real_5: 0.23269346356391907 (0.23338915259877513) + | > loss_0: 2.449164390563965 (2.4584030316880923) + | > grad_norm_0: tensor(17.6279, device='cuda:0') (tensor(12.6101, device='cuda:0')) + | > loss_gen: 2.355069160461426 (2.36104402266258) + | > loss_kl: 1.9191460609436035 (1.8644474991096938) + | > loss_feat: 7.4572882652282715 (7.151637463530233) + | > loss_mel: 18.506898880004883 (17.551493313686912) + | > loss_duration: 1.5048631429672241 (1.4480410173904796) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.743267059326172 (30.376663144954964) + | > grad_norm_1: tensor(345.6348, device='cuda:0') (tensor(301.2976, device='cuda:0')) + | > current_lr_0: 0.00018868212590794116 + | > current_lr_1: 0.00018868212590794116 + | > step_time: 0.4412 (0.37784106278222457) + | > loader_time: 0.0067 (0.006042476527947037) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.620272159576416 (2.620272159576416) + | > loss_disc_real_0: 0.24800868332386017 (0.24800868332386017) + | > loss_disc_real_1: 0.3005566895008087 (0.3005566895008087) + | > loss_disc_real_2: 0.23980456590652466 (0.23980456590652466) + | > loss_disc_real_3: 0.24285277724266052 (0.24285277724266052) + | > loss_disc_real_4: 0.23462145030498505 (0.23462145030498505) + | > loss_disc_real_5: 0.25364768505096436 (0.25364768505096436) + | > loss_0: 2.620272159576416 (2.620272159576416) + | > loss_gen: 2.7084782123565674 (2.7084782123565674) + | > loss_kl: 2.520551919937134 (2.520551919937134) + | > loss_feat: 6.563010215759277 (6.563010215759277) + | > loss_mel: 17.60352325439453 (17.60352325439453) + | > loss_duration: 1.7585818767547607 (1.7585818767547607) + | > loss_1: 31.154144287109375 (31.154144287109375) + + --> STEP: 1 + | > loss_disc: 2.5792555809020996 (2.5792555809020996) + | > loss_disc_real_0: 0.23625384271144867 (0.23625384271144867) + | > loss_disc_real_1: 0.2749795615673065 (0.2749795615673065) + | > loss_disc_real_2: 0.23668771982192993 (0.23668771982192993) + | > loss_disc_real_3: 0.23646420240402222 (0.23646420240402222) + | > loss_disc_real_4: 0.24079972505569458 (0.24079972505569458) + | > loss_disc_real_5: 0.2472338080406189 (0.2472338080406189) + | > loss_0: 2.5792555809020996 (2.5792555809020996) + | > loss_gen: 2.498652935028076 (2.498652935028076) + | > loss_kl: 2.6223199367523193 (2.6223199367523193) + | > loss_feat: 6.655394554138184 (6.655394554138184) + | > loss_mel: 18.171241760253906 (18.171241760253906) + | > loss_duration: 1.8248546123504639 (1.8248546123504639) + | > loss_1: 31.772462844848633 (31.772462844848633) + + --> STEP: 2 + | > loss_disc: 2.413675308227539 (2.4964654445648193) + | > loss_disc_real_0: 0.18208327889442444 (0.20916856080293655) + | > loss_disc_real_1: 0.29140716791152954 (0.28319336473941803) + | > loss_disc_real_2: 0.22015276551246643 (0.22842024266719818) + | > loss_disc_real_3: 0.23365479707717896 (0.23505949974060059) + | > loss_disc_real_4: 0.20448219776153564 (0.2226409614086151) + | > loss_disc_real_5: 0.23946182429790497 (0.24334781616926193) + | > loss_0: 2.413675308227539 (2.4964654445648193) + | > loss_gen: 2.709825038909912 (2.604238986968994) + | > loss_kl: 2.3839147090911865 (2.503117322921753) + | > loss_feat: 7.662558555603027 (7.1589765548706055) + | > loss_mel: 18.69212532043457 (18.43168354034424) + | > loss_duration: 1.7523062229156494 (1.7885804176330566) + | > loss_1: 33.20072937011719 (32.48659610748291) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036743879318237305 (+0.00021505355834960938) + | > avg_loss_disc: 2.4964654445648193 (-0.003662586212158203) + | > avg_loss_disc_real_0: 0.20916856080293655 (+0.032124266028404236) + | > avg_loss_disc_real_1: 0.28319336473941803 (+0.024259179830551147) + | > avg_loss_disc_real_2: 0.22842024266719818 (-0.013948440551757812) + | > avg_loss_disc_real_3: 0.23505949974060059 (+0.016082763671875) + | > avg_loss_disc_real_4: 0.2226409614086151 (-0.012044556438922882) + | > avg_loss_disc_real_5: 0.24334781616926193 (+0.01692943274974823) + | > avg_loss_0: 2.4964654445648193 (-0.003662586212158203) + | > avg_loss_gen: 2.604238986968994 (+0.1779160499572754) + | > avg_loss_kl: 2.503117322921753 (-0.16783833503723145) + | > avg_loss_feat: 7.1589765548706055 (+0.24680018424987793) + | > avg_loss_mel: 18.43168354034424 (+0.7477750778198242) + | > avg_loss_duration: 1.7885804176330566 (+0.009016156196594238) + | > avg_loss_1: 32.48659610748291 (+1.0136690139770508) + + + > EPOCH: 467/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:38:51)  + + --> TIME: 2025-04-19 06:38:54 -- STEP: 4/142 -- GLOBAL_STEP: 445600 + | > loss_disc: 2.3902337551116943 (2.3755953311920166) + | > loss_disc_real_0: 0.11501749604940414 (0.08781596645712852) + | > loss_disc_real_1: 0.19870302081108093 (0.2012476697564125) + | > loss_disc_real_2: 0.22066393494606018 (0.21774184331297874) + | > loss_disc_real_3: 0.23979416489601135 (0.2397795505821705) + | > loss_disc_real_4: 0.21806050837039948 (0.22021255269646645) + | > loss_disc_real_5: 0.25064903497695923 (0.22445664182305336) + | > loss_0: 2.3902337551116943 (2.3755953311920166) + | > grad_norm_0: tensor(7.8534, device='cuda:0') (tensor(8.1611, device='cuda:0')) + | > loss_gen: 2.520249605178833 (2.4497070908546448) + | > loss_kl: 1.6655919551849365 (1.8334217071533203) + | > loss_feat: 7.3265700340271 (7.724494934082031) + | > loss_mel: 17.982284545898438 (18.361021995544434) + | > loss_duration: 1.4061601161956787 (1.4058203101158142) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.900856018066406 (31.774466514587402) + | > grad_norm_1: tensor(133.0758, device='cuda:0') (tensor(226.9428, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.3205 (0.33174169063568115) + | > loader_time: 0.0054 (0.004534602165222168) + + + --> TIME: 2025-04-19 06:39:02 -- STEP: 29/142 -- GLOBAL_STEP: 445625 + | > loss_disc: 2.6128828525543213 (2.4064289619182717) + | > loss_disc_real_0: 0.13709810376167297 (0.10978351032425618) + | > loss_disc_real_1: 0.20877331495285034 (0.201728453923916) + | > loss_disc_real_2: 0.2721957862377167 (0.21984754245856714) + | > loss_disc_real_3: 0.2151583731174469 (0.2258891149841506) + | > loss_disc_real_4: 0.26866695284843445 (0.22485432337070332) + | > loss_disc_real_5: 0.22491435706615448 (0.22798983752727509) + | > loss_0: 2.6128828525543213 (2.4064289619182717) + | > grad_norm_0: tensor(4.1832, device='cuda:0') (tensor(9.9929, device='cuda:0')) + | > loss_gen: 2.3764162063598633 (2.3980095633145027) + | > loss_kl: 1.7853924036026 (1.817201334854652) + | > loss_feat: 7.818506240844727 (7.49746271659588) + | > loss_mel: 17.79526138305664 (17.938434206206225) + | > loss_duration: 1.4164254665374756 (1.4157900769135048) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.19200325012207 (31.06689821440598) + | > grad_norm_1: tensor(263.7310, device='cuda:0') (tensor(264.0388, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.3541 (0.33981403811224575) + | > loader_time: 0.0053 (0.005038836906696188) + + + --> TIME: 2025-04-19 06:39:12 -- STEP: 54/142 -- GLOBAL_STEP: 445650 + | > loss_disc: 2.5833280086517334 (2.4202454664089053) + | > loss_disc_real_0: 0.23246455192565918 (0.11538847101231416) + | > loss_disc_real_1: 0.21034206449985504 (0.20268684101325493) + | > loss_disc_real_2: 0.2396778166294098 (0.2216555636238169) + | > loss_disc_real_3: 0.22052040696144104 (0.22708153421128238) + | > loss_disc_real_4: 0.24873973429203033 (0.22613138843465735) + | > loss_disc_real_5: 0.2534385621547699 (0.2319060023184176) + | > loss_0: 2.5833280086517334 (2.4202454664089053) + | > grad_norm_0: tensor(14.0869, device='cuda:0') (tensor(10.8699, device='cuda:0')) + | > loss_gen: 2.4125359058380127 (2.402131601616187) + | > loss_kl: 1.8398970365524292 (1.8298904520493966) + | > loss_feat: 7.0560078620910645 (7.438899296301383) + | > loss_mel: 16.70831871032715 (17.938274118635395) + | > loss_duration: 1.452313780784607 (1.42447433869044) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.469074249267578 (31.033669966238516) + | > grad_norm_1: tensor(268.7496, device='cuda:0') (tensor(278.4503, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.3806 (0.3540017604827881) + | > loader_time: 0.0064 (0.005447581962302879) + + + --> TIME: 2025-04-19 06:39:22 -- STEP: 79/142 -- GLOBAL_STEP: 445675 + | > loss_disc: 2.4808804988861084 (2.4222406345077694) + | > loss_disc_real_0: 0.1346968412399292 (0.11606792082326321) + | > loss_disc_real_1: 0.2213251292705536 (0.20337098289894148) + | > loss_disc_real_2: 0.25784555077552795 (0.22251352516910697) + | > loss_disc_real_3: 0.25096961855888367 (0.22661024388633197) + | > loss_disc_real_4: 0.2549605965614319 (0.22604864835739136) + | > loss_disc_real_5: 0.244611456990242 (0.23160083195831202) + | > loss_0: 2.4808804988861084 (2.4222406345077694) + | > grad_norm_0: tensor(4.0224, device='cuda:0') (tensor(11.2076, device='cuda:0')) + | > loss_gen: 2.4478189945220947 (2.3991248366198943) + | > loss_kl: 1.9239224195480347 (1.8419895368286325) + | > loss_feat: 6.339730739593506 (7.387868802758712) + | > loss_mel: 17.041046142578125 (17.864317531827137) + | > loss_duration: 1.49319589138031 (1.4357892184317866) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.24571418762207 (30.929089992861204) + | > grad_norm_1: tensor(181.8600, device='cuda:0') (tensor(285.7641, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.3974 (0.36821672886232787) + | > loader_time: 0.007 (0.005802281295196918) + + + --> TIME: 2025-04-19 06:39:32 -- STEP: 104/142 -- GLOBAL_STEP: 445700 + | > loss_disc: 2.4011824131011963 (2.4262395432362163) + | > loss_disc_real_0: 0.10770044475793839 (0.11746225430845068) + | > loss_disc_real_1: 0.19535161554813385 (0.2037041459519129) + | > loss_disc_real_2: 0.1849919706583023 (0.2226827317992082) + | > loss_disc_real_3: 0.20406654477119446 (0.22706375958827826) + | > loss_disc_real_4: 0.2541623115539551 (0.22681644487266356) + | > loss_disc_real_5: 0.24391046166419983 (0.23235901516790575) + | > loss_0: 2.4011824131011963 (2.4262395432362163) + | > grad_norm_0: tensor(9.2521, device='cuda:0') (tensor(10.4762, device='cuda:0')) + | > loss_gen: 2.367321729660034 (2.3876169736568733) + | > loss_kl: 1.968808889389038 (1.853861092374875) + | > loss_feat: 7.452824115753174 (7.338900685310364) + | > loss_mel: 18.16727638244629 (17.78957476065709) + | > loss_duration: 1.496464490890503 (1.4451404660940173) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.452693939208984 (30.81509412251986) + | > grad_norm_1: tensor(219.6375, device='cuda:0') (tensor(266.4426, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.4102 (0.37891600223687977) + | > loader_time: 0.0069 (0.006055591198114247) + + + --> TIME: 2025-04-19 06:39:43 -- STEP: 129/142 -- GLOBAL_STEP: 445725 + | > loss_disc: 2.37137508392334 (2.427577514057009) + | > loss_disc_real_0: 0.09797129780054092 (0.11879020822487137) + | > loss_disc_real_1: 0.14980526268482208 (0.20388130716575206) + | > loss_disc_real_2: 0.17995496094226837 (0.22271856492341952) + | > loss_disc_real_3: 0.20420072972774506 (0.22665297996628192) + | > loss_disc_real_4: 0.2156122773885727 (0.2258495337279268) + | > loss_disc_real_5: 0.23176336288452148 (0.2316817399836326) + | > loss_0: 2.37137508392334 (2.427577514057009) + | > grad_norm_0: tensor(5.6363, device='cuda:0') (tensor(10.0167, device='cuda:0')) + | > loss_gen: 2.377825975418091 (2.380578447681988) + | > loss_kl: 1.9321314096450806 (1.8610617236573568) + | > loss_feat: 7.6346917152404785 (7.2898856318274206) + | > loss_mel: 18.085847854614258 (17.719122036482933) + | > loss_duration: 1.4725457429885864 (1.451586027478063) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.50304412841797 (30.70223398726116) + | > grad_norm_1: tensor(84.2728, device='cuda:0') (tensor(255.9457, device='cuda:0')) + | > current_lr_0: 0.00018865854064220266 + | > current_lr_1: 0.00018865854064220266 + | > step_time: 0.4446 (0.3887731160304343) + | > loader_time: 0.0071 (0.006277890168419179) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.4153966903686523 (2.4153966903686523) + | > loss_disc_real_0: 0.1811686009168625 (0.1811686009168625) + | > loss_disc_real_1: 0.1361245959997177 (0.1361245959997177) + | > loss_disc_real_2: 0.2508084774017334 (0.2508084774017334) + | > loss_disc_real_3: 0.24860399961471558 (0.24860399961471558) + | > loss_disc_real_4: 0.20072099566459656 (0.20072099566459656) + | > loss_disc_real_5: 0.3026319742202759 (0.3026319742202759) + | > loss_0: 2.4153966903686523 (2.4153966903686523) + | > loss_gen: 2.549856662750244 (2.549856662750244) + | > loss_kl: 2.5091428756713867 (2.5091428756713867) + | > loss_feat: 7.332428932189941 (7.332428932189941) + | > loss_mel: 19.736921310424805 (19.736921310424805) + | > loss_duration: 1.7674485445022583 (1.7674485445022583) + | > loss_1: 33.89579772949219 (33.89579772949219) + + --> STEP: 1 + | > loss_disc: 2.46602463722229 (2.46602463722229) + | > loss_disc_real_0: 0.16910849511623383 (0.16910849511623383) + | > loss_disc_real_1: 0.13940349221229553 (0.13940349221229553) + | > loss_disc_real_2: 0.24598446488380432 (0.24598446488380432) + | > loss_disc_real_3: 0.23882396519184113 (0.23882396519184113) + | > loss_disc_real_4: 0.2718386650085449 (0.2718386650085449) + | > loss_disc_real_5: 0.2696842849254608 (0.2696842849254608) + | > loss_0: 2.46602463722229 (2.46602463722229) + | > loss_gen: 2.442054510116577 (2.442054510116577) + | > loss_kl: 2.8466007709503174 (2.8466007709503174) + | > loss_feat: 7.413734436035156 (7.413734436035156) + | > loss_mel: 18.6944637298584 (18.6944637298584) + | > loss_duration: 1.8395392894744873 (1.8395392894744873) + | > loss_1: 33.236392974853516 (33.236392974853516) + + --> STEP: 2 + | > loss_disc: 2.565531015396118 (2.515777826309204) + | > loss_disc_real_0: 0.299333393573761 (0.2342209443449974) + | > loss_disc_real_1: 0.17527174949645996 (0.15733762085437775) + | > loss_disc_real_2: 0.2593822479248047 (0.2526833564043045) + | > loss_disc_real_3: 0.2204340398311615 (0.2296290025115013) + | > loss_disc_real_4: 0.23369987308979034 (0.25276926904916763) + | > loss_disc_real_5: 0.24805064499378204 (0.25886746495962143) + | > loss_0: 2.565531015396118 (2.515777826309204) + | > loss_gen: 2.543795585632324 (2.4929250478744507) + | > loss_kl: 2.491692543029785 (2.6691466569900513) + | > loss_feat: 7.975666046142578 (7.694700241088867) + | > loss_mel: 18.221982955932617 (18.458223342895508) + | > loss_duration: 1.7563481330871582 (1.7979437112808228) + | > loss_1: 32.98948669433594 (33.11293983459473) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00328981876373291 (-0.0003845691680908203) + | > avg_loss_disc: 2.515777826309204 (+0.019312381744384766) + | > avg_loss_disc_real_0: 0.2342209443449974 (+0.025052383542060852) + | > avg_loss_disc_real_1: 0.15733762085437775 (-0.12585574388504028) + | > avg_loss_disc_real_2: 0.2526833564043045 (+0.024263113737106323) + | > avg_loss_disc_real_3: 0.2296290025115013 (-0.005430497229099274) + | > avg_loss_disc_real_4: 0.25276926904916763 (+0.03012830764055252) + | > avg_loss_disc_real_5: 0.25886746495962143 (+0.015519648790359497) + | > avg_loss_0: 2.515777826309204 (+0.019312381744384766) + | > avg_loss_gen: 2.4929250478744507 (-0.11131393909454346) + | > avg_loss_kl: 2.6691466569900513 (+0.16602933406829834) + | > avg_loss_feat: 7.694700241088867 (+0.5357236862182617) + | > avg_loss_mel: 18.458223342895508 (+0.02653980255126953) + | > avg_loss_duration: 1.7979437112808228 (+0.009363293647766113) + | > avg_loss_1: 33.11293983459473 (+0.6263437271118164) + + + > EPOCH: 468/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:39:51)  + + --> TIME: 2025-04-19 06:39:56 -- STEP: 12/142 -- GLOBAL_STEP: 445750 + | > loss_disc: 2.4141063690185547 (2.3954362074534097) + | > loss_disc_real_0: 0.1542624533176422 (0.1258854027837515) + | > loss_disc_real_1: 0.18507862091064453 (0.2054960529009501) + | > loss_disc_real_2: 0.18783706426620483 (0.21386563902099928) + | > loss_disc_real_3: 0.21356259286403656 (0.219730656594038) + | > loss_disc_real_4: 0.23079891502857208 (0.2135691319902738) + | > loss_disc_real_5: 0.2150598168373108 (0.2256025212506453) + | > loss_0: 2.4141063690185547 (2.3954362074534097) + | > grad_norm_0: tensor(11.8247, device='cuda:0') (tensor(11.6143, device='cuda:0')) + | > loss_gen: 2.3549797534942627 (2.391623934110006) + | > loss_kl: 1.8188179731369019 (1.8141990303993225) + | > loss_feat: 7.593255043029785 (7.454975525538127) + | > loss_mel: 18.16131019592285 (17.606080849965412) + | > loss_duration: 1.4180419445037842 (1.4048365354537964) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.346405029296875 (30.67171573638916) + | > grad_norm_1: tensor(402.6034, device='cuda:0') (tensor(224.2674, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.3384 (0.3271695176760356) + | > loader_time: 0.0051 (0.004464407761891683) + + + --> TIME: 2025-04-19 06:40:04 -- STEP: 37/142 -- GLOBAL_STEP: 445775 + | > loss_disc: 2.3423285484313965 (2.4314018842336296) + | > loss_disc_real_0: 0.10881534963846207 (0.12004283573981878) + | > loss_disc_real_1: 0.19464342296123505 (0.20693862116014636) + | > loss_disc_real_2: 0.2177310585975647 (0.2240792976037876) + | > loss_disc_real_3: 0.20925605297088623 (0.22480505060505224) + | > loss_disc_real_4: 0.203542098402977 (0.22318559562837756) + | > loss_disc_real_5: 0.1919540911912918 (0.2288695448959196) + | > loss_0: 2.3423285484313965 (2.4314018842336296) + | > grad_norm_0: tensor(10.7382, device='cuda:0') (tensor(9.7062, device='cuda:0')) + | > loss_gen: 2.4692044258117676 (2.3724745093165214) + | > loss_kl: 1.8706178665161133 (1.8245700694419242) + | > loss_feat: 7.554477691650391 (7.362440405665217) + | > loss_mel: 18.713518142700195 (17.465539545626246) + | > loss_duration: 1.4372570514678955 (1.4173226259850167) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 32.045074462890625 (30.442347036825645) + | > grad_norm_1: tensor(220.6137, device='cuda:0') (tensor(227.1605, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.3534 (0.3392266389485952) + | > loader_time: 0.0065 (0.005105753202696104) + + + --> TIME: 2025-04-19 06:40:14 -- STEP: 62/142 -- GLOBAL_STEP: 445800 + | > loss_disc: 2.3830173015594482 (2.44061593855581) + | > loss_disc_real_0: 0.09583800286054611 (0.11860568268645194) + | > loss_disc_real_1: 0.19362826645374298 (0.20477348181509203) + | > loss_disc_real_2: 0.21858984231948853 (0.22453507925233535) + | > loss_disc_real_3: 0.18325652182102203 (0.2276332926365637) + | > loss_disc_real_4: 0.21492402255535126 (0.22778708151271265) + | > loss_disc_real_5: 0.23751620948314667 (0.23371335599691637) + | > loss_0: 2.3830173015594482 (2.44061593855581) + | > grad_norm_0: tensor(7.5917, device='cuda:0') (tensor(9.7306, device='cuda:0')) + | > loss_gen: 2.3004238605499268 (2.3739237170065595) + | > loss_kl: 1.947925090789795 (1.8430971003347827) + | > loss_feat: 7.583428859710693 (7.306946662164504) + | > loss_mel: 19.1666202545166 (17.612020892481645) + | > loss_duration: 1.4726529121398926 (1.4281915041708177) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 32.47105407714844 (30.564179758871756) + | > grad_norm_1: tensor(60.7616, device='cuda:0') (tensor(220.3440, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.3818 (0.35061499380296285) + | > loader_time: 0.006 (0.005441731022250268) + + + --> TIME: 2025-04-19 06:40:24 -- STEP: 87/142 -- GLOBAL_STEP: 445825 + | > loss_disc: 2.515331983566284 (2.4523567577888237) + | > loss_disc_real_0: 0.2189432680606842 (0.12288327540816932) + | > loss_disc_real_1: 0.22584936022758484 (0.20656324511971966) + | > loss_disc_real_2: 0.25607046484947205 (0.2262700504955204) + | > loss_disc_real_3: 0.260796457529068 (0.22848684332151523) + | > loss_disc_real_4: 0.2768711745738983 (0.22826920038667217) + | > loss_disc_real_5: 0.26358893513679504 (0.23283083839663143) + | > loss_0: 2.515331983566284 (2.4523567577888237) + | > grad_norm_0: tensor(26.2232, device='cuda:0') (tensor(10.4445, device='cuda:0')) + | > loss_gen: 2.479358673095703 (2.3667080950462953) + | > loss_kl: 2.0117838382720947 (1.8512079647217674) + | > loss_feat: 6.961767196655273 (7.237843102422254) + | > loss_mel: 18.617843627929688 (17.647502471660736) + | > loss_duration: 1.4723385572433472 (1.4389591408872056) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.5430908203125 (30.54222063086499) + | > grad_norm_1: tensor(655.1867, device='cuda:0') (tensor(233.2878, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.3995 (0.3625772657065558) + | > loader_time: 0.0064 (0.005709237065808526) + + + --> TIME: 2025-04-19 06:40:34 -- STEP: 112/142 -- GLOBAL_STEP: 445850 + | > loss_disc: 2.359072685241699 (2.4554699148450583) + | > loss_disc_real_0: 0.09991612285375595 (0.12274332085092153) + | > loss_disc_real_1: 0.19583044946193695 (0.20555612458182232) + | > loss_disc_real_2: 0.17864622175693512 (0.22584971692413092) + | > loss_disc_real_3: 0.18523216247558594 (0.22832985634782485) + | > loss_disc_real_4: 0.19886118173599243 (0.2284797481926424) + | > loss_disc_real_5: 0.19594673812389374 (0.2329996783020241) + | > loss_0: 2.359072685241699 (2.4554699148450583) + | > grad_norm_0: tensor(9.0519, device='cuda:0') (tensor(10.1983, device='cuda:0')) + | > loss_gen: 2.4504685401916504 (2.358859947749546) + | > loss_kl: 1.856256365776062 (1.8582827129534312) + | > loss_feat: 7.760218620300293 (7.186831678662982) + | > loss_mel: 18.12917709350586 (17.575787689004624) + | > loss_duration: 1.4498754739761353 (1.4464079045823641) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.645994186401367 (30.426169838224137) + | > grad_norm_1: tensor(285.6587, device='cuda:0') (tensor(231.4633, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.4254 (0.37331805910382965) + | > loader_time: 0.007 (0.0059650008167539325) + + + --> TIME: 2025-04-19 06:40:45 -- STEP: 137/142 -- GLOBAL_STEP: 445875 + | > loss_disc: 2.4797017574310303 (2.458094649071241) + | > loss_disc_real_0: 0.1308983564376831 (0.12424539649573556) + | > loss_disc_real_1: 0.25570571422576904 (0.2069859709182795) + | > loss_disc_real_2: 0.22843757271766663 (0.22606881811235943) + | > loss_disc_real_3: 0.24065878987312317 (0.22806050797013475) + | > loss_disc_real_4: 0.2257029414176941 (0.22745583429388758) + | > loss_disc_real_5: 0.21700866520404816 (0.23293342870952435) + | > loss_0: 2.4797017574310303 (2.458094649071241) + | > grad_norm_0: tensor(7.2672, device='cuda:0') (tensor(10.2494, device='cuda:0')) + | > loss_gen: 2.247562885284424 (2.3520366919301723) + | > loss_kl: 2.006737470626831 (1.8670593265199313) + | > loss_feat: 5.844531059265137 (7.14606307022763) + | > loss_mel: 15.834980010986328 (17.5696513322148) + | > loss_duration: 1.4710972309112549 (1.4527614882392605) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 27.404909133911133 (30.387571822117714) + | > grad_norm_1: tensor(165.1080, device='cuda:0') (tensor(235.8137, device='cuda:0')) + | > current_lr_0: 0.00018863495832462237 + | > current_lr_1: 0.00018863495832462237 + | > step_time: 0.4572 (0.3852813574519472) + | > loader_time: 0.0078 (0.006196276114804901) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.591758966445923 (2.591758966445923) + | > loss_disc_real_0: 0.2623266875743866 (0.2623266875743866) + | > loss_disc_real_1: 0.20911671221256256 (0.20911671221256256) + | > loss_disc_real_2: 0.2035709023475647 (0.2035709023475647) + | > loss_disc_real_3: 0.21603205800056458 (0.21603205800056458) + | > loss_disc_real_4: 0.21603919565677643 (0.21603919565677643) + | > loss_disc_real_5: 0.2844916880130768 (0.2844916880130768) + | > loss_0: 2.591758966445923 (2.591758966445923) + | > loss_gen: 2.450921058654785 (2.450921058654785) + | > loss_kl: 2.5849266052246094 (2.5849266052246094) + | > loss_feat: 6.6566901206970215 (6.6566901206970215) + | > loss_mel: 18.38494110107422 (18.38494110107422) + | > loss_duration: 1.7707233428955078 (1.7707233428955078) + | > loss_1: 31.848203659057617 (31.848203659057617) + + --> STEP: 1 + | > loss_disc: 2.417799234390259 (2.417799234390259) + | > loss_disc_real_0: 0.17842529714107513 (0.17842529714107513) + | > loss_disc_real_1: 0.24452194571495056 (0.24452194571495056) + | > loss_disc_real_2: 0.21058806777000427 (0.21058806777000427) + | > loss_disc_real_3: 0.20911969244480133 (0.20911969244480133) + | > loss_disc_real_4: 0.22007940709590912 (0.22007940709590912) + | > loss_disc_real_5: 0.2750782370567322 (0.2750782370567322) + | > loss_0: 2.417799234390259 (2.417799234390259) + | > loss_gen: 2.539389133453369 (2.539389133453369) + | > loss_kl: 2.8595762252807617 (2.8595762252807617) + | > loss_feat: 7.1460041999816895 (7.1460041999816895) + | > loss_mel: 17.70026969909668 (17.70026969909668) + | > loss_duration: 1.8163909912109375 (1.8163909912109375) + | > loss_1: 32.06163024902344 (32.06163024902344) + + --> STEP: 2 + | > loss_disc: 2.466541051864624 (2.4421701431274414) + | > loss_disc_real_0: 0.14404024183750153 (0.16123276948928833) + | > loss_disc_real_1: 0.26579463481903076 (0.25515829026699066) + | > loss_disc_real_2: 0.1960919350385666 (0.20334000140428543) + | > loss_disc_real_3: 0.22100640833377838 (0.21506305038928986) + | > loss_disc_real_4: 0.2099096179008484 (0.21499451249837875) + | > loss_disc_real_5: 0.2336355596780777 (0.25435689836740494) + | > loss_0: 2.466541051864624 (2.4421701431274414) + | > loss_gen: 2.476714611053467 (2.508051872253418) + | > loss_kl: 2.209407091140747 (2.5344916582107544) + | > loss_feat: 7.5995330810546875 (7.3727686405181885) + | > loss_mel: 18.865001678466797 (18.28263568878174) + | > loss_duration: 1.7585911750793457 (1.7874910831451416) + | > loss_1: 32.90924835205078 (32.48543930053711) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003297090530395508 (+7.271766662597656e-06) + | > avg_loss_disc: 2.4421701431274414 (-0.0736076831817627) + | > avg_loss_disc_real_0: 0.16123276948928833 (-0.07298817485570908) + | > avg_loss_disc_real_1: 0.25515829026699066 (+0.09782066941261292) + | > avg_loss_disc_real_2: 0.20334000140428543 (-0.049343355000019073) + | > avg_loss_disc_real_3: 0.21506305038928986 (-0.014565952122211456) + | > avg_loss_disc_real_4: 0.21499451249837875 (-0.03777475655078888) + | > avg_loss_disc_real_5: 0.25435689836740494 (-0.004510566592216492) + | > avg_loss_0: 2.4421701431274414 (-0.0736076831817627) + | > avg_loss_gen: 2.508051872253418 (+0.015126824378967285) + | > avg_loss_kl: 2.5344916582107544 (-0.13465499877929688) + | > avg_loss_feat: 7.3727686405181885 (-0.3219316005706787) + | > avg_loss_mel: 18.28263568878174 (-0.17558765411376953) + | > avg_loss_duration: 1.7874910831451416 (-0.010452628135681152) + | > avg_loss_1: 32.48543930053711 (-0.6275005340576172) + + + > EPOCH: 469/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:40:49)  + + --> TIME: 2025-04-19 06:40:57 -- STEP: 20/142 -- GLOBAL_STEP: 445900 + | > loss_disc: 2.360797166824341 (2.419428288936615) + | > loss_disc_real_0: 0.12448544800281525 (0.11179340668022633) + | > loss_disc_real_1: 0.1802353858947754 (0.20387205779552459) + | > loss_disc_real_2: 0.22220861911773682 (0.22243575528264045) + | > loss_disc_real_3: 0.22963528335094452 (0.22678659483790398) + | > loss_disc_real_4: 0.22191780805587769 (0.22441729605197908) + | > loss_disc_real_5: 0.240666002035141 (0.2334293968975544) + | > loss_0: 2.360797166824341 (2.419428288936615) + | > grad_norm_0: tensor(13.5738, device='cuda:0') (tensor(10.7533, device='cuda:0')) + | > loss_gen: 2.49662709236145 (2.4230984330177305) + | > loss_kl: 1.776881456375122 (1.769182300567627) + | > loss_feat: 7.556019306182861 (7.563748526573181) + | > loss_mel: 17.94096565246582 (17.677901935577392) + | > loss_duration: 1.4615395069122314 (1.4087981820106505) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 31.23203468322754 (30.842729473114012) + | > grad_norm_1: tensor(275.9532, device='cuda:0') (tensor(280.9159, device='cuda:0')) + | > current_lr_0: 0.00018861137895483178 + | > current_lr_1: 0.00018861137895483178 + | > step_time: 0.3434 (0.33529850244522097) + | > loader_time: 0.0056 (0.004890716075897217) + + + --> TIME: 2025-04-19 06:41:06 -- STEP: 45/142 -- GLOBAL_STEP: 445925 + | > loss_disc: 2.453766107559204 (2.4064518345726866) + | > loss_disc_real_0: 0.08290908485651016 (0.11036576446559694) + | > loss_disc_real_1: 0.17479154467582703 (0.20111090176635318) + | > loss_disc_real_2: 0.2369861751794815 (0.22201147708627914) + | > loss_disc_real_3: 0.22321440279483795 (0.22647153205341763) + | > loss_disc_real_4: 0.20218896865844727 (0.22446692387262981) + | > loss_disc_real_5: 0.21192485094070435 (0.23168885111808776) + | > loss_0: 2.453766107559204 (2.4064518345726866) + | > grad_norm_0: tensor(12.1044, device='cuda:0') (tensor(10.6353, device='cuda:0')) + | > loss_gen: 2.3209404945373535 (2.4179460525512697) + | > loss_kl: 1.7452061176300049 (1.7852466556761) + | > loss_feat: 7.410450458526611 (7.409019014570448) + | > loss_mel: 17.302732467651367 (17.625023078918453) + | > loss_duration: 1.4438692331314087 (1.4203073554568824) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.22319793701172 (30.657542165120443) + | > grad_norm_1: tensor(169.4399, device='cuda:0') (tensor(292.9622, device='cuda:0')) + | > current_lr_0: 0.00018861137895483178 + | > current_lr_1: 0.00018861137895483178 + | > step_time: 0.3803 (0.35041666030883795) + | > loader_time: 0.0064 (0.005373568005032008) + + + --> TIME: 2025-04-19 06:41:16 -- STEP: 70/142 -- GLOBAL_STEP: 445950 + | > loss_disc: 2.501028060913086 (2.4226733616420204) + | > loss_disc_real_0: 0.13208265602588654 (0.11505669419254576) + | > loss_disc_real_1: 0.2202146202325821 (0.20403602165835247) + | > loss_disc_real_2: 0.23702429234981537 (0.22395455794675012) + | > loss_disc_real_3: 0.2525521516799927 (0.22785229214600156) + | > loss_disc_real_4: 0.240286722779274 (0.22554500741618017) + | > loss_disc_real_5: 0.2508217990398407 (0.23156470784119199) + | > loss_0: 2.501028060913086 (2.4226733616420204) + | > grad_norm_0: tensor(9.7029, device='cuda:0') (tensor(10.6414, device='cuda:0')) + | > loss_gen: 2.3738977909088135 (2.4063005822045467) + | > loss_kl: 1.9297558069229126 (1.802036566393716) + | > loss_feat: 6.52770471572876 (7.321759142194476) + | > loss_mel: 16.717607498168945 (17.55723851067678) + | > loss_duration: 1.4697790145874023 (1.4308195011956353) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 29.01874542236328 (30.518154335021972) + | > grad_norm_1: tensor(236.6055, device='cuda:0') (tensor(298.6476, device='cuda:0')) + | > current_lr_0: 0.00018861137895483178 + | > current_lr_1: 0.00018861137895483178 + | > step_time: 0.3925 (0.36383233070373533) + | > loader_time: 0.0064 (0.005678050858633858) + + + --> TIME: 2025-04-19 06:41:26 -- STEP: 95/142 -- GLOBAL_STEP: 445975 + | > loss_disc: 2.439065933227539 (2.431389048225001) + | > loss_disc_real_0: 0.1774177998304367 (0.11688701353575054) + | > loss_disc_real_1: 0.1671602576971054 (0.20376739000019276) + | > loss_disc_real_2: 0.21552301943302155 (0.22582081289667832) + | > loss_disc_real_3: 0.2354285567998886 (0.2280068532416695) + | > loss_disc_real_4: 0.2387031763792038 (0.2271143875624004) + | > loss_disc_real_5: 0.22953708469867706 (0.23177309098996615) + | > loss_0: 2.439065933227539 (2.431389048225001) + | > grad_norm_0: tensor(14.1865, device='cuda:0') (tensor(10.7781, device='cuda:0')) + | > loss_gen: 2.3832197189331055 (2.397283988249932) + | > loss_kl: 1.8274399042129517 (1.810257517664056) + | > loss_feat: 6.822844505310059 (7.3107329970911925) + | > loss_mel: 18.309459686279297 (17.682823934053108) + | > loss_duration: 1.4776344299316406 (1.441403194477684) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.820598602294922 (30.64250169051321) + | > grad_norm_1: tensor(295.3382, device='cuda:0') (tensor(299.2422, device='cuda:0')) + | > current_lr_0: 0.00018861137895483178 + | > current_lr_1: 0.00018861137895483178 + | > step_time: 0.4072 (0.37276696154945776) + | > loader_time: 0.0067 (0.005925562507227846) + + + --> TIME: 2025-04-19 06:41:37 -- STEP: 120/142 -- GLOBAL_STEP: 446000 + | > loss_disc: 2.517423391342163 (2.4358783821264893) + | > loss_disc_real_0: 0.09646026045084 (0.11881676266590754) + | > loss_disc_real_1: 0.22365227341651917 (0.20455089484651884) + | > loss_disc_real_2: 0.21911613643169403 (0.22596657623847324) + | > loss_disc_real_3: 0.271381139755249 (0.2278488554060459) + | > loss_disc_real_4: 0.2499830275774002 (0.22687426048020523) + | > loss_disc_real_5: 0.2704554796218872 (0.23110364750027657) + | > loss_0: 2.517423391342163 (2.4358783821264893) + | > grad_norm_0: tensor(4.1167, device='cuda:0') (tensor(11.4976, device='cuda:0')) + | > loss_gen: 2.356173515319824 (2.3849764188130727) + | > loss_kl: 1.7900307178497314 (1.8246491650740306) + | > loss_feat: 6.696159839630127 (7.259733947118123) + | > loss_mel: 18.17612075805664 (17.628469896316513) + | > loss_duration: 1.4884295463562012 (1.4481177210807803) + | > amp_scaler: 32.0 (32.0) + | > loss_1: 30.506916046142578 (30.545947233835857) + | > grad_norm_1: tensor(214.6509, device='cuda:0') (tensor(301.1183, device='cuda:0')) + | > current_lr_0: 0.00018861137895483178 + | > current_lr_1: 0.00018861137895483178 + | > step_time: 0.4315 (0.38244728446006776) + | > loader_time: 0.0075 (0.006176451841990152) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.46279239654541 (2.46279239654541) + | > loss_disc_real_0: 0.12420257180929184 (0.12420257180929184) + | > loss_disc_real_1: 0.2008817195892334 (0.2008817195892334) + | > loss_disc_real_2: 0.2457684576511383 (0.2457684576511383) + | > loss_disc_real_3: 0.24337157607078552 (0.24337157607078552) + | > loss_disc_real_4: 0.18619926273822784 (0.18619926273822784) + | > loss_disc_real_5: 0.2943371534347534 (0.2943371534347534) + | > loss_0: 2.46279239654541 (2.46279239654541) + | > loss_gen: 2.294670581817627 (2.294670581817627) + | > loss_kl: 2.380042791366577 (2.380042791366577) + | > loss_feat: 6.974713325500488 (6.974713325500488) + | > loss_mel: 18.69518280029297 (18.69518280029297) + | > loss_duration: 1.7605817317962646 (1.7605817317962646) + | > loss_1: 32.10519027709961 (32.10519027709961) + + --> STEP: 1 + | > loss_disc: 2.3776516914367676 (2.3776516914367676) + | > loss_disc_real_0: 0.10963761061429977 (0.10963761061429977) + | > loss_disc_real_1: 0.2309824526309967 (0.2309824526309967) + | > loss_disc_real_2: 0.21639478206634521 (0.21639478206634521) + | > loss_disc_real_3: 0.2121691256761551 (0.2121691256761551) + | > loss_disc_real_4: 0.2096118927001953 (0.2096118927001953) + | > loss_disc_real_5: 0.20413024723529816 (0.20413024723529816) + | > loss_0: 2.3776516914367676 (2.3776516914367676) + | > loss_gen: 2.247483253479004 (2.247483253479004) + | > loss_kl: 2.671872138977051 (2.671872138977051) + | > loss_feat: 7.02993631362915 (7.02993631362915) + | > loss_mel: 19.246070861816406 (19.246070861816406) + | > loss_duration: 1.8478996753692627 (1.8478996753692627) + | > loss_1: 33.04326248168945 (33.04326248168945) + + --> STEP: 2 + | > loss_disc: 2.5082273483276367 (2.442939519882202) + | > loss_disc_real_0: 0.08520559966564178 (0.09742160513997078) + | > loss_disc_real_1: 0.2865843176841736 (0.25878338515758514) + | > loss_disc_real_2: 0.2635434865951538 (0.2399691343307495) + | > loss_disc_real_3: 0.2023445963859558 (0.20725686103105545) + | > loss_disc_real_4: 0.24258942902088165 (0.22610066086053848) + | > loss_disc_real_5: 0.26653537154197693 (0.23533280938863754) + | > loss_0: 2.5082273483276367 (2.442939519882202) + | > loss_gen: 2.2800538539886475 (2.2637685537338257) + | > loss_kl: 2.1977686882019043 (2.4348204135894775) + | > loss_feat: 7.456120491027832 (7.243028402328491) + | > loss_mel: 18.335691452026367 (18.790881156921387) + | > loss_duration: 1.7450212240219116 (1.7964604496955872) + | > loss_1: 32.01465606689453 (32.52895927429199) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035539865493774414 (+0.0002568960189819336) + | > avg_loss_disc: 2.442939519882202 (+0.0007693767547607422) + | > avg_loss_disc_real_0: 0.09742160513997078 (-0.06381116434931755) + | > avg_loss_disc_real_1: 0.25878338515758514 (+0.0036250948905944824) + | > avg_loss_disc_real_2: 0.2399691343307495 (+0.03662913292646408) + | > avg_loss_disc_real_3: 0.20725686103105545 (-0.0078061893582344055) + | > avg_loss_disc_real_4: 0.22610066086053848 (+0.011106148362159729) + | > avg_loss_disc_real_5: 0.23533280938863754 (-0.019024088978767395) + | > avg_loss_0: 2.442939519882202 (+0.0007693767547607422) + | > avg_loss_gen: 2.2637685537338257 (-0.24428331851959229) + | > avg_loss_kl: 2.4348204135894775 (-0.09967124462127686) + | > avg_loss_feat: 7.243028402328491 (-0.12974023818969727) + | > avg_loss_mel: 18.790881156921387 (+0.5082454681396484) + | > avg_loss_duration: 1.7964604496955872 (+0.008969366550445557) + | > avg_loss_1: 32.52895927429199 (+0.04351997375488281) + + + > EPOCH: 470/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:41:48)  + + --> TIME: 2025-04-19 06:41:50 -- STEP: 3/142 -- GLOBAL_STEP: 446025 + | > loss_disc: 2.4678845405578613 (2.494783639907837) + | > loss_disc_real_0: 0.15456335246562958 (0.19630805651346842) + | > loss_disc_real_1: 0.16500620543956757 (0.21109423538049063) + | > loss_disc_real_2: 0.2263014018535614 (0.231868177652359) + | > loss_disc_real_3: 0.25742799043655396 (0.24133098622163138) + | > loss_disc_real_4: 0.2734561264514923 (0.238298237323761) + | > loss_disc_real_5: 0.2638061046600342 (0.2563423216342926) + | > loss_0: 2.4678845405578613 (2.494783639907837) + | > grad_norm_0: tensor(25.6506, device='cuda:0') (tensor(30.5211, device='cuda:0')) + | > loss_gen: 2.2946856021881104 (2.489838202794393) + | > loss_kl: 1.8572190999984741 (1.738300085067749) + | > loss_feat: 7.631375312805176 (7.766525745391846) + | > loss_mel: 18.895498275756836 (18.439942677815754) + | > loss_duration: 1.4036760330200195 (1.3973316351572673) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.082454681396484 (31.83193842569987) + | > grad_norm_1: tensor(123.8022, device='cuda:0') (tensor(164.6750, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.3237 (0.32306090990702313) + | > loader_time: 0.0036 (0.004284461339314778) + + + --> TIME: 2025-04-19 06:41:58 -- STEP: 28/142 -- GLOBAL_STEP: 446050 + | > loss_disc: 2.5320515632629395 (2.476220965385437) + | > loss_disc_real_0: 0.18689166009426117 (0.11533737328967877) + | > loss_disc_real_1: 0.20016208291053772 (0.2039331336106573) + | > loss_disc_real_2: 0.22775377333164215 (0.22955803839223726) + | > loss_disc_real_3: 0.2360425740480423 (0.2368947012083871) + | > loss_disc_real_4: 0.2586214542388916 (0.24478243717125484) + | > loss_disc_real_5: 0.26969316601753235 (0.2328151815703937) + | > loss_0: 2.5320515632629395 (2.476220965385437) + | > grad_norm_0: tensor(20.1166, device='cuda:0') (tensor(12.0889, device='cuda:0')) + | > loss_gen: 2.34354829788208 (2.432837401117598) + | > loss_kl: 1.7387648820877075 (1.7823714443615504) + | > loss_feat: 6.3794169425964355 (7.411741478102548) + | > loss_mel: 16.145570755004883 (17.716630050114222) + | > loss_duration: 1.4381738901138306 (1.4139468712466103) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.045473098754883 (30.757527147020614) + | > grad_norm_1: tensor(68.8330, device='cuda:0') (tensor(244.1172, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.3459 (0.33476412296295166) + | > loader_time: 0.0051 (0.004793022360120501) + + + --> TIME: 2025-04-19 06:42:08 -- STEP: 53/142 -- GLOBAL_STEP: 446075 + | > loss_disc: 2.4512102603912354 (2.4566014217880543) + | > loss_disc_real_0: 0.1519704908132553 (0.11617055429883723) + | > loss_disc_real_1: 0.1979263424873352 (0.20667882882199198) + | > loss_disc_real_2: 0.20229284465312958 (0.2275636558825115) + | > loss_disc_real_3: 0.19339783489704132 (0.23103509702772465) + | > loss_disc_real_4: 0.1983085721731186 (0.23485988800255758) + | > loss_disc_real_5: 0.1953609436750412 (0.23088452163732276) + | > loss_0: 2.4512102603912354 (2.4566014217880543) + | > grad_norm_0: tensor(5.0224, device='cuda:0') (tensor(9.8291, device='cuda:0')) + | > loss_gen: 2.1875712871551514 (2.3856437431191497) + | > loss_kl: 1.848447561264038 (1.8079590212624028) + | > loss_feat: 7.242587566375732 (7.252106072767726) + | > loss_mel: 16.247718811035156 (17.548127678205383) + | > loss_duration: 1.4412915706634521 (1.4213021566283028) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 28.967615127563477 (30.415138496542877) + | > grad_norm_1: tensor(36.8384, device='cuda:0') (tensor(199.5860, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.3675 (0.3464478771641569) + | > loader_time: 0.0059 (0.005184789873519033) + + + --> TIME: 2025-04-19 06:42:17 -- STEP: 78/142 -- GLOBAL_STEP: 446100 + | > loss_disc: 2.394956350326538 (2.4530648268186135) + | > loss_disc_real_0: 0.08252378553152084 (0.11866253495025329) + | > loss_disc_real_1: 0.1821705400943756 (0.2049110610133562) + | > loss_disc_real_2: 0.23199371993541718 (0.2265058981302457) + | > loss_disc_real_3: 0.24891342222690582 (0.23124905771169907) + | > loss_disc_real_4: 0.22554858028888702 (0.23353286641530502) + | > loss_disc_real_5: 0.2260628193616867 (0.23195940867448464) + | > loss_0: 2.394956350326538 (2.4530648268186135) + | > grad_norm_0: tensor(6.5105, device='cuda:0') (tensor(8.9417, device='cuda:0')) + | > loss_gen: 2.422135829925537 (2.384248583744734) + | > loss_kl: 1.867740511894226 (1.827791990377964) + | > loss_feat: 7.641704559326172 (7.267628877590864) + | > loss_mel: 18.698312759399414 (17.659494192172314) + | > loss_duration: 1.473848581314087 (1.4325886368751521) + | > amp_scaler: 16.0 (16.0) + | > loss_1: 32.10374450683594 (30.571752205873146) + | > grad_norm_1: tensor(245.7706, device='cuda:0') (tensor(170.1045, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.3932 (0.35858927017603154) + | > loader_time: 0.0061 (0.005498595726795684) + + + --> TIME: 2025-04-19 06:42:28 -- STEP: 103/142 -- GLOBAL_STEP: 446125 + | > loss_disc: 2.429155111312866 (2.458586938172868) + | > loss_disc_real_0: 0.15021052956581116 (0.11981393516208362) + | > loss_disc_real_1: 0.23198507726192474 (0.2062735247959211) + | > loss_disc_real_2: 0.22837044298648834 (0.22764403307900846) + | > loss_disc_real_3: 0.22098439931869507 (0.23083762520725287) + | > loss_disc_real_4: 0.22725939750671387 (0.23256713585946168) + | > loss_disc_real_5: 0.21669408679008484 (0.23213495690267064) + | > loss_0: 2.429155111312866 (2.458586938172868) + | > grad_norm_0: tensor(5.3379, device='cuda:0') (tensor(8.8206, device='cuda:0')) + | > loss_gen: 2.423898935317993 (2.3692772214852504) + | > loss_kl: 1.6767936944961548 (nan) + | > loss_feat: 6.6280035972595215 (7.148817511438166) + | > loss_mel: 16.899564743041992 (17.521257724576774) + | > loss_duration: 1.4908010959625244 (nan) + | > amp_scaler: 4.0 (13.941747572815533) + | > loss_1: 29.119060516357422 (nan) + | > grad_norm_1: tensor(180.6596, device='cuda:0') (tensor(170.8112, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.4102 (0.36885844156580067) + | > loader_time: 0.0069 (0.005764167285659938) + + + --> TIME: 2025-04-19 06:42:39 -- STEP: 128/142 -- GLOBAL_STEP: 446150 + | > loss_disc: 2.4496688842773438 (2.4553221575915813) + | > loss_disc_real_0: 0.1304028481245041 (0.12077890775981359) + | > loss_disc_real_1: 0.19361214339733124 (0.2060496772173792) + | > loss_disc_real_2: 0.2317965179681778 (0.22745491471141577) + | > loss_disc_real_3: 0.23509587347507477 (0.22950035997200757) + | > loss_disc_real_4: 0.2193743884563446 (0.2314524605171755) + | > loss_disc_real_5: 0.23669706284999847 (0.23089386092033237) + | > loss_0: 2.4496688842773438 (2.4553221575915813) + | > grad_norm_0: tensor(5.5467, device='cuda:0') (tensor(8.8566, device='cuda:0')) + | > loss_gen: 2.4516994953155518 (2.367868079803885) + | > loss_kl: 1.8951457738876343 (nan) + | > loss_feat: 7.073929309844971 (7.164942089468241) + | > loss_mel: 17.742889404296875 (17.50944531708955) + | > loss_duration: 1.4799920320510864 (nan) + | > amp_scaler: 4.0 (12.0) + | > loss_1: 30.64365577697754 (nan) + | > grad_norm_1: tensor(191.4029, device='cuda:0') (tensor(175.2202, device='cuda:0')) + | > current_lr_0: 0.0001885878025324624 + | > current_lr_1: 0.0001885878025324624 + | > step_time: 0.4352 (0.38040980882942677) + | > loader_time: 0.0074 (0.0059835780411958694) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.525020122528076 (2.525020122528076) + | > loss_disc_real_0: 0.17908862233161926 (0.17908862233161926) + | > loss_disc_real_1: 0.20104660093784332 (0.20104660093784332) + | > loss_disc_real_2: 0.23034097254276276 (0.23034097254276276) + | > loss_disc_real_3: 0.24471290409564972 (0.24471290409564972) + | > loss_disc_real_4: 0.2390163540840149 (0.2390163540840149) + | > loss_disc_real_5: 0.2976442575454712 (0.2976442575454712) + | > loss_0: 2.525020122528076 (2.525020122528076) + | > loss_gen: 2.430975914001465 (2.430975914001465) + | > loss_kl: 2.6654460430145264 (2.6654460430145264) + | > loss_feat: 7.171064376831055 (7.171064376831055) + | > loss_mel: 19.131166458129883 (19.131166458129883) + | > loss_duration: 1.7677252292633057 (1.7677252292633057) + | > loss_1: 33.1663818359375 (33.1663818359375) + + --> STEP: 1 + | > loss_disc: 2.4291281700134277 (2.4291281700134277) + | > loss_disc_real_0: 0.17106950283050537 (0.17106950283050537) + | > loss_disc_real_1: 0.22735270857810974 (0.22735270857810974) + | > loss_disc_real_2: 0.208076611161232 (0.208076611161232) + | > loss_disc_real_3: 0.23325331509113312 (0.23325331509113312) + | > loss_disc_real_4: 0.25652796030044556 (0.25652796030044556) + | > loss_disc_real_5: 0.2338383048772812 (0.2338383048772812) + | > loss_0: 2.4291281700134277 (2.4291281700134277) + | > loss_gen: 2.535861015319824 (2.535861015319824) + | > loss_kl: 2.7782962322235107 (2.7782962322235107) + | > loss_feat: 7.209992408752441 (7.209992408752441) + | > loss_mel: 18.84483528137207 (18.84483528137207) + | > loss_duration: 1.8572334051132202 (1.8572334051132202) + | > loss_1: 33.226219177246094 (33.226219177246094) + + --> STEP: 2 + | > loss_disc: 2.705625057220459 (2.5673766136169434) + | > loss_disc_real_0: 0.30601900815963745 (0.2385442554950714) + | > loss_disc_real_1: 0.2171003818511963 (0.22222654521465302) + | > loss_disc_real_2: 0.24150221049785614 (0.22478941082954407) + | > loss_disc_real_3: 0.2736837863922119 (0.2534685507416725) + | > loss_disc_real_4: 0.28206866979599 (0.2692983150482178) + | > loss_disc_real_5: 0.2851893901824951 (0.25951384752988815) + | > loss_0: 2.705625057220459 (2.5673766136169434) + | > loss_gen: 2.3225412368774414 (2.429201126098633) + | > loss_kl: 2.313934803009033 (2.546115517616272) + | > loss_feat: 4.760962009429932 (5.9854772090911865) + | > loss_mel: 13.57678508758545 (16.21081018447876) + | > loss_duration: 1.743767499923706 (1.8005004525184631) + | > loss_1: 24.71799087524414 (28.972105026245117) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003559589385986328 (+5.602836608886719e-06) + | > avg_loss_disc: 2.5673766136169434 (+0.12443709373474121) + | > avg_loss_disc_real_0: 0.2385442554950714 (+0.14112265035510063) + | > avg_loss_disc_real_1: 0.22222654521465302 (-0.03655683994293213) + | > avg_loss_disc_real_2: 0.22478941082954407 (-0.015179723501205444) + | > avg_loss_disc_real_3: 0.2534685507416725 (+0.046211689710617065) + | > avg_loss_disc_real_4: 0.2692983150482178 (+0.04319765418767929) + | > avg_loss_disc_real_5: 0.25951384752988815 (+0.02418103814125061) + | > avg_loss_0: 2.5673766136169434 (+0.12443709373474121) + | > avg_loss_gen: 2.429201126098633 (+0.16543257236480713) + | > avg_loss_kl: 2.546115517616272 (+0.11129510402679443) + | > avg_loss_feat: 5.9854772090911865 (-1.2575511932373047) + | > avg_loss_mel: 16.21081018447876 (-2.580070972442627) + | > avg_loss_duration: 1.8005004525184631 (+0.0040400028228759766) + | > avg_loss_1: 28.972105026245117 (-3.556854248046875) + + + > EPOCH: 471/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:42:46)  + + --> TIME: 2025-04-19 06:42:51 -- STEP: 11/142 -- GLOBAL_STEP: 446175 + | > loss_disc: 2.3373923301696777 (2.4048523902893066) + | > loss_disc_real_0: 0.11302962154150009 (0.11664318970658562) + | > loss_disc_real_1: 0.2047671526670456 (0.206562802195549) + | > loss_disc_real_2: 0.22979584336280823 (0.21734080666845496) + | > loss_disc_real_3: 0.23290275037288666 (0.22246579825878143) + | > loss_disc_real_4: 0.20651797950267792 (0.21727397089654749) + | > loss_disc_real_5: 0.23117904365062714 (0.22549250451001254) + | > loss_0: 2.3373923301696777 (2.4048523902893066) + | > grad_norm_0: tensor(10.2513, device='cuda:0') (tensor(8.9077, device='cuda:0')) + | > loss_gen: 2.6020853519439697 (2.3881435827775435) + | > loss_kl: 1.7995368242263794 (1.7799150076779453) + | > loss_feat: 7.928264617919922 (7.476535927165639) + | > loss_mel: 19.333635330200195 (17.87271395596591) + | > loss_duration: 1.418762445449829 (1.4061715711246838) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 33.08228302001953 (30.923480120572176) + | > grad_norm_1: tensor(197.8157, device='cuda:0') (tensor(202.2554, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.3305 (0.32592886144464667) + | > loader_time: 0.0046 (0.004401337016712536) + + + --> TIME: 2025-04-19 06:43:00 -- STEP: 36/142 -- GLOBAL_STEP: 446200 + | > loss_disc: 2.3133792877197266 (2.406338943375482) + | > loss_disc_real_0: 0.11760365217924118 (0.1137541967133681) + | > loss_disc_real_1: 0.21714754402637482 (0.20346943454609978) + | > loss_disc_real_2: 0.2169814258813858 (0.22247058360113037) + | > loss_disc_real_3: 0.22286905348300934 (0.22085297190480763) + | > loss_disc_real_4: 0.2131582498550415 (0.2201063090728389) + | > loss_disc_real_5: 0.2649751603603363 (0.22768236613935894) + | > loss_0: 2.3133792877197266 (2.406338943375482) + | > grad_norm_0: tensor(11.5414, device='cuda:0') (tensor(9.3093, device='cuda:0')) + | > loss_gen: 2.5920841693878174 (2.4103344281514487) + | > loss_kl: 1.898637056350708 (1.8107393483320873) + | > loss_feat: 7.827691078186035 (7.430852519141303) + | > loss_mel: 18.01040267944336 (17.75933949152628) + | > loss_duration: 1.4206006526947021 (1.4173322982258267) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 31.74941635131836 (30.828598022460938) + | > grad_norm_1: tensor(386.9280, device='cuda:0') (tensor(276.3587, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.3586 (0.34017692671881783) + | > loader_time: 0.0083 (0.004975736141204834) + + + --> TIME: 2025-04-19 06:43:09 -- STEP: 61/142 -- GLOBAL_STEP: 446225 + | > loss_disc: 2.3948614597320557 (2.416093638685883) + | > loss_disc_real_0: 0.09556280821561813 (0.11460616390724651) + | > loss_disc_real_1: 0.18974252045154572 (0.20385246125401044) + | > loss_disc_real_2: 0.23763452470302582 (0.2219529782162338) + | > loss_disc_real_3: 0.21941353380680084 (0.22536876064832093) + | > loss_disc_real_4: 0.21632210910320282 (0.2234391281839277) + | > loss_disc_real_5: 0.19895294308662415 (0.229155604223736) + | > loss_0: 2.3948614597320557 (2.416093638685883) + | > grad_norm_0: tensor(12.2185, device='cuda:0') (tensor(10.6765, device='cuda:0')) + | > loss_gen: 2.5843939781188965 (2.410196929681496) + | > loss_kl: 1.9141851663589478 (1.837654999045075) + | > loss_feat: 7.718173980712891 (7.3765104715941385) + | > loss_mel: 17.822824478149414 (17.716313237049533) + | > loss_duration: 1.4414253234863281 (1.4288682918079565) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 31.48100471496582 (30.769543944812213) + | > grad_norm_1: tensor(477.8987, device='cuda:0') (tensor(289.8618, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.3805 (0.35357830954379726) + | > loader_time: 0.006 (0.005297578749109487) + + + --> TIME: 2025-04-19 06:43:19 -- STEP: 86/142 -- GLOBAL_STEP: 446250 + | > loss_disc: 2.4283087253570557 (2.4337325317915095) + | > loss_disc_real_0: 0.15244555473327637 (0.11866314934436666) + | > loss_disc_real_1: 0.19819729030132294 (0.2059253402227579) + | > loss_disc_real_2: 0.2508482336997986 (0.2243634620031645) + | > loss_disc_real_3: 0.19746974110603333 (0.22669490946586743) + | > loss_disc_real_4: 0.1925840973854065 (0.2256167118632516) + | > loss_disc_real_5: 0.20182670652866364 (0.23032107949256897) + | > loss_0: 2.4283087253570557 (2.4337325317915095) + | > grad_norm_0: tensor(17.3194, device='cuda:0') (tensor(12.4955, device='cuda:0')) + | > loss_gen: 2.234020948410034 (2.403536646865134) + | > loss_kl: 1.8230769634246826 (1.8541380305622899) + | > loss_feat: 7.300601482391357 (7.301399458286374) + | > loss_mel: 17.59442138671875 (17.660203467967893) + | > loss_duration: 1.4878277778625488 (1.4404218487961347) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 30.4399471282959 (30.659699373467024) + | > grad_norm_1: tensor(255.7132, device='cuda:0') (tensor(303.7775, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.4019 (0.3650468366090642) + | > loader_time: 0.0066 (0.0056394809900328175) + + + --> TIME: 2025-04-19 06:43:30 -- STEP: 111/142 -- GLOBAL_STEP: 446275 + | > loss_disc: 2.4937686920166016 (2.4439982745024547) + | > loss_disc_real_0: 0.1670776903629303 (0.11977105064166559) + | > loss_disc_real_1: 0.19196760654449463 (0.20672616456543003) + | > loss_disc_real_2: 0.21573857963085175 (0.2252928147444854) + | > loss_disc_real_3: 0.23948031663894653 (0.22727022568384805) + | > loss_disc_real_4: 0.26847410202026367 (0.22685165542203026) + | > loss_disc_real_5: 0.24655739963054657 (0.23095163179410472) + | > loss_0: 2.4937686920166016 (2.4439982745024547) + | > grad_norm_0: tensor(10.5195, device='cuda:0') (tensor(13.5647, device='cuda:0')) + | > loss_gen: 2.5046186447143555 (2.391141642321336) + | > loss_kl: 1.8605691194534302 (1.8659446035419498) + | > loss_feat: 7.072390556335449 (7.238665241378921) + | > loss_mel: 18.91529655456543 (17.64628394874367) + | > loss_duration: 1.4926836490631104 (1.4484339855812687) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 31.845558166503906 (30.59046942908485) + | > grad_norm_1: tensor(484.9380, device='cuda:0') (tensor(317.3786, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.4279 (0.37593690339509434) + | > loader_time: 0.007 (0.005919477961084864) + + + --> TIME: 2025-04-19 06:43:41 -- STEP: 136/142 -- GLOBAL_STEP: 446300 + | > loss_disc: 2.3290605545043945 (2.447583500076744) + | > loss_disc_real_0: 0.07205437868833542 (0.12099575202035553) + | > loss_disc_real_1: 0.1875585913658142 (0.20651698287795572) + | > loss_disc_real_2: 0.22633691132068634 (0.225376278600272) + | > loss_disc_real_3: 0.2221127450466156 (0.22809768555795445) + | > loss_disc_real_4: 0.21199889481067657 (0.2269377183607396) + | > loss_disc_real_5: 0.2311377227306366 (0.23122635439914815) + | > loss_0: 2.3290605545043945 (2.447583500076744) + | > grad_norm_0: tensor(6.2227, device='cuda:0') (tensor(13.0643, device='cuda:0')) + | > loss_gen: 2.5342233180999756 (2.3781044448123247) + | > loss_kl: 1.8069783449172974 (1.868757591528051) + | > loss_feat: 7.659868240356445 (7.195261268054738) + | > loss_mel: 18.564868927001953 (17.640064884634587) + | > loss_duration: 1.503458023071289 (1.4553733406697997) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.06939697265625 (30.537561514798334) + | > grad_norm_1: tensor(208.2339, device='cuda:0') (tensor(306.9498, device='cuda:0')) + | > current_lr_0: 0.00018856422905714584 + | > current_lr_1: 0.00018856422905714584 + | > step_time: 0.4498 (0.3874838720349705) + | > loader_time: 0.0077 (0.006159503670299754) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.459665298461914 (2.459665298461914) + | > loss_disc_real_0: 0.14550316333770752 (0.14550316333770752) + | > loss_disc_real_1: 0.18803009390830994 (0.18803009390830994) + | > loss_disc_real_2: 0.211007758975029 (0.211007758975029) + | > loss_disc_real_3: 0.2601805329322815 (0.2601805329322815) + | > loss_disc_real_4: 0.24637892842292786 (0.24637892842292786) + | > loss_disc_real_5: 0.24525824189186096 (0.24525824189186096) + | > loss_0: 2.459665298461914 (2.459665298461914) + | > loss_gen: 2.3633387088775635 (2.3633387088775635) + | > loss_kl: 2.562959671020508 (2.562959671020508) + | > loss_feat: 7.09763765335083 (7.09763765335083) + | > loss_mel: 19.314983367919922 (19.314983367919922) + | > loss_duration: 1.7806921005249023 (1.7806921005249023) + | > loss_1: 33.11960983276367 (33.11960983276367) + + --> STEP: 1 + | > loss_disc: 2.527355670928955 (2.527355670928955) + | > loss_disc_real_0: 0.15440094470977783 (0.15440094470977783) + | > loss_disc_real_1: 0.25467851758003235 (0.25467851758003235) + | > loss_disc_real_2: 0.25451576709747314 (0.25451576709747314) + | > loss_disc_real_3: 0.2433379739522934 (0.2433379739522934) + | > loss_disc_real_4: 0.2033185064792633 (0.2033185064792633) + | > loss_disc_real_5: 0.2420583814382553 (0.2420583814382553) + | > loss_0: 2.527355670928955 (2.527355670928955) + | > loss_gen: 2.401724100112915 (2.401724100112915) + | > loss_kl: 2.6619620323181152 (2.6619620323181152) + | > loss_feat: 7.38837194442749 (7.38837194442749) + | > loss_mel: 18.63938331604004 (18.63938331604004) + | > loss_duration: 1.7972548007965088 (1.7972548007965088) + | > loss_1: 32.888694763183594 (32.888694763183594) + + --> STEP: 2 + | > loss_disc: 2.444563150405884 (2.4859594106674194) + | > loss_disc_real_0: 0.16409289836883545 (0.15924692153930664) + | > loss_disc_real_1: 0.24641583859920502 (0.2505471780896187) + | > loss_disc_real_2: 0.2733552157878876 (0.26393549144268036) + | > loss_disc_real_3: 0.25202271342277527 (0.24768034368753433) + | > loss_disc_real_4: 0.264139324426651 (0.23372891545295715) + | > loss_disc_real_5: 0.24459034204483032 (0.24332436174154282) + | > loss_0: 2.444563150405884 (2.4859594106674194) + | > loss_gen: 2.592217445373535 (2.496970772743225) + | > loss_kl: 2.2165873050689697 (2.4392746686935425) + | > loss_feat: 7.2307257652282715 (7.309548854827881) + | > loss_mel: 19.009376525878906 (18.824379920959473) + | > loss_duration: 1.7479932308197021 (1.7726240158081055) + | > loss_1: 32.79690170288086 (32.84279823303223) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032857656478881836 (-0.00027382373809814453) + | > avg_loss_disc: 2.4859594106674194 (-0.08141720294952393) + | > avg_loss_disc_real_0: 0.15924692153930664 (-0.07929733395576477) + | > avg_loss_disc_real_1: 0.2505471780896187 (+0.028320632874965668) + | > avg_loss_disc_real_2: 0.26393549144268036 (+0.03914608061313629) + | > avg_loss_disc_real_3: 0.24768034368753433 (-0.005788207054138184) + | > avg_loss_disc_real_4: 0.23372891545295715 (-0.03556939959526062) + | > avg_loss_disc_real_5: 0.24332436174154282 (-0.016189485788345337) + | > avg_loss_0: 2.4859594106674194 (-0.08141720294952393) + | > avg_loss_gen: 2.496970772743225 (+0.06776964664459229) + | > avg_loss_kl: 2.4392746686935425 (-0.10684084892272949) + | > avg_loss_feat: 7.309548854827881 (+1.3240716457366943) + | > avg_loss_mel: 18.824379920959473 (+2.613569736480713) + | > avg_loss_duration: 1.7726240158081055 (-0.027876436710357666) + | > avg_loss_1: 32.84279823303223 (+3.8706932067871094) + + + > EPOCH: 472/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:43:45)  + + --> TIME: 2025-04-19 06:43:52 -- STEP: 19/142 -- GLOBAL_STEP: 446325 + | > loss_disc: 2.3203670978546143 (2.4345789206655404) + | > loss_disc_real_0: 0.1395384818315506 (0.11669144230453592) + | > loss_disc_real_1: 0.18209080398082733 (0.2051338890665456) + | > loss_disc_real_2: 0.22356410324573517 (0.22605445432035545) + | > loss_disc_real_3: 0.21064165234565735 (0.2266266887125216) + | > loss_disc_real_4: 0.22784598171710968 (0.2296327846614938) + | > loss_disc_real_5: 0.20126044750213623 (0.23011879230800428) + | > loss_0: 2.3203670978546143 (2.4345789206655404) + | > grad_norm_0: tensor(9.6697, device='cuda:0') (tensor(7.7053, device='cuda:0')) + | > loss_gen: 2.483577251434326 (2.388775950983951) + | > loss_kl: 1.8912239074707031 (1.8604311378378617) + | > loss_feat: 7.946105480194092 (7.372770936865556) + | > loss_mel: 19.160320281982422 (17.898183923018607) + | > loss_duration: 1.403464913368225 (1.4093405259282965) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.88469314575195 (30.929502487182617) + | > grad_norm_1: tensor(299.0426, device='cuda:0') (tensor(200.9944, device='cuda:0')) + | > current_lr_0: 0.00018854065852851368 + | > current_lr_1: 0.00018854065852851368 + | > step_time: 0.3373 (0.33056160023337916) + | > loader_time: 0.0046 (0.004645008789865593) + + + --> TIME: 2025-04-19 06:44:01 -- STEP: 44/142 -- GLOBAL_STEP: 446350 + | > loss_disc: 2.294602394104004 (2.4169696894559) + | > loss_disc_real_0: 0.10276158899068832 (0.11388222009620884) + | > loss_disc_real_1: 0.1681053787469864 (0.2042282965372909) + | > loss_disc_real_2: 0.2643060088157654 (0.22687250511212784) + | > loss_disc_real_3: 0.2266968935728073 (0.22612726417454806) + | > loss_disc_real_4: 0.2509123384952545 (0.2264858169311827) + | > loss_disc_real_5: 0.19995851814746857 (0.2265581519088962) + | > loss_0: 2.294602394104004 (2.4169696894559) + | > grad_norm_0: tensor(5.7284, device='cuda:0') (tensor(7.1579, device='cuda:0')) + | > loss_gen: 2.4440460205078125 (2.388236040418798) + | > loss_kl: 1.9375433921813965 (1.8690229843963275) + | > loss_feat: 7.864108562469482 (7.432651205496355) + | > loss_mel: 18.87139320373535 (17.811429587277498) + | > loss_duration: 1.4553840160369873 (1.4219384437257594) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.57247543334961 (30.923278505151924) + | > grad_norm_1: tensor(196.3819, device='cuda:0') (tensor(190.4034, device='cuda:0')) + | > current_lr_0: 0.00018854065852851368 + | > current_lr_1: 0.00018854065852851368 + | > step_time: 0.358 (0.3430349176580256) + | > loader_time: 0.0057 (0.004988675767725163) + + + --> TIME: 2025-04-19 06:44:11 -- STEP: 69/142 -- GLOBAL_STEP: 446375 + | > loss_disc: 2.4889822006225586 (2.4342674241549727) + | > loss_disc_real_0: 0.12940220534801483 (0.11688378669213557) + | > loss_disc_real_1: 0.21560709178447723 (0.20641577934873276) + | > loss_disc_real_2: 0.23626165091991425 (0.22724023526129516) + | > loss_disc_real_3: 0.24365592002868652 (0.22765230571014294) + | > loss_disc_real_4: 0.24032437801361084 (0.2281888954449391) + | > loss_disc_real_5: 0.2258463203907013 (0.22837691851284192) + | > loss_0: 2.4889822006225586 (2.4342674241549727) + | > grad_norm_0: tensor(5.3204, device='cuda:0') (tensor(7.6517, device='cuda:0')) + | > loss_gen: 2.3594002723693848 (2.3823628494704976) + | > loss_kl: 1.935415506362915 (1.8768521720084592) + | > loss_feat: 6.497068881988525 (7.369998938795449) + | > loss_mel: 17.560701370239258 (17.754273290219516) + | > loss_duration: 1.4998195171356201 (1.4319026314693948) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 29.852405548095703 (30.815390075462453) + | > grad_norm_1: tensor(204.6344, device='cuda:0') (tensor(203.8161, device='cuda:0')) + | > current_lr_0: 0.00018854065852851368 + | > current_lr_1: 0.00018854065852851368 + | > step_time: 0.3985 (0.3547428448994954) + | > loader_time: 0.006 (0.0053628700366918575) + + + --> TIME: 2025-04-19 06:44:21 -- STEP: 94/142 -- GLOBAL_STEP: 446400 + | > loss_disc: 2.423304557800293 (2.4393426915432546) + | > loss_disc_real_0: 0.1306999772787094 (0.11891417688828834) + | > loss_disc_real_1: 0.19846978783607483 (0.20699843732600517) + | > loss_disc_real_2: 0.2050158679485321 (0.22631428850458024) + | > loss_disc_real_3: 0.20139124989509583 (0.22765985448309717) + | > loss_disc_real_4: 0.21366257965564728 (0.2270673413542991) + | > loss_disc_real_5: 0.21952420473098755 (0.2295611744865458) + | > loss_0: 2.423304557800293 (2.4393426915432546) + | > grad_norm_0: tensor(6.1862, device='cuda:0') (tensor(8.5482, device='cuda:0')) + | > loss_gen: 2.2663207054138184 (2.36857979602002) + | > loss_kl: 1.7940620183944702 (1.8757954683709652) + | > loss_feat: 6.671681880950928 (7.277950855011635) + | > loss_mel: 17.042634963989258 (17.65640410971134) + | > loss_duration: 1.485809087753296 (1.4433604935382272) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 29.260509490966797 (30.62209078606139) + | > grad_norm_1: tensor(134.2444, device='cuda:0') (tensor(219.6054, device='cuda:0')) + | > current_lr_0: 0.00018854065852851368 + | > current_lr_1: 0.00018854065852851368 + | > step_time: 0.3983 (0.3662582737334231) + | > loader_time: 0.0066 (0.005694853498580605) + + + --> TIME: 2025-04-19 06:44:32 -- STEP: 119/142 -- GLOBAL_STEP: 446425 + | > loss_disc: 2.4399795532226562 (2.445877820503812) + | > loss_disc_real_0: 0.06932758539915085 (0.12008832370032783) + | > loss_disc_real_1: 0.2185114026069641 (0.20698948855660543) + | > loss_disc_real_2: 0.20906910300254822 (0.22656223553569377) + | > loss_disc_real_3: 0.22879210114479065 (0.22818653698728866) + | > loss_disc_real_4: 0.23072358965873718 (0.22785515544795187) + | > loss_disc_real_5: 0.2525264620780945 (0.23034224239718012) + | > loss_0: 2.4399795532226562 (2.445877820503812) + | > grad_norm_0: tensor(15.1734, device='cuda:0') (tensor(9.4576, device='cuda:0')) + | > loss_gen: 2.327418327331543 (2.36626131594682) + | > loss_kl: 1.856784462928772 (1.878411366158173) + | > loss_feat: 8.063188552856445 (7.23688866110409) + | > loss_mel: 19.051301956176758 (17.647062501987488) + | > loss_duration: 1.517540454864502 (1.4507478245166168) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.81623458862305 (30.579371716795848) + | > grad_norm_1: tensor(468.0654, device='cuda:0') (tensor(233.3900, device='cuda:0')) + | > current_lr_0: 0.00018854065852851368 + | > current_lr_1: 0.00018854065852851368 + | > step_time: 0.4298 (0.3778799802315335) + | > loader_time: 0.0066 (0.005964148946169042) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.404963493347168 (2.404963493347168) + | > loss_disc_real_0: 0.06470063328742981 (0.06470063328742981) + | > loss_disc_real_1: 0.22672276198863983 (0.22672276198863983) + | > loss_disc_real_2: 0.24933934211730957 (0.24933934211730957) + | > loss_disc_real_3: 0.20118942856788635 (0.20118942856788635) + | > loss_disc_real_4: 0.23646773397922516 (0.23646773397922516) + | > loss_disc_real_5: 0.22037920355796814 (0.22037920355796814) + | > loss_0: 2.404963493347168 (2.404963493347168) + | > loss_gen: 2.1781225204467773 (2.1781225204467773) + | > loss_kl: 2.4895260334014893 (2.4895260334014893) + | > loss_feat: 7.316705703735352 (7.316705703735352) + | > loss_mel: 19.204294204711914 (19.204294204711914) + | > loss_duration: 1.7674561738967896 (1.7674561738967896) + | > loss_1: 32.95610427856445 (32.95610427856445) + + --> STEP: 1 + | > loss_disc: 2.473994493484497 (2.473994493484497) + | > loss_disc_real_0: 0.14039960503578186 (0.14039960503578186) + | > loss_disc_real_1: 0.23408134281635284 (0.23408134281635284) + | > loss_disc_real_2: 0.22435621917247772 (0.22435621917247772) + | > loss_disc_real_3: 0.22782233357429504 (0.22782233357429504) + | > loss_disc_real_4: 0.20014755427837372 (0.20014755427837372) + | > loss_disc_real_5: 0.235189288854599 (0.235189288854599) + | > loss_0: 2.473994493484497 (2.473994493484497) + | > loss_gen: 2.1778247356414795 (2.1778247356414795) + | > loss_kl: 2.9420127868652344 (2.9420127868652344) + | > loss_feat: 5.890483379364014 (5.890483379364014) + | > loss_mel: 17.146589279174805 (17.146589279174805) + | > loss_duration: 1.80946946144104 (1.80946946144104) + | > loss_1: 29.966379165649414 (29.966379165649414) + + --> STEP: 2 + | > loss_disc: 2.3928375244140625 (2.43341600894928) + | > loss_disc_real_0: 0.15713688731193542 (0.14876824617385864) + | > loss_disc_real_1: 0.20841816067695618 (0.2212497517466545) + | > loss_disc_real_2: 0.24794922769069672 (0.23615272343158722) + | > loss_disc_real_3: 0.2177012413740158 (0.22276178747415543) + | > loss_disc_real_4: 0.20764216780662537 (0.20389486104249954) + | > loss_disc_real_5: 0.20872262120246887 (0.22195595502853394) + | > loss_0: 2.3928375244140625 (2.43341600894928) + | > loss_gen: 2.551201105117798 (2.3645129203796387) + | > loss_kl: 2.1736490726470947 (2.5578309297561646) + | > loss_feat: 8.068241119384766 (6.97936224937439) + | > loss_mel: 19.676944732666016 (18.41176700592041) + | > loss_duration: 1.754056453704834 (1.781762957572937) + | > loss_1: 34.224090576171875 (32.095234870910645) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004285097122192383 (+0.0009993314743041992) + | > avg_loss_disc: 2.43341600894928 (-0.05254340171813965) + | > avg_loss_disc_real_0: 0.14876824617385864 (-0.010478675365447998) + | > avg_loss_disc_real_1: 0.2212497517466545 (-0.029297426342964172) + | > avg_loss_disc_real_2: 0.23615272343158722 (-0.02778276801109314) + | > avg_loss_disc_real_3: 0.22276178747415543 (-0.024918556213378906) + | > avg_loss_disc_real_4: 0.20389486104249954 (-0.02983405441045761) + | > avg_loss_disc_real_5: 0.22195595502853394 (-0.02136840671300888) + | > avg_loss_0: 2.43341600894928 (-0.05254340171813965) + | > avg_loss_gen: 2.3645129203796387 (-0.13245785236358643) + | > avg_loss_kl: 2.5578309297561646 (+0.11855626106262207) + | > avg_loss_feat: 6.97936224937439 (-0.3301866054534912) + | > avg_loss_mel: 18.41176700592041 (-0.4126129150390625) + | > avg_loss_duration: 1.781762957572937 (+0.009138941764831543) + | > avg_loss_1: 32.095234870910645 (-0.747563362121582) + + + > EPOCH: 473/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:44:44)  + + --> TIME: 2025-04-19 06:44:45 -- STEP: 2/142 -- GLOBAL_STEP: 446450 + | > loss_disc: 2.4157283306121826 (2.416743516921997) + | > loss_disc_real_0: 0.23154273629188538 (0.21824052929878235) + | > loss_disc_real_1: 0.19247369468212128 (0.21461042016744614) + | > loss_disc_real_2: 0.23047202825546265 (0.24000193178653717) + | > loss_disc_real_3: 0.26137775182724 (0.27010925114154816) + | > loss_disc_real_4: 0.19496311247348785 (0.20214961469173431) + | > loss_disc_real_5: 0.2281077802181244 (0.22328802198171616) + | > loss_0: 2.4157283306121826 (2.416743516921997) + | > grad_norm_0: tensor(25.1347, device='cuda:0') (tensor(24.2106, device='cuda:0')) + | > loss_gen: 2.5550835132598877 (2.630255699157715) + | > loss_kl: 1.732396125793457 (1.797364592552185) + | > loss_feat: 7.659319877624512 (7.580620288848877) + | > loss_mel: 17.372941970825195 (17.665507316589355) + | > loss_duration: 1.443450927734375 (1.4153224229812622) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 30.763193130493164 (31.089070320129395) + | > grad_norm_1: tensor(401.3364, device='cuda:0') (tensor(297.1671, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.331 (0.33672308921813965) + | > loader_time: 0.0042 (0.004529118537902832) + + + --> TIME: 2025-04-19 06:44:54 -- STEP: 27/142 -- GLOBAL_STEP: 446475 + | > loss_disc: 2.5053393840789795 (2.462792626133671) + | > loss_disc_real_0: 0.09367603063583374 (0.1279550538294845) + | > loss_disc_real_1: 0.17794537544250488 (0.20874320908829017) + | > loss_disc_real_2: 0.20575523376464844 (0.22770637273788452) + | > loss_disc_real_3: 0.2127198427915573 (0.23035743998156655) + | > loss_disc_real_4: 0.2340611070394516 (0.2290796196019208) + | > loss_disc_real_5: 0.23139531910419464 (0.2329769101407793) + | > loss_0: 2.5053393840789795 (2.462792626133671) + | > grad_norm_0: tensor(16.3976, device='cuda:0') (tensor(17.3339, device='cuda:0')) + | > loss_gen: 2.1457343101501465 (2.382705909234506) + | > loss_kl: 1.8993247747421265 (1.7950117764649567) + | > loss_feat: 7.695970058441162 (7.293781368820755) + | > loss_mel: 18.178388595581055 (17.699280067726416) + | > loss_duration: 1.435167670249939 (1.418326077637849) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 31.354585647583008 (30.589105040938765) + | > grad_norm_1: tensor(418.1658, device='cuda:0') (tensor(288.3921, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.3497 (0.3367599734553584) + | > loader_time: 0.0048 (0.004841018606115271) + + + --> TIME: 2025-04-19 06:45:03 -- STEP: 52/142 -- GLOBAL_STEP: 446500 + | > loss_disc: 2.3703598976135254 (2.450252918096689) + | > loss_disc_real_0: 0.0992368757724762 (0.12202714791951268) + | > loss_disc_real_1: 0.20284943282604218 (0.20849628259356207) + | > loss_disc_real_2: 0.1963597983121872 (0.22681182651565626) + | > loss_disc_real_3: 0.22356940805912018 (0.2307236951131087) + | > loss_disc_real_4: 0.2362632155418396 (0.22861161426855967) + | > loss_disc_real_5: 0.23547662794589996 (0.23179089048734078) + | > loss_0: 2.3703598976135254 (2.450252918096689) + | > grad_norm_0: tensor(10.8099, device='cuda:0') (tensor(15.4766, device='cuda:0')) + | > loss_gen: 2.3954427242279053 (2.381115844616523) + | > loss_kl: 1.9075223207473755 (1.826732215973047) + | > loss_feat: 7.569399833679199 (7.249062418937683) + | > loss_mel: 19.27872657775879 (17.604883964245136) + | > loss_duration: 1.4851250648498535 (1.4265552575771625) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.6362190246582 (30.48834962111253) + | > grad_norm_1: tensor(407.4923, device='cuda:0') (tensor(334.5163, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.3667 (0.3475082745918861) + | > loader_time: 0.0054 (0.00512622411434467) + + + --> TIME: 2025-04-19 06:45:13 -- STEP: 77/142 -- GLOBAL_STEP: 446525 + | > loss_disc: 2.5146098136901855 (2.449059334668246) + | > loss_disc_real_0: 0.11091317981481552 (0.12059165832477728) + | > loss_disc_real_1: 0.21225851774215698 (0.20774749424550443) + | > loss_disc_real_2: 0.2203802764415741 (0.22587826286817525) + | > loss_disc_real_3: 0.24386471509933472 (0.22959204327750515) + | > loss_disc_real_4: 0.19880123436450958 (0.22824395070602368) + | > loss_disc_real_5: 0.2271188199520111 (0.2322992349599863) + | > loss_0: 2.5146098136901855 (2.449059334668246) + | > grad_norm_0: tensor(6.8876, device='cuda:0') (tensor(12.6944, device='cuda:0')) + | > loss_gen: 2.2714345455169678 (2.3774740788843727) + | > loss_kl: 1.9733160734176636 (1.85405496498207) + | > loss_feat: 7.644920825958252 (7.28116292458076) + | > loss_mel: 18.006027221679688 (17.699057690508955) + | > loss_duration: 1.4469457864761353 (1.4368507506011368) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 31.34264373779297 (30.64860031821511) + | > grad_norm_1: tensor(148.4227, device='cuda:0') (tensor(269.2501, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.3929 (0.3598866493670971) + | > loader_time: 0.0065 (0.00545377545542531) + + + --> TIME: 2025-04-19 06:45:23 -- STEP: 102/142 -- GLOBAL_STEP: 446550 + | > loss_disc: 2.6141855716705322 (2.451917194852642) + | > loss_disc_real_0: 0.14925093948841095 (0.12155579421304019) + | > loss_disc_real_1: 0.20584608614444733 (0.20716691806035883) + | > loss_disc_real_2: 0.25829827785491943 (0.22592933008483812) + | > loss_disc_real_3: 0.24390235543251038 (0.22924229210498287) + | > loss_disc_real_4: 0.29150697588920593 (0.22846820453802744) + | > loss_disc_real_5: 0.2471258044242859 (0.23241660539426057) + | > loss_0: 2.6141855716705322 (2.451917194852642) + | > grad_norm_0: tensor(5.3043, device='cuda:0') (tensor(11.0533, device='cuda:0')) + | > loss_gen: 2.2870383262634277 (2.3655183245154) + | > loss_kl: 1.807295322418213 (1.8658778071403503) + | > loss_feat: 6.492201805114746 (7.226172063864913) + | > loss_mel: 17.376279830932617 (17.679947488448196) + | > loss_duration: 1.446542501449585 (1.4466216832983727) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 29.409358978271484 (30.58413729948156) + | > grad_norm_1: tensor(66.0227, device='cuda:0') (tensor(233.0285, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.4177 (0.370346705118815) + | > loader_time: 0.0071 (0.00576662549785539) + + + --> TIME: 2025-04-19 06:45:34 -- STEP: 127/142 -- GLOBAL_STEP: 446575 + | > loss_disc: 2.4421582221984863 (2.456911417442983) + | > loss_disc_real_0: 0.08597379922866821 (0.12349188371788797) + | > loss_disc_real_1: 0.20547831058502197 (0.20723521157981842) + | > loss_disc_real_2: 0.24321869015693665 (0.2260380261760997) + | > loss_disc_real_3: 0.2644595801830292 (0.22984769083852843) + | > loss_disc_real_4: 0.2765688896179199 (0.2289457911346841) + | > loss_disc_real_5: 0.22519609332084656 (0.2324529517823317) + | > loss_0: 2.4421582221984863 (2.456911417442983) + | > grad_norm_0: tensor(10.5806, device='cuda:0') (tensor(10.8393, device='cuda:0')) + | > loss_gen: 2.3285036087036133 (2.359421985355889) + | > loss_kl: 1.9263863563537598 (1.8767630997605211) + | > loss_feat: 6.825439453125 (7.148651539810061) + | > loss_mel: 16.98021697998047 (17.56118158655842) + | > loss_duration: 1.4905695915222168 (1.4543712157902755) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 29.551116943359375 (30.400389363446575) + | > grad_norm_1: tensor(61.6855, device='cuda:0') (tensor(227.5012, device='cuda:0')) + | > current_lr_0: 0.00018851709094619762 + | > current_lr_1: 0.00018851709094619762 + | > step_time: 0.4379 (0.3814997297572338) + | > loader_time: 0.0075 (0.006015368333951693) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.493130683898926 (2.493130683898926) + | > loss_disc_real_0: 0.1741330772638321 (0.1741330772638321) + | > loss_disc_real_1: 0.2789258062839508 (0.2789258062839508) + | > loss_disc_real_2: 0.2419821172952652 (0.2419821172952652) + | > loss_disc_real_3: 0.2722885012626648 (0.2722885012626648) + | > loss_disc_real_4: 0.2619895040988922 (0.2619895040988922) + | > loss_disc_real_5: 0.28116855025291443 (0.28116855025291443) + | > loss_0: 2.493130683898926 (2.493130683898926) + | > loss_gen: 2.825749397277832 (2.825749397277832) + | > loss_kl: 2.5784378051757812 (2.5784378051757812) + | > loss_feat: 7.8445634841918945 (7.8445634841918945) + | > loss_mel: 19.17821502685547 (19.17821502685547) + | > loss_duration: 1.7467402219772339 (1.7467402219772339) + | > loss_1: 34.173702239990234 (34.173702239990234) + + --> STEP: 1 + | > loss_disc: 2.5548086166381836 (2.5548086166381836) + | > loss_disc_real_0: 0.17388683557510376 (0.17388683557510376) + | > loss_disc_real_1: 0.31010064482688904 (0.31010064482688904) + | > loss_disc_real_2: 0.25609955191612244 (0.25609955191612244) + | > loss_disc_real_3: 0.25066375732421875 (0.25066375732421875) + | > loss_disc_real_4: 0.23220188915729523 (0.23220188915729523) + | > loss_disc_real_5: 0.21543873846530914 (0.21543873846530914) + | > loss_0: 2.5548086166381836 (2.5548086166381836) + | > loss_gen: 2.5727875232696533 (2.5727875232696533) + | > loss_kl: 2.9834742546081543 (2.9834742546081543) + | > loss_feat: 6.640623092651367 (6.640623092651367) + | > loss_mel: 18.307748794555664 (18.307748794555664) + | > loss_duration: 1.8331992626190186 (1.8331992626190186) + | > loss_1: 32.337833404541016 (32.337833404541016) + + --> STEP: 2 + | > loss_disc: 2.5838518142700195 (2.5693302154541016) + | > loss_disc_real_0: 0.25452107191085815 (0.21420395374298096) + | > loss_disc_real_1: 0.2657020092010498 (0.2879013270139694) + | > loss_disc_real_2: 0.22529028356075287 (0.24069491773843765) + | > loss_disc_real_3: 0.2495373785495758 (0.2501005679368973) + | > loss_disc_real_4: 0.26002728939056396 (0.2461145892739296) + | > loss_disc_real_5: 0.2386181801557541 (0.22702845931053162) + | > loss_0: 2.5838518142700195 (2.5693302154541016) + | > loss_gen: 2.632232427597046 (2.6025099754333496) + | > loss_kl: 2.4372243881225586 (2.7103493213653564) + | > loss_feat: 7.541072845458984 (7.090847969055176) + | > loss_mel: 16.731468200683594 (17.51960849761963) + | > loss_duration: 1.7534326314926147 (1.7933159470558167) + | > loss_1: 31.09543228149414 (31.716632843017578) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036858320236206055 (-0.0005992650985717773) + | > avg_loss_disc: 2.5693302154541016 (+0.13591420650482178) + | > avg_loss_disc_real_0: 0.21420395374298096 (+0.06543570756912231) + | > avg_loss_disc_real_1: 0.2879013270139694 (+0.06665157526731491) + | > avg_loss_disc_real_2: 0.24069491773843765 (+0.004542194306850433) + | > avg_loss_disc_real_3: 0.2501005679368973 (+0.027338780462741852) + | > avg_loss_disc_real_4: 0.2461145892739296 (+0.042219728231430054) + | > avg_loss_disc_real_5: 0.22702845931053162 (+0.005072504281997681) + | > avg_loss_0: 2.5693302154541016 (+0.13591420650482178) + | > avg_loss_gen: 2.6025099754333496 (+0.23799705505371094) + | > avg_loss_kl: 2.7103493213653564 (+0.1525183916091919) + | > avg_loss_feat: 7.090847969055176 (+0.11148571968078613) + | > avg_loss_mel: 17.51960849761963 (-0.8921585083007812) + | > avg_loss_duration: 1.7933159470558167 (+0.011552989482879639) + | > avg_loss_1: 31.716632843017578 (-0.3786020278930664) + + + > EPOCH: 474/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:45:42)  + + --> TIME: 2025-04-19 06:45:46 -- STEP: 10/142 -- GLOBAL_STEP: 446600 + | > loss_disc: 2.5558722019195557 (2.499116086959839) + | > loss_disc_real_0: 0.16881035268306732 (0.1236930288374424) + | > loss_disc_real_1: 0.21004125475883484 (0.22003545016050338) + | > loss_disc_real_2: 0.23503345251083374 (0.23227709382772446) + | > loss_disc_real_3: 0.22565428912639618 (0.22411088943481444) + | > loss_disc_real_4: 0.2618674337863922 (0.24252667874097825) + | > loss_disc_real_5: 0.2403930127620697 (0.24144596755504608) + | > loss_0: 2.5558722019195557 (2.499116086959839) + | > grad_norm_0: tensor(12.8903, device='cuda:0') (tensor(10.8019, device='cuda:0')) + | > loss_gen: 2.5304293632507324 (2.4098630428314207) + | > loss_kl: 1.8244229555130005 (1.8432949542999268) + | > loss_feat: 7.395832538604736 (7.34274468421936) + | > loss_mel: 18.881147384643555 (18.512854385375977) + | > loss_duration: 1.3982391357421875 (1.4125885367393494) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 32.03007125854492 (31.52134609222412) + | > grad_norm_1: tensor(358.4536, device='cuda:0') (tensor(175.3687, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.338 (0.3251122713088989) + | > loader_time: 0.0047 (0.004427170753479004) + + + --> TIME: 2025-04-19 06:45:55 -- STEP: 35/142 -- GLOBAL_STEP: 446625 + | > loss_disc: 2.495088815689087 (2.4640060765402656) + | > loss_disc_real_0: 0.09111156314611435 (0.11902532130479812) + | > loss_disc_real_1: 0.22220659255981445 (0.21145675012043544) + | > loss_disc_real_2: 0.21381334960460663 (0.22862175234726498) + | > loss_disc_real_3: 0.22207793593406677 (0.22902058831283023) + | > loss_disc_real_4: 0.22613699734210968 (0.23201695936066763) + | > loss_disc_real_5: 0.2703477442264557 (0.23748162814549037) + | > loss_0: 2.495088815689087 (2.4640060765402656) + | > grad_norm_0: tensor(9.8039, device='cuda:0') (tensor(8.7161, device='cuda:0')) + | > loss_gen: 2.3288638591766357 (2.370052078792027) + | > loss_kl: 1.8889546394348145 (1.8489728246416364) + | > loss_feat: 7.022206783294678 (7.328702190944127) + | > loss_mel: 17.37335968017578 (17.976760591779435) + | > loss_duration: 1.4181978702545166 (1.4235919850213188) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 30.03158187866211 (30.94807951790946) + | > grad_norm_1: tensor(247.9197, device='cuda:0') (tensor(191.9537, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.3557 (0.33756074905395506) + | > loader_time: 0.0054 (0.00499241692679269) + + + --> TIME: 2025-04-19 06:46:04 -- STEP: 60/142 -- GLOBAL_STEP: 446650 + | > loss_disc: 2.50601863861084 (2.440269867579143) + | > loss_disc_real_0: 0.15892313420772552 (0.12156958344082038) + | > loss_disc_real_1: 0.1954420953989029 (0.2076934151351452) + | > loss_disc_real_2: 0.20638225972652435 (0.22334993705153466) + | > loss_disc_real_3: 0.22043119370937347 (0.2269549585878849) + | > loss_disc_real_4: 0.2110806405544281 (0.22689484904209772) + | > loss_disc_real_5: 0.18525627255439758 (0.2331280951698621) + | > loss_0: 2.50601863861084 (2.440269867579143) + | > grad_norm_0: tensor(8.9012, device='cuda:0') (tensor(8.2582, device='cuda:0')) + | > loss_gen: 2.3066844940185547 (2.3699935555458067) + | > loss_kl: 1.733267903327942 (1.8489878177642822) + | > loss_feat: 7.242825031280518 (7.261142714818319) + | > loss_mel: 16.562149047851562 (17.699475495020547) + | > loss_duration: 1.4815301895141602 (1.4317907492319744) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 29.326457977294922 (30.611390336354575) + | > grad_norm_1: tensor(120.2735, device='cuda:0') (tensor(193.2818, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.374 (0.34911556243896485) + | > loader_time: 0.0073 (0.005275479952494304) + + + --> TIME: 2025-04-19 06:46:14 -- STEP: 85/142 -- GLOBAL_STEP: 446675 + | > loss_disc: 2.556339740753174 (2.4586329375996314) + | > loss_disc_real_0: 0.13583682477474213 (0.12375719442087062) + | > loss_disc_real_1: 0.22656947374343872 (0.20871715580715852) + | > loss_disc_real_2: 0.2446209192276001 (0.22637679313912112) + | > loss_disc_real_3: 0.2390434294939041 (0.2289906803299399) + | > loss_disc_real_4: 0.23959942162036896 (0.22837765847935396) + | > loss_disc_real_5: 0.2079000324010849 (0.2336327142575208) + | > loss_0: 2.556339740753174 (2.4586329375996314) + | > grad_norm_0: tensor(14.5661, device='cuda:0') (tensor(8.9304, device='cuda:0')) + | > loss_gen: 2.139590263366699 (2.3594438244314753) + | > loss_kl: 1.8121249675750732 (1.8506189809126012) + | > loss_feat: 5.52710485458374 (7.140101090599509) + | > loss_mel: 14.776679992675781 (17.596031357260316) + | > loss_duration: 1.4774231910705566 (1.442170109468348) + | > amp_scaler: 4.0 (4.0) + | > loss_1: 25.73292350769043 (30.388365352855008) + | > grad_norm_1: tensor(306.9416, device='cuda:0') (tensor(204.1304, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.3956 (0.36113044794868027) + | > loader_time: 0.0064 (0.005551680396584904) + + + --> TIME: 2025-04-19 06:46:25 -- STEP: 110/142 -- GLOBAL_STEP: 446700 + | > loss_disc: 2.3597464561462402 (2.4575203700499113) + | > loss_disc_real_0: 0.09876960515975952 (0.12367678338831121) + | > loss_disc_real_1: 0.1852312982082367 (0.2073837849226865) + | > loss_disc_real_2: 0.2039669305086136 (0.22744615497914228) + | > loss_disc_real_3: 0.2270580381155014 (0.2286128355698152) + | > loss_disc_real_4: 0.22161412239074707 (0.22894241931763562) + | > loss_disc_real_5: 0.21750320494174957 (0.23369428583166815) + | > loss_0: 2.3597464561462402 (2.4575203700499113) + | > grad_norm_0: tensor(14.9605, device='cuda:0') (tensor(9.1826, device='cuda:0')) + | > loss_gen: 2.408982038497925 (2.362363483689048) + | > loss_kl: 1.7504751682281494 (nan) + | > loss_feat: 7.456638813018799 (7.150888802788474) + | > loss_mel: 18.854753494262695 (17.615721399133875) + | > loss_duration: 1.4741606712341309 (nan) + | > amp_scaler: 2.0 (3.7818181818181817) + | > loss_1: 31.945009231567383 (nan) + | > grad_norm_1: tensor(295.6115, device='cuda:0') (tensor(211.1579, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.4306 (0.3715570558201183) + | > loader_time: 0.0072 (0.005859236283735796) + + + --> TIME: 2025-04-19 06:46:36 -- STEP: 135/142 -- GLOBAL_STEP: 446725 + | > loss_disc: 2.47428560256958 (2.4575506316290965) + | > loss_disc_real_0: 0.1088552251458168 (0.12373103466298845) + | > loss_disc_real_1: 0.21427655220031738 (0.20659007606682955) + | > loss_disc_real_2: 0.19517691433429718 (0.2259753344235597) + | > loss_disc_real_3: 0.21840526163578033 (0.2288241645804158) + | > loss_disc_real_4: 0.2071492224931717 (0.22879892101994267) + | > loss_disc_real_5: 0.26163312792778015 (0.234495198947412) + | > loss_0: 2.47428560256958 (2.4575506316290965) + | > grad_norm_0: tensor(4.9720, device='cuda:0') (tensor(9.8894, device='cuda:0')) + | > loss_gen: 2.5130057334899902 (2.368029710981581) + | > loss_kl: 1.9781540632247925 (nan) + | > loss_feat: 7.169887542724609 (7.158152544939959) + | > loss_mel: 17.457155227661133 (17.651994881806566) + | > loss_duration: 1.4674592018127441 (nan) + | > amp_scaler: 1.0 (3.4074074074074074) + | > loss_1: 30.585662841796875 (nan) + | > grad_norm_1: tensor(362.3135, device='cuda:0') (tensor(227.7502, device='cuda:0')) + | > current_lr_0: 0.00018849352630982935 + | > current_lr_1: 0.00018849352630982935 + | > step_time: 0.4523 (0.38344569736056855) + | > loader_time: 0.0078 (0.006115201667503074) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.6341986656188965 (2.6341986656188965) + | > loss_disc_real_0: 0.21965189278125763 (0.21965189278125763) + | > loss_disc_real_1: 0.2322668880224228 (0.2322668880224228) + | > loss_disc_real_2: 0.26286718249320984 (0.26286718249320984) + | > loss_disc_real_3: 0.23743082582950592 (0.23743082582950592) + | > loss_disc_real_4: 0.22736689448356628 (0.22736689448356628) + | > loss_disc_real_5: 0.23988142609596252 (0.23988142609596252) + | > loss_0: 2.6341986656188965 (2.6341986656188965) + | > loss_gen: 2.339524507522583 (2.339524507522583) + | > loss_kl: 2.676898717880249 (2.676898717880249) + | > loss_feat: 6.42481803894043 (6.42481803894043) + | > loss_mel: 17.7450008392334 (17.7450008392334) + | > loss_duration: 1.7880576848983765 (1.7880576848983765) + | > loss_1: 30.974300384521484 (30.974300384521484) + + --> STEP: 1 + | > loss_disc: 2.6144542694091797 (2.6144542694091797) + | > loss_disc_real_0: 0.2670478820800781 (0.2670478820800781) + | > loss_disc_real_1: 0.2082752138376236 (0.2082752138376236) + | > loss_disc_real_2: 0.27532050013542175 (0.27532050013542175) + | > loss_disc_real_3: 0.21961775422096252 (0.21961775422096252) + | > loss_disc_real_4: 0.1938285231590271 (0.1938285231590271) + | > loss_disc_real_5: 0.24182575941085815 (0.24182575941085815) + | > loss_0: 2.6144542694091797 (2.6144542694091797) + | > loss_gen: 2.4337234497070312 (2.4337234497070312) + | > loss_kl: 2.840801477432251 (2.840801477432251) + | > loss_feat: 6.934678554534912 (6.934678554534912) + | > loss_mel: 19.20615005493164 (19.20615005493164) + | > loss_duration: 1.8142591714859009 (1.8142591714859009) + | > loss_1: 33.2296142578125 (33.2296142578125) + + --> STEP: 2 + | > loss_disc: 2.490112543106079 (2.5522834062576294) + | > loss_disc_real_0: 0.26691734790802 (0.2669826149940491) + | > loss_disc_real_1: 0.21915921568870544 (0.21371721476316452) + | > loss_disc_real_2: 0.2511639893054962 (0.263242244720459) + | > loss_disc_real_3: 0.246860072016716 (0.23323891311883926) + | > loss_disc_real_4: 0.18360407650470734 (0.18871629983186722) + | > loss_disc_real_5: 0.22357358038425446 (0.2326996698975563) + | > loss_0: 2.490112543106079 (2.5522834062576294) + | > loss_gen: 2.702946424484253 (2.568334937095642) + | > loss_kl: 2.5649921894073486 (2.7028968334198) + | > loss_feat: 7.647919178009033 (7.291298866271973) + | > loss_mel: 18.807117462158203 (19.006633758544922) + | > loss_duration: 1.7551937103271484 (1.7847264409065247) + | > loss_1: 33.47816848754883 (33.353891372680664) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034407377243041992 (-0.00024509429931640625) + | > avg_loss_disc: 2.5522834062576294 (-0.017046809196472168) + | > avg_loss_disc_real_0: 0.2669826149940491 (+0.052778661251068115) + | > avg_loss_disc_real_1: 0.21371721476316452 (-0.0741841122508049) + | > avg_loss_disc_real_2: 0.263242244720459 (+0.022547326982021332) + | > avg_loss_disc_real_3: 0.23323891311883926 (-0.016861654818058014) + | > avg_loss_disc_real_4: 0.18871629983186722 (-0.05739828944206238) + | > avg_loss_disc_real_5: 0.2326996698975563 (+0.005671210587024689) + | > avg_loss_0: 2.5522834062576294 (-0.017046809196472168) + | > avg_loss_gen: 2.568334937095642 (-0.03417503833770752) + | > avg_loss_kl: 2.7028968334198 (-0.007452487945556641) + | > avg_loss_feat: 7.291298866271973 (+0.20045089721679688) + | > avg_loss_mel: 19.006633758544922 (+1.487025260925293) + | > avg_loss_duration: 1.7847264409065247 (-0.008589506149291992) + | > avg_loss_1: 33.353891372680664 (+1.637258529663086) + + + > EPOCH: 475/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:46:41)  + + --> TIME: 2025-04-19 06:46:47 -- STEP: 18/142 -- GLOBAL_STEP: 446750 + | > loss_disc: 2.259338617324829 (2.3953683508767023) + | > loss_disc_real_0: 0.06885036826133728 (0.10160201808644666) + | > loss_disc_real_1: 0.2030634731054306 (0.20667865872383118) + | > loss_disc_real_2: 0.21378348767757416 (0.22483441481987634) + | > loss_disc_real_3: 0.22364696860313416 (0.22590955595175424) + | > loss_disc_real_4: 0.23292528092861176 (0.2248945708076159) + | > loss_disc_real_5: 0.23651544749736786 (0.2295545587937037) + | > loss_0: 2.259338617324829 (2.3953683508767023) + | > grad_norm_0: tensor(11.0248, device='cuda:0') (tensor(11.6743, device='cuda:0')) + | > loss_gen: 2.668745279312134 (2.4392328792148166) + | > loss_kl: 1.8360556364059448 (nan) + | > loss_feat: 7.989823818206787 (7.501312096913655) + | > loss_mel: 18.825931549072266 (17.770826763576924) + | > loss_duration: 1.4652601480484009 (nan) + | > amp_scaler: 0.5 (0.8888888888888888) + | > loss_1: 32.78581619262695 (nan) + | > grad_norm_1: tensor(420.8310, device='cuda:0') (tensor(280.3794, device='cuda:0')) + | > current_lr_0: 0.00018846996461904062 + | > current_lr_1: 0.00018846996461904062 + | > step_time: 0.337 (0.3281801806555854) + | > loader_time: 0.0051 (0.00463096300760905) + + + --> TIME: 2025-04-19 06:46:56 -- STEP: 43/142 -- GLOBAL_STEP: 446775 + | > loss_disc: 2.3576741218566895 (2.4148140951644543) + | > loss_disc_real_0: 0.11866263300180435 (0.10750636588348898) + | > loss_disc_real_1: 0.26096493005752563 (0.20368155282597208) + | > loss_disc_real_2: 0.21229763329029083 (0.22430989146232605) + | > loss_disc_real_3: 0.22814536094665527 (0.22681644212367924) + | > loss_disc_real_4: 0.2484809160232544 (0.22798382127007774) + | > loss_disc_real_5: 0.23820850253105164 (0.2334382936011913) + | > loss_0: 2.3576741218566895 (2.4148140951644543) + | > grad_norm_0: tensor(6.2360, device='cuda:0') (tensor(11.2873, device='cuda:0')) + | > loss_gen: 2.524099826812744 (2.421594081923019) + | > loss_kl: 1.9682563543319702 (nan) + | > loss_feat: 7.405034065246582 (7.3841706431189245) + | > loss_mel: 18.485279083251953 (17.84187013049458) + | > loss_duration: 1.4867488145828247 (nan) + | > amp_scaler: 0.25 (0.5290697674418605) + | > loss_1: 31.86941909790039 (nan) + | > grad_norm_1: tensor(330.5930, device='cuda:0') (tensor(280.2832, device='cuda:0')) + | > current_lr_0: 0.00018846996461904062 + | > current_lr_1: 0.00018846996461904062 + | > step_time: 0.3613 (0.34096837043762207) + | > loader_time: 0.0056 (0.0050561594408611915) + + + --> TIME: 2025-04-19 06:47:06 -- STEP: 68/142 -- GLOBAL_STEP: 446800 + | > loss_disc: 2.27571177482605 (2.415267600732691) + | > loss_disc_real_0: 0.11123144626617432 (0.11177714338854831) + | > loss_disc_real_1: 0.21819300949573517 (0.20529233313658657) + | > loss_disc_real_2: 0.2125188261270523 (0.22226877050364718) + | > loss_disc_real_3: 0.211600661277771 (0.22606129659449353) + | > loss_disc_real_4: 0.21825948357582092 (0.22529631918844054) + | > loss_disc_real_5: 0.21940946578979492 (0.23154112499426394) + | > loss_0: 2.27571177482605 (2.415267600732691) + | > grad_norm_0: tensor(6.6817, device='cuda:0') (tensor(10.9938, device='cuda:0')) + | > loss_gen: 2.6025612354278564 (2.412887015763451) + | > loss_kl: nan (nan) + | > loss_feat: 7.936851978302002 (7.339864927179673) + | > loss_mel: 18.172012329101562 (17.738159347982965) + | > loss_duration: nan (nan) + | > amp_scaler: 0.001953125 (0.39208984375) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(256.4741, device='cuda:0')) + | > current_lr_0: 0.00018846996461904062 + | > current_lr_1: 0.00018846996461904062 + | > step_time: 0.3788 (0.35212910175323486) + | > loader_time: 0.0059 (0.005409110994899974) + + + --> TIME: 2025-04-19 06:47:16 -- STEP: 93/142 -- GLOBAL_STEP: 446825 + | > loss_disc: 2.3647727966308594 (2.3972067166400217) + | > loss_disc_real_0: 0.12148522585630417 (0.11208939428130786) + | > loss_disc_real_1: 0.1982548087835312 (0.20297868005050126) + | > loss_disc_real_2: 0.19152405858039856 (0.21701236325566486) + | > loss_disc_real_3: 0.18877476453781128 (0.22109273957308903) + | > loss_disc_real_4: 0.16979201138019562 (0.21532107569197173) + | > loss_disc_real_5: 0.2290891855955124 (0.23334472954914134) + | > loss_0: 2.3647727966308594 (2.3972067166400217) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(8.6150, device='cuda:0')) + | > loss_gen: 2.6882729530334473 (2.4518918837270425) + | > loss_kl: 2.4384090900421143 (nan) + | > loss_feat: 8.322985649108887 (7.375559709405386) + | > loss_mel: 19.26567268371582 (17.741904925274593) + | > loss_duration: 1.6247336864471436 (nan) + | > amp_scaler: 1.4901161193847656e-08 (0.2867117851011216) + | > loss_1: 34.34007263183594 (nan) + | > grad_norm_1: tensor(8.4470, device='cuda:0') (tensor(192.8938, device='cuda:0')) + | > current_lr_0: 0.00018846996461904062 + | > current_lr_1: 0.00018846996461904062 + | > step_time: 0.4032 (0.3636662857506865) + | > loader_time: 0.0064 (0.005734338555284726) + + + --> TIME: 2025-04-19 06:47:26 -- STEP: 118/142 -- GLOBAL_STEP: 446850 + | > loss_disc: 2.4606804847717285 (2.40646568799423) + | > loss_disc_real_0: 0.11506688594818115 (0.1127334226049104) + | > loss_disc_real_1: 0.21045181155204773 (0.2058481225522898) + | > loss_disc_real_2: 0.19778987765312195 (0.21402116448192274) + | > loss_disc_real_3: 0.1873031109571457 (0.21666644084251532) + | > loss_disc_real_4: 0.18098288774490356 (0.20518230735245396) + | > loss_disc_real_5: 0.25417155027389526 (0.23355235727661747) + | > loss_0: 2.4606804847717285 (2.40646568799423) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(6.7898, device='cuda:0')) + | > loss_gen: 2.72273850440979 (2.502029259326094) + | > loss_kl: 2.568356513977051 (nan) + | > loss_feat: 7.145018577575684 (7.37215879812079) + | > loss_mel: 17.076147079467773 (17.69045627723306) + | > loss_duration: 1.6405556201934814 (nan) + | > amp_scaler: 3.552713678800501e-15 (0.22596776299179006) + | > loss_1: 31.152816772460938 (nan) + | > grad_norm_1: tensor(2.5150, device='cuda:0') (tensor(152.0961, device='cuda:0')) + | > current_lr_0: 0.00018846996461904062 + | > current_lr_1: 0.00018846996461904062 + | > step_time: 0.4515 (0.37290466437905523) + | > loader_time: 0.0076 (0.005986268237485724) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: 2.34603214263916 (2.34603214263916) + | > loss_disc_real_0: 0.15544922649860382 (0.15544922649860382) + | > loss_disc_real_1: 0.18082159757614136 (0.18082159757614136) + | > loss_disc_real_2: 0.17414747178554535 (0.17414747178554535) + | > loss_disc_real_3: 0.17798545956611633 (0.17798545956611633) + | > loss_disc_real_4: 0.16931851208209991 (0.16931851208209991) + | > loss_disc_real_5: 0.21819743514060974 (0.21819743514060974) + | > loss_0: 2.34603214263916 (2.34603214263916) + | > loss_gen: 2.674053430557251 (2.674053430557251) + | > loss_kl: 2.81589674949646 (2.81589674949646) + | > loss_feat: 7.4047088623046875 (7.4047088623046875) + | > loss_mel: 19.394895553588867 (19.394895553588867) + | > loss_duration: 1.8480583429336548 (1.8480583429336548) + | > loss_1: 34.137611389160156 (34.137611389160156) + + --> STEP: 1 + | > loss_disc: 2.6344897747039795 (2.6344897747039795) + | > loss_disc_real_0: 0.14455793797969818 (0.14455793797969818) + | > loss_disc_real_1: 0.21757271885871887 (0.21757271885871887) + | > loss_disc_real_2: 0.1924280822277069 (0.1924280822277069) + | > loss_disc_real_3: 0.20549295842647552 (0.20549295842647552) + | > loss_disc_real_4: 0.16746821999549866 (0.16746821999549866) + | > loss_disc_real_5: 0.22059309482574463 (0.22059309482574463) + | > loss_0: 2.6344897747039795 (2.6344897747039795) + | > loss_gen: 2.3639373779296875 (2.3639373779296875) + | > loss_kl: 3.251765727996826 (3.251765727996826) + | > loss_feat: 6.2619853019714355 (6.2619853019714355) + | > loss_mel: 17.17561912536621 (17.17561912536621) + | > loss_duration: 1.9088371992111206 (1.9088371992111206) + | > loss_1: 30.962142944335938 (30.962142944335938) + + --> STEP: 2 + | > loss_disc: 2.776263952255249 (2.7053768634796143) + | > loss_disc_real_0: 0.19480594992637634 (0.16968194395303726) + | > loss_disc_real_1: 0.24658088386058807 (0.23207680135965347) + | > loss_disc_real_2: 0.2172587364912033 (0.2048434093594551) + | > loss_disc_real_3: 0.18720565736293793 (0.19634930789470673) + | > loss_disc_real_4: 0.19762298464775085 (0.18254560232162476) + | > loss_disc_real_5: 0.2325269728899002 (0.22656003385782242) + | > loss_0: 2.776263952255249 (2.7053768634796143) + | > loss_gen: 2.404048204421997 (2.3839927911758423) + | > loss_kl: 2.8887650966644287 (3.0702654123306274) + | > loss_feat: 5.6655731201171875 (5.9637792110443115) + | > loss_mel: 14.971869468688965 (16.073744297027588) + | > loss_duration: 1.7980897426605225 (1.8534634709358215) + | > loss_1: 27.72834587097168 (29.34524440765381) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034928321838378906 (+5.2094459533691406e-05) + | > avg_loss_disc: 2.7053768634796143 (+0.15309345722198486) + | > avg_loss_disc_real_0: 0.16968194395303726 (-0.09730067104101181) + | > avg_loss_disc_real_1: 0.23207680135965347 (+0.018359586596488953) + | > avg_loss_disc_real_2: 0.2048434093594551 (-0.058398835361003876) + | > avg_loss_disc_real_3: 0.19634930789470673 (-0.03688960522413254) + | > avg_loss_disc_real_4: 0.18254560232162476 (-0.006170697510242462) + | > avg_loss_disc_real_5: 0.22656003385782242 (-0.006139636039733887) + | > avg_loss_0: 2.7053768634796143 (+0.15309345722198486) + | > avg_loss_gen: 2.3839927911758423 (-0.1843421459197998) + | > avg_loss_kl: 3.0702654123306274 (+0.36736857891082764) + | > avg_loss_feat: 5.9637792110443115 (-1.3275196552276611) + | > avg_loss_mel: 16.073744297027588 (-2.932889461517334) + | > avg_loss_duration: 1.8534634709358215 (+0.06873703002929688) + | > avg_loss_1: 29.34524440765381 (-4.0086469650268555) + + + > EPOCH: 476/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:47:39)  + + --> TIME: 2025-04-19 06:47:40 -- STEP: 1/142 -- GLOBAL_STEP: 446875 + | > loss_disc: 2.3322532176971436 (2.3322532176971436) + | > loss_disc_real_0: 0.09346149116754532 (0.09346149116754532) + | > loss_disc_real_1: 0.18508277833461761 (0.18508277833461761) + | > loss_disc_real_2: 0.18965479731559753 (0.18965479731559753) + | > loss_disc_real_3: 0.2228848934173584 (0.2228848934173584) + | > loss_disc_real_4: 0.1633385866880417 (0.1633385866880417) + | > loss_disc_real_5: 0.22313575446605682 (0.22313575446605682) + | > loss_0: 2.3322532176971436 (2.3322532176971436) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(0., device='cuda:0')) + | > loss_gen: 2.886519193649292 (2.886519193649292) + | > loss_kl: nan (nan) + | > loss_feat: 8.104804992675781 (8.104804992675781) + | > loss_mel: 18.70924949645996 (18.70924949645996) + | > loss_duration: nan (nan) + | > amp_scaler: 1.6940658945086007e-21 (1.6940658945086007e-21) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.3136 (0.31357240676879883) + | > loader_time: 0.0041 (0.00409698486328125) + + + --> TIME: 2025-04-19 06:47:48 -- STEP: 26/142 -- GLOBAL_STEP: 446900 + | > loss_disc: 2.521827459335327 (2.380777019720811) + | > loss_disc_real_0: 0.11309058964252472 (0.10473004069465858) + | > loss_disc_real_1: 0.21509429812431335 (0.20811080302183443) + | > loss_disc_real_2: 0.2055022418498993 (0.19641938690955824) + | > loss_disc_real_3: 0.215797558426857 (0.192312736923878) + | > loss_disc_real_4: 0.17262081801891327 (0.16379092576412058) + | > loss_disc_real_5: 0.25035950541496277 (0.2271656972857622) + | > loss_0: 2.521827459335327 (2.380777019720811) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(0., device='cuda:0')) + | > loss_gen: 2.856895923614502 (2.82563841342926) + | > loss_kl: nan (nan) + | > loss_feat: 7.864439964294434 (7.948820664332463) + | > loss_mel: 17.69280433654785 (17.926919790414665) + | > loss_duration: nan (nan) + | > amp_scaler: 2.5849394142282115e-26 (2.0291873822438162e-22) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(0.5698, device='cuda:0')) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.3363 (0.3242972997518686) + | > loader_time: 0.0052 (0.00481589023883526) + + + --> TIME: 2025-04-19 06:47:57 -- STEP: 51/142 -- GLOBAL_STEP: 446925 + | > loss_disc: 2.4059109687805176 (2.395600795745849) + | > loss_disc_real_0: 0.12940065562725067 (0.11012407494526286) + | > loss_disc_real_1: 0.2202751189470291 (0.2104173501332601) + | > loss_disc_real_2: 0.20164187252521515 (0.19670797884464264) + | > loss_disc_real_3: 0.19652369618415833 (0.19472725689411163) + | > loss_disc_real_4: 0.1716616153717041 (0.16496663029287378) + | > loss_disc_real_5: 0.22101563215255737 (0.22863737099310932) + | > loss_0: 2.4059109687805176 (2.395600795745849) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(0., device='cuda:0')) + | > loss_gen: 2.8597540855407715 (2.811614920111264) + | > loss_kl: nan (nan) + | > loss_feat: 8.364344596862793 (7.829950351341098) + | > loss_mel: 17.892086029052734 (17.89225333344703) + | > loss_duration: nan (nan) + | > amp_scaler: 4.930380657631324e-32 (1.0344928034869838e-22) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(0.4768, device='cuda:0')) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.3557 (0.3360726272358615) + | > loader_time: 0.0061 (0.005211348627127852) + + + --> TIME: 2025-04-19 06:48:06 -- STEP: 76/142 -- GLOBAL_STEP: 446950 + | > loss_disc: 2.3597331047058105 (2.4038671694303813) + | > loss_disc_real_0: 0.12059904634952545 (0.11152461612303007) + | > loss_disc_real_1: 0.2157147377729416 (0.21184463230402847) + | > loss_disc_real_2: 0.2224126011133194 (0.19759850949048996) + | > loss_disc_real_3: 0.2072269767522812 (0.19534056516070114) + | > loss_disc_real_4: 0.16422928869724274 (0.16432738755094378) + | > loss_disc_real_5: 0.20054899156093597 (0.22651072219014168) + | > loss_0: 2.3597331047058105 (2.4038671694303813) + | > grad_norm_0: tensor(0., device='cuda:0') (tensor(0., device='cuda:0')) + | > loss_gen: 2.6774895191192627 (2.7957611523176493) + | > loss_kl: nan (nan) + | > loss_feat: 7.504086494445801 (7.775541236526088) + | > loss_mel: 18.601757049560547 (17.867156216972752) + | > loss_duration: nan (nan) + | > amp_scaler: 1.1754943508222875e-38 (6.941991181380393e-23) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(0.3879, device='cuda:0')) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.3777 (0.3477243467381127) + | > loader_time: 0.0063 (0.005524927063992149) + + + --> TIME: 2025-04-19 06:48:16 -- STEP: 101/142 -- GLOBAL_STEP: 446975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (tensor(0., device='cuda:0')) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.967140197753906 (17.8670789038781) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (5.2236765325238616e-23) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(0.2919, device='cuda:0')) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.3986 (0.3550245856294539) + | > loader_time: 0.01 (0.005837973981800646) + + + --> TIME: 2025-04-19 06:48:26 -- STEP: 126/142 -- GLOBAL_STEP: 447000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (tensor(0., device='cuda:0')) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.626605033874512 (17.889878227597194) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (4.1872327760707144e-23) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (tensor(0.2340, device='cuda:0')) + | > current_lr_0: 0.00018844640587346325 + | > current_lr_1: 0.00018844640587346325 + | > step_time: 0.4202 (0.3650135842580645) + | > loader_time: 0.0074 (0.006109994555276538) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.889700412750244 (2.889700412750244) + | > loss_feat: nan (nan) + | > loss_mel: 18.349735260009766 (18.349735260009766) + | > loss_duration: 1.828188180923462 (1.828188180923462) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3916845321655273 (3.3916845321655273) + | > loss_feat: nan (nan) + | > loss_mel: 18.41075325012207 (18.41075325012207) + | > loss_duration: 1.9126895666122437 (1.9126895666122437) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9326882362365723 (3.16218638420105) + | > loss_feat: nan (nan) + | > loss_mel: 19.63104820251465 (19.02090072631836) + | > loss_duration: 1.791012167930603 (1.8518508672714233) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0041217803955078125 (+0.0006289482116699219) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.16218638420105 (+0.09192097187042236) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.02090072631836 (+2.9471564292907715) + | > avg_loss_duration: 1.8518508672714233 (-0.0016126036643981934) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 477/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:48:35)  + + --> TIME: 2025-04-19 06:48:38 -- STEP: 9/142 -- GLOBAL_STEP: 447025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.582334280014038 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.135875701904297 (18.346099005805122) + | > loss_duration: 1.5963289737701416 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.2986 (0.2986002498202854) + | > loader_time: 0.0047 (0.004204882515801324) + + + --> TIME: 2025-04-19 06:48:46 -- STEP: 34/142 -- GLOBAL_STEP: 447050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6344289779663086 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.478904724121094 (17.856959988089177) + | > loss_duration: 1.6185739040374756 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.3294 (0.31247510629541736) + | > loader_time: 0.0055 (0.005013318622813505) + + + --> TIME: 2025-04-19 06:48:55 -- STEP: 59/142 -- GLOBAL_STEP: 447075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.141971588134766 (17.969864667472198) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.3488 (0.32402799897274737) + | > loader_time: 0.0061 (0.0053612458503852455) + + + --> TIME: 2025-04-19 06:49:05 -- STEP: 84/142 -- GLOBAL_STEP: 447100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.469540596008301 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.803844451904297 (17.904804547627773) + | > loss_duration: 1.7108901739120483 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.3691 (0.33642846345901495) + | > loader_time: 0.0065 (0.005621717089698428) + + + --> TIME: 2025-04-19 06:49:14 -- STEP: 109/142 -- GLOBAL_STEP: 447125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.512311935424805 (17.85165488391842) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.3953 (0.3472613566512362) + | > loader_time: 0.0066 (0.005889177322387695) + + + --> TIME: 2025-04-19 06:49:25 -- STEP: 134/142 -- GLOBAL_STEP: 447150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.800981521606445 (17.837559415333313) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018842285007272905 + | > current_lr_1: 0.00018842285007272905 + | > step_time: 0.417 (0.3596377265987112) + | > loader_time: 0.0074 (0.006159008438907453) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.842689037322998 (2.842689037322998) + | > loss_feat: nan (nan) + | > loss_mel: 19.247037887573242 (19.247037887573242) + | > loss_duration: 1.809633731842041 (1.809633731842041) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4125852584838867 (3.4125852584838867) + | > loss_feat: nan (nan) + | > loss_mel: 18.565139770507812 (18.565139770507812) + | > loss_duration: 1.9192241430282593 (1.9192241430282593) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.765761613845825 (3.089173436164856) + | > loss_feat: nan (nan) + | > loss_mel: 18.467966079711914 (18.516552925109863) + | > loss_duration: 1.809440016746521 (1.8643320798873901) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034574270248413086 (-0.0006643533706665039) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.089173436164856 (-0.07301294803619385) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.516552925109863 (-0.5043478012084961) + | > avg_loss_duration: 1.8643320798873901 (+0.012481212615966797) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 478/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:49:30)  + + --> TIME: 2025-04-19 06:49:36 -- STEP: 17/142 -- GLOBAL_STEP: 447175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.21722412109375 (17.810425253475415) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018839929721646995 + | > current_lr_1: 0.00018839929721646995 + | > step_time: 0.3168 (0.30477623378529267) + | > loader_time: 0.005 (0.004721445195815142) + + + --> TIME: 2025-04-19 06:49:44 -- STEP: 42/142 -- GLOBAL_STEP: 447200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.30477523803711 (17.985939343770344) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018839929721646995 + | > current_lr_1: 0.00018839929721646995 + | > step_time: 0.3455 (0.3211992808750698) + | > loader_time: 0.0059 (0.005231885682968866) + + + --> TIME: 2025-04-19 06:49:53 -- STEP: 67/142 -- GLOBAL_STEP: 447225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.353618621826172 (18.050817802770815) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018839929721646995 + | > current_lr_1: 0.00018839929721646995 + | > step_time: 0.368 (0.33558776841234805) + | > loader_time: 0.0062 (0.005578065986063943) + + + --> TIME: 2025-04-19 06:50:03 -- STEP: 92/142 -- GLOBAL_STEP: 447250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.168922424316406 (18.06230129366336) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018839929721646995 + | > current_lr_1: 0.00018839929721646995 + | > step_time: 0.386 (0.3480175707651222) + | > loader_time: 0.0063 (0.005776737047278363) + + + --> TIME: 2025-04-19 06:50:13 -- STEP: 117/142 -- GLOBAL_STEP: 447275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.665977478027344 (17.99560671586258) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018839929721646995 + | > current_lr_1: 0.00018839929721646995 + | > step_time: 0.4088 (0.35709081348190963) + | > loader_time: 0.007 (0.006038366219936273) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.710137128829956 (2.710137128829956) + | > loss_feat: nan (nan) + | > loss_mel: 18.08147621154785 (18.08147621154785) + | > loss_duration: 1.8196948766708374 (1.8196948766708374) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.167971611022949 (3.167971611022949) + | > loss_feat: nan (nan) + | > loss_mel: 17.422012329101562 (17.422012329101562) + | > loss_duration: 1.8820651769638062 (1.8820651769638062) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7244460582733154 (2.9462088346481323) + | > loss_feat: nan (nan) + | > loss_mel: 16.591747283935547 (17.006879806518555) + | > loss_duration: 1.7806464433670044 (1.8313558101654053) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003240346908569336 (-0.00021708011627197266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9462088346481323 (-0.14296460151672363) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.006879806518555 (-1.5096731185913086) + | > avg_loss_duration: 1.8313558101654053 (-0.03297626972198486) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 479/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:50:25)  + + --> TIME: 2025-04-19 06:50:26 -- STEP: 0/142 -- GLOBAL_STEP: 447300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.086406707763672 (18.086406707763672) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.3281 (0.3281214237213135) + | > loader_time: 0.3523 (0.3522622585296631) + + + --> TIME: 2025-04-19 06:50:34 -- STEP: 25/142 -- GLOBAL_STEP: 447325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4530913829803467 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.854040145874023 (18.35777000427246) + | > loss_duration: 1.6645328998565674 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.319 (0.3090826892852783) + | > loader_time: 0.0055 (0.005015287399291992) + + + --> TIME: 2025-04-19 06:50:42 -- STEP: 50/142 -- GLOBAL_STEP: 447350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.736989974975586 (18.102666587829596) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.3435 (0.32067051887512205) + | > loader_time: 0.0057 (0.005417876243591309) + + + --> TIME: 2025-04-19 06:50:51 -- STEP: 75/142 -- GLOBAL_STEP: 447375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.80963897705078 (18.05414599100749) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.3706 (0.3325132687886556) + | > loader_time: 0.0065 (0.005717598597208658) + + + --> TIME: 2025-04-19 06:51:01 -- STEP: 100/142 -- GLOBAL_STEP: 447400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.490406036376953 (17.97878774642945) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.3877 (0.34311222314834594) + | > loader_time: 0.0067 (0.005942590236663818) + + + --> TIME: 2025-04-19 06:51:11 -- STEP: 125/142 -- GLOBAL_STEP: 447425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5440673828125 (17.96611639404298) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018837574730431789 + | > current_lr_1: 0.00018837574730431789 + | > step_time: 0.4119 (0.354688404083252) + | > loader_time: 0.0074 (0.0061930065155029294) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9908206462860107 (2.9908206462860107) + | > loss_feat: nan (nan) + | > loss_mel: 19.851533889770508 (19.851533889770508) + | > loss_duration: 1.8425118923187256 (1.8425118923187256) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2633440494537354 (3.2633440494537354) + | > loss_feat: nan (nan) + | > loss_mel: 18.445329666137695 (18.445329666137695) + | > loss_duration: 1.894453525543213 (1.894453525543213) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.794391393661499 (3.028867721557617) + | > loss_feat: nan (nan) + | > loss_mel: 19.199190139770508 (18.8222599029541) + | > loss_duration: 1.8032898902893066 (1.8488717079162598) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004087090492248535 (+0.0008467435836791992) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.028867721557617 (+0.08265888690948486) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.8222599029541 (+1.8153800964355469) + | > avg_loss_duration: 1.8488717079162598 (+0.017515897750854492) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 480/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:51:20)  + + --> TIME: 2025-04-19 06:51:23 -- STEP: 8/142 -- GLOBAL_STEP: 447450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.55756378173828 (17.94588017463684) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.2986 (0.29855138063430786) + | > loader_time: 0.0044 (0.0043678879737854) + + + --> TIME: 2025-04-19 06:51:31 -- STEP: 33/142 -- GLOBAL_STEP: 447475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.018075942993164 (18.18787430271959) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.34 (0.31904275489575934) + | > loader_time: 0.0052 (0.005189686110525419) + + + --> TIME: 2025-04-19 06:51:40 -- STEP: 58/142 -- GLOBAL_STEP: 447500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.342424392700195 (18.17550494753082) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.3607 (0.33172867626979435) + | > loader_time: 0.0064 (0.005540387383822735) + + + --> TIME: 2025-04-19 06:51:50 -- STEP: 83/142 -- GLOBAL_STEP: 447525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.578795909881592 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.593706130981445 (18.032189185360835) + | > loss_duration: 1.7119765281677246 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.3861 (0.3450954305120262) + | > loader_time: 0.0065 (0.005886635148381611) + + + --> TIME: 2025-04-19 06:52:00 -- STEP: 108/142 -- GLOBAL_STEP: 447550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.452417373657227 (17.947898979540234) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.4126 (0.3566534938635649) + | > loader_time: 0.0077 (0.00617624212194372) + + + --> TIME: 2025-04-19 06:52:11 -- STEP: 133/142 -- GLOBAL_STEP: 447575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.177812576293945 (17.93516301750241) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018835220033590485 + | > current_lr_1: 0.00018835220033590485 + | > step_time: 0.4115 (0.37028156904349646) + | > loader_time: 0.0074 (0.006450378805174861) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.975578546524048 (2.975578546524048) + | > loss_feat: nan (nan) + | > loss_mel: 18.621292114257812 (18.621292114257812) + | > loss_duration: 1.8214178085327148 (1.8214178085327148) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3897504806518555 (3.3897504806518555) + | > loss_feat: nan (nan) + | > loss_mel: 17.68704605102539 (17.68704605102539) + | > loss_duration: 1.9162824153900146 (1.9162824153900146) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8676631450653076 (3.1287068128585815) + | > loss_feat: nan (nan) + | > loss_mel: 14.583913803100586 (16.13547992706299) + | > loss_duration: 1.8022434711456299 (1.8592629432678223) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036602020263671875 (-0.00042688846588134766) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1287068128585815 (+0.09983909130096436) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.13547992706299 (-2.6867799758911133) + | > avg_loss_duration: 1.8592629432678223 (+0.0103912353515625) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 481/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:52:16)  + + --> TIME: 2025-04-19 06:52:22 -- STEP: 16/142 -- GLOBAL_STEP: 447600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.318819046020508 (18.22243678569794) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.3138 (0.3025565445423126) + | > loader_time: 0.005 (0.004529222846031189) + + + --> TIME: 2025-04-19 06:52:30 -- STEP: 41/142 -- GLOBAL_STEP: 447625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.74459457397461 (18.23523986630323) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.3439 (0.3159559645303866) + | > loader_time: 0.0056 (0.0051089089091231185) + + + --> TIME: 2025-04-19 06:52:39 -- STEP: 66/142 -- GLOBAL_STEP: 447650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.766204833984375 (18.12061095960212) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.3577 (0.3279766928065907) + | > loader_time: 0.0064 (0.00542959299954501) + + + --> TIME: 2025-04-19 06:52:49 -- STEP: 91/142 -- GLOBAL_STEP: 447675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.412240982055664 (18.027888874431216) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.3771 (0.33916317499600934) + | > loader_time: 0.0065 (0.005724720902495332) + + + --> TIME: 2025-04-19 06:52:58 -- STEP: 116/142 -- GLOBAL_STEP: 447700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.919992446899414 (18.005630756246624) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.4031 (0.3499447538934905) + | > loader_time: 0.0069 (0.005977860812483163) + + + --> TIME: 2025-04-19 06:53:09 -- STEP: 141/142 -- GLOBAL_STEP: 447725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.339404106140137 (17.99500384229294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018832865631086286 + | > current_lr_1: 0.00018832865631086286 + | > step_time: 0.1786 (0.3604946068838133) + | > loader_time: 0.0026 (0.00621435320969169) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.988354444503784 (2.988354444503784) + | > loss_feat: nan (nan) + | > loss_mel: 19.39516830444336 (19.39516830444336) + | > loss_duration: 1.8190383911132812 (1.8190383911132812) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.346585988998413 (3.346585988998413) + | > loss_feat: nan (nan) + | > loss_mel: 17.29359245300293 (17.29359245300293) + | > loss_duration: 1.9043339490890503 (1.9043339490890503) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7958502769470215 (3.0712181329727173) + | > loss_feat: nan (nan) + | > loss_mel: 18.046960830688477 (17.670276641845703) + | > loss_duration: 1.806236743927002 (1.8552853465080261) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033795833587646484 (-0.00028061866760253906) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0712181329727173 (-0.05748867988586426) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.670276641845703 (+1.5347967147827148) + | > avg_loss_duration: 1.8552853465080261 (-0.003977596759796143) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 482/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:53:11)  + + --> TIME: 2025-04-19 06:53:19 -- STEP: 24/142 -- GLOBAL_STEP: 447750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.954423904418945 (17.91553505261739) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000188305115228824 + | > current_lr_1: 0.000188305115228824 + | > step_time: 0.3235 (0.3087523281574249) + | > loader_time: 0.005 (0.004507869482040405) + + + --> TIME: 2025-04-19 06:53:28 -- STEP: 49/142 -- GLOBAL_STEP: 447775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.613598585128784 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.629924774169922 (17.974567763659415) + | > loss_duration: 1.6036207675933838 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000188305115228824 + | > current_lr_1: 0.000188305115228824 + | > step_time: 0.3483 (0.3261354650769915) + | > loader_time: 0.0059 (0.004958610145413145) + + + --> TIME: 2025-04-19 06:53:37 -- STEP: 74/142 -- GLOBAL_STEP: 447800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.749959707260132 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.74428367614746 (17.977760650016165) + | > loss_duration: 1.642977237701416 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000188305115228824 + | > current_lr_1: 0.000188305115228824 + | > step_time: 0.3628 (0.3385698602006242) + | > loader_time: 0.0061 (0.005403792535936509) + + + --> TIME: 2025-04-19 06:53:47 -- STEP: 99/142 -- GLOBAL_STEP: 447825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7076220512390137 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.29144859313965 (17.969401157263547) + | > loss_duration: 1.688578486442566 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000188305115228824 + | > current_lr_1: 0.000188305115228824 + | > step_time: 0.3836 (0.3485684394836426) + | > loader_time: 0.0065 (0.005680452693592419) + + + --> TIME: 2025-04-19 06:53:57 -- STEP: 124/142 -- GLOBAL_STEP: 447850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.614028930664062 (17.95105258880122) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000188305115228824 + | > current_lr_1: 0.000188305115228824 + | > step_time: 0.4147 (0.35929077286874095) + | > loader_time: 0.0074 (0.0059451614656756005) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9602043628692627 (2.9602043628692627) + | > loss_feat: nan (nan) + | > loss_mel: 17.81167984008789 (17.81167984008789) + | > loss_duration: 1.841737151145935 (1.841737151145935) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.365839719772339 (3.365839719772339) + | > loss_feat: nan (nan) + | > loss_mel: 17.788368225097656 (17.788368225097656) + | > loss_duration: 1.8962328433990479 (1.8962328433990479) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6621382236480713 (3.013988971710205) + | > loss_feat: nan (nan) + | > loss_mel: 19.346187591552734 (18.567277908325195) + | > loss_duration: 1.8052202463150024 (1.8507265448570251) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032558441162109375 (-0.00012373924255371094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.013988971710205 (-0.05722916126251221) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.567277908325195 (+0.8970012664794922) + | > avg_loss_duration: 1.8507265448570251 (-0.0045588016510009766) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 483/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:54:06)  + + --> TIME: 2025-04-19 06:54:09 -- STEP: 7/142 -- GLOBAL_STEP: 447875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.165361404418945 (18.085270200456893) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.298 (0.2984489713396345) + | > loader_time: 0.0041 (0.0044029780796595985) + + + --> TIME: 2025-04-19 06:54:17 -- STEP: 32/142 -- GLOBAL_STEP: 447900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.930923461914062 (17.852804601192474) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.3357 (0.3118964210152626) + | > loader_time: 0.0077 (0.0051128864288330095) + + + --> TIME: 2025-04-19 06:54:26 -- STEP: 57/142 -- GLOBAL_STEP: 447925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.647897243499756 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.217876434326172 (17.877526484037702) + | > loss_duration: 1.6744741201400757 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.3486 (0.3232420787476657) + | > loader_time: 0.0059 (0.005354350073295728) + + + --> TIME: 2025-04-19 06:54:35 -- STEP: 82/142 -- GLOBAL_STEP: 447950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.788429260253906 (17.919882344036573) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.3671 (0.33524470794491645) + | > loader_time: 0.0063 (0.005641146403987233) + + + --> TIME: 2025-04-19 06:54:45 -- STEP: 107/142 -- GLOBAL_STEP: 447975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.108980178833008 (17.932276654466293) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.3891 (0.34583217406941347) + | > loader_time: 0.0064 (0.00589560125475732) + + + --> TIME: 2025-04-19 06:54:55 -- STEP: 132/142 -- GLOBAL_STEP: 448000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.459842681884766 (17.853221452597417) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882815770894204 + | > current_lr_1: 0.0001882815770894204 + | > step_time: 0.4216 (0.35806887438803003) + | > loader_time: 0.0068 (0.006144409829919989) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9613230228424072 (2.9613230228424072) + | > loss_feat: nan (nan) + | > loss_mel: 19.88041877746582 (19.88041877746582) + | > loss_duration: 1.8259117603302002 (1.8259117603302002) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.308427333831787 (3.308427333831787) + | > loss_feat: nan (nan) + | > loss_mel: 18.373441696166992 (18.373441696166992) + | > loss_duration: 1.9164037704467773 (1.9164037704467773) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.946138381958008 (3.1272828578948975) + | > loss_feat: nan (nan) + | > loss_mel: 17.472591400146484 (17.92301654815674) + | > loss_duration: 1.806402325630188 (1.8614030480384827) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0041942596435546875 (+0.00093841552734375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1272828578948975 (+0.11329388618469238) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.92301654815674 (-0.644261360168457) + | > avg_loss_duration: 1.8614030480384827 (+0.01067650318145752) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 484/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:55:01)  + + --> TIME: 2025-04-19 06:55:06 -- STEP: 15/142 -- GLOBAL_STEP: 448025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.820039749145508 (17.978928248087563) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.307 (0.3020073254903158) + | > loader_time: 0.0052 (0.004811716079711914) + + + --> TIME: 2025-04-19 06:55:15 -- STEP: 40/142 -- GLOBAL_STEP: 448050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.447994232177734 (17.718321442604065) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.3494 (0.3193256616592407) + | > loader_time: 0.006 (0.005211085081100466) + + + --> TIME: 2025-04-19 06:55:24 -- STEP: 65/142 -- GLOBAL_STEP: 448075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.121164321899414 (17.789358637883108) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.3771 (0.33358560708852913) + | > loader_time: 0.0062 (0.005547241064218376) + + + --> TIME: 2025-04-19 06:55:34 -- STEP: 90/142 -- GLOBAL_STEP: 448100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.361635208129883 (17.798554865519197) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.3783 (0.34811560577816436) + | > loader_time: 0.0071 (0.005829623010423451) + + + --> TIME: 2025-04-19 06:55:44 -- STEP: 115/142 -- GLOBAL_STEP: 448125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.59736442565918 (17.7554131715194) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.4013 (0.35857222805852473) + | > loader_time: 0.0068 (0.0061062957929528296) + + + --> TIME: 2025-04-19 06:55:55 -- STEP: 140/142 -- GLOBAL_STEP: 448150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.676401138305664 (17.756458364214215) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018825804189228422 + | > current_lr_1: 0.00018825804189228422 + | > step_time: 0.432 (0.36933794702802386) + | > loader_time: 0.0069 (0.006310159819466729) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8204843997955322 (2.8204843997955322) + | > loss_feat: nan (nan) + | > loss_mel: 17.796852111816406 (17.796852111816406) + | > loss_duration: 1.8354206085205078 (1.8354206085205078) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.240946054458618 (3.240946054458618) + | > loss_feat: nan (nan) + | > loss_mel: 16.73955535888672 (16.73955535888672) + | > loss_duration: 1.8921847343444824 (1.8921847343444824) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8087680339813232 (3.0248570442199707) + | > loss_feat: nan (nan) + | > loss_mel: 17.909605026245117 (17.324580192565918) + | > loss_duration: 1.7935333251953125 (1.8428590297698975) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035663843154907227 (-0.0006278753280639648) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0248570442199707 (-0.10242581367492676) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.324580192565918 (-0.5984363555908203) + | > avg_loss_duration: 1.8428590297698975 (-0.018544018268585205) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 485/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:55:57)  + + --> TIME: 2025-04-19 06:56:05 -- STEP: 23/142 -- GLOBAL_STEP: 448175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.02239990234375 (18.264401228531547) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882345096370477 + | > current_lr_1: 0.0001882345096370477 + | > step_time: 0.3199 (0.30760473790376086) + | > loader_time: 0.0052 (0.00462428383205248) + + + --> TIME: 2025-04-19 06:56:13 -- STEP: 48/142 -- GLOBAL_STEP: 448200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.986202239990234 (18.028212149937943) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882345096370477 + | > current_lr_1: 0.0001882345096370477 + | > step_time: 0.3377 (0.31938070555528003) + | > loader_time: 0.0057 (0.005138203501701355) + + + --> TIME: 2025-04-19 06:56:22 -- STEP: 73/142 -- GLOBAL_STEP: 448225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.2639102935791 (17.946213369500146) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882345096370477 + | > current_lr_1: 0.0001882345096370477 + | > step_time: 0.3704 (0.3325324907694777) + | > loader_time: 0.0063 (0.005480371109426838) + + + --> TIME: 2025-04-19 06:56:32 -- STEP: 98/142 -- GLOBAL_STEP: 448250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.196880340576172 (17.941623940759772) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882345096370477 + | > current_lr_1: 0.0001882345096370477 + | > step_time: 0.384 (0.3435554431409252) + | > loader_time: 0.007 (0.005762963878865145) + + + --> TIME: 2025-04-19 06:56:42 -- STEP: 123/142 -- GLOBAL_STEP: 448275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.471595287322998 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.86359214782715 (17.958338962337827) + | > loss_duration: 1.660564661026001 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001882345096370477 + | > current_lr_1: 0.0001882345096370477 + | > step_time: 0.4132 (0.3547357291710086) + | > loader_time: 0.0074 (0.006031156555423892) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.095017671585083 (3.095017671585083) + | > loss_feat: nan (nan) + | > loss_mel: 18.260787963867188 (18.260787963867188) + | > loss_duration: 1.8120620250701904 (1.8120620250701904) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0890114307403564 (3.0890114307403564) + | > loss_feat: nan (nan) + | > loss_mel: 18.880802154541016 (18.880802154541016) + | > loss_duration: 1.9053069353103638 (1.9053069353103638) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7734286785125732 (2.931220054626465) + | > loss_feat: nan (nan) + | > loss_mel: 16.14240074157715 (17.511601448059082) + | > loss_duration: 1.792135238647461 (1.8487210869789124) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003307938575744629 (-0.00025844573974609375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.931220054626465 (-0.09363698959350586) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.511601448059082 (+0.18702125549316406) + | > avg_loss_duration: 1.8487210869789124 (+0.005862057209014893) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 486/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:56:52)  + + --> TIME: 2025-04-19 06:56:54 -- STEP: 6/142 -- GLOBAL_STEP: 448300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.665759325027466 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.3068790435791 (17.75409762064616) + | > loss_duration: 1.6376347541809082 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.3022 (0.29790302117665607) + | > loader_time: 0.0037 (0.0040209293365478516) + + + --> TIME: 2025-04-19 06:57:02 -- STEP: 31/142 -- GLOBAL_STEP: 448325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.55588722229004 (18.226416926230158) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.3345 (0.3142161061686854) + | > loader_time: 0.0059 (0.004863546740624214) + + + --> TIME: 2025-04-19 06:57:11 -- STEP: 56/142 -- GLOBAL_STEP: 448350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.61197853088379 (18.125423771994456) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.3478 (0.33170398218291136) + | > loader_time: 0.0058 (0.005296157939093453) + + + --> TIME: 2025-04-19 06:57:21 -- STEP: 81/142 -- GLOBAL_STEP: 448375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.997201919555664 (18.054718417885862) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.3668 (0.341423299577501) + | > loader_time: 0.0067 (0.0055354671713746615) + + + --> TIME: 2025-04-19 06:57:30 -- STEP: 106/142 -- GLOBAL_STEP: 448400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.13348388671875 (18.03779879156148) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.3836 (0.350510131638005) + | > loader_time: 0.0068 (0.005775537130967627) + + + --> TIME: 2025-04-19 06:57:41 -- STEP: 131/142 -- GLOBAL_STEP: 448425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.83711051940918 (17.94008640842583) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018821098032334305 + | > current_lr_1: 0.00018821098032334305 + | > step_time: 0.4222 (0.3615666946382012) + | > loader_time: 0.0071 (0.006024899373527702) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9905338287353516 (2.9905338287353516) + | > loss_feat: nan (nan) + | > loss_mel: 18.543521881103516 (18.543521881103516) + | > loss_duration: 1.8394925594329834 (1.8394925594329834) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3212549686431885 (3.3212549686431885) + | > loss_feat: nan (nan) + | > loss_mel: 16.907672882080078 (16.907672882080078) + | > loss_duration: 1.894304871559143 (1.894304871559143) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8880653381347656 (3.104660153388977) + | > loss_feat: nan (nan) + | > loss_mel: 19.44357681274414 (18.17562484741211) + | > loss_duration: 1.7918251752853394 (1.8430650234222412) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035181045532226562 (+0.00021016597747802734) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.104660153388977 (+0.1734400987625122) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.17562484741211 (+0.6640233993530273) + | > avg_loss_duration: 1.8430650234222412 (-0.005656063556671143) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 487/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:57:47)  + + --> TIME: 2025-04-19 06:57:52 -- STEP: 14/142 -- GLOBAL_STEP: 448450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.495854139328003 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.778120040893555 (18.19507666996547) + | > loss_duration: 1.6344225406646729 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.3081 (0.30470303126743864) + | > loader_time: 0.0048 (0.0045479706355503625) + + + --> TIME: 2025-04-19 06:58:00 -- STEP: 39/142 -- GLOBAL_STEP: 448475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.603423595428467 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.126964569091797 (17.991146772335735) + | > loss_duration: 1.6187032461166382 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.3299 (0.31598748916234737) + | > loader_time: 0.005 (0.005041287495539739) + + + --> TIME: 2025-04-19 06:58:09 -- STEP: 64/142 -- GLOBAL_STEP: 448500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.75701141357422 (17.888190209865563) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.3572 (0.3274229466915131) + | > loader_time: 0.006 (0.00540560483932495) + + + --> TIME: 2025-04-19 06:58:19 -- STEP: 89/142 -- GLOBAL_STEP: 448525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.55797004699707 (17.8256426929088) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.3736 (0.3387944323293279) + | > loader_time: 0.0064 (0.005691903360774007) + + + --> TIME: 2025-04-19 06:58:28 -- STEP: 114/142 -- GLOBAL_STEP: 448550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.13670539855957 (17.857592340101263) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.4022 (0.3495501844506516) + | > loader_time: 0.0068 (0.005923848403127569) + + + --> TIME: 2025-04-19 06:58:39 -- STEP: 139/142 -- GLOBAL_STEP: 448575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.400142669677734 (17.81664106657178) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018818745395080263 + | > current_lr_1: 0.00018818745395080263 + | > step_time: 0.4256 (0.36148138355008136) + | > loader_time: 0.0074 (0.006182680884711176) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.015300989151001 (3.015300989151001) + | > loss_feat: nan (nan) + | > loss_mel: 17.735424041748047 (17.735424041748047) + | > loss_duration: 1.8323519229888916 (1.8323519229888916) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.430460214614868 (3.430460214614868) + | > loss_feat: nan (nan) + | > loss_mel: 16.75198745727539 (16.75198745727539) + | > loss_duration: 1.8958395719528198 (1.8958395719528198) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.729238271713257 (3.0798492431640625) + | > loss_feat: nan (nan) + | > loss_mel: 18.33602523803711 (17.54400634765625) + | > loss_duration: 1.799800157546997 (1.8478198647499084) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032799243927001953 (-0.00023818016052246094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0798492431640625 (-0.02481091022491455) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.54400634765625 (-0.6316184997558594) + | > avg_loss_duration: 1.8478198647499084 (+0.004754841327667236) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 488/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:58:42)  + + --> TIME: 2025-04-19 06:58:49 -- STEP: 22/142 -- GLOBAL_STEP: 448600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.438920974731445 (17.996811736713756) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018816393051905876 + | > current_lr_1: 0.00018816393051905876 + | > step_time: 0.3256 (0.30729748985984107) + | > loader_time: 0.0054 (0.004718780517578125) + + + --> TIME: 2025-04-19 06:58:58 -- STEP: 47/142 -- GLOBAL_STEP: 448625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.04035758972168 (17.944403810704) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018816393051905876 + | > current_lr_1: 0.00018816393051905876 + | > step_time: 0.3481 (0.32375130247562495) + | > loader_time: 0.0059 (0.005208527788202813) + + + --> TIME: 2025-04-19 06:59:07 -- STEP: 72/142 -- GLOBAL_STEP: 448650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.946956634521484 (17.93821717633141) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018816393051905876 + | > current_lr_1: 0.00018816393051905876 + | > step_time: 0.3789 (0.3378509316179488) + | > loader_time: 0.0065 (0.005588548050986396) + + + --> TIME: 2025-04-19 06:59:17 -- STEP: 97/142 -- GLOBAL_STEP: 448675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.866533279418945 (17.971939116409146) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018816393051905876 + | > current_lr_1: 0.00018816393051905876 + | > step_time: 0.3981 (0.3501582637275618) + | > loader_time: 0.0072 (0.005907842793415501) + + + --> TIME: 2025-04-19 06:59:28 -- STEP: 122/142 -- GLOBAL_STEP: 448700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.286684036254883 (18.019110077717272) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018816393051905876 + | > current_lr_1: 0.00018816393051905876 + | > step_time: 0.4274 (0.3633300065994263) + | > loader_time: 0.0068 (0.006154115082787685) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8263802528381348 (2.8263802528381348) + | > loss_feat: nan (nan) + | > loss_mel: 18.25287437438965 (18.25287437438965) + | > loss_duration: 1.8410435914993286 (1.8410435914993286) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4114198684692383 (3.4114198684692383) + | > loss_feat: nan (nan) + | > loss_mel: 19.03034210205078 (19.03034210205078) + | > loss_duration: 1.9017058610916138 (1.9017058610916138) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8672828674316406 (3.1393513679504395) + | > loss_feat: nan (nan) + | > loss_mel: 14.62267017364502 (16.8265061378479) + | > loss_duration: 1.7995660305023193 (1.8506359457969666) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003723621368408203 (+0.0004436969757080078) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1393513679504395 (+0.05950212478637695) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.8265061378479 (-0.7175002098083496) + | > avg_loss_duration: 1.8506359457969666 (+0.0028160810470581055) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 489/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 06:59:38)  + + --> TIME: 2025-04-19 06:59:40 -- STEP: 5/142 -- GLOBAL_STEP: 448725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.192214965820312 (18.69285888671875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.3001 (0.30074496269226075) + | > loader_time: 0.0045 (0.004310894012451172) + + + --> TIME: 2025-04-19 06:59:48 -- STEP: 30/142 -- GLOBAL_STEP: 448750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.285810470581055 (18.118714872996005) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.3251 (0.3106834332148234) + | > loader_time: 0.006 (0.00494850476582845) + + + --> TIME: 2025-04-19 06:59:57 -- STEP: 55/142 -- GLOBAL_STEP: 448775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.919097900390625 (18.03028590462424) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.3492 (0.32783293290571736) + | > loader_time: 0.0056 (0.005349414998834783) + + + --> TIME: 2025-04-19 07:00:06 -- STEP: 80/142 -- GLOBAL_STEP: 448800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7671685218811035 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.061264038085938 (17.950889837741848) + | > loss_duration: 1.672835350036621 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.3834 (0.3392819344997406) + | > loader_time: 0.0066 (0.005653285980224609) + + + --> TIME: 2025-04-19 07:00:16 -- STEP: 105/142 -- GLOBAL_STEP: 448825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.272172927856445 (17.948619751703166) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.3918 (0.34854069891430084) + | > loader_time: 0.0071 (0.005909903844197591) + + + --> TIME: 2025-04-19 07:00:26 -- STEP: 130/142 -- GLOBAL_STEP: 448850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.200101852416992 (17.8857371697059) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018814041002774388 + | > current_lr_1: 0.00018814041002774388 + | > step_time: 0.4229 (0.35963733929854175) + | > loader_time: 0.0076 (0.006182129566486065) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8903796672821045 (2.8903796672821045) + | > loss_feat: nan (nan) + | > loss_mel: 19.607820510864258 (19.607820510864258) + | > loss_duration: 1.8261125087738037 (1.8261125087738037) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3957390785217285 (3.3957390785217285) + | > loss_feat: nan (nan) + | > loss_mel: 17.74856948852539 (17.74856948852539) + | > loss_duration: 1.8983535766601562 (1.8983535766601562) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.884931802749634 (3.140335440635681) + | > loss_feat: nan (nan) + | > loss_mel: 15.614850044250488 (16.68170976638794) + | > loss_duration: 1.8029227256774902 (1.8506381511688232) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003702878952026367 (-2.0742416381835938e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.140335440635681 (+0.0009840726852416992) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.68170976638794 (-0.14479637145996094) + | > avg_loss_duration: 1.8506381511688232 (+2.205371856689453e-06) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 490/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:00:33)  + + --> TIME: 2025-04-19 07:00:38 -- STEP: 13/142 -- GLOBAL_STEP: 448875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.47544264793396 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.416528701782227 (18.108361317561222) + | > loss_duration: 1.6225606203079224 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.3078 (0.31957424603975737) + | > loader_time: 0.0049 (0.0046385985154372) + + + --> TIME: 2025-04-19 07:00:46 -- STEP: 38/142 -- GLOBAL_STEP: 448900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.6561336517334 (18.013830887643916) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.328 (0.32205208351737574) + | > loader_time: 0.0061 (0.005091911868045204) + + + --> TIME: 2025-04-19 07:00:55 -- STEP: 63/142 -- GLOBAL_STEP: 448925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.606027603149414 (17.988937014625186) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.3671 (0.3328824383871896) + | > loader_time: 0.0062 (0.005407462044367713) + + + --> TIME: 2025-04-19 07:01:04 -- STEP: 88/142 -- GLOBAL_STEP: 448950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.383302688598633 (18.015524430708457) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.3721 (0.3441100418567657) + | > loader_time: 0.0069 (0.0056987892497669555) + + + --> TIME: 2025-04-19 07:01:14 -- STEP: 113/142 -- GLOBAL_STEP: 448975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.663654327392578 (18.0095599520523) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.3988 (0.35352402028784286) + | > loader_time: 0.0072 (0.005950594370344043) + + + --> TIME: 2025-04-19 07:01:25 -- STEP: 138/142 -- GLOBAL_STEP: 449000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.446348190307617 (17.922355313231986) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018811689247649042 + | > current_lr_1: 0.00018811689247649042 + | > step_time: 0.4297 (0.36485551923945336) + | > loader_time: 0.0078 (0.006211533062699912) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0507519245147705 (3.0507519245147705) + | > loss_feat: nan (nan) + | > loss_mel: 18.216428756713867 (18.216428756713867) + | > loss_duration: 1.8336906433105469 (1.8336906433105469) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.340830087661743 (3.340830087661743) + | > loss_feat: nan (nan) + | > loss_mel: 19.730783462524414 (19.730783462524414) + | > loss_duration: 1.903329849243164 (1.903329849243164) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.728609800338745 (3.034719944000244) + | > loss_feat: nan (nan) + | > loss_mel: 17.615089416503906 (18.67293643951416) + | > loss_duration: 1.8054051399230957 (1.8543674945831299) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037164688110351562 (+1.3589859008789062e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.034719944000244 (-0.10561549663543701) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.67293643951416 (+1.9912266731262207) + | > avg_loss_duration: 1.8543674945831299 (+0.0037293434143066406) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 491/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:01:28)  + + --> TIME: 2025-04-19 07:01:35 -- STEP: 21/142 -- GLOBAL_STEP: 449025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.07353973388672 (18.325676963442852) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018809337786493085 + | > current_lr_1: 0.00018809337786493085 + | > step_time: 0.3152 (0.30626055172511507) + | > loader_time: 0.0052 (0.00482442265465146) + + + --> TIME: 2025-04-19 07:01:44 -- STEP: 46/142 -- GLOBAL_STEP: 449050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.768802642822266 (18.078906059265137) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018809337786493085 + | > current_lr_1: 0.00018809337786493085 + | > step_time: 0.3416 (0.3201446636863376) + | > loader_time: 0.0058 (0.00520461020262345) + + + --> TIME: 2025-04-19 07:01:53 -- STEP: 71/142 -- GLOBAL_STEP: 449075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.328922271728516 (18.03596888797384) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018809337786493085 + | > current_lr_1: 0.00018809337786493085 + | > step_time: 0.3607 (0.3331162392253606) + | > loader_time: 0.0065 (0.0055233626298501455) + + + --> TIME: 2025-04-19 07:02:03 -- STEP: 96/142 -- GLOBAL_STEP: 449100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.554542064666748 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.327306747436523 (18.06299620866775) + | > loss_duration: 1.71376633644104 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018809337786493085 + | > current_lr_1: 0.00018809337786493085 + | > step_time: 0.385 (0.34426504870255775) + | > loader_time: 0.0072 (0.005805770556131999) + + + --> TIME: 2025-04-19 07:02:13 -- STEP: 121/142 -- GLOBAL_STEP: 449125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.811180114746094 (18.06194406304477) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018809337786493085 + | > current_lr_1: 0.00018809337786493085 + | > step_time: 0.4151 (0.3549151361481216) + | > loader_time: 0.0076 (0.00604027164869072) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.089982032775879 (3.089982032775879) + | > loss_feat: nan (nan) + | > loss_mel: 18.16341209411621 (18.16341209411621) + | > loss_duration: 1.8451892137527466 (1.8451892137527466) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2767598628997803 (3.2767598628997803) + | > loss_feat: nan (nan) + | > loss_mel: 17.36487579345703 (17.36487579345703) + | > loss_duration: 1.8902862071990967 (1.8902862071990967) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.864757537841797 (3.0707587003707886) + | > loss_feat: nan (nan) + | > loss_mel: 19.769664764404297 (18.567270278930664) + | > loss_duration: 1.8227835893630981 (1.8565348982810974) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034475326538085938 (-0.0002689361572265625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0707587003707886 (+0.036038756370544434) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.567270278930664 (-0.1056661605834961) + | > avg_loss_duration: 1.8565348982810974 (+0.0021674036979675293) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 492/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:02:23)  + + --> TIME: 2025-04-19 07:02:25 -- STEP: 4/142 -- GLOBAL_STEP: 449150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.11298370361328 (18.330377101898193) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.2953 (0.298088014125824) + | > loader_time: 0.0053 (0.004246771335601807) + + + --> TIME: 2025-04-19 07:02:33 -- STEP: 29/142 -- GLOBAL_STEP: 449175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.234155654907227 (17.8537018874596) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.3228 (0.3107965321376406) + | > loader_time: 0.0054 (0.004966818053146888) + + + --> TIME: 2025-04-19 07:02:42 -- STEP: 54/142 -- GLOBAL_STEP: 449200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5584518909454346 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.32113265991211 (17.988650251317917) + | > loss_duration: 1.624806523323059 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.3436 (0.32248235190356217) + | > loader_time: 0.0061 (0.005344046486748589) + + + --> TIME: 2025-04-19 07:02:51 -- STEP: 79/142 -- GLOBAL_STEP: 449225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3979389667510986 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.593595504760742 (17.980135519293295) + | > loss_duration: 1.6794095039367676 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.3644 (0.3352907730054252) + | > loader_time: 0.0065 (0.005652020249185682) + + + --> TIME: 2025-04-19 07:03:01 -- STEP: 104/142 -- GLOBAL_STEP: 449250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.735639572143555 (17.945047635298508) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.3882 (0.34691811524904687) + | > loader_time: 0.0067 (0.005914672062947198) + + + --> TIME: 2025-04-19 07:03:11 -- STEP: 129/142 -- GLOBAL_STEP: 449275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.598857879638672 (17.894513884256053) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018806986619269772 + | > current_lr_1: 0.00018806986619269772 + | > step_time: 0.4192 (0.35902041982310684) + | > loader_time: 0.0075 (0.006191131680510763) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9581873416900635 (2.9581873416900635) + | > loss_feat: nan (nan) + | > loss_mel: 19.11713981628418 (19.11713981628418) + | > loss_duration: 1.8392446041107178 (1.8392446041107178) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3223493099212646 (3.3223493099212646) + | > loss_feat: nan (nan) + | > loss_mel: 15.836566925048828 (15.836566925048828) + | > loss_duration: 1.913527488708496 (1.913527488708496) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8842573165893555 (3.10330331325531) + | > loss_feat: nan (nan) + | > loss_mel: 18.117523193359375 (16.9770450592041) + | > loss_duration: 1.7929173707962036 (1.8532224297523499) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035783052444458008 (+0.00013077259063720703) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.10330331325531 (+0.032544612884521484) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.9770450592041 (-1.5902252197265625) + | > avg_loss_duration: 1.8532224297523499 (-0.0033124685287475586) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 493/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:03:18)  + + --> TIME: 2025-04-19 07:03:23 -- STEP: 12/142 -- GLOBAL_STEP: 449300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.50764274597168 (18.209511121114094) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.3047 (0.30232354005177814) + | > loader_time: 0.0052 (0.004514336585998535) + + + --> TIME: 2025-04-19 07:03:31 -- STEP: 37/142 -- GLOBAL_STEP: 449325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.499799728393555 (17.979817777066618) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.3278 (0.31544695029387604) + | > loader_time: 0.0058 (0.005128628498799092) + + + --> TIME: 2025-04-19 07:03:40 -- STEP: 62/142 -- GLOBAL_STEP: 449350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.00412368774414 (17.91408671102217) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.3559 (0.32613175915133563) + | > loader_time: 0.0059 (0.0054209847604074785) + + + --> TIME: 2025-04-19 07:03:49 -- STEP: 87/142 -- GLOBAL_STEP: 449375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.293209075927734 (17.945600597337744) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.3817 (0.33856480149016993) + | > loader_time: 0.0067 (0.005734999974568685) + + + --> TIME: 2025-04-19 07:03:59 -- STEP: 112/142 -- GLOBAL_STEP: 449400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.195960998535156 (17.899824466024125) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.4054 (0.34998823063714174) + | > loader_time: 0.0075 (0.006012727107320513) + + + --> TIME: 2025-04-19 07:04:10 -- STEP: 137/142 -- GLOBAL_STEP: 449425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5742568969726562 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.980117797851562 (17.851055061730154) + | > loss_duration: 1.6613978147506714 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018804635745942361 + | > current_lr_1: 0.00018804635745942361 + | > step_time: 0.4289 (0.3626525158429668) + | > loader_time: 0.0074 (0.006330296941047167) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0744450092315674 (3.0744450092315674) + | > loss_feat: nan (nan) + | > loss_mel: 19.502613067626953 (19.502613067626953) + | > loss_duration: 1.8435921669006348 (1.8435921669006348) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.512706995010376 (3.512706995010376) + | > loss_feat: nan (nan) + | > loss_mel: 17.893030166625977 (17.893030166625977) + | > loss_duration: 1.8943610191345215 (1.8943610191345215) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.84302020072937 (3.177863597869873) + | > loss_feat: nan (nan) + | > loss_mel: 16.583539962768555 (17.238285064697266) + | > loss_duration: 1.7722629308700562 (1.8333119750022888) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034035444259643555 (-0.0001747608184814453) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.177863597869873 (+0.07456028461456299) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.238285064697266 (+0.26124000549316406) + | > avg_loss_duration: 1.8333119750022888 (-0.019910454750061035) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 494/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:04:13)  + + --> TIME: 2025-04-19 07:04:20 -- STEP: 20/142 -- GLOBAL_STEP: 449450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.441249370574951 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.58695411682129 (17.790423679351807) + | > loss_duration: 1.6724294424057007 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018802285166474118 + | > current_lr_1: 0.00018802285166474118 + | > step_time: 0.318 (0.30626180171966555) + | > loader_time: 0.0055 (0.004866957664489746) + + + --> TIME: 2025-04-19 07:04:29 -- STEP: 45/142 -- GLOBAL_STEP: 449475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.463056564331055 (17.761768786112466) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018802285166474118 + | > current_lr_1: 0.00018802285166474118 + | > step_time: 0.3431 (0.3239331245422364) + | > loader_time: 0.0061 (0.0052988953060574) + + + --> TIME: 2025-04-19 07:04:38 -- STEP: 70/142 -- GLOBAL_STEP: 449500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.052522659301758 (17.77755231857299) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018802285166474118 + | > current_lr_1: 0.00018802285166474118 + | > step_time: 0.3653 (0.33431973457336434) + | > loader_time: 0.0068 (0.005623044286455426) + + + --> TIME: 2025-04-19 07:04:47 -- STEP: 95/142 -- GLOBAL_STEP: 449525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.78461265563965 (17.83262293966193) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018802285166474118 + | > current_lr_1: 0.00018802285166474118 + | > step_time: 0.3792 (0.3443067651045952) + | > loader_time: 0.007 (0.00590795215807463) + + + --> TIME: 2025-04-19 07:04:58 -- STEP: 120/142 -- GLOBAL_STEP: 449550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.568552017211914 (17.753502281506865) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018802285166474118 + | > current_lr_1: 0.00018802285166474118 + | > step_time: 0.4112 (0.35470120310783404) + | > loader_time: 0.0076 (0.006194468339284261) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.910273313522339 (2.910273313522339) + | > loss_feat: nan (nan) + | > loss_mel: 15.48023796081543 (15.48023796081543) + | > loss_duration: 1.8007701635360718 (1.8007701635360718) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.371122121810913 (3.371122121810913) + | > loss_feat: nan (nan) + | > loss_mel: 17.710079193115234 (17.710079193115234) + | > loss_duration: 1.897676706314087 (1.897676706314087) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.837021589279175 (3.104071855545044) + | > loss_feat: nan (nan) + | > loss_mel: 19.18929100036621 (18.449685096740723) + | > loss_duration: 1.7997969388961792 (1.848736822605133) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035042762756347656 (+0.00010073184967041016) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.104071855545044 (-0.0737917423248291) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.449685096740723 (+1.211400032043457) + | > avg_loss_duration: 1.848736822605133 (+0.015424847602844238) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 495/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:05:08)  + + --> TIME: 2025-04-19 07:05:10 -- STEP: 3/142 -- GLOBAL_STEP: 449575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.683446884155273 (17.302098592122395) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.3 (0.2989030679066976) + | > loader_time: 0.0043 (0.004201571146647136) + + + --> TIME: 2025-04-19 07:05:18 -- STEP: 28/142 -- GLOBAL_STEP: 449600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.550471544265747 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.987722396850586 (17.846773147583) + | > loss_duration: 1.644277572631836 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.323 (0.3094988635608128) + | > loader_time: 0.0054 (0.005089248929704938) + + + --> TIME: 2025-04-19 07:05:26 -- STEP: 53/142 -- GLOBAL_STEP: 449625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.534915924072266 (18.053413391113278) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.3415 (0.3216184040285507) + | > loader_time: 0.0061 (0.005379672320383902) + + + --> TIME: 2025-04-19 07:05:36 -- STEP: 78/142 -- GLOBAL_STEP: 449650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6476247310638428 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.665565490722656 (18.051837395399044) + | > loss_duration: 1.669301986694336 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.3669 (0.3337687712449294) + | > loader_time: 0.0062 (0.0056561506711519695) + + + --> TIME: 2025-04-19 07:05:45 -- STEP: 103/142 -- GLOBAL_STEP: 449675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.818256378173828 (18.037582369684017) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.3872 (0.3443212740629622) + | > loader_time: 0.0065 (0.0058753143236475085) + + + --> TIME: 2025-04-19 07:05:56 -- STEP: 128/142 -- GLOBAL_STEP: 449700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.672473907470703 (18.00312855839729) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001879993488082831 + | > current_lr_1: 0.0001879993488082831 + | > step_time: 0.4105 (0.35586318001151085) + | > loader_time: 0.0075 (0.0061402954161167145) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9453532695770264 (2.9453532695770264) + | > loss_feat: nan (nan) + | > loss_mel: 17.352197647094727 (17.352197647094727) + | > loss_duration: 1.8419640064239502 (1.8419640064239502) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3308444023132324 (3.3308444023132324) + | > loss_feat: nan (nan) + | > loss_mel: 17.534000396728516 (17.534000396728516) + | > loss_duration: 1.8977690935134888 (1.8977690935134888) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7836482524871826 (3.0572463274002075) + | > loss_feat: nan (nan) + | > loss_mel: 16.91065788269043 (17.222329139709473) + | > loss_duration: 1.7857742309570312 (1.84177166223526) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003186464309692383 (-0.0003178119659423828) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0572463274002075 (-0.046825528144836426) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.222329139709473 (-1.22735595703125) + | > avg_loss_duration: 1.84177166223526 (-0.006965160369873047) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 496/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:06:03)  + + --> TIME: 2025-04-19 07:06:07 -- STEP: 11/142 -- GLOBAL_STEP: 449725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.484601974487305 (17.950197740034625) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.3056 (0.30121478167447174) + | > loader_time: 0.0051 (0.004648100246082653) + + + --> TIME: 2025-04-19 07:06:16 -- STEP: 36/142 -- GLOBAL_STEP: 449750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.97650909423828 (17.800087107552425) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.3391 (0.31721272733476424) + | > loader_time: 0.006 (0.005137701829274495) + + + --> TIME: 2025-04-19 07:06:24 -- STEP: 61/142 -- GLOBAL_STEP: 449775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.952951431274414 (17.830934946654278) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.3591 (0.32824614790619405) + | > loader_time: 0.0059 (0.005460012154500993) + + + --> TIME: 2025-04-19 07:06:34 -- STEP: 86/142 -- GLOBAL_STEP: 449800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4883532524108887 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.65645980834961 (17.790249879970112) + | > loss_duration: 1.6856865882873535 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.3777 (0.3405720910360648) + | > loader_time: 0.0071 (0.005792828493340071) + + + --> TIME: 2025-04-19 07:06:44 -- STEP: 111/142 -- GLOBAL_STEP: 449825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.099456787109375 (17.791582373885426) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.4039 (0.3520969549814861) + | > loader_time: 0.0074 (0.006074735710212776) + + + --> TIME: 2025-04-19 07:06:55 -- STEP: 136/142 -- GLOBAL_STEP: 449850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.743236541748047 (17.80533540950102) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018797584888968205 + | > current_lr_1: 0.00018797584888968205 + | > step_time: 0.438 (0.3653524780974669) + | > loader_time: 0.0073 (0.006358285160625682) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.871350049972534 (2.871350049972534) + | > loss_feat: nan (nan) + | > loss_mel: 19.097402572631836 (19.097402572631836) + | > loss_duration: 1.8472708463668823 (1.8472708463668823) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.438143730163574 (3.438143730163574) + | > loss_feat: nan (nan) + | > loss_mel: 18.861295700073242 (18.861295700073242) + | > loss_duration: 1.9015815258026123 (1.9015815258026123) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8854074478149414 (3.161775588989258) + | > loss_feat: nan (nan) + | > loss_mel: 17.963321685791016 (18.41230869293213) + | > loss_duration: 1.7862508296966553 (1.8439161777496338) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004132866859436035 (+0.0009464025497436523) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.161775588989258 (+0.10452926158905029) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.41230869293213 (+1.1899795532226562) + | > avg_loss_duration: 1.8439161777496338 (+0.0021445155143737793) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 497/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:06:59)  + + --> TIME: 2025-04-19 07:07:05 -- STEP: 19/142 -- GLOBAL_STEP: 449875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.264650344848633 (17.35248605828536) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018795235190857083 + | > current_lr_1: 0.00018795235190857083 + | > step_time: 0.3125 (0.3060886232476485) + | > loader_time: 0.005 (0.004925702747545745) + + + --> TIME: 2025-04-19 07:07:14 -- STEP: 44/142 -- GLOBAL_STEP: 449900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.218835830688477 (17.573864590037953) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018795235190857083 + | > current_lr_1: 0.00018795235190857083 + | > step_time: 0.334 (0.3173164237629284) + | > loader_time: 0.0058 (0.005258142948150635) + + + --> TIME: 2025-04-19 07:07:23 -- STEP: 69/142 -- GLOBAL_STEP: 449925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.52333641052246 (17.700581509134043) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018795235190857083 + | > current_lr_1: 0.00018795235190857083 + | > step_time: 0.3722 (0.32948153606359504) + | > loader_time: 0.0063 (0.005555073420206706) + + + --> TIME: 2025-04-19 07:07:32 -- STEP: 94/142 -- GLOBAL_STEP: 449950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 14.836853981018066 (17.76504099622686) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018795235190857083 + | > current_lr_1: 0.00018795235190857083 + | > step_time: 0.3792 (0.34177615541092893) + | > loader_time: 0.0081 (0.00585613859460709) + + + --> TIME: 2025-04-19 07:07:43 -- STEP: 119/142 -- GLOBAL_STEP: 449975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.26753044128418 (17.75485222279526) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018795235190857083 + | > current_lr_1: 0.00018795235190857083 + | > step_time: 0.4031 (0.35381055679641854) + | > loader_time: 0.0076 (0.006114270506786699) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9702489376068115 (2.9702489376068115) + | > loss_feat: nan (nan) + | > loss_mel: 17.861804962158203 (17.861804962158203) + | > loss_duration: 1.8064355850219727 (1.8064355850219727) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3348140716552734 (3.3348140716552734) + | > loss_feat: nan (nan) + | > loss_mel: 17.07494354248047 (17.07494354248047) + | > loss_duration: 1.8958334922790527 (1.8958334922790527) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7264020442962646 (3.030608057975769) + | > loss_feat: nan (nan) + | > loss_mel: 19.263227462768555 (18.16908550262451) + | > loss_duration: 1.7992384433746338 (1.8475359678268433) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037555694580078125 (-0.00037729740142822266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.030608057975769 (-0.13116753101348877) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.16908550262451 (-0.2432231903076172) + | > avg_loss_duration: 1.8475359678268433 (+0.0036197900772094727) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 498/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:07:54)  + + --> TIME: 2025-04-19 07:07:55 -- STEP: 2/142 -- GLOBAL_STEP: 450000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6436264514923096 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.78166389465332 (18.513635635375977) + | > loss_duration: 1.67527437210083 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.2966 (0.2995877265930176) + | > loader_time: 0.0048 (0.004413127899169922) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_450000.pth + + --> TIME: 2025-04-19 07:08:05 -- STEP: 27/142 -- GLOBAL_STEP: 450025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.28139877319336 (18.12215706154152) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.3246 (0.30970519560354715) + | > loader_time: 0.0054 (0.005036663126062464) + + + --> TIME: 2025-04-19 07:08:13 -- STEP: 52/142 -- GLOBAL_STEP: 450050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.38644790649414 (18.042140190417946) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.3426 (0.32203884766652036) + | > loader_time: 0.0056 (0.0053691955713125366) + + + --> TIME: 2025-04-19 07:08:22 -- STEP: 77/142 -- GLOBAL_STEP: 450075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.396705627441406 (17.99994422863055) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.3685 (0.334291021545212) + | > loader_time: 0.0058 (0.00566805802382432) + + + --> TIME: 2025-04-19 07:08:32 -- STEP: 102/142 -- GLOBAL_STEP: 450100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.652854919433594 (17.987317319009808) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.3946 (0.34582972292806585) + | > loader_time: 0.0066 (0.0059320809794407275) + + + --> TIME: 2025-04-19 07:08:43 -- STEP: 127/142 -- GLOBAL_STEP: 450125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.16396141052246 (17.988112569793934) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018792885786458223 + | > current_lr_1: 0.00018792885786458223 + | > step_time: 0.4142 (0.3570711406197135) + | > loader_time: 0.0077 (0.006209302136278529) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8672940731048584 (2.8672940731048584) + | > loss_feat: nan (nan) + | > loss_mel: 18.782766342163086 (18.782766342163086) + | > loss_duration: 1.8343050479888916 (1.8343050479888916) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.373838186264038 (3.373838186264038) + | > loss_feat: nan (nan) + | > loss_mel: 19.472803115844727 (19.472803115844727) + | > loss_duration: 1.8976885080337524 (1.8976885080337524) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.841552972793579 (3.1076955795288086) + | > loss_feat: nan (nan) + | > loss_mel: 17.942211151123047 (18.707507133483887) + | > loss_duration: 1.788995623588562 (1.8433420658111572) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003363490104675293 (-0.00039207935333251953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1076955795288086 (+0.07708752155303955) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.707507133483887 (+0.538421630859375) + | > avg_loss_duration: 1.8433420658111572 (-0.004193902015686035) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 499/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:08:50)  + + --> TIME: 2025-04-19 07:08:54 -- STEP: 10/142 -- GLOBAL_STEP: 450150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.641799926757812 (18.331548309326173) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.3104 (0.3010704755783081) + | > loader_time: 0.0046 (0.004134130477905273) + + + --> TIME: 2025-04-19 07:09:02 -- STEP: 35/142 -- GLOBAL_STEP: 450175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.783553123474121 (17.99079499925886) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.3319 (0.31713756833757684) + | > loader_time: 0.0053 (0.004783439636230471) + + + --> TIME: 2025-04-19 07:09:11 -- STEP: 60/142 -- GLOBAL_STEP: 450200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.514331340789795 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.307926177978516 (17.9294692516327) + | > loss_duration: 1.6603738069534302 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.3497 (0.32795621951421106) + | > loader_time: 0.006 (0.005208579699198406) + + + --> TIME: 2025-04-19 07:09:21 -- STEP: 85/142 -- GLOBAL_STEP: 450225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.556879997253418 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.212675094604492 (17.927747266432824) + | > loss_duration: 1.6786866188049316 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.3699 (0.3394756373237161) + | > loader_time: 0.0061 (0.005528615502750174) + + + --> TIME: 2025-04-19 07:09:30 -- STEP: 110/142 -- GLOBAL_STEP: 450250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6714913845062256 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.408374786376953 (17.96487581946634) + | > loss_duration: 1.6572209596633911 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.4079 (0.3503239219838923) + | > loader_time: 0.0072 (0.005818507888100365) + + + --> TIME: 2025-04-19 07:09:41 -- STEP: 135/142 -- GLOBAL_STEP: 450275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4066829681396484 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.75249671936035 (17.92312416500516) + | > loss_duration: 1.6261472702026367 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018790536675734917 + | > current_lr_1: 0.00018790536675734917 + | > step_time: 0.4271 (0.36191714074876574) + | > loader_time: 0.0077 (0.006085717236554181) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.083672046661377 (3.083672046661377) + | > loss_feat: nan (nan) + | > loss_mel: 18.617979049682617 (18.617979049682617) + | > loss_duration: 1.8199775218963623 (1.8199775218963623) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4438962936401367 (3.4438962936401367) + | > loss_feat: nan (nan) + | > loss_mel: 17.79174041748047 (17.79174041748047) + | > loss_duration: 1.887638807296753 (1.887638807296753) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7505624294281006 (3.0972293615341187) + | > loss_feat: nan (nan) + | > loss_mel: 18.147096633911133 (17.9694185256958) + | > loss_duration: 1.8105146884918213 (1.849076747894287) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003319263458251953 (-4.4226646423339844e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0972293615341187 (-0.010466217994689941) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.9694185256958 (-0.7380886077880859) + | > avg_loss_duration: 1.849076747894287 (+0.005734682083129883) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 500/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:09:46)  + + --> TIME: 2025-04-19 07:09:52 -- STEP: 18/142 -- GLOBAL_STEP: 450300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5327980518341064 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.689022064208984 (18.07472758822971) + | > loss_duration: 1.6032274961471558 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018788187858650448 + | > current_lr_1: 0.00018788187858650448 + | > step_time: 0.314 (0.30501333872477215) + | > loader_time: 0.0044 (0.004532429907057021) + + + --> TIME: 2025-04-19 07:10:00 -- STEP: 43/142 -- GLOBAL_STEP: 450325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.04296112060547 (17.95937591375307) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018788187858650448 + | > current_lr_1: 0.00018788187858650448 + | > step_time: 0.3373 (0.31768954077432326) + | > loader_time: 0.0057 (0.004891179328741029) + + + --> TIME: 2025-04-19 07:10:09 -- STEP: 68/142 -- GLOBAL_STEP: 450350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.38677978515625 (17.97023254282335) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018788187858650448 + | > current_lr_1: 0.00018788187858650448 + | > step_time: 0.3629 (0.32957855392904845) + | > loader_time: 0.0058 (0.005254903260399313) + + + --> TIME: 2025-04-19 07:10:19 -- STEP: 93/142 -- GLOBAL_STEP: 450375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.07207489013672 (17.917668311826645) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018788187858650448 + | > current_lr_1: 0.00018788187858650448 + | > step_time: 0.378 (0.3413021820847706) + | > loader_time: 0.0067 (0.005574505816223802) + + + --> TIME: 2025-04-19 07:10:29 -- STEP: 118/142 -- GLOBAL_STEP: 450400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.688572406768799 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.46965789794922 (17.93845256708436) + | > loss_duration: 1.6310155391693115 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018788187858650448 + | > current_lr_1: 0.00018788187858650448 + | > step_time: 0.429 (0.35326875468431895) + | > loader_time: 0.0082 (0.00586420802746789) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8918135166168213 (2.8918135166168213) + | > loss_feat: nan (nan) + | > loss_mel: 18.141887664794922 (18.141887664794922) + | > loss_duration: 1.8465415239334106 (1.8465415239334106) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.396272897720337 (3.396272897720337) + | > loss_feat: nan (nan) + | > loss_mel: 17.145315170288086 (17.145315170288086) + | > loss_duration: 1.91994309425354 (1.91994309425354) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.874539613723755 (3.135406255722046) + | > loss_feat: nan (nan) + | > loss_mel: 19.817543029785156 (18.48142910003662) + | > loss_duration: 1.7767186164855957 (1.8483308553695679) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003917574882507324 (+0.0005983114242553711) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.135406255722046 (+0.038176894187927246) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.48142910003662 (+0.5120105743408203) + | > avg_loss_duration: 1.8483308553695679 (-0.0007458925247192383) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 501/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:10:41)  + + --> TIME: 2025-04-19 07:10:42 -- STEP: 1/142 -- GLOBAL_STEP: 450425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.413052797317505 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.218420028686523 (17.218420028686523) + | > loss_duration: 1.6092041730880737 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.302 (0.3020319938659668) + | > loader_time: 0.0041 (0.00410151481628418) + + + --> TIME: 2025-04-19 07:10:50 -- STEP: 26/142 -- GLOBAL_STEP: 450450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.499563455581665 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.1298885345459 (18.064209424532375) + | > loss_duration: 1.6490930318832397 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.3329 (0.3114488675044133) + | > loader_time: 0.0055 (0.004855797840998724) + + + --> TIME: 2025-04-19 07:10:59 -- STEP: 51/142 -- GLOBAL_STEP: 450475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.646461486816406 (17.85499383888993) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.3534 (0.3264950069726683) + | > loader_time: 0.0059 (0.005321666306140376) + + + --> TIME: 2025-04-19 07:11:08 -- STEP: 76/142 -- GLOBAL_STEP: 450500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.414722442626953 (17.99105457255715) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.3795 (0.34064974910334545) + | > loader_time: 0.0067 (0.005686439965900622) + + + --> TIME: 2025-04-19 07:11:18 -- STEP: 101/142 -- GLOBAL_STEP: 450525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.290138244628906 (17.923683119292306) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.4051 (0.35247219435059196) + | > loader_time: 0.0071 (0.005993954025872863) + + + --> TIME: 2025-04-19 07:11:29 -- STEP: 126/142 -- GLOBAL_STEP: 450550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.226430892944336 (17.833382591368668) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018785839335168115 + | > current_lr_1: 0.00018785839335168115 + | > step_time: 0.4267 (0.36509943576086135) + | > loader_time: 0.0075 (0.006254237795632983) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0122127532958984 (3.0122127532958984) + | > loss_feat: nan (nan) + | > loss_mel: 18.68340492248535 (18.68340492248535) + | > loss_duration: 1.8661041259765625 (1.8661041259765625) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2837798595428467 (3.2837798595428467) + | > loss_feat: nan (nan) + | > loss_mel: 19.410247802734375 (19.410247802734375) + | > loss_duration: 1.8833224773406982 (1.8833224773406982) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9283337593078613 (3.106056809425354) + | > loss_feat: nan (nan) + | > loss_mel: 18.89872932434082 (19.154488563537598) + | > loss_duration: 1.8084805011749268 (1.8459014892578125) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037251710891723633 (-0.00019240379333496094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.106056809425354 (-0.029349446296691895) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.154488563537598 (+0.6730594635009766) + | > avg_loss_duration: 1.8459014892578125 (-0.002429366111755371) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 502/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:11:37)  + + --> TIME: 2025-04-19 07:11:41 -- STEP: 9/142 -- GLOBAL_STEP: 450575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.47577667236328 (17.887595494588215) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.3 (0.30010035302903915) + | > loader_time: 0.0044 (0.004265414343939887) + + + --> TIME: 2025-04-19 07:11:49 -- STEP: 34/142 -- GLOBAL_STEP: 450600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.501614809036255 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.538930892944336 (18.02590336519129) + | > loss_duration: 1.6335420608520508 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.3292 (0.31300892549402576) + | > loader_time: 0.0057 (0.004915581030004166) + + + --> TIME: 2025-04-19 07:11:58 -- STEP: 59/142 -- GLOBAL_STEP: 450625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.74281883239746 (18.007976402670653) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.3533 (0.32475033048856067) + | > loader_time: 0.0058 (0.005324145494881324) + + + --> TIME: 2025-04-19 07:12:07 -- STEP: 84/142 -- GLOBAL_STEP: 450650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.492241859436035 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.270273208618164 (17.946674562635877) + | > loss_duration: 1.7118737697601318 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.3697 (0.33700306074959907) + | > loader_time: 0.0063 (0.005583953289758592) + + + --> TIME: 2025-04-19 07:12:17 -- STEP: 109/142 -- GLOBAL_STEP: 450675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.728084564208984 (17.957133275653234) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.3887 (0.34750699778215616) + | > loader_time: 0.0066 (0.0058154731715491055) + + + --> TIME: 2025-04-19 07:12:27 -- STEP: 134/142 -- GLOBAL_STEP: 450700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.5056209564209 (17.985991342743826) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878349110525122 + | > current_lr_1: 0.0001878349110525122 + | > step_time: 0.4112 (0.35879934723697504) + | > loader_time: 0.0068 (0.006049419516947732) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.006162166595459 (3.006162166595459) + | > loss_feat: nan (nan) + | > loss_mel: 18.288597106933594 (18.288597106933594) + | > loss_duration: 1.8273448944091797 (1.8273448944091797) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.22952938079834 (3.22952938079834) + | > loss_feat: nan (nan) + | > loss_mel: 17.826833724975586 (17.826833724975586) + | > loss_duration: 1.8955342769622803 (1.8955342769622803) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9454736709594727 (3.0875015258789062) + | > loss_feat: nan (nan) + | > loss_mel: 19.569528579711914 (18.69818115234375) + | > loss_duration: 1.8110475540161133 (1.8532909154891968) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003447890281677246 (-0.0002772808074951172) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0875015258789062 (-0.018555283546447754) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.69818115234375 (-0.45630741119384766) + | > avg_loss_duration: 1.8532909154891968 (+0.007389426231384277) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 503/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:12:32)  + + --> TIME: 2025-04-19 07:12:38 -- STEP: 17/142 -- GLOBAL_STEP: 450725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.430591583251953 (18.05552650900448) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878114316886306 + | > current_lr_1: 0.0001878114316886306 + | > step_time: 0.3129 (0.30780849737279553) + | > loader_time: 0.0055 (0.004656833760878619) + + + --> TIME: 2025-04-19 07:12:46 -- STEP: 42/142 -- GLOBAL_STEP: 450750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5728540420532227 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.967247009277344 (18.003894715082083) + | > loss_duration: 1.6300311088562012 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878114316886306 + | > current_lr_1: 0.0001878114316886306 + | > step_time: 0.3378 (0.3201265732447306) + | > loader_time: 0.0059 (0.005196077483040946) + + + --> TIME: 2025-04-19 07:12:55 -- STEP: 67/142 -- GLOBAL_STEP: 450775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.94439125061035 (18.082308811927916) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878114316886306 + | > current_lr_1: 0.0001878114316886306 + | > step_time: 0.3561 (0.3307061088618948) + | > loader_time: 0.0064 (0.005552113945804425) + + + --> TIME: 2025-04-19 07:13:05 -- STEP: 92/142 -- GLOBAL_STEP: 450800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.68545150756836 (18.035534506258752) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878114316886306 + | > current_lr_1: 0.0001878114316886306 + | > step_time: 0.3871 (0.3412642789923625) + | > loader_time: 0.007 (0.0058125490727631915) + + + --> TIME: 2025-04-19 07:13:15 -- STEP: 117/142 -- GLOBAL_STEP: 450825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.508373260498047 (18.00176061320507) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001878114316886306 + | > current_lr_1: 0.0001878114316886306 + | > step_time: 0.4059 (0.35161820639911867) + | > loader_time: 0.0071 (0.00605097591367542) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.93742036819458 (2.93742036819458) + | > loss_feat: nan (nan) + | > loss_mel: 18.715984344482422 (18.715984344482422) + | > loss_duration: 1.8371680974960327 (1.8371680974960327) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3577029705047607 (3.3577029705047607) + | > loss_feat: nan (nan) + | > loss_mel: 16.291290283203125 (16.291290283203125) + | > loss_duration: 1.8961714506149292 (1.8961714506149292) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.107966661453247 (3.232834815979004) + | > loss_feat: nan (nan) + | > loss_mel: 19.9771785736084 (18.13423442840576) + | > loss_duration: 1.8017079830169678 (1.8489397168159485) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004036068916320801 (+0.0005881786346435547) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.232834815979004 (+0.14533329010009766) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.13423442840576 (-0.5639467239379883) + | > avg_loss_duration: 1.8489397168159485 (-0.004351198673248291) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 504/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:13:27)  + + --> TIME: 2025-04-19 07:13:28 -- STEP: 0/142 -- GLOBAL_STEP: 450850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.667783737182617 (16.667783737182617) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.3402 (0.3402278423309326) + | > loader_time: 0.3345 (0.3344731330871582) + + + --> TIME: 2025-04-19 07:13:35 -- STEP: 25/142 -- GLOBAL_STEP: 450875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.628045082092285 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.880346298217773 (18.06498001098633) + | > loss_duration: 1.6662721633911133 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.3207 (0.30802135467529296) + | > loader_time: 0.0055 (0.0048597145080566395) + + + --> TIME: 2025-04-19 07:13:44 -- STEP: 50/142 -- GLOBAL_STEP: 450900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.10985565185547 (17.920760097503663) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.3443 (0.32049574375152595) + | > loader_time: 0.006 (0.005281457901000977) + + + --> TIME: 2025-04-19 07:13:53 -- STEP: 75/142 -- GLOBAL_STEP: 450925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.903200149536133 (17.91150160471599) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.3719 (0.33334525744120286) + | > loader_time: 0.007 (0.005572026570638021) + + + --> TIME: 2025-04-19 07:14:03 -- STEP: 100/142 -- GLOBAL_STEP: 450950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.290584564208984 (17.90613026618958) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.3922 (0.34442216396331793) + | > loader_time: 0.0072 (0.005833714008331299) + + + --> TIME: 2025-04-19 07:14:13 -- STEP: 125/142 -- GLOBAL_STEP: 450975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.597137451171875 (17.934520523071296) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877879552596695 + | > current_lr_1: 0.0001877879552596695 + | > step_time: 0.415 (0.3571702003479004) + | > loader_time: 0.0074 (0.00607646369934082) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8732619285583496 (2.8732619285583496) + | > loss_feat: nan (nan) + | > loss_mel: 18.736051559448242 (18.736051559448242) + | > loss_duration: 1.8244786262512207 (1.8244786262512207) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2963638305664062 (3.2963638305664062) + | > loss_feat: nan (nan) + | > loss_mel: 18.814828872680664 (18.814828872680664) + | > loss_duration: 1.9019627571105957 (1.9019627571105957) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8811352252960205 (3.0887495279312134) + | > loss_feat: nan (nan) + | > loss_mel: 17.44120979309082 (18.128019332885742) + | > loss_duration: 1.801800012588501 (1.8518813848495483) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003715991973876953 (-0.00032007694244384766) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0887495279312134 (-0.14408528804779053) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.128019332885742 (-0.006215095520019531) + | > avg_loss_duration: 1.8518813848495483 (+0.0029416680335998535) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 505/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:14:22)  + + --> TIME: 2025-04-19 07:14:25 -- STEP: 8/142 -- GLOBAL_STEP: 451000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.137296676635742 (17.99898338317871) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.2988 (0.29844048619270325) + | > loader_time: 0.0049 (0.0044824182987213135) + + + --> TIME: 2025-04-19 07:14:33 -- STEP: 33/142 -- GLOBAL_STEP: 451025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.77071762084961 (17.869015664765328) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.329 (0.3118323412808505) + | > loader_time: 0.0054 (0.005123412970340613) + + + --> TIME: 2025-04-19 07:14:42 -- STEP: 58/142 -- GLOBAL_STEP: 451050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.243480682373047 (17.919059917844574) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.3491 (0.32333367857439765) + | > loader_time: 0.006 (0.005423455402768892) + + + --> TIME: 2025-04-19 07:14:51 -- STEP: 83/142 -- GLOBAL_STEP: 451075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.587718963623047 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.560222625732422 (17.9827728616186) + | > loss_duration: 1.7209278345108032 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.3742 (0.3383646700755659) + | > loader_time: 0.0066 (0.00581639358796269) + + + --> TIME: 2025-04-19 07:15:01 -- STEP: 108/142 -- GLOBAL_STEP: 451100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.459564208984375 (18.011990291100965) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.3995 (0.3493183762938888) + | > loader_time: 0.0074 (0.006069724206571226) + + + --> TIME: 2025-04-19 07:15:12 -- STEP: 133/142 -- GLOBAL_STEP: 451125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.847441673278809 (17.955223226905773) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018776448176526206 + | > current_lr_1: 0.00018776448176526206 + | > step_time: 0.4218 (0.3626006599655725) + | > loader_time: 0.0072 (0.006330187159373348) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8495993614196777 (2.8495993614196777) + | > loss_feat: nan (nan) + | > loss_mel: 18.335172653198242 (18.335172653198242) + | > loss_duration: 1.8193644285202026 (1.8193644285202026) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.289505958557129 (3.289505958557129) + | > loss_feat: nan (nan) + | > loss_mel: 16.81501579284668 (16.81501579284668) + | > loss_duration: 1.9034807682037354 (1.9034807682037354) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7409422397613525 (3.0152240991592407) + | > loss_feat: nan (nan) + | > loss_mel: 20.007036209106445 (18.411026000976562) + | > loss_duration: 1.808797001838684 (1.8561388850212097) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038962364196777344 (+0.00018024444580078125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0152240991592407 (-0.07352542877197266) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.411026000976562 (+0.2830066680908203) + | > avg_loss_duration: 1.8561388850212097 (+0.004257500171661377) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 506/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:15:18)  + + --> TIME: 2025-04-19 07:15:23 -- STEP: 16/142 -- GLOBAL_STEP: 451150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.697757720947266 (18.0936484336853) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.3252 (0.3119702935218811) + | > loader_time: 0.005 (0.004754066467285156) + + + --> TIME: 2025-04-19 07:15:32 -- STEP: 41/142 -- GLOBAL_STEP: 451175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.909088134765625 (18.114805244817966) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.3369 (0.3199860293690751) + | > loader_time: 0.0058 (0.005257042442880024) + + + --> TIME: 2025-04-19 07:15:41 -- STEP: 66/142 -- GLOBAL_STEP: 451200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.681344985961914 (17.902015844980873) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.3601 (0.33070342468492914) + | > loader_time: 0.0064 (0.005582361510305694) + + + --> TIME: 2025-04-19 07:15:50 -- STEP: 91/142 -- GLOBAL_STEP: 451225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.736764907836914 (17.857897748003946) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.3836 (0.34225521768842426) + | > loader_time: 0.0065 (0.005835580301808787) + + + --> TIME: 2025-04-19 07:16:00 -- STEP: 116/142 -- GLOBAL_STEP: 451250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.40395736694336 (17.816265476161036) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.4084 (0.3534290913877816) + | > loader_time: 0.0069 (0.006063120118502913) + + + --> TIME: 2025-04-19 07:16:11 -- STEP: 141/142 -- GLOBAL_STEP: 451275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 12.478280067443848 (17.769940836209777) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001877410112050414 + | > current_lr_1: 0.0001877410112050414 + | > step_time: 0.1822 (0.36349461264644106) + | > loader_time: 0.003 (0.006264818475601521) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0499937534332275 (3.0499937534332275) + | > loss_feat: nan (nan) + | > loss_mel: 18.802927017211914 (18.802927017211914) + | > loss_duration: 1.8511712551116943 (1.8511712551116943) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4211807250976562 (3.4211807250976562) + | > loss_feat: nan (nan) + | > loss_mel: 19.738866806030273 (19.738866806030273) + | > loss_duration: 1.8956891298294067 (1.8956891298294067) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.5642666816711426 (2.9927237033843994) + | > loss_feat: nan (nan) + | > loss_mel: 16.977367401123047 (18.35811710357666) + | > loss_duration: 1.7790582180023193 (1.837373673915863) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037616491317749023 (-0.00013458728790283203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9927237033843994 (-0.02250039577484131) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.35811710357666 (-0.052908897399902344) + | > avg_loss_duration: 1.837373673915863 (-0.01876521110534668) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 507/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:16:13)  + + --> TIME: 2025-04-19 07:16:21 -- STEP: 24/142 -- GLOBAL_STEP: 451300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.152307510375977 (17.77742664019267) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018771754357864076 + | > current_lr_1: 0.00018771754357864076 + | > step_time: 0.3179 (0.3072671294212341) + | > loader_time: 0.0053 (0.004937092463175455) + + + --> TIME: 2025-04-19 07:16:29 -- STEP: 49/142 -- GLOBAL_STEP: 451325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.546107053756714 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.193023681640625 (17.670337365598105) + | > loss_duration: 1.6046948432922363 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018771754357864076 + | > current_lr_1: 0.00018771754357864076 + | > step_time: 0.3432 (0.32007815886516955) + | > loader_time: 0.0059 (0.005289671372394173) + + + --> TIME: 2025-04-19 07:16:39 -- STEP: 74/142 -- GLOBAL_STEP: 451350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4896340370178223 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.22191619873047 (17.716068731771937) + | > loss_duration: 1.6381027698516846 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018771754357864076 + | > current_lr_1: 0.00018771754357864076 + | > step_time: 0.3611 (0.3322739504479072) + | > loader_time: 0.0066 (0.005593483512466018) + + + --> TIME: 2025-04-19 07:16:48 -- STEP: 99/142 -- GLOBAL_STEP: 451375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.063426971435547 (17.754319441438934) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018771754357864076 + | > current_lr_1: 0.00018771754357864076 + | > step_time: 0.3823 (0.3438307781412143) + | > loader_time: 0.0068 (0.005820291210906674) + + + --> TIME: 2025-04-19 07:16:59 -- STEP: 124/142 -- GLOBAL_STEP: 451400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.208602905273438 (17.798575670488425) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018771754357864076 + | > current_lr_1: 0.00018771754357864076 + | > step_time: 0.4138 (0.35663317288121865) + | > loader_time: 0.0071 (0.006127369019293016) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0285484790802 (3.0285484790802) + | > loss_feat: nan (nan) + | > loss_mel: 18.68418312072754 (18.68418312072754) + | > loss_duration: 1.822780966758728 (1.822780966758728) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.300043821334839 (3.300043821334839) + | > loss_feat: nan (nan) + | > loss_mel: 18.048683166503906 (18.048683166503906) + | > loss_duration: 1.900846242904663 (1.900846242904663) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8993186950683594 (3.099681258201599) + | > loss_feat: nan (nan) + | > loss_mel: 17.619287490844727 (17.833985328674316) + | > loss_duration: 1.7782795429229736 (1.8395628929138184) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003396272659301758 (-0.00036537647247314453) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.099681258201599 (+0.10695755481719971) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.833985328674316 (-0.5241317749023438) + | > avg_loss_duration: 1.8395628929138184 (+0.0021892189979553223) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 508/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:17:08)  + + --> TIME: 2025-04-19 07:17:10 -- STEP: 7/142 -- GLOBAL_STEP: 451425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.951021194458008 (18.091953005109513) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.2961 (0.2969263621738979) + | > loader_time: 0.0042 (0.004390137536185128) + + + --> TIME: 2025-04-19 07:17:19 -- STEP: 32/142 -- GLOBAL_STEP: 451450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.23423957824707 (18.1672078371048) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.3362 (0.314374677836895) + | > loader_time: 0.0058 (0.005201973021030426) + + + --> TIME: 2025-04-19 07:17:28 -- STEP: 57/142 -- GLOBAL_STEP: 451475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.799384593963623 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.403074264526367 (18.053575984218668) + | > loss_duration: 1.673686146736145 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.3511 (0.33149651895489607) + | > loader_time: 0.006 (0.005496317880195484) + + + --> TIME: 2025-04-19 07:17:37 -- STEP: 82/142 -- GLOBAL_STEP: 451500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.21988296508789 (17.999719073132777) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.3691 (0.34145483447284236) + | > loader_time: 0.0066 (0.0057347256962845955) + + + --> TIME: 2025-04-19 07:17:47 -- STEP: 107/142 -- GLOBAL_STEP: 451525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.20764923095703 (17.986148183590895) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.3908 (0.35102630107202276) + | > loader_time: 0.0069 (0.00597759496385806) + + + --> TIME: 2025-04-19 07:17:57 -- STEP: 132/142 -- GLOBAL_STEP: 451550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.878159523010254 (17.89266872406005) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018769407888569344 + | > current_lr_1: 0.00018769407888569344 + | > step_time: 0.4203 (0.3621778903585493) + | > loader_time: 0.0072 (0.006209055582682292) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8274731636047363 (2.8274731636047363) + | > loss_feat: nan (nan) + | > loss_mel: 17.482328414916992 (17.482328414916992) + | > loss_duration: 1.83953058719635 (1.83953058719635) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1705377101898193 (3.1705377101898193) + | > loss_feat: nan (nan) + | > loss_mel: 18.276216506958008 (18.276216506958008) + | > loss_duration: 1.9109413623809814 (1.9109413623809814) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8552472591400146 (3.012892484664917) + | > loss_feat: nan (nan) + | > loss_mel: 17.360261917114258 (17.818239212036133) + | > loss_duration: 1.8132838010787964 (1.862112581729889) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037537813186645508 (+0.00035750865936279297) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.012892484664917 (-0.08678877353668213) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.818239212036133 (-0.015746116638183594) + | > avg_loss_duration: 1.862112581729889 (+0.022549688816070557) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 509/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:18:03)  + + --> TIME: 2025-04-19 07:18:08 -- STEP: 15/142 -- GLOBAL_STEP: 451575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.94135093688965 (18.1621950785319) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.3091 (0.30164141654968263) + | > loader_time: 0.0056 (0.0050221443176269535) + + + --> TIME: 2025-04-19 07:18:17 -- STEP: 40/142 -- GLOBAL_STEP: 451600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.323291778564453 (18.130042791366577) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.3384 (0.31484987735748293) + | > loader_time: 0.0061 (0.005250716209411621) + + + --> TIME: 2025-04-19 07:18:25 -- STEP: 65/142 -- GLOBAL_STEP: 451625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.6102237701416 (18.18176516019381) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.3645 (0.32672370763925407) + | > loader_time: 0.0061 (0.005537674977229191) + + + --> TIME: 2025-04-19 07:18:35 -- STEP: 90/142 -- GLOBAL_STEP: 451650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.429168701171875 (18.16385295655993) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.3777 (0.3385537068049112) + | > loader_time: 0.0066 (0.005813015831841363) + + + --> TIME: 2025-04-19 07:18:45 -- STEP: 115/142 -- GLOBAL_STEP: 451675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.31572914123535 (18.107059619737704) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.4001 (0.34934286863907515) + | > loader_time: 0.0083 (0.006067528931990914) + + + --> TIME: 2025-04-19 07:18:56 -- STEP: 140/142 -- GLOBAL_STEP: 451700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.468711853027344 (18.09309443746294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018767061712583272 + | > current_lr_1: 0.00018767061712583272 + | > step_time: 0.4317 (0.36228886672428673) + | > loader_time: 0.0073 (0.006327986717224121) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0029468536376953 (3.0029468536376953) + | > loss_feat: nan (nan) + | > loss_mel: 19.509178161621094 (19.509178161621094) + | > loss_duration: 1.8265457153320312 (1.8265457153320312) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1707746982574463 (3.1707746982574463) + | > loss_feat: nan (nan) + | > loss_mel: 19.65294075012207 (19.65294075012207) + | > loss_duration: 1.9311192035675049 (1.9311192035675049) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6321847438812256 (2.901479721069336) + | > loss_feat: nan (nan) + | > loss_mel: 16.966896057128906 (18.30991840362549) + | > loss_duration: 1.7872815132141113 (1.859200358390808) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003163576126098633 (-0.000590205192565918) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.901479721069336 (-0.11141276359558105) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.30991840362549 (+0.49167919158935547) + | > avg_loss_duration: 1.859200358390808 (-0.0029122233390808105) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 510/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:18:58)  + + --> TIME: 2025-04-19 07:19:06 -- STEP: 23/142 -- GLOBAL_STEP: 451725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6233184337615967 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.006500244140625 (17.976988668027133) + | > loss_duration: 1.6167845726013184 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018764715829869199 + | > current_lr_1: 0.00018764715829869199 + | > step_time: 0.3267 (0.30923373802848475) + | > loader_time: 0.0054 (0.004862961561783501) + + + --> TIME: 2025-04-19 07:19:14 -- STEP: 48/142 -- GLOBAL_STEP: 451750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.62809944152832 (17.871730486551918) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018764715829869199 + | > current_lr_1: 0.00018764715829869199 + | > step_time: 0.3462 (0.32522161304950703) + | > loader_time: 0.006 (0.005352179209391276) + + + --> TIME: 2025-04-19 07:19:24 -- STEP: 73/142 -- GLOBAL_STEP: 451775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.584023475646973 (17.815553051151642) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018764715829869199 + | > current_lr_1: 0.00018764715829869199 + | > step_time: 0.3798 (0.3394650106560693) + | > loader_time: 0.0066 (0.005683030167671099) + + + --> TIME: 2025-04-19 07:19:34 -- STEP: 98/142 -- GLOBAL_STEP: 451800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.50377655029297 (17.877988309276343) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018764715829869199 + | > current_lr_1: 0.00018764715829869199 + | > step_time: 0.3972 (0.3518255705736121) + | > loader_time: 0.008 (0.006006659293661312) + + + --> TIME: 2025-04-19 07:19:44 -- STEP: 123/142 -- GLOBAL_STEP: 451825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.11332130432129 (17.841040642280877) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018764715829869199 + | > current_lr_1: 0.00018764715829869199 + | > step_time: 0.4136 (0.363439598703772) + | > loader_time: 0.0075 (0.006275014179508861) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7755377292633057 (2.7755377292633057) + | > loss_feat: nan (nan) + | > loss_mel: 19.619943618774414 (19.619943618774414) + | > loss_duration: 1.8425716161727905 (1.8425716161727905) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.384079933166504 (3.384079933166504) + | > loss_feat: nan (nan) + | > loss_mel: 18.209121704101562 (18.209121704101562) + | > loss_duration: 1.881499171257019 (1.881499171257019) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9236209392547607 (3.1538504362106323) + | > loss_feat: nan (nan) + | > loss_mel: 18.949562072753906 (18.579341888427734) + | > loss_duration: 1.7856383323669434 (1.8335687518119812) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035066604614257812 (+0.00034308433532714844) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1538504362106323 (+0.2523707151412964) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.579341888427734 (+0.2694234848022461) + | > avg_loss_duration: 1.8335687518119812 (-0.025631606578826904) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 511/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:19:54)  + + --> TIME: 2025-04-19 07:19:56 -- STEP: 6/142 -- GLOBAL_STEP: 451850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.567396879196167 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.830747604370117 (18.28473885854085) + | > loss_duration: 1.6346275806427002 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.3027 (0.30002884070078534) + | > loader_time: 0.0043 (0.0043265024820963545) + + + --> TIME: 2025-04-19 07:20:04 -- STEP: 31/142 -- GLOBAL_STEP: 451875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.229772567749023 (18.131318369219372) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.3261 (0.31088240685001495) + | > loader_time: 0.0052 (0.004973003941197549) + + + --> TIME: 2025-04-19 07:20:13 -- STEP: 56/142 -- GLOBAL_STEP: 451900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.67731475830078 (18.1354318005698) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.3502 (0.3230520316532681) + | > loader_time: 0.0071 (0.005279558045523507) + + + --> TIME: 2025-04-19 07:20:22 -- STEP: 81/142 -- GLOBAL_STEP: 451925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.065004348754883 (18.044171062516572) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.3694 (0.3352404082262959) + | > loader_time: 0.0064 (0.00555620664431725) + + + --> TIME: 2025-04-19 07:20:32 -- STEP: 106/142 -- GLOBAL_STEP: 451950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.987375259399414 (17.97760187904789) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.3835 (0.34555593751511504) + | > loader_time: 0.007 (0.005803648030982827) + + + --> TIME: 2025-04-19 07:20:42 -- STEP: 131/142 -- GLOBAL_STEP: 451975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.356348037719727 (17.956088830496515) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018762370240390464 + | > current_lr_1: 0.00018762370240390464 + | > step_time: 0.4232 (0.35774396030047484) + | > loader_time: 0.0071 (0.006059013250219913) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.904759645462036 (2.904759645462036) + | > loss_feat: nan (nan) + | > loss_mel: 17.842308044433594 (17.842308044433594) + | > loss_duration: 1.8391072750091553 (1.8391072750091553) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4670586585998535 (3.4670586585998535) + | > loss_feat: nan (nan) + | > loss_mel: 18.273942947387695 (18.273942947387695) + | > loss_duration: 1.8983328342437744 (1.8983328342437744) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9829092025756836 (3.2249839305877686) + | > loss_feat: nan (nan) + | > loss_mel: 17.416332244873047 (17.84513759613037) + | > loss_duration: 1.7776731252670288 (1.8380029797554016) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031881332397460938 (-0.0003185272216796875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2249839305877686 (+0.07113349437713623) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.84513759613037 (-0.7342042922973633) + | > avg_loss_duration: 1.8380029797554016 (+0.00443422794342041) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 512/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:20:49)  + + --> TIME: 2025-04-19 07:20:53 -- STEP: 14/142 -- GLOBAL_STEP: 452000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.602792501449585 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.24917221069336 (17.784046854291642) + | > loss_duration: 1.6467705965042114 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.306 (0.300829359463283) + | > loader_time: 0.0051 (0.004621318408421108) + + + --> TIME: 2025-04-19 07:21:02 -- STEP: 39/142 -- GLOBAL_STEP: 452025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.16158103942871 (17.795170710637016) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.3405 (0.3206028510362675) + | > loader_time: 0.0057 (0.005302692070985451) + + + --> TIME: 2025-04-19 07:21:11 -- STEP: 64/142 -- GLOBAL_STEP: 452050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.527576446533203 (17.84762966632843) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.3691 (0.33497102558612823) + | > loader_time: 0.0062 (0.005642645061016083) + + + --> TIME: 2025-04-19 07:21:21 -- STEP: 89/142 -- GLOBAL_STEP: 452075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.975101470947266 (17.90416835399157) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.3867 (0.3478869936439428) + | > loader_time: 0.0067 (0.005941549043976859) + + + --> TIME: 2025-04-19 07:21:31 -- STEP: 114/142 -- GLOBAL_STEP: 452100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.033315658569336 (17.918911415233953) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.4094 (0.359867861396388) + | > loader_time: 0.0071 (0.006196620171530205) + + + --> TIME: 2025-04-19 07:21:42 -- STEP: 139/142 -- GLOBAL_STEP: 452125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.72922134399414 (17.9289215828875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018760024944110414 + | > current_lr_1: 0.00018760024944110414 + | > step_time: 0.4393 (0.3723016985886389) + | > loader_time: 0.0079 (0.006453673616587687) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.952829599380493 (2.952829599380493) + | > loss_feat: nan (nan) + | > loss_mel: 19.08832550048828 (19.08832550048828) + | > loss_duration: 1.824352741241455 (1.824352741241455) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4254541397094727 (3.4254541397094727) + | > loss_feat: nan (nan) + | > loss_mel: 19.436899185180664 (19.436899185180664) + | > loss_duration: 1.9022562503814697 (1.9022562503814697) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8765673637390137 (3.151010751724243) + | > loss_feat: nan (nan) + | > loss_mel: 18.01112174987793 (18.724010467529297) + | > loss_duration: 1.7929844856262207 (1.8476203680038452) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036766529083251953 (+0.0004885196685791016) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.151010751724243 (-0.07397317886352539) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.724010467529297 (+0.8788728713989258) + | > avg_loss_duration: 1.8476203680038452 (+0.009617388248443604) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 513/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:21:45)  + + --> TIME: 2025-04-19 07:21:52 -- STEP: 22/142 -- GLOBAL_STEP: 452150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.78192901611328 (17.800087148492988) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000187576799409924 + | > current_lr_1: 0.000187576799409924 + | > step_time: 0.3173 (0.3076684258200906) + | > loader_time: 0.0049 (0.004709482192993164) + + + --> TIME: 2025-04-19 07:22:01 -- STEP: 47/142 -- GLOBAL_STEP: 452175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5594124794006348 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.536380767822266 (18.028741593056548) + | > loss_duration: 1.602832555770874 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000187576799409924 + | > current_lr_1: 0.000187576799409924 + | > step_time: 0.3389 (0.3192496553380438) + | > loader_time: 0.0058 (0.005139619746106735) + + + --> TIME: 2025-04-19 07:22:10 -- STEP: 72/142 -- GLOBAL_STEP: 452200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.104652404785156 (18.030110531383098) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000187576799409924 + | > current_lr_1: 0.000187576799409924 + | > step_time: 0.3725 (0.3312382002671559) + | > loader_time: 0.0062 (0.005417717827690971) + + + --> TIME: 2025-04-19 07:22:20 -- STEP: 97/142 -- GLOBAL_STEP: 452225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.425871849060059 (18.0068663370978) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000187576799409924 + | > current_lr_1: 0.000187576799409924 + | > step_time: 0.3845 (0.34202157099222386) + | > loader_time: 0.0065 (0.005692228828508827) + + + --> TIME: 2025-04-19 07:22:30 -- STEP: 122/142 -- GLOBAL_STEP: 452250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.649293899536133 (17.95936723615303) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000187576799409924 + | > current_lr_1: 0.000187576799409924 + | > step_time: 0.4263 (0.3533534573726966) + | > loader_time: 0.007 (0.005975691998591186) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9396965503692627 (2.9396965503692627) + | > loss_feat: nan (nan) + | > loss_mel: 17.62298583984375 (17.62298583984375) + | > loss_duration: 1.8181121349334717 (1.8181121349334717) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.392601251602173 (3.392601251602173) + | > loss_feat: nan (nan) + | > loss_mel: 18.783294677734375 (18.783294677734375) + | > loss_duration: 1.9150058031082153 (1.9150058031082153) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8973162174224854 (3.144958734512329) + | > loss_feat: nan (nan) + | > loss_mel: 18.307905197143555 (18.545599937438965) + | > loss_duration: 1.77065110206604 (1.8428284525871277) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003441333770751953 (-0.0002353191375732422) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.144958734512329 (-0.0060520172119140625) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.545599937438965 (-0.17841053009033203) + | > avg_loss_duration: 1.8428284525871277 (-0.004791915416717529) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 514/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:22:40)  + + --> TIME: 2025-04-19 07:22:42 -- STEP: 5/142 -- GLOBAL_STEP: 452275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.331087112426758 (18.55272102355957) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.2989 (0.297857666015625) + | > loader_time: 0.0041 (0.004218530654907226) + + + --> TIME: 2025-04-19 07:22:50 -- STEP: 30/142 -- GLOBAL_STEP: 452300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.278165817260742 (18.034786287943522) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.3322 (0.31293771266937254) + | > loader_time: 0.0055 (0.004791625340779622) + + + --> TIME: 2025-04-19 07:22:59 -- STEP: 55/142 -- GLOBAL_STEP: 452325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.47410774230957 (17.937362289428716) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.3563 (0.3277458624406294) + | > loader_time: 0.0058 (0.005175872282548383) + + + --> TIME: 2025-04-19 07:23:08 -- STEP: 80/142 -- GLOBAL_STEP: 452350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.773350954055786 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.80133628845215 (17.937423515319825) + | > loss_duration: 1.665266990661621 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.4009 (0.3433790624141693) + | > loader_time: 0.0079 (0.005579340457916259) + + + --> TIME: 2025-04-19 07:23:18 -- STEP: 105/142 -- GLOBAL_STEP: 452375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6974592208862305 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.61382293701172 (17.936813844953267) + | > loss_duration: 1.6701467037200928 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.4059 (0.3552225930350167) + | > loader_time: 0.0069 (0.005938929603213354) + + + --> TIME: 2025-04-19 07:23:29 -- STEP: 130/142 -- GLOBAL_STEP: 452400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.350364685058594 (17.8714142872737) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018755335230999777 + | > current_lr_1: 0.00018755335230999777 + | > step_time: 0.4364 (0.3676620116600623) + | > loader_time: 0.0078 (0.006242629197927621) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.851015329360962 (2.851015329360962) + | > loss_feat: nan (nan) + | > loss_mel: 18.629552841186523 (18.629552841186523) + | > loss_duration: 1.8198367357254028 (1.8198367357254028) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4184677600860596 (3.4184677600860596) + | > loss_feat: nan (nan) + | > loss_mel: 18.753610610961914 (18.753610610961914) + | > loss_duration: 1.9129794836044312 (1.9129794836044312) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7773382663726807 (3.09790301322937) + | > loss_feat: nan (nan) + | > loss_mel: 16.98304557800293 (17.868328094482422) + | > loss_duration: 1.7981688976287842 (1.8555741906166077) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003302335739135742 (-0.00013899803161621094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.09790301322937 (-0.047055721282958984) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.868328094482422 (-0.677271842956543) + | > avg_loss_duration: 1.8555741906166077 (+0.01274573802947998) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 515/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:23:36)  + + --> TIME: 2025-04-19 07:23:41 -- STEP: 13/142 -- GLOBAL_STEP: 452425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.941429138183594 (18.13277479318472) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.3043 (0.3011151643899771) + | > loader_time: 0.0052 (0.0044468182783860425) + + + --> TIME: 2025-04-19 07:23:49 -- STEP: 38/142 -- GLOBAL_STEP: 452450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.74239730834961 (17.997940314443486) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.327 (0.31400427692814875) + | > loader_time: 0.0057 (0.004962400386207984) + + + --> TIME: 2025-04-19 07:23:58 -- STEP: 63/142 -- GLOBAL_STEP: 452475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.296621322631836 (18.063994755820627) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.3647 (0.3259929891616579) + | > loader_time: 0.0058 (0.005277001668536475) + + + --> TIME: 2025-04-19 07:24:07 -- STEP: 88/142 -- GLOBAL_STEP: 452500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.39195442199707 (17.957456653768368) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.3699 (0.3375972969965502) + | > loader_time: 0.0067 (0.005584193901582198) + + + --> TIME: 2025-04-19 07:24:17 -- STEP: 113/142 -- GLOBAL_STEP: 452525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.463769912719727 (17.972606625177164) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.4004 (0.34876555468128856) + | > loader_time: 0.0069 (0.00581379274351407) + + + --> TIME: 2025-04-19 07:24:28 -- STEP: 138/142 -- GLOBAL_STEP: 452550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7687277793884277 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.16838836669922 (17.915362413378727) + | > loss_duration: 1.734179973602295 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018752990814095903 + | > current_lr_1: 0.00018752990814095903 + | > step_time: 0.4286 (0.3607853789260423) + | > loader_time: 0.0075 (0.006089473116225091) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.795315742492676 (2.795315742492676) + | > loss_feat: nan (nan) + | > loss_mel: 17.451379776000977 (17.451379776000977) + | > loss_duration: 1.8574678897857666 (1.8574678897857666) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.42141056060791 (3.42141056060791) + | > loss_feat: nan (nan) + | > loss_mel: 17.52813148498535 (17.52813148498535) + | > loss_duration: 1.8909884691238403 (1.8909884691238403) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7085275650024414 (3.064969062805176) + | > loss_feat: nan (nan) + | > loss_mel: 18.07168960571289 (17.79991054534912) + | > loss_duration: 1.7958488464355469 (1.8434186577796936) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003543853759765625 (+0.0002415180206298828) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.064969062805176 (-0.032933950424194336) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.79991054534912 (-0.06841754913330078) + | > avg_loss_duration: 1.8434186577796936 (-0.012155532836914062) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 516/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:24:31)  + + --> TIME: 2025-04-19 07:24:38 -- STEP: 21/142 -- GLOBAL_STEP: 452575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.01753044128418 (17.905976704188756) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001875064669024414 + | > current_lr_1: 0.0001875064669024414 + | > step_time: 0.3172 (0.3054931390853155) + | > loader_time: 0.0052 (0.004819234212239583) + + + --> TIME: 2025-04-19 07:24:46 -- STEP: 46/142 -- GLOBAL_STEP: 452600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.808637619018555 (17.917633181032922) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001875064669024414 + | > current_lr_1: 0.0001875064669024414 + | > step_time: 0.3386 (0.3174961390702621) + | > loader_time: 0.006 (0.0051881178565647305) + + + --> TIME: 2025-04-19 07:24:55 -- STEP: 71/142 -- GLOBAL_STEP: 452625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.742015838623047 (17.98366815271512) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001875064669024414 + | > current_lr_1: 0.0001875064669024414 + | > step_time: 0.3628 (0.33019240473357725) + | > loader_time: 0.0065 (0.005514198625591439) + + + --> TIME: 2025-04-19 07:25:05 -- STEP: 96/142 -- GLOBAL_STEP: 452650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.63383150100708 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.142288208007812 (17.93581427137057) + | > loss_duration: 1.6822172403335571 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001875064669024414 + | > current_lr_1: 0.0001875064669024414 + | > step_time: 0.3994 (0.3440245613455774) + | > loader_time: 0.007 (0.00580300639073054) + + + --> TIME: 2025-04-19 07:25:16 -- STEP: 121/142 -- GLOBAL_STEP: 452675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.539159774780273 (17.879810648516187) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001875064669024414 + | > current_lr_1: 0.0001875064669024414 + | > step_time: 0.4294 (0.3588042653296606) + | > loader_time: 0.0073 (0.006135010522259168) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.997833013534546 (2.997833013534546) + | > loss_feat: nan (nan) + | > loss_mel: 17.695159912109375 (17.695159912109375) + | > loss_duration: 1.8450541496276855 (1.8450541496276855) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.427557945251465 (3.427557945251465) + | > loss_feat: nan (nan) + | > loss_mel: 16.679529190063477 (16.679529190063477) + | > loss_duration: 1.9381911754608154 (1.9381911754608154) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8988759517669678 (3.1632169485092163) + | > loss_feat: nan (nan) + | > loss_mel: 19.548377990722656 (18.113953590393066) + | > loss_duration: 1.8181298971176147 (1.878160536289215) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003461122512817383 (-8.273124694824219e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1632169485092163 (+0.09824788570404053) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.113953590393066 (+0.3140430450439453) + | > avg_loss_duration: 1.878160536289215 (+0.034741878509521484) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 517/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:25:27)  + + --> TIME: 2025-04-19 07:25:28 -- STEP: 4/142 -- GLOBAL_STEP: 452700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.420297622680664 (17.987648963928223) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.2945 (0.29741883277893066) + | > loader_time: 0.0059 (0.004542052745819092) + + + --> TIME: 2025-04-19 07:25:37 -- STEP: 29/142 -- GLOBAL_STEP: 452725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.876663208007812 (18.150779658350448) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.3294 (0.31313817254428206) + | > loader_time: 0.0055 (0.0048895457695270405) + + + --> TIME: 2025-04-19 07:25:45 -- STEP: 54/142 -- GLOBAL_STEP: 452750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.226266860961914 (18.0149903120818) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.3443 (0.32481209878568296) + | > loader_time: 0.006 (0.00529625239195647) + + + --> TIME: 2025-04-19 07:25:54 -- STEP: 79/142 -- GLOBAL_STEP: 452775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.34156036376953 (17.921963317484803) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.3612 (0.33650535873219933) + | > loader_time: 0.0064 (0.005602474454083021) + + + --> TIME: 2025-04-19 07:26:04 -- STEP: 104/142 -- GLOBAL_STEP: 452800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.100248336791992 (17.949267277350803) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.3839 (0.34670990246992833) + | > loader_time: 0.0069 (0.005868996565158551) + + + --> TIME: 2025-04-19 07:26:15 -- STEP: 129/142 -- GLOBAL_STEP: 452825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.17205810546875 (17.93277001196101) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018748302859407858 + | > current_lr_1: 0.00018748302859407858 + | > step_time: 0.4223 (0.35818408810815133) + | > loader_time: 0.0075 (0.006160405255103296) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9778292179107666 (2.9778292179107666) + | > loss_feat: nan (nan) + | > loss_mel: 19.163591384887695 (19.163591384887695) + | > loss_duration: 1.8451179265975952 (1.8451179265975952) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3168959617614746 (3.3168959617614746) + | > loss_feat: nan (nan) + | > loss_mel: 18.706743240356445 (18.706743240356445) + | > loss_duration: 1.939022183418274 (1.939022183418274) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.754725456237793 (3.035810708999634) + | > loss_feat: nan (nan) + | > loss_mel: 16.995832443237305 (17.851287841796875) + | > loss_duration: 1.7828717231750488 (1.8609469532966614) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003796815872192383 (+0.000335693359375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.035810708999634 (-0.12740623950958252) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.851287841796875 (-0.2626657485961914) + | > avg_loss_duration: 1.8609469532966614 (-0.01721358299255371) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 518/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:26:21)  + + --> TIME: 2025-04-19 07:26:26 -- STEP: 12/142 -- GLOBAL_STEP: 452850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.477537155151367 (18.0659925142924) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.3058 (0.3053257664044698) + | > loader_time: 0.0051 (0.0044987400372823085) + + + --> TIME: 2025-04-19 07:26:34 -- STEP: 37/142 -- GLOBAL_STEP: 452875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6145272254943848 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.26541519165039 (17.74990179732039) + | > loss_duration: 1.613511085510254 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.3383 (0.32013449797759186) + | > loader_time: 0.0063 (0.00499358692684689) + + + --> TIME: 2025-04-19 07:26:43 -- STEP: 62/142 -- GLOBAL_STEP: 452900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.311519622802734 (17.81799664035921) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.375 (0.33343514319389095) + | > loader_time: 0.0058 (0.005434839956222041) + + + --> TIME: 2025-04-19 07:26:53 -- STEP: 87/142 -- GLOBAL_STEP: 452925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.059280395507812 (17.829277597624685) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.3984 (0.3480678646043799) + | > loader_time: 0.0067 (0.005728529787611687) + + + --> TIME: 2025-04-19 07:27:03 -- STEP: 112/142 -- GLOBAL_STEP: 452950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.017690658569336 (17.876198930399767) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.4139 (0.3595550187996455) + | > loader_time: 0.0072 (0.0060084377016339984) + + + --> TIME: 2025-04-19 07:27:14 -- STEP: 137/142 -- GLOBAL_STEP: 452975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.885942459106445 (17.897462719548365) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018745959321550432 + | > current_lr_1: 0.00018745959321550432 + | > step_time: 0.4388 (0.3719962026080946) + | > loader_time: 0.0078 (0.006269112120579629) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0033109188079834 (3.0033109188079834) + | > loss_feat: nan (nan) + | > loss_mel: 20.059276580810547 (20.059276580810547) + | > loss_duration: 1.83243989944458 (1.83243989944458) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.953733205795288 (2.953733205795288) + | > loss_feat: nan (nan) + | > loss_mel: 17.901870727539062 (17.901870727539062) + | > loss_duration: 1.922032356262207 (1.922032356262207) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.799518346786499 (2.8766257762908936) + | > loss_feat: nan (nan) + | > loss_mel: 17.044275283813477 (17.47307300567627) + | > loss_duration: 1.8039920330047607 (1.8630121946334839) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003300309181213379 (-0.0004965066909790039) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.8766257762908936 (-0.15918493270874023) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.47307300567627 (-0.37821483612060547) + | > avg_loss_duration: 1.8630121946334839 (+0.0020652413368225098) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 519/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:27:18)  + + --> TIME: 2025-04-19 07:27:25 -- STEP: 20/142 -- GLOBAL_STEP: 453000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6477153301239014 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.520254135131836 (17.636221265792848) + | > loss_duration: 1.6800425052642822 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018743616076635238 + | > current_lr_1: 0.00018743616076635238 + | > step_time: 0.3156 (0.3094288229942322) + | > loader_time: 0.005 (0.004890239238739014) + + + --> TIME: 2025-04-19 07:27:33 -- STEP: 45/142 -- GLOBAL_STEP: 453025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.478755950927734 (17.846711497836644) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018743616076635238 + | > current_lr_1: 0.00018743616076635238 + | > step_time: 0.3413 (0.31984107229444714) + | > loader_time: 0.006 (0.005259508556789822) + + + --> TIME: 2025-04-19 07:27:42 -- STEP: 70/142 -- GLOBAL_STEP: 453050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.342039108276367 (17.922038705008372) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018743616076635238 + | > current_lr_1: 0.00018743616076635238 + | > step_time: 0.3645 (0.33122884546007425) + | > loader_time: 0.0062 (0.005530187061854771) + + + --> TIME: 2025-04-19 07:27:52 -- STEP: 95/142 -- GLOBAL_STEP: 453075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.012258529663086 (17.962752191643972) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018743616076635238 + | > current_lr_1: 0.00018743616076635238 + | > step_time: 0.3788 (0.34171075318989014) + | > loader_time: 0.0067 (0.00584490173741391) + + + --> TIME: 2025-04-19 07:28:02 -- STEP: 120/142 -- GLOBAL_STEP: 453100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.6823787689209 (17.945885165532438) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018743616076635238 + | > current_lr_1: 0.00018743616076635238 + | > step_time: 0.4094 (0.3526818990707399) + | > loader_time: 0.0075 (0.006104463338851929) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.050952434539795 (3.050952434539795) + | > loss_feat: nan (nan) + | > loss_mel: 18.933456420898438 (18.933456420898438) + | > loss_duration: 1.831366777420044 (1.831366777420044) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3392980098724365 (3.3392980098724365) + | > loss_feat: nan (nan) + | > loss_mel: 18.177776336669922 (18.177776336669922) + | > loss_duration: 1.9106626510620117 (1.9106626510620117) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.80715012550354 (3.0732240676879883) + | > loss_feat: nan (nan) + | > loss_mel: 18.996665954589844 (18.587221145629883) + | > loss_duration: 1.8045403957366943 (1.857601523399353) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036063194274902344 (+0.00030601024627685547) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0732240676879883 (+0.19659829139709473) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.587221145629883 (+1.1141481399536133) + | > avg_loss_duration: 1.857601523399353 (-0.005410671234130859) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 520/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:28:13)  + + --> TIME: 2025-04-19 07:28:14 -- STEP: 3/142 -- GLOBAL_STEP: 453125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.621727705001831 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.170848846435547 (17.885125478108723) + | > loss_duration: 1.5966730117797852 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.2952 (0.3038516839345296) + | > loader_time: 0.0036 (0.004018704096476237) + + + --> TIME: 2025-04-19 07:28:22 -- STEP: 28/142 -- GLOBAL_STEP: 453150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5275237560272217 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.81329917907715 (17.70750856399536) + | > loss_duration: 1.6463508605957031 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.3236 (0.3104851841926576) + | > loader_time: 0.0055 (0.00477550710950579) + + + --> TIME: 2025-04-19 07:28:31 -- STEP: 53/142 -- GLOBAL_STEP: 453175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.150779724121094 (17.771366119384762) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.3518 (0.3258544364065495) + | > loader_time: 0.006 (0.00521757017891362) + + + --> TIME: 2025-04-19 07:28:41 -- STEP: 78/142 -- GLOBAL_STEP: 453200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.971023559570312 (17.781553977575058) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.3784 (0.34035203701410544) + | > loader_time: 0.0061 (0.005537781959924945) + + + --> TIME: 2025-04-19 07:28:51 -- STEP: 103/142 -- GLOBAL_STEP: 453225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.05974578857422 (17.773299096857464) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.4 (0.35254900895276114) + | > loader_time: 0.0071 (0.005838910352836538) + + + --> TIME: 2025-04-19 07:29:01 -- STEP: 128/142 -- GLOBAL_STEP: 453250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.186141967773438 (17.794584959745414) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018741273124625658 + | > current_lr_1: 0.00018741273124625658 + | > step_time: 0.4203 (0.3651934340596199) + | > loader_time: 0.0076 (0.006141515448689463) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.705533027648926 (2.705533027648926) + | > loss_feat: nan (nan) + | > loss_mel: 19.147369384765625 (19.147369384765625) + | > loss_duration: 1.8269704580307007 (1.8269704580307007) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3555312156677246 (3.3555312156677246) + | > loss_feat: nan (nan) + | > loss_mel: 17.39242935180664 (17.39242935180664) + | > loss_duration: 1.8771779537200928 (1.8771779537200928) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7889702320098877 (3.072250723838806) + | > loss_feat: nan (nan) + | > loss_mel: 16.50695037841797 (16.949689865112305) + | > loss_duration: 1.7972025871276855 (1.8371902704238892) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037795305252075195 (+0.00017321109771728516) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.072250723838806 (-0.0009733438491821289) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.949689865112305 (-1.6375312805175781) + | > avg_loss_duration: 1.8371902704238892 (-0.020411252975463867) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 521/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:29:09)  + + --> TIME: 2025-04-19 07:29:13 -- STEP: 11/142 -- GLOBAL_STEP: 453275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.498230218887329 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.005029678344727 (17.85481071472168) + | > loss_duration: 1.5876386165618896 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.3054 (0.3025654229250821) + | > loader_time: 0.0047 (0.004308158701116388) + + + --> TIME: 2025-04-19 07:29:21 -- STEP: 36/142 -- GLOBAL_STEP: 453300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.30217742919922 (17.998213211695354) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.329 (0.3146567543347676) + | > loader_time: 0.0055 (0.004847897423638238) + + + --> TIME: 2025-04-19 07:29:30 -- STEP: 61/142 -- GLOBAL_STEP: 453325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.16954231262207 (17.912244593510863) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.3594 (0.32878113574669005) + | > loader_time: 0.0061 (0.005237696600742026) + + + --> TIME: 2025-04-19 07:29:40 -- STEP: 86/142 -- GLOBAL_STEP: 453350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5620341300964355 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.536861419677734 (18.023263010867804) + | > loss_duration: 1.6805074214935303 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.3762 (0.34002197897711467) + | > loader_time: 0.0067 (0.005553109701289684) + + + --> TIME: 2025-04-19 07:29:50 -- STEP: 111/142 -- GLOBAL_STEP: 453375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.609294891357422 (18.008002298372286) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.4035 (0.3511160257700327) + | > loader_time: 0.0074 (0.005841465683670728) + + + --> TIME: 2025-04-19 07:30:00 -- STEP: 136/142 -- GLOBAL_STEP: 453400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.71380615234375 (17.94110350047841) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001873893046548508 + | > current_lr_1: 0.0001873893046548508 + | > step_time: 0.4361 (0.3643137584714329) + | > loader_time: 0.0073 (0.0061134538229774) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6354947090148926 (2.6354947090148926) + | > loss_feat: nan (nan) + | > loss_mel: 20.019121170043945 (20.019121170043945) + | > loss_duration: 1.847456932067871 (1.847456932067871) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3654510974884033 (3.3654510974884033) + | > loss_feat: nan (nan) + | > loss_mel: 19.36773681640625 (19.36773681640625) + | > loss_duration: 1.8962831497192383 (1.8962831497192383) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.837582588195801 (3.101516842842102) + | > loss_feat: nan (nan) + | > loss_mel: 17.57853126525879 (18.47313404083252) + | > loss_duration: 1.8047223091125488 (1.8505027294158936) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035341978073120117 (-0.0002453327178955078) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.101516842842102 (+0.0292661190032959) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.47313404083252 (+1.5234441757202148) + | > avg_loss_duration: 1.8505027294158936 (+0.013312458992004395) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 522/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:30:04)  + + --> TIME: 2025-04-19 07:30:11 -- STEP: 19/142 -- GLOBAL_STEP: 453425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.32741928100586 (18.345902995059365) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018736588099176892 + | > current_lr_1: 0.00018736588099176892 + | > step_time: 0.3124 (0.3058442567524157) + | > loader_time: 0.0047 (0.004757554907547801) + + + --> TIME: 2025-04-19 07:30:20 -- STEP: 44/142 -- GLOBAL_STEP: 453450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.336833953857422 (18.116623141548846) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018736588099176892 + | > current_lr_1: 0.00018736588099176892 + | > step_time: 0.3402 (0.3221958117051558) + | > loader_time: 0.0061 (0.005230903625488281) + + + --> TIME: 2025-04-19 07:30:29 -- STEP: 69/142 -- GLOBAL_STEP: 453475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.928482055664062 (18.133777286695395) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018736588099176892 + | > current_lr_1: 0.00018736588099176892 + | > step_time: 0.391 (0.33637822538182355) + | > loader_time: 0.0058 (0.0055561169334079905) + + + --> TIME: 2025-04-19 07:30:39 -- STEP: 94/142 -- GLOBAL_STEP: 453500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.170490264892578 (18.049010317376318) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018736588099176892 + | > current_lr_1: 0.00018736588099176892 + | > step_time: 0.3822 (0.3488097114765898) + | > loader_time: 0.0068 (0.005859321736274881) + + + --> TIME: 2025-04-19 07:30:49 -- STEP: 119/142 -- GLOBAL_STEP: 453525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.96555519104004 (17.99419250808845) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018736588099176892 + | > current_lr_1: 0.00018736588099176892 + | > step_time: 0.4136 (0.36118775656243335) + | > loader_time: 0.0072 (0.006107324311713211) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8343398571014404 (2.8343398571014404) + | > loss_feat: nan (nan) + | > loss_mel: 18.216920852661133 (18.216920852661133) + | > loss_duration: 1.8327429294586182 (1.8327429294586182) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4317286014556885 (3.4317286014556885) + | > loss_feat: nan (nan) + | > loss_mel: 19.1367130279541 (19.1367130279541) + | > loss_duration: 1.876708745956421 (1.876708745956421) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9554452896118164 (3.1935869455337524) + | > loss_feat: nan (nan) + | > loss_mel: 17.446990966796875 (18.29185199737549) + | > loss_duration: 1.8096826076507568 (1.8431956768035889) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038933753967285156 (+0.0003591775894165039) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1935869455337524 (+0.09207010269165039) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.29185199737549 (-0.18128204345703125) + | > avg_loss_duration: 1.8431956768035889 (-0.0073070526123046875) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 523/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:31:01)  + + --> TIME: 2025-04-19 07:31:02 -- STEP: 2/142 -- GLOBAL_STEP: 453550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6664412021636963 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.2662353515625 (18.22186851501465) + | > loss_duration: 1.6804120540618896 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.2954 (0.29867422580718994) + | > loader_time: 0.004 (0.0041255950927734375) + + + --> TIME: 2025-04-19 07:31:10 -- STEP: 27/142 -- GLOBAL_STEP: 453575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.639044761657715 (18.01067518304895) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.3268 (0.3092793182090477) + | > loader_time: 0.0051 (0.004872542840463146) + + + --> TIME: 2025-04-19 07:31:18 -- STEP: 52/142 -- GLOBAL_STEP: 453600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.53982162475586 (18.03039673658517) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.3406 (0.321463493200449) + | > loader_time: 0.0066 (0.005237460136413574) + + + --> TIME: 2025-04-19 07:31:28 -- STEP: 77/142 -- GLOBAL_STEP: 453625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.64214515686035 (18.009383251140637) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.3665 (0.333990933059098) + | > loader_time: 0.0065 (0.005508723197045264) + + + --> TIME: 2025-04-19 07:31:37 -- STEP: 102/142 -- GLOBAL_STEP: 453650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.1358642578125 (17.901689968857102) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.3945 (0.34642532526278036) + | > loader_time: 0.0068 (0.005837363355300004) + + + --> TIME: 2025-04-19 07:31:48 -- STEP: 127/142 -- GLOBAL_STEP: 453675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.393545150756836 (17.782964398541782) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018734246025664495 + | > current_lr_1: 0.00018734246025664495 + | > step_time: 0.4145 (0.3575287240696705) + | > loader_time: 0.0074 (0.006079446612380619) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.101902484893799 (3.101902484893799) + | > loss_feat: nan (nan) + | > loss_mel: 17.513273239135742 (17.513273239135742) + | > loss_duration: 1.83328115940094 (1.83328115940094) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.33186936378479 (3.33186936378479) + | > loss_feat: nan (nan) + | > loss_mel: 19.160587310791016 (19.160587310791016) + | > loss_duration: 1.9133760929107666 (1.9133760929107666) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9236767292022705 (3.1277730464935303) + | > loss_feat: nan (nan) + | > loss_mel: 17.332813262939453 (18.246700286865234) + | > loss_duration: 1.807802438735962 (1.8605892658233643) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003440380096435547 (-0.00045299530029296875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1277730464935303 (-0.06581389904022217) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.246700286865234 (-0.045151710510253906) + | > avg_loss_duration: 1.8605892658233643 (+0.01739358901977539) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 524/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:31:56)  + + --> TIME: 2025-04-19 07:31:59 -- STEP: 10/142 -- GLOBAL_STEP: 453700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.735233306884766 (18.294559669494628) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.3098 (0.29935650825500487) + | > loader_time: 0.0048 (0.004252338409423828) + + + --> TIME: 2025-04-19 07:32:07 -- STEP: 35/142 -- GLOBAL_STEP: 453725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.609053134918213 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.611642837524414 (18.04586990901402) + | > loss_duration: 1.6032816171646118 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.3299 (0.3119678974151611) + | > loader_time: 0.0055 (0.004964665004185268) + + + --> TIME: 2025-04-19 07:32:16 -- STEP: 60/142 -- GLOBAL_STEP: 453750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.44670295715332 (18.00772926012676) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.3486 (0.3240222493807474) + | > loader_time: 0.0061 (0.005361815293629964) + + + --> TIME: 2025-04-19 07:32:26 -- STEP: 85/142 -- GLOBAL_STEP: 453775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6422173976898193 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.375076293945312 (17.949621290319108) + | > loss_duration: 1.6531742811203003 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.3722 (0.3366087548873004) + | > loader_time: 0.0063 (0.005698310627656823) + + + --> TIME: 2025-04-19 07:32:35 -- STEP: 110/142 -- GLOBAL_STEP: 453800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.745621681213379 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.83131980895996 (17.88986289284446) + | > loss_duration: 1.6671833992004395 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.406 (0.3482136076146906) + | > loader_time: 0.0072 (0.005972573973915793) + + + --> TIME: 2025-04-19 07:32:46 -- STEP: 135/142 -- GLOBAL_STEP: 453825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.625889778137207 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.665674209594727 (17.899471777456775) + | > loss_duration: 1.6294078826904297 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018731904244911286 + | > current_lr_1: 0.00018731904244911286 + | > step_time: 0.4285 (0.3601823065016005) + | > loader_time: 0.0073 (0.006256682784469038) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0101327896118164 (3.0101327896118164) + | > loss_feat: nan (nan) + | > loss_mel: 16.5489501953125 (16.5489501953125) + | > loss_duration: 1.8284448385238647 (1.8284448385238647) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.422794818878174 (3.422794818878174) + | > loss_feat: nan (nan) + | > loss_mel: 18.647130966186523 (18.647130966186523) + | > loss_duration: 1.917084813117981 (1.917084813117981) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.990541934967041 (3.2066683769226074) + | > loss_feat: nan (nan) + | > loss_mel: 18.828821182250977 (18.73797607421875) + | > loss_duration: 1.8022416830062866 (1.8596632480621338) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003207683563232422 (-0.000232696533203125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2066683769226074 (+0.07889533042907715) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.73797607421875 (+0.4912757873535156) + | > avg_loss_duration: 1.8596632480621338 (-0.0009260177612304688) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 525/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:32:51)  + + --> TIME: 2025-04-19 07:32:57 -- STEP: 18/142 -- GLOBAL_STEP: 453850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.23810577392578 (17.77325513627794) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872956275688067 + | > current_lr_1: 0.0001872956275688067 + | > step_time: 0.3151 (0.3044382863574558) + | > loader_time: 0.0048 (0.0045880211724175345) + + + --> TIME: 2025-04-19 07:33:05 -- STEP: 43/142 -- GLOBAL_STEP: 453875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.961889266967773 (17.89649326856746) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872956275688067 + | > current_lr_1: 0.0001872956275688067 + | > step_time: 0.3363 (0.3174622003422227) + | > loader_time: 0.0053 (0.005007705023122387) + + + --> TIME: 2025-04-19 07:33:14 -- STEP: 68/142 -- GLOBAL_STEP: 453900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.842472076416016 (18.03469959427328) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872956275688067 + | > current_lr_1: 0.0001872956275688067 + | > step_time: 0.3649 (0.3296267179881825) + | > loader_time: 0.0064 (0.005349411683924057) + + + --> TIME: 2025-04-19 07:33:24 -- STEP: 93/142 -- GLOBAL_STEP: 453925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.418468475341797 (18.031989548795956) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872956275688067 + | > current_lr_1: 0.0001872956275688067 + | > step_time: 0.3751 (0.34092207365138555) + | > loader_time: 0.0063 (0.005572113939510879) + + + --> TIME: 2025-04-19 07:33:34 -- STEP: 118/142 -- GLOBAL_STEP: 453950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.542666435241699 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.661283493041992 (18.004612154879798) + | > loss_duration: 1.6398203372955322 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872956275688067 + | > current_lr_1: 0.0001872956275688067 + | > step_time: 0.4118 (0.35170663817454195) + | > loader_time: 0.007 (0.005802558640302238) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.968050479888916 (2.968050479888916) + | > loss_feat: nan (nan) + | > loss_mel: 18.794614791870117 (18.794614791870117) + | > loss_duration: 1.8315799236297607 (1.8315799236297607) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.247915744781494 (3.247915744781494) + | > loss_feat: nan (nan) + | > loss_mel: 16.406030654907227 (16.406030654907227) + | > loss_duration: 1.9037467241287231 (1.9037467241287231) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7945749759674072 (3.0212453603744507) + | > loss_feat: nan (nan) + | > loss_mel: 19.25421905517578 (17.830124855041504) + | > loss_duration: 1.8119227886199951 (1.8578347563743591) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031777620315551758 (-2.9921531677246094e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0212453603744507 (-0.18542301654815674) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.830124855041504 (-0.9078512191772461) + | > avg_loss_duration: 1.8578347563743591 (-0.0018284916877746582) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 526/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:33:45)  + + --> TIME: 2025-04-19 07:33:46 -- STEP: 1/142 -- GLOBAL_STEP: 453975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.685640335083008 (17.685640335083008) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.3015 (0.3014659881591797) + | > loader_time: 0.0045 (0.004477977752685547) + + + --> TIME: 2025-04-19 07:33:54 -- STEP: 26/142 -- GLOBAL_STEP: 454000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.70949363708496 (17.88936607654278) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.3225 (0.307446002960205) + | > loader_time: 0.0054 (0.0050553083419799805) + + + --> TIME: 2025-04-19 07:34:03 -- STEP: 51/142 -- GLOBAL_STEP: 454025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.698795318603516 (17.826258883756747) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.344 (0.3201124153885187) + | > loader_time: 0.0059 (0.005492916294172698) + + + --> TIME: 2025-04-19 07:34:12 -- STEP: 76/142 -- GLOBAL_STEP: 454050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.915374755859375 (17.803435112300672) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.3677 (0.3327677689100567) + | > loader_time: 0.0063 (0.005771611866198088) + + + --> TIME: 2025-04-19 07:34:22 -- STEP: 101/142 -- GLOBAL_STEP: 454075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.94355010986328 (17.8966346778492) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.3887 (0.34351479180968636) + | > loader_time: 0.0068 (0.006009361531474803) + + + --> TIME: 2025-04-19 07:34:32 -- STEP: 126/142 -- GLOBAL_STEP: 454100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.82640266418457 (17.831901081024668) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872722156153606 + | > current_lr_1: 0.0001872722156153606 + | > step_time: 0.4103 (0.35473309244428364) + | > loader_time: 0.0073 (0.006230844391716851) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0014138221740723 (3.0014138221740723) + | > loss_feat: nan (nan) + | > loss_mel: 17.79623794555664 (17.79623794555664) + | > loss_duration: 1.8320388793945312 (1.8320388793945312) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.222177028656006 (3.222177028656006) + | > loss_feat: nan (nan) + | > loss_mel: 19.687204360961914 (19.687204360961914) + | > loss_duration: 1.9091434478759766 (1.9091434478759766) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9550366401672363 (3.088606834411621) + | > loss_feat: nan (nan) + | > loss_mel: 20.199329376220703 (19.94326686859131) + | > loss_duration: 1.7906757593154907 (1.8499096035957336) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003329753875732422 (+0.0001519918441772461) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.088606834411621 (+0.06736147403717041) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.94326686859131 (+2.1131420135498047) + | > avg_loss_duration: 1.8499096035957336 (-0.007925152778625488) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 527/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:34:40)  + + --> TIME: 2025-04-19 07:34:44 -- STEP: 9/142 -- GLOBAL_STEP: 454125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.82651138305664 (17.940291510687935) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.2993 (0.32891689406500924) + | > loader_time: 0.0048 (0.004528045654296875) + + + --> TIME: 2025-04-19 07:34:52 -- STEP: 34/142 -- GLOBAL_STEP: 454150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6095621585845947 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.23824119567871 (18.011296047883874) + | > loss_duration: 1.6283737421035767 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.3292 (0.32150767130010266) + | > loader_time: 0.0059 (0.00516657969530891) + + + --> TIME: 2025-04-19 07:35:01 -- STEP: 59/142 -- GLOBAL_STEP: 454175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.140291213989258 (18.054115844985184) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.3533 (0.3295889345266052) + | > loader_time: 0.0061 (0.005494566287024547) + + + --> TIME: 2025-04-19 07:35:10 -- STEP: 84/142 -- GLOBAL_STEP: 454200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.495844841003418 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.899635314941406 (18.054146017347065) + | > loss_duration: 1.7064015865325928 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.3713 (0.34014875264394845) + | > loader_time: 0.0071 (0.0057653387387593584) + + + --> TIME: 2025-04-19 07:35:20 -- STEP: 109/142 -- GLOBAL_STEP: 454225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.831571578979492 (18.027803596006613) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.3873 (0.35008248276666765) + | > loader_time: 0.007 (0.0060329437255859375) + + + --> TIME: 2025-04-19 07:35:30 -- STEP: 134/142 -- GLOBAL_STEP: 454250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.551454544067383 (17.965108472909492) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018724880658840866 + | > current_lr_1: 0.00018724880658840866 + | > step_time: 0.4134 (0.3615484913783287) + | > loader_time: 0.0078 (0.006268191693434075) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7775189876556396 (2.7775189876556396) + | > loss_feat: nan (nan) + | > loss_mel: 19.107084274291992 (19.107084274291992) + | > loss_duration: 1.8465654850006104 (1.8465654850006104) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2757813930511475 (3.2757813930511475) + | > loss_feat: nan (nan) + | > loss_mel: 18.068206787109375 (18.068206787109375) + | > loss_duration: 1.9014146327972412 (1.9014146327972412) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.794189453125 (3.0349854230880737) + | > loss_feat: nan (nan) + | > loss_mel: 18.458812713623047 (18.26350975036621) + | > loss_duration: 1.8038692474365234 (1.8526419401168823) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003240227699279785 (-8.952617645263672e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0349854230880737 (-0.05362141132354736) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.26350975036621 (-1.6797571182250977) + | > avg_loss_duration: 1.8526419401168823 (+0.0027323365211486816) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 528/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:35:35)  + + --> TIME: 2025-04-19 07:35:41 -- STEP: 17/142 -- GLOBAL_STEP: 454275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.511542558670044 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.68012237548828 (18.124132492963007) + | > loss_duration: 1.5929737091064453 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872254004875851 + | > current_lr_1: 0.0001872254004875851 + | > step_time: 0.3156 (0.3090820592992446) + | > loader_time: 0.0052 (0.004750223720774931) + + + --> TIME: 2025-04-19 07:35:49 -- STEP: 42/142 -- GLOBAL_STEP: 454300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.692004442214966 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.46112632751465 (18.043475923084078) + | > loss_duration: 1.6545805931091309 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872254004875851 + | > current_lr_1: 0.0001872254004875851 + | > step_time: 0.3357 (0.319883284114656) + | > loader_time: 0.0058 (0.005165168217250279) + + + --> TIME: 2025-04-19 07:35:58 -- STEP: 67/142 -- GLOBAL_STEP: 454325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.000911712646484 (18.066502841550914) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872254004875851 + | > current_lr_1: 0.0001872254004875851 + | > step_time: 0.359 (0.3315336455160113) + | > loader_time: 0.0062 (0.005463493404103748) + + + --> TIME: 2025-04-19 07:36:08 -- STEP: 92/142 -- GLOBAL_STEP: 454350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.363374710083008 (17.96660154798757) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872254004875851 + | > current_lr_1: 0.0001872254004875851 + | > step_time: 0.3855 (0.3427225066267926) + | > loader_time: 0.0062 (0.00575846692790156) + + + --> TIME: 2025-04-19 07:36:18 -- STEP: 117/142 -- GLOBAL_STEP: 454375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.26477813720703 (17.89473218999358) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001872254004875851 + | > current_lr_1: 0.0001872254004875851 + | > step_time: 0.4078 (0.3530737355224088) + | > loader_time: 0.0074 (0.006024093709440314) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.839970350265503 (2.839970350265503) + | > loss_feat: nan (nan) + | > loss_mel: 19.430068969726562 (19.430068969726562) + | > loss_duration: 1.822234869003296 (1.822234869003296) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.208524227142334 (3.208524227142334) + | > loss_feat: nan (nan) + | > loss_mel: 19.357912063598633 (19.357912063598633) + | > loss_duration: 1.8887724876403809 (1.8887724876403809) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.683856725692749 (2.9461904764175415) + | > loss_feat: nan (nan) + | > loss_mel: 19.511014938354492 (19.434463500976562) + | > loss_duration: 1.7924551963806152 (1.840613842010498) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003414630889892578 (+0.00017440319061279297) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9461904764175415 (-0.08879494667053223) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.434463500976562 (+1.1709537506103516) + | > avg_loss_duration: 1.840613842010498 (-0.012028098106384277) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 529/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:36:30)  + + --> TIME: 2025-04-19 07:36:31 -- STEP: 0/142 -- GLOBAL_STEP: 454400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.167034149169922 (18.167034149169922) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.321 (0.3210432529449463) + | > loader_time: 0.3194 (0.3193976879119873) + + + --> TIME: 2025-04-19 07:36:39 -- STEP: 25/142 -- GLOBAL_STEP: 454425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.53290057182312 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.71345329284668 (17.769009628295898) + | > loss_duration: 1.6696211099624634 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.3264 (0.30996291160583495) + | > loader_time: 0.006 (0.004922246932983399) + + + --> TIME: 2025-04-19 07:36:47 -- STEP: 50/142 -- GLOBAL_STEP: 454450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.625795364379883 (17.846915340423575) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.3435 (0.32668460845947267) + | > loader_time: 0.0058 (0.005290107727050781) + + + --> TIME: 2025-04-19 07:36:57 -- STEP: 75/142 -- GLOBAL_STEP: 454475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.709796905517578 (17.756423149108887) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.3727 (0.33682534535725916) + | > loader_time: 0.0064 (0.005548445383707684) + + + --> TIME: 2025-04-19 07:37:06 -- STEP: 100/142 -- GLOBAL_STEP: 454500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.18345832824707 (17.798218784332278) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.3896 (0.3466882848739624) + | > loader_time: 0.007 (0.00578697204589844) + + + --> TIME: 2025-04-19 07:37:16 -- STEP: 125/142 -- GLOBAL_STEP: 454525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.771724700927734 (17.80988813781738) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018720199731252416 + | > current_lr_1: 0.00018720199731252416 + | > step_time: 0.4109 (0.35768172645568846) + | > loader_time: 0.007 (0.006032775878906251) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7434394359588623 (2.7434394359588623) + | > loss_feat: nan (nan) + | > loss_mel: 18.91934585571289 (18.91934585571289) + | > loss_duration: 1.8225637674331665 (1.8225637674331665) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4466845989227295 (3.4466845989227295) + | > loss_feat: nan (nan) + | > loss_mel: 18.86908721923828 (18.86908721923828) + | > loss_duration: 1.897829532623291 (1.897829532623291) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9368553161621094 (3.1917699575424194) + | > loss_feat: nan (nan) + | > loss_mel: 18.599124908447266 (18.734106063842773) + | > loss_duration: 1.8030288219451904 (1.8504291772842407) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033655166625976562 (-4.9114227294921875e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1917699575424194 (+0.24557948112487793) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.734106063842773 (-0.7003574371337891) + | > avg_loss_duration: 1.8504291772842407 (+0.009815335273742676) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 530/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:37:25)  + + --> TIME: 2025-04-19 07:37:28 -- STEP: 8/142 -- GLOBAL_STEP: 454550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5812575817108154 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.50442123413086 (17.993749856948853) + | > loss_duration: 1.644829511642456 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.299 (0.29877692461013794) + | > loader_time: 0.0044 (0.0042556822299957275) + + + --> TIME: 2025-04-19 07:37:36 -- STEP: 33/142 -- GLOBAL_STEP: 454575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.933168411254883 (18.120787476048324) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.3299 (0.3118911150730017) + | > loader_time: 0.005 (0.005029071461070667) + + + --> TIME: 2025-04-19 07:37:45 -- STEP: 58/142 -- GLOBAL_STEP: 454600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.001583099365234 (17.987303470743118) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.3578 (0.324623103799491) + | > loader_time: 0.0067 (0.0053662554971103) + + + --> TIME: 2025-04-19 07:37:55 -- STEP: 83/142 -- GLOBAL_STEP: 454625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.647373676300049 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.67879295349121 (18.00904962240932) + | > loss_duration: 1.695122480392456 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.3728 (0.33837863910629074) + | > loader_time: 0.0066 (0.005676950316831291) + + + --> TIME: 2025-04-19 07:38:04 -- STEP: 108/142 -- GLOBAL_STEP: 454650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.720661163330078 (17.91709751553006) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.3935 (0.34861344761318613) + | > loader_time: 0.0073 (0.005952404605017768) + + + --> TIME: 2025-04-19 07:38:15 -- STEP: 133/142 -- GLOBAL_STEP: 454675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.715726852416992 (17.87827308195875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001871785970628601 + | > current_lr_1: 0.0001871785970628601 + | > step_time: 0.4145 (0.3602301877244073) + | > loader_time: 0.0076 (0.00622902418437757) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.930955171585083 (2.930955171585083) + | > loss_feat: nan (nan) + | > loss_mel: 18.054428100585938 (18.054428100585938) + | > loss_duration: 1.8315999507904053 (1.8315999507904053) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4065377712249756 (3.4065377712249756) + | > loss_feat: nan (nan) + | > loss_mel: 16.807931900024414 (16.807931900024414) + | > loss_duration: 1.8834320306777954 (1.8834320306777954) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.723374128341675 (3.064955949783325) + | > loss_feat: nan (nan) + | > loss_mel: 20.011320114135742 (18.409626007080078) + | > loss_duration: 1.779068112373352 (1.8312500715255737) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034787654876708984 (+0.00011324882507324219) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.064955949783325 (-0.12681400775909424) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.409626007080078 (-0.3244800567626953) + | > avg_loss_duration: 1.8312500715255737 (-0.019179105758666992) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 531/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:38:20)  + + --> TIME: 2025-04-19 07:38:26 -- STEP: 16/142 -- GLOBAL_STEP: 454700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.616304397583008 (18.176480174064636) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.3133 (0.30293284356594086) + | > loader_time: 0.0054 (0.004792630672454834) + + + --> TIME: 2025-04-19 07:38:34 -- STEP: 41/142 -- GLOBAL_STEP: 454725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5521507263183594 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.17082977294922 (18.070679850694617) + | > loss_duration: 1.6084529161453247 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.344 (0.3198553352821164) + | > loader_time: 0.0059 (0.005264636946887505) + + + --> TIME: 2025-04-19 07:38:43 -- STEP: 66/142 -- GLOBAL_STEP: 454750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 14.306609153747559 (18.00085206465288) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.3578 (0.3338686372294571) + | > loader_time: 0.0067 (0.005523399873213334) + + + --> TIME: 2025-04-19 07:38:53 -- STEP: 91/142 -- GLOBAL_STEP: 454775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.78737449645996 (17.930262743771735) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.377 (0.3438292545276684) + | > loader_time: 0.0063 (0.005791721763191642) + + + --> TIME: 2025-04-19 07:39:03 -- STEP: 116/142 -- GLOBAL_STEP: 454800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.396404266357422 (17.978635952390473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.4045 (0.3539317949064847) + | > loader_time: 0.0071 (0.00603065614042611) + + + --> TIME: 2025-04-19 07:39:13 -- STEP: 141/142 -- GLOBAL_STEP: 454825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.215797424316406 (17.886995369660937) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018715519973822723 + | > current_lr_1: 0.00018715519973822723 + | > step_time: 0.1807 (0.36381706785648427) + | > loader_time: 0.003 (0.006244576569144608) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.906740188598633 (2.906740188598633) + | > loss_feat: nan (nan) + | > loss_mel: 14.566884994506836 (14.566884994506836) + | > loss_duration: 1.8143689632415771 (1.8143689632415771) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.071993589401245 (3.071993589401245) + | > loss_feat: nan (nan) + | > loss_mel: 16.934520721435547 (16.934520721435547) + | > loss_duration: 1.8933390378952026 (1.8933390378952026) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.764284372329712 (2.9181389808654785) + | > loss_feat: nan (nan) + | > loss_mel: 18.45473861694336 (17.694629669189453) + | > loss_duration: 1.7960290908813477 (1.8446840643882751) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033578872680664062 (-0.00012087821960449219) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9181389808654785 (-0.14681696891784668) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.694629669189453 (-0.714996337890625) + | > avg_loss_duration: 1.8446840643882751 (+0.013433992862701416) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 532/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:39:15)  + + --> TIME: 2025-04-19 07:39:24 -- STEP: 24/142 -- GLOBAL_STEP: 454850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.544684886932373 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.855076789855957 (17.944216052691143) + | > loss_duration: 1.6157753467559814 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018713180533825994 + | > current_lr_1: 0.00018713180533825994 + | > step_time: 0.3194 (0.3094051579634349) + | > loader_time: 0.0055 (0.004871845245361328) + + + --> TIME: 2025-04-19 07:39:32 -- STEP: 49/142 -- GLOBAL_STEP: 454875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3883678913116455 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.56169891357422 (17.954669563137756) + | > loss_duration: 1.6147198677062988 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018713180533825994 + | > current_lr_1: 0.00018713180533825994 + | > step_time: 0.3455 (0.32143752428950095) + | > loader_time: 0.0063 (0.005336299234507035) + + + --> TIME: 2025-04-19 07:39:41 -- STEP: 74/142 -- GLOBAL_STEP: 454900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.456799268722534 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.46916961669922 (17.887901319039837) + | > loss_duration: 1.6495561599731445 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018713180533825994 + | > current_lr_1: 0.00018713180533825994 + | > step_time: 0.3618 (0.3327988933872532) + | > loader_time: 0.0064 (0.0056199157560193865) + + + --> TIME: 2025-04-19 07:39:51 -- STEP: 99/142 -- GLOBAL_STEP: 454925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7020881175994873 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.22637939453125 (17.893867579373453) + | > loss_duration: 1.6856906414031982 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018713180533825994 + | > current_lr_1: 0.00018713180533825994 + | > step_time: 0.3803 (0.343390500906742) + | > loader_time: 0.0065 (0.005859189563327365) + + + --> TIME: 2025-04-19 07:40:01 -- STEP: 124/142 -- GLOBAL_STEP: 454950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.223957061767578 (17.874102915486983) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018713180533825994 + | > current_lr_1: 0.00018713180533825994 + | > step_time: 0.4158 (0.3554280919413413) + | > loader_time: 0.0074 (0.0061525164111968) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1097605228424072 (3.1097605228424072) + | > loss_feat: nan (nan) + | > loss_mel: 16.735551834106445 (16.735551834106445) + | > loss_duration: 1.8339364528656006 (1.8339364528656006) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2639577388763428 (3.2639577388763428) + | > loss_feat: nan (nan) + | > loss_mel: 18.638057708740234 (18.638057708740234) + | > loss_duration: 1.9071733951568604 (1.9071733951568604) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9607653617858887 (3.1123615503311157) + | > loss_feat: nan (nan) + | > loss_mel: 17.84836196899414 (18.243209838867188) + | > loss_duration: 1.796684741973877 (1.8519290685653687) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003682255744934082 (+0.0003243684768676758) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1123615503311157 (+0.1942225694656372) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.243209838867188 (+0.5485801696777344) + | > avg_loss_duration: 1.8519290685653687 (+0.007245004177093506) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 533/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:40:10)  + + --> TIME: 2025-04-19 07:40:13 -- STEP: 7/142 -- GLOBAL_STEP: 454975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.19269371032715 (18.452362605503627) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.2972 (0.2991487298692976) + | > loader_time: 0.0044 (0.004576172147478376) + + + --> TIME: 2025-04-19 07:40:21 -- STEP: 32/142 -- GLOBAL_STEP: 455000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.053054809570312 (18.061702251434323) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.3379 (0.3144841939210892) + | > loader_time: 0.0054 (0.005139902234077454) + + + --> TIME: 2025-04-19 07:40:30 -- STEP: 57/142 -- GLOBAL_STEP: 455025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5216777324676514 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.671857833862305 (18.065562666508185) + | > loss_duration: 1.693098783493042 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.3714 (0.3290671131067107) + | > loader_time: 0.0062 (0.005479754063121061) + + + --> TIME: 2025-04-19 07:40:40 -- STEP: 82/142 -- GLOBAL_STEP: 455050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.36374282836914 (17.978805158196423) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.379 (0.3440553037131704) + | > loader_time: 0.0067 (0.005780432282424556) + + + --> TIME: 2025-04-19 07:40:50 -- STEP: 107/142 -- GLOBAL_STEP: 455075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.110872268676758 (17.99931837242341) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.3908 (0.35633976660042166) + | > loader_time: 0.0066 (0.006025349982431002) + + + --> TIME: 2025-04-19 07:41:00 -- STEP: 132/142 -- GLOBAL_STEP: 455100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.839582443237305 (17.98811127200272) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018710841386259265 + | > current_lr_1: 0.00018710841386259265 + | > step_time: 0.4229 (0.36655676545518817) + | > loader_time: 0.0079 (0.0062417117032137785) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.879973888397217 (2.879973888397217) + | > loss_feat: nan (nan) + | > loss_mel: 16.399364471435547 (16.399364471435547) + | > loss_duration: 1.8466486930847168 (1.8466486930847168) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.462219715118408 (3.462219715118408) + | > loss_feat: nan (nan) + | > loss_mel: 19.297279357910156 (19.297279357910156) + | > loss_duration: 1.9074820280075073 (1.9074820280075073) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.91131591796875 (3.186767816543579) + | > loss_feat: nan (nan) + | > loss_mel: 19.456260681152344 (19.37677001953125) + | > loss_duration: 1.7907946109771729 (1.84913831949234) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003712177276611328 (+2.9921531677246094e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.186767816543579 (+0.07440626621246338) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.37677001953125 (+1.1335601806640625) + | > avg_loss_duration: 1.84913831949234 (-0.0027907490730285645) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 534/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:41:06)  + + --> TIME: 2025-04-19 07:41:11 -- STEP: 15/142 -- GLOBAL_STEP: 455125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.521475076675415 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.38688087463379 (18.345360056559244) + | > loss_duration: 1.624511480331421 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.3087 (0.30390338897705077) + | > loader_time: 0.0053 (0.004664103190104167) + + + --> TIME: 2025-04-19 07:41:20 -- STEP: 40/142 -- GLOBAL_STEP: 455150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.316349029541016 (18.13530511856079) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.3358 (0.316341382265091) + | > loader_time: 0.0061 (0.005160897970199586) + + + --> TIME: 2025-04-19 07:41:28 -- STEP: 65/142 -- GLOBAL_STEP: 455175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.014802932739258 (18.1440895667443) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.3655 (0.3280830383300781) + | > loader_time: 0.0061 (0.005534832294170673) + + + --> TIME: 2025-04-19 07:41:38 -- STEP: 90/142 -- GLOBAL_STEP: 455200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.216476440429688 (18.092605061001255) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.3769 (0.339225016699897) + | > loader_time: 0.0069 (0.005862871805826823) + + + --> TIME: 2025-04-19 07:41:48 -- STEP: 115/142 -- GLOBAL_STEP: 455225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.68144416809082 (17.98474198216979) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.3989 (0.3497064673382304) + | > loader_time: 0.0073 (0.006075436135996943) + + + --> TIME: 2025-04-19 07:41:58 -- STEP: 140/142 -- GLOBAL_STEP: 455250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.833505630493164 (17.92185826301576) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001870850253108598 + | > current_lr_1: 0.0001870850253108598 + | > step_time: 0.4375 (0.3622339623314995) + | > loader_time: 0.0073 (0.006318088940211705) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.971367359161377 (2.971367359161377) + | > loss_feat: nan (nan) + | > loss_mel: 20.323392868041992 (20.323392868041992) + | > loss_duration: 1.8524835109710693 (1.8524835109710693) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1629724502563477 (3.1629724502563477) + | > loss_feat: nan (nan) + | > loss_mel: 20.13495635986328 (20.13495635986328) + | > loss_duration: 1.9073415994644165 (1.9073415994644165) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.927494764328003 (3.0452336072921753) + | > loss_feat: nan (nan) + | > loss_mel: 18.3221435546875 (19.22854995727539) + | > loss_duration: 1.7820208072662354 (1.844681203365326) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003445863723754883 (-0.0002663135528564453) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0452336072921753 (-0.1415342092514038) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.22854995727539 (-0.14822006225585938) + | > avg_loss_duration: 1.844681203365326 (-0.00445711612701416) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 535/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:42:01)  + + --> TIME: 2025-04-19 07:42:09 -- STEP: 23/142 -- GLOBAL_STEP: 455275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5307986736297607 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.29600715637207 (18.226663921190344) + | > loss_duration: 1.6166754961013794 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018706163968269592 + | > current_lr_1: 0.00018706163968269592 + | > step_time: 0.3278 (0.30755243094071094) + | > loader_time: 0.005 (0.004872643429300059) + + + --> TIME: 2025-04-19 07:42:17 -- STEP: 48/142 -- GLOBAL_STEP: 455300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.447954177856445 (18.073467532793675) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018706163968269592 + | > current_lr_1: 0.00018706163968269592 + | > step_time: 0.3469 (0.3244222849607468) + | > loader_time: 0.0062 (0.005492781599362691) + + + --> TIME: 2025-04-19 07:42:27 -- STEP: 73/142 -- GLOBAL_STEP: 455325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.370800018310547 (18.097094444379415) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018706163968269592 + | > current_lr_1: 0.00018706163968269592 + | > step_time: 0.3797 (0.33863419702608294) + | > loader_time: 0.0062 (0.005837146550008695) + + + --> TIME: 2025-04-19 07:42:37 -- STEP: 98/142 -- GLOBAL_STEP: 455350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.664398193359375 (17.987218282660656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018706163968269592 + | > current_lr_1: 0.00018706163968269592 + | > step_time: 0.3967 (0.35067697933741976) + | > loader_time: 0.0069 (0.006077104685257892) + + + --> TIME: 2025-04-19 07:42:47 -- STEP: 123/142 -- GLOBAL_STEP: 455375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6670546531677246 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.261760711669922 (18.030733224822246) + | > loss_duration: 1.6452903747558594 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018706163968269592 + | > current_lr_1: 0.00018706163968269592 + | > step_time: 0.4243 (0.3632589975992839) + | > loader_time: 0.0072 (0.0063510503226179415) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8783726692199707 (2.8783726692199707) + | > loss_feat: nan (nan) + | > loss_mel: 19.46800422668457 (19.46800422668457) + | > loss_duration: 1.8328306674957275 (1.8328306674957275) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4085006713867188 (3.4085006713867188) + | > loss_feat: nan (nan) + | > loss_mel: 16.81017303466797 (16.81017303466797) + | > loss_duration: 1.9143955707550049 (1.9143955707550049) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9523184299468994 (3.180409550666809) + | > loss_feat: nan (nan) + | > loss_mel: 20.044710159301758 (18.427441596984863) + | > loss_duration: 1.7980042695999146 (1.8561999201774597) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003722667694091797 (+0.00027680397033691406) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.180409550666809 (+0.1351759433746338) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.427441596984863 (-0.8011083602905273) + | > avg_loss_duration: 1.8561999201774597 (+0.011518716812133789) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 536/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:42:57)  + + --> TIME: 2025-04-19 07:42:59 -- STEP: 6/142 -- GLOBAL_STEP: 455400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.688405990600586 (18.32308578491211) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.3039 (0.29928914705912274) + | > loader_time: 0.0041 (0.00428934892018636) + + + --> TIME: 2025-04-19 07:43:07 -- STEP: 31/142 -- GLOBAL_STEP: 455425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.733327865600586 (17.75782092925041) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.3269 (0.31118985145322736) + | > loader_time: 0.0053 (0.0050109201861966044) + + + --> TIME: 2025-04-19 07:43:16 -- STEP: 56/142 -- GLOBAL_STEP: 455450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.215330123901367 (17.847928966794704) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.3523 (0.3232267200946808) + | > loader_time: 0.0058 (0.0053700847285134455) + + + --> TIME: 2025-04-19 07:43:25 -- STEP: 81/142 -- GLOBAL_STEP: 455475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.710330963134766 (17.765851479989518) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.3685 (0.33543418660575963) + | > loader_time: 0.0069 (0.005640865843973042) + + + --> TIME: 2025-04-19 07:43:35 -- STEP: 106/142 -- GLOBAL_STEP: 455500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.704713821411133 (17.791560604887188) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.3842 (0.34585785640860506) + | > loader_time: 0.0068 (0.005900796854271079) + + + --> TIME: 2025-04-19 07:43:46 -- STEP: 131/142 -- GLOBAL_STEP: 455525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.29364776611328 (17.768511124239623) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018703825697773558 + | > current_lr_1: 0.00018703825697773558 + | > step_time: 0.4226 (0.35769482605329905) + | > loader_time: 0.0075 (0.006190487446675774) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9311106204986572 (2.9311106204986572) + | > loss_feat: nan (nan) + | > loss_mel: 18.242904663085938 (18.242904663085938) + | > loss_duration: 1.8486799001693726 (1.8486799001693726) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.202934980392456 (3.202934980392456) + | > loss_feat: nan (nan) + | > loss_mel: 17.96932029724121 (17.96932029724121) + | > loss_duration: 1.9106945991516113 (1.9106945991516113) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7346153259277344 (2.968775153160095) + | > loss_feat: nan (nan) + | > loss_mel: 16.92011260986328 (17.444716453552246) + | > loss_duration: 1.791586995124817 (1.8511407971382141) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003307819366455078 (-0.00041484832763671875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.968775153160095 (-0.21163439750671387) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.444716453552246 (-0.9827251434326172) + | > avg_loss_duration: 1.8511407971382141 (-0.0050591230392456055) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 537/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:43:52)  + + --> TIME: 2025-04-19 07:43:57 -- STEP: 14/142 -- GLOBAL_STEP: 455550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5896506309509277 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.114225387573242 (17.63006857463292) + | > loss_duration: 1.6210585832595825 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.3055 (0.30139379841940744) + | > loader_time: 0.0047 (0.004724213055201939) + + + --> TIME: 2025-04-19 07:44:05 -- STEP: 39/142 -- GLOBAL_STEP: 455575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.971982955932617 (17.787224671779533) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.3461 (0.32114576682066304) + | > loader_time: 0.0056 (0.005240801053169446) + + + --> TIME: 2025-04-19 07:44:14 -- STEP: 64/142 -- GLOBAL_STEP: 455600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.25153160095215 (17.88118414580822) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.3679 (0.33477367833256716) + | > loader_time: 0.0065 (0.005613300949335098) + + + --> TIME: 2025-04-19 07:44:24 -- STEP: 89/142 -- GLOBAL_STEP: 455625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.060062408447266 (17.885973180277958) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.3893 (0.34800045677785124) + | > loader_time: 0.0076 (0.005914988142720768) + + + --> TIME: 2025-04-19 07:44:34 -- STEP: 114/142 -- GLOBAL_STEP: 455650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.174610137939453 (17.86349586854901) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.4087 (0.3598206733402453) + | > loader_time: 0.0073 (0.006168365478515624) + + + --> TIME: 2025-04-19 07:44:45 -- STEP: 139/142 -- GLOBAL_STEP: 455675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.041778564453125 (17.833146726484774) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018701487719561334 + | > current_lr_1: 0.00018701487719561334 + | > step_time: 0.4409 (0.3723473514584328) + | > loader_time: 0.0078 (0.0064158199502409785) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9036941528320312 (2.9036941528320312) + | > loss_feat: nan (nan) + | > loss_mel: 18.742918014526367 (18.742918014526367) + | > loss_duration: 1.844522476196289 (1.844522476196289) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.298460006713867 (3.298460006713867) + | > loss_feat: nan (nan) + | > loss_mel: 19.062833786010742 (19.062833786010742) + | > loss_duration: 1.9034236669540405 (1.9034236669540405) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6525988578796387 (2.975529432296753) + | > loss_feat: nan (nan) + | > loss_mel: 16.958023071289062 (18.010428428649902) + | > loss_duration: 1.7884042263031006 (1.8459139466285706) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034482479095458984 (+0.0001404285430908203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.975529432296753 (+0.006754279136657715) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.010428428649902 (+0.5657119750976562) + | > avg_loss_duration: 1.8459139466285706 (-0.005226850509643555) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 538/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:44:48)  + + --> TIME: 2025-04-19 07:44:56 -- STEP: 22/142 -- GLOBAL_STEP: 455700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.41300392150879 (18.2701169794256) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869915003359639 + | > current_lr_1: 0.0001869915003359639 + | > step_time: 0.3172 (0.31016846136613324) + | > loader_time: 0.0055 (0.004826415668834339) + + + --> TIME: 2025-04-19 07:45:04 -- STEP: 47/142 -- GLOBAL_STEP: 455725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.37405776977539 (18.200795072190306) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869915003359639 + | > current_lr_1: 0.0001869915003359639 + | > step_time: 0.3393 (0.3216929334275266) + | > loader_time: 0.0058 (0.005255536830171627) + + + --> TIME: 2025-04-19 07:45:13 -- STEP: 72/142 -- GLOBAL_STEP: 455750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.382997512817383 (18.06635492377811) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869915003359639 + | > current_lr_1: 0.0001869915003359639 + | > step_time: 0.3691 (0.33275792996088666) + | > loader_time: 0.0059 (0.005590236849255034) + + + --> TIME: 2025-04-19 07:45:23 -- STEP: 97/142 -- GLOBAL_STEP: 455775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.419832229614258 (18.014267016931907) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869915003359639 + | > current_lr_1: 0.0001869915003359639 + | > step_time: 0.3829 (0.34318987610413854) + | > loader_time: 0.0065 (0.005824288142096138) + + + --> TIME: 2025-04-19 07:45:33 -- STEP: 122/142 -- GLOBAL_STEP: 455800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.234704971313477 (18.01609757689178) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869915003359639 + | > current_lr_1: 0.0001869915003359639 + | > step_time: 0.4281 (0.35429617615996817) + | > loader_time: 0.0077 (0.0060886828625788475) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8688466548919678 (2.8688466548919678) + | > loss_feat: nan (nan) + | > loss_mel: 18.34687614440918 (18.34687614440918) + | > loss_duration: 1.8546963930130005 (1.8546963930130005) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2798640727996826 (3.2798640727996826) + | > loss_feat: nan (nan) + | > loss_mel: 17.391796112060547 (17.391796112060547) + | > loss_duration: 1.907273292541504 (1.907273292541504) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9332151412963867 (3.1065396070480347) + | > loss_feat: nan (nan) + | > loss_mel: 17.596542358398438 (17.494169235229492) + | > loss_duration: 1.807948350906372 (1.857610821723938) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00390625 (+0.00045800209045410156) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1065396070480347 (+0.13101017475128174) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.494169235229492 (-0.5162591934204102) + | > avg_loss_duration: 1.857610821723938 (+0.011696875095367432) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 539/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:45:43)  + + --> TIME: 2025-04-19 07:45:45 -- STEP: 5/142 -- GLOBAL_STEP: 455825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.677532196044922 (18.020903396606446) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.2986 (0.29754462242126467) + | > loader_time: 0.0043 (0.004437446594238281) + + + --> TIME: 2025-04-19 07:45:53 -- STEP: 30/142 -- GLOBAL_STEP: 455850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.070520401000977 (18.133572896321613) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.3284 (0.3104989926020304) + | > loader_time: 0.0055 (0.005083711942036947) + + + --> TIME: 2025-04-19 07:46:02 -- STEP: 55/142 -- GLOBAL_STEP: 455875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.60245132446289 (18.104481714422054) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.3558 (0.32666469487276945) + | > loader_time: 0.0062 (0.0054573969407515095) + + + --> TIME: 2025-04-19 07:46:12 -- STEP: 80/142 -- GLOBAL_STEP: 455900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.638401508331299 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.737915992736816 (18.00477616786957) + | > loss_duration: 1.6611601114273071 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.4025 (0.34268070459365846) + | > loader_time: 0.0064 (0.005829989910125733) + + + --> TIME: 2025-04-19 07:46:22 -- STEP: 105/142 -- GLOBAL_STEP: 455925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6677706241607666 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.915061950683594 (18.01005730401902) + | > loss_duration: 1.6963746547698975 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.4078 (0.35468317667643245) + | > loader_time: 0.0071 (0.0060983294532412574) + + + --> TIME: 2025-04-19 07:46:32 -- STEP: 130/142 -- GLOBAL_STEP: 455950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.87594223022461 (17.949726134080155) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869681263984219 + | > current_lr_1: 0.0001869681263984219 + | > step_time: 0.4331 (0.3672067532172571) + | > loader_time: 0.0077 (0.006377790524409368) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.181057929992676 (3.181057929992676) + | > loss_feat: nan (nan) + | > loss_mel: 19.932756423950195 (19.932756423950195) + | > loss_duration: 1.8220455646514893 (1.8220455646514893) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3195888996124268 (3.3195888996124268) + | > loss_feat: nan (nan) + | > loss_mel: 17.623058319091797 (17.623058319091797) + | > loss_duration: 1.8960349559783936 (1.8960349559783936) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.87119722366333 (3.0953930616378784) + | > loss_feat: nan (nan) + | > loss_mel: 19.429386138916016 (18.526222229003906) + | > loss_duration: 1.7872889041900635 (1.8416619300842285) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036242008209228516 (-0.00028204917907714844) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0953930616378784 (-0.01114654541015625) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.526222229003906 (+1.032052993774414) + | > avg_loss_duration: 1.8416619300842285 (-0.015948891639709473) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 540/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:46:39)  + + --> TIME: 2025-04-19 07:46:44 -- STEP: 13/142 -- GLOBAL_STEP: 455975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.4124698638916 (17.884712072519157) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.306 (0.30182968653165376) + | > loader_time: 0.005 (0.004639423810518705) + + + --> TIME: 2025-04-19 07:46:52 -- STEP: 38/142 -- GLOBAL_STEP: 456000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.968194961547852 (17.846709552564118) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.3274 (0.3146656061473645) + | > loader_time: 0.0056 (0.005203447843852796) + + + --> TIME: 2025-04-19 07:47:01 -- STEP: 63/142 -- GLOBAL_STEP: 456025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6890065670013428 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.512836456298828 (17.822693234398244) + | > loss_duration: 1.6524438858032227 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.3625 (0.32655394644964303) + | > loader_time: 0.0059 (0.0054531967829144195) + + + --> TIME: 2025-04-19 07:47:10 -- STEP: 88/142 -- GLOBAL_STEP: 456050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.652326583862305 (17.9088721925562) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.3695 (0.3384498168121685) + | > loader_time: 0.0066 (0.005711035294966266) + + + --> TIME: 2025-04-19 07:47:20 -- STEP: 113/142 -- GLOBAL_STEP: 456075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.894943237304688 (17.89101690106688) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.3998 (0.3492315300797994) + | > loader_time: 0.0069 (0.005962293759911463) + + + --> TIME: 2025-04-19 07:47:31 -- STEP: 138/142 -- GLOBAL_STEP: 456100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7392120361328125 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.94205665588379 (17.899703309155896) + | > loss_duration: 1.7290712594985962 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001869447553826221 + | > current_lr_1: 0.0001869447553826221 + | > step_time: 0.4323 (0.36134909892427747) + | > loader_time: 0.0072 (0.006208171015200409) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9269514083862305 (2.9269514083862305) + | > loss_feat: nan (nan) + | > loss_mel: 19.804231643676758 (19.804231643676758) + | > loss_duration: 1.8523727655410767 (1.8523727655410767) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1839404106140137 (3.1839404106140137) + | > loss_feat: nan (nan) + | > loss_mel: 18.824594497680664 (18.824594497680664) + | > loss_duration: 1.9225623607635498 (1.9225623607635498) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8422763347625732 (3.0131083726882935) + | > loss_feat: nan (nan) + | > loss_mel: 19.816099166870117 (19.32034683227539) + | > loss_duration: 1.7909245491027832 (1.8567434549331665) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031409263610839844 (-0.0004832744598388672) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0131083726882935 (-0.08228468894958496) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.32034683227539 (+0.7941246032714844) + | > avg_loss_duration: 1.8567434549331665 (+0.015081524848937988) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 541/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:47:34)  + + --> TIME: 2025-04-19 07:47:41 -- STEP: 21/142 -- GLOBAL_STEP: 456125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.186250686645508 (17.685892241341726) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018692138728819925 + | > current_lr_1: 0.00018692138728819925 + | > step_time: 0.332 (0.30772939182463144) + | > loader_time: 0.0055 (0.00482789675394694) + + + --> TIME: 2025-04-19 07:47:50 -- STEP: 46/142 -- GLOBAL_STEP: 456150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.805259704589844 (17.7898330066515) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018692138728819925 + | > current_lr_1: 0.00018692138728819925 + | > step_time: 0.3423 (0.3189199489095937) + | > loader_time: 0.0058 (0.005163472631703253) + + + --> TIME: 2025-04-19 07:47:59 -- STEP: 71/142 -- GLOBAL_STEP: 456175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.74855613708496 (17.90789108545008) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018692138728819925 + | > current_lr_1: 0.00018692138728819925 + | > step_time: 0.3613 (0.33082532546889626) + | > loader_time: 0.0064 (0.005499890152837189) + + + --> TIME: 2025-04-19 07:48:08 -- STEP: 96/142 -- GLOBAL_STEP: 456200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6369404792785645 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.80055046081543 (17.958879560232173) + | > loss_duration: 1.6818199157714844 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018692138728819925 + | > current_lr_1: 0.00018692138728819925 + | > step_time: 0.3816 (0.3424238115549089) + | > loader_time: 0.0071 (0.0058102260033289594) + + + --> TIME: 2025-04-19 07:48:19 -- STEP: 121/142 -- GLOBAL_STEP: 456225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.10523223876953 (17.899572608884707) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018692138728819925 + | > current_lr_1: 0.00018692138728819925 + | > step_time: 0.4262 (0.35533361198488356) + | > loader_time: 0.0071 (0.006071616795437395) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.841773748397827 (2.841773748397827) + | > loss_feat: nan (nan) + | > loss_mel: 15.590392112731934 (15.590392112731934) + | > loss_duration: 1.8385812044143677 (1.8385812044143677) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.315938949584961 (3.315938949584961) + | > loss_feat: nan (nan) + | > loss_mel: 17.10572624206543 (17.10572624206543) + | > loss_duration: 1.893860101699829 (1.893860101699829) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.745990514755249 (3.030964732170105) + | > loss_feat: nan (nan) + | > loss_mel: 18.976879119873047 (18.04130268096924) + | > loss_duration: 1.8067326545715332 (1.8502963781356812) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003579258918762207 (+0.00043833255767822266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.030964732170105 (+0.017856359481811523) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.04130268096924 (-1.2790441513061523) + | > avg_loss_duration: 1.8502963781356812 (-0.0064470767974853516) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 542/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:48:29)  + + --> TIME: 2025-04-19 07:48:31 -- STEP: 4/142 -- GLOBAL_STEP: 456250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.03672981262207 (17.629153728485107) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.2934 (0.29730719327926636) + | > loader_time: 0.0053 (0.004535973072052002) + + + --> TIME: 2025-04-19 07:48:39 -- STEP: 29/142 -- GLOBAL_STEP: 456275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.169414520263672 (18.11066568308863) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.3196 (0.3099401819294897) + | > loader_time: 0.0056 (0.00502734348691743) + + + --> TIME: 2025-04-19 07:48:48 -- STEP: 54/142 -- GLOBAL_STEP: 456300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.165422439575195 (18.10487886711404) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.3437 (0.3226003646850587) + | > loader_time: 0.0055 (0.005379248548437047) + + + --> TIME: 2025-04-19 07:48:57 -- STEP: 79/142 -- GLOBAL_STEP: 456325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.551603317260742 (17.96929063676279) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.365 (0.33517722842059566) + | > loader_time: 0.0069 (0.005726959131941011) + + + --> TIME: 2025-04-19 07:49:07 -- STEP: 104/142 -- GLOBAL_STEP: 456350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.046615600585938 (18.031416682096626) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.3823 (0.34642530633853047) + | > loader_time: 0.0067 (0.005995851296644943) + + + --> TIME: 2025-04-19 07:49:17 -- STEP: 129/142 -- GLOBAL_STEP: 456375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.359770774841309 (17.9591784514198) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018689802211478823 + | > current_lr_1: 0.00018689802211478823 + | > step_time: 0.4194 (0.35838707657747504) + | > loader_time: 0.0071 (0.006247642428375953) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9917705059051514 (2.9917705059051514) + | > loss_feat: nan (nan) + | > loss_mel: 17.634401321411133 (17.634401321411133) + | > loss_duration: 1.8352446556091309 (1.8352446556091309) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1039083003997803 (3.1039083003997803) + | > loss_feat: nan (nan) + | > loss_mel: 17.479761123657227 (17.479761123657227) + | > loss_duration: 1.89756441116333 (1.89756441116333) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9215869903564453 (3.012747645378113) + | > loss_feat: nan (nan) + | > loss_mel: 19.806228637695312 (18.64299488067627) + | > loss_duration: 1.796250343322754 (1.846907377243042) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004037141799926758 (+0.0004578828811645508) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.012747645378113 (-0.018217086791992188) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.64299488067627 (+0.6016921997070312) + | > avg_loss_duration: 1.846907377243042 (-0.00338900089263916) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 543/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:49:24)  + + --> TIME: 2025-04-19 07:49:29 -- STEP: 12/142 -- GLOBAL_STEP: 456400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.364046096801758 (17.547134081522625) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.3041 (0.30115662018458045) + | > loader_time: 0.005 (0.0048444271087646484) + + + --> TIME: 2025-04-19 07:49:37 -- STEP: 37/142 -- GLOBAL_STEP: 456425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6302170753479004 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.418365478515625 (17.645410872794486) + | > loss_duration: 1.6134669780731201 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.3349 (0.31780560596569163) + | > loader_time: 0.0059 (0.005225078479663745) + + + --> TIME: 2025-04-19 07:49:46 -- STEP: 62/142 -- GLOBAL_STEP: 456450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.621383666992188 (17.63096875529136) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.3734 (0.3323417402082874) + | > loader_time: 0.0064 (0.005540767023640294) + + + --> TIME: 2025-04-19 07:49:56 -- STEP: 87/142 -- GLOBAL_STEP: 456475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.368911743164062 (17.799157778422032) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.4006 (0.34672942928884215) + | > loader_time: 0.0067 (0.005820455222294246) + + + --> TIME: 2025-04-19 07:50:06 -- STEP: 112/142 -- GLOBAL_STEP: 456500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.8167781829834 (17.948256143501833) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.417 (0.3585156798362731) + | > loader_time: 0.0069 (0.006091392466000147) + + + --> TIME: 2025-04-19 07:50:17 -- STEP: 137/142 -- GLOBAL_STEP: 456525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.587112426757812 (17.869336371874294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018687465986202387 + | > current_lr_1: 0.00018687465986202387 + | > step_time: 0.4418 (0.3709210357526792) + | > loader_time: 0.0076 (0.006384085564717757) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.905019521713257 (2.905019521713257) + | > loss_feat: nan (nan) + | > loss_mel: 19.301382064819336 (19.301382064819336) + | > loss_duration: 1.8400342464447021 (1.8400342464447021) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.443964719772339 (3.443964719772339) + | > loss_feat: nan (nan) + | > loss_mel: 18.8010196685791 (18.8010196685791) + | > loss_duration: 1.9126296043395996 (1.9126296043395996) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.963188409805298 (3.2035765647888184) + | > loss_feat: nan (nan) + | > loss_mel: 18.74374771118164 (18.77238368988037) + | > loss_duration: 1.7752690315246582 (1.843949317932129) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033577680587768555 (-0.0006793737411499023) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2035765647888184 (+0.19082891941070557) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.77238368988037 (+0.12938880920410156) + | > avg_loss_duration: 1.843949317932129 (-0.002958059310913086) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 544/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:50:21)  + + --> TIME: 2025-04-19 07:50:28 -- STEP: 20/142 -- GLOBAL_STEP: 456550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.8326456546783447 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.124242782592773 (18.12089385986328) + | > loss_duration: 1.685729742050171 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018685130052954112 + | > current_lr_1: 0.00018685130052954112 + | > step_time: 0.3149 (0.30901923179626467) + | > loader_time: 0.0055 (0.004871332645416259) + + + --> TIME: 2025-04-19 07:50:36 -- STEP: 45/142 -- GLOBAL_STEP: 456575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.18998146057129 (17.94437245262994) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018685130052954112 + | > current_lr_1: 0.00018685130052954112 + | > step_time: 0.3422 (0.3196759382883708) + | > loader_time: 0.0062 (0.00527860853407118) + + + --> TIME: 2025-04-19 07:50:45 -- STEP: 70/142 -- GLOBAL_STEP: 456600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.960962295532227 (17.997753293173655) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018685130052954112 + | > current_lr_1: 0.00018685130052954112 + | > step_time: 0.3656 (0.33101621695927214) + | > loader_time: 0.007 (0.005596446990966796) + + + --> TIME: 2025-04-19 07:50:54 -- STEP: 95/142 -- GLOBAL_STEP: 456625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.527437210083008 (17.949847783540438) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018685130052954112 + | > current_lr_1: 0.00018685130052954112 + | > step_time: 0.3789 (0.34179514332821503) + | > loader_time: 0.0064 (0.005857515335083006) + + + --> TIME: 2025-04-19 07:51:05 -- STEP: 120/142 -- GLOBAL_STEP: 456650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.205106735229492 (17.911909270286575) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018685130052954112 + | > current_lr_1: 0.00018685130052954112 + | > step_time: 0.4093 (0.35279686649640407) + | > loader_time: 0.0074 (0.00613223115603129) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8506760597229004 (2.8506760597229004) + | > loss_feat: nan (nan) + | > loss_mel: 17.023508071899414 (17.023508071899414) + | > loss_duration: 1.8344871997833252 (1.8344871997833252) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.374345541000366 (3.374345541000366) + | > loss_feat: nan (nan) + | > loss_mel: 17.807403564453125 (17.807403564453125) + | > loss_duration: 1.9045538902282715 (1.9045538902282715) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.928762197494507 (3.1515538692474365) + | > loss_feat: nan (nan) + | > loss_mel: 18.133867263793945 (17.970635414123535) + | > loss_duration: 1.7864235639572144 (1.845488727092743) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035337209701538086 (+0.00017595291137695312) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1515538692474365 (-0.052022695541381836) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.970635414123535 (-0.8017482757568359) + | > avg_loss_duration: 1.845488727092743 (+0.0015394091606140137) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 545/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:51:15)  + + --> TIME: 2025-04-19 07:51:17 -- STEP: 3/142 -- GLOBAL_STEP: 456675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.586977243423462 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.07722282409668 (18.41391944885254) + | > loss_duration: 1.6019964218139648 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.2959 (0.2970222632090251) + | > loader_time: 0.0041 (0.004281838734944661) + + + --> TIME: 2025-04-19 07:51:25 -- STEP: 28/142 -- GLOBAL_STEP: 456700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.328170776367188 (17.91827426637922) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.3209 (0.30950840030397675) + | > loader_time: 0.0052 (0.004877311842782157) + + + --> TIME: 2025-04-19 07:51:34 -- STEP: 53/142 -- GLOBAL_STEP: 456725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.214855194091797 (17.83924109980746) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.3538 (0.3264184852816024) + | > loader_time: 0.006 (0.005403023845744583) + + + --> TIME: 2025-04-19 07:51:43 -- STEP: 78/142 -- GLOBAL_STEP: 456750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.36162567138672 (17.7757517129947) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.3813 (0.3410636400565123) + | > loader_time: 0.0066 (0.005731659057812813) + + + --> TIME: 2025-04-19 07:51:53 -- STEP: 103/142 -- GLOBAL_STEP: 456775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5263867378234863 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.222761154174805 (17.840664178422355) + | > loss_duration: 1.6834819316864014 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.4011 (0.3532241362969853) + | > loader_time: 0.0069 (0.006041198100858521) + + + --> TIME: 2025-04-19 07:52:04 -- STEP: 128/142 -- GLOBAL_STEP: 456800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.485929489135742 (17.848054789006717) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018682794411697493 + | > current_lr_1: 0.00018682794411697493 + | > step_time: 0.4209 (0.3658067714422942) + | > loader_time: 0.0079 (0.006304509937763214) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9087002277374268 (2.9087002277374268) + | > loss_feat: nan (nan) + | > loss_mel: 19.712024688720703 (19.712024688720703) + | > loss_duration: 1.8333916664123535 (1.8333916664123535) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2694685459136963 (3.2694685459136963) + | > loss_feat: nan (nan) + | > loss_mel: 16.431058883666992 (16.431058883666992) + | > loss_duration: 1.911771297454834 (1.911771297454834) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8319544792175293 (3.050711512565613) + | > loss_feat: nan (nan) + | > loss_mel: 17.708961486816406 (17.0700101852417) + | > loss_duration: 1.794090747833252 (1.852931022644043) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034296512603759766 (-0.00010406970977783203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.050711512565613 (-0.10084235668182373) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.0700101852417 (-0.9006252288818359) + | > avg_loss_duration: 1.852931022644043 (+0.007442295551300049) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 546/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:52:12)  + + --> TIME: 2025-04-19 07:52:16 -- STEP: 11/142 -- GLOBAL_STEP: 456825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.769761085510254 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.821727752685547 (17.83179942044345) + | > loss_duration: 1.607452630996704 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.3088 (0.30146182667125354) + | > loader_time: 0.0046 (0.004309979352084073) + + + --> TIME: 2025-04-19 07:52:24 -- STEP: 36/142 -- GLOBAL_STEP: 456850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.377979278564453 (18.041973802778458) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.3314 (0.3144741588168674) + | > loader_time: 0.0054 (0.004836618900299072) + + + --> TIME: 2025-04-19 07:52:33 -- STEP: 61/142 -- GLOBAL_STEP: 456875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.096920013427734 (17.89107435257708) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.3565 (0.3270593510299433) + | > loader_time: 0.0062 (0.005205236497472544) + + + --> TIME: 2025-04-19 07:52:42 -- STEP: 86/142 -- GLOBAL_STEP: 456900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.713226795196533 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.073686599731445 (17.931805754816807) + | > loss_duration: 1.688552737236023 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.373 (0.33870728348576756) + | > loader_time: 0.0064 (0.005535178406294003) + + + --> TIME: 2025-04-19 07:52:52 -- STEP: 111/142 -- GLOBAL_STEP: 456925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.95755386352539 (17.83869081789309) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.4036 (0.3491995893083176) + | > loader_time: 0.0072 (0.005811167192888689) + + + --> TIME: 2025-04-19 07:53:02 -- STEP: 136/142 -- GLOBAL_STEP: 456950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.071531295776367 (17.856251155628875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001868045906239603 + | > current_lr_1: 0.0001868045906239603 + | > step_time: 0.4171 (0.3602871631874757) + | > loader_time: 0.0079 (0.006082636468550738) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.854618549346924 (2.854618549346924) + | > loss_feat: nan (nan) + | > loss_mel: 18.7724666595459 (18.7724666595459) + | > loss_duration: 1.8221021890640259 (1.8221021890640259) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.384887933731079 (3.384887933731079) + | > loss_feat: nan (nan) + | > loss_mel: 18.924575805664062 (18.924575805664062) + | > loss_duration: 1.9194657802581787 (1.9194657802581787) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8242862224578857 (3.1045870780944824) + | > loss_feat: nan (nan) + | > loss_mel: 18.976675033569336 (18.9506254196167) + | > loss_duration: 1.7951909303665161 (1.8573283553123474) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033545494079589844 (-7.510185241699219e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1045870780944824 (+0.05387556552886963) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.9506254196167 (+1.880615234375) + | > avg_loss_duration: 1.8573283553123474 (+0.004397332668304443) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 547/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:53:06)  + + --> TIME: 2025-04-19 07:53:13 -- STEP: 19/142 -- GLOBAL_STEP: 456975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.445524215698242 (17.952491157933284) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001867812400501323 + | > current_lr_1: 0.0001867812400501323 + | > step_time: 0.3117 (0.30587395868803324) + | > loader_time: 0.0048 (0.004621028900146484) + + + --> TIME: 2025-04-19 07:53:21 -- STEP: 44/142 -- GLOBAL_STEP: 457000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.767837524414062 (17.837206537073307) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001867812400501323 + | > current_lr_1: 0.0001867812400501323 + | > step_time: 0.3325 (0.3180077617818659) + | > loader_time: 0.0054 (0.005036354064941407) + + + --> TIME: 2025-04-19 07:53:30 -- STEP: 69/142 -- GLOBAL_STEP: 457025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.13457679748535 (17.799751530522883) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001867812400501323 + | > current_lr_1: 0.0001867812400501323 + | > step_time: 0.3947 (0.3302808464437291) + | > loader_time: 0.006 (0.005363816800324813) + + + --> TIME: 2025-04-19 07:53:40 -- STEP: 94/142 -- GLOBAL_STEP: 457050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.889122009277344 (17.765942888056973) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001867812400501323 + | > current_lr_1: 0.0001867812400501323 + | > step_time: 0.3856 (0.34106656845579764) + | > loader_time: 0.0072 (0.005671825814754406) + + + --> TIME: 2025-04-19 07:53:50 -- STEP: 119/142 -- GLOBAL_STEP: 457075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.119264602661133 (17.809361810443775) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001867812400501323 + | > current_lr_1: 0.0001867812400501323 + | > step_time: 0.4051 (0.35196698813879196) + | > loader_time: 0.0065 (0.005917927798102885) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.885157585144043 (2.885157585144043) + | > loss_feat: nan (nan) + | > loss_mel: 17.22848129272461 (17.22848129272461) + | > loss_duration: 1.8321300745010376 (1.8321300745010376) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4510841369628906 (3.4510841369628906) + | > loss_feat: nan (nan) + | > loss_mel: 20.129499435424805 (20.129499435424805) + | > loss_duration: 1.9061639308929443 (1.9061639308929443) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7763028144836426 (3.1136934757232666) + | > loss_feat: nan (nan) + | > loss_mel: 17.523120880126953 (18.82631015777588) + | > loss_duration: 1.7960224151611328 (1.8510931730270386) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034360885620117188 (+8.153915405273438e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1136934757232666 (+0.00910639762878418) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.82631015777588 (-0.12431526184082031) + | > avg_loss_duration: 1.8510931730270386 (-0.006235182285308838) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 548/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:54:01)  + + --> TIME: 2025-04-19 07:54:02 -- STEP: 2/142 -- GLOBAL_STEP: 457100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5391368865966797 (2.5752570629119873) + | > loss_feat: nan (nan) + | > loss_mel: 18.900039672851562 (18.94496440887451) + | > loss_duration: 1.6389656066894531 (1.6192227602005005) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.2982 (0.3028395175933838) + | > loader_time: 0.0041 (0.003980755805969238) + + + --> TIME: 2025-04-19 07:54:10 -- STEP: 27/142 -- GLOBAL_STEP: 457125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.88872528076172 (18.378885057237408) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.3281 (0.30976355517352083) + | > loader_time: 0.0053 (0.004776071619104455) + + + --> TIME: 2025-04-19 07:54:19 -- STEP: 52/142 -- GLOBAL_STEP: 457150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.35633087158203 (18.165742030510543) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.3607 (0.3220648215367244) + | > loader_time: 0.0063 (0.0051773694845346305) + + + --> TIME: 2025-04-19 07:54:28 -- STEP: 77/142 -- GLOBAL_STEP: 457175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.059425354003906 (18.062346755684203) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.3707 (0.3347272934851708) + | > loader_time: 0.006 (0.005442647190837116) + + + --> TIME: 2025-04-19 07:54:38 -- STEP: 102/142 -- GLOBAL_STEP: 457200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.812725067138672 (18.040499621746584) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.4008 (0.3480784776164036) + | > loader_time: 0.0067 (0.0058295469658047544) + + + --> TIME: 2025-04-19 07:54:49 -- STEP: 127/142 -- GLOBAL_STEP: 457225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.964523315429688 (18.034953162426078) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018675789239512602 + | > current_lr_1: 0.00018675789239512602 + | > step_time: 0.4147 (0.35954298935537266) + | > loader_time: 0.0069 (0.006095073354525829) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.127124309539795 (3.127124309539795) + | > loss_feat: nan (nan) + | > loss_mel: 18.55959129333496 (18.55959129333496) + | > loss_duration: 1.835359811782837 (1.835359811782837) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5769715309143066 (3.5769715309143066) + | > loss_feat: nan (nan) + | > loss_mel: 18.912321090698242 (18.912321090698242) + | > loss_duration: 1.8932604789733887 (1.8932604789733887) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.750471353530884 (3.163721442222595) + | > loss_feat: nan (nan) + | > loss_mel: 18.25720977783203 (18.584765434265137) + | > loss_duration: 1.799661636352539 (1.8464610576629639) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003375530242919922 (-6.0558319091796875e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.163721442222595 (+0.05002796649932861) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.584765434265137 (-0.2415447235107422) + | > avg_loss_duration: 1.8464610576629639 (-0.004632115364074707) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 549/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:54:56)  + + --> TIME: 2025-04-19 07:55:00 -- STEP: 10/142 -- GLOBAL_STEP: 457250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.01793670654297 (18.085078620910643) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.3096 (0.2997953176498413) + | > loader_time: 0.0049 (0.004467368125915527) + + + --> TIME: 2025-04-19 07:55:08 -- STEP: 35/142 -- GLOBAL_STEP: 457275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.201396942138672 (17.943551063537598) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.3393 (0.3162648405347551) + | > loader_time: 0.0059 (0.005100475038800921) + + + --> TIME: 2025-04-19 07:55:17 -- STEP: 60/142 -- GLOBAL_STEP: 457300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6776974201202393 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.80986213684082 (17.867227538426718) + | > loss_duration: 1.6644904613494873 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.3598 (0.3303544044494629) + | > loader_time: 0.0066 (0.005446290969848633) + + + --> TIME: 2025-04-19 07:55:27 -- STEP: 85/142 -- GLOBAL_STEP: 457325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6360862255096436 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.190149307250977 (17.84984762528363) + | > loss_duration: 1.6455068588256836 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.3811 (0.3443196661332074) + | > loader_time: 0.0068 (0.005731961306403665) + + + --> TIME: 2025-04-19 07:55:37 -- STEP: 110/142 -- GLOBAL_STEP: 457350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.717803716659546 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.668663024902344 (17.915898574482306) + | > loss_duration: 1.6598196029663086 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.4169 (0.3563379742882469) + | > loader_time: 0.0071 (0.006024202433499423) + + + --> TIME: 2025-04-19 07:55:48 -- STEP: 135/142 -- GLOBAL_STEP: 457375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.62928009033203 (17.836899347658505) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018673454765857661 + | > current_lr_1: 0.00018673454765857661 + | > step_time: 0.4436 (0.36901990219398784) + | > loader_time: 0.0078 (0.0063107066684299044) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.876955986022949 (2.876955986022949) + | > loss_feat: nan (nan) + | > loss_mel: 18.19220542907715 (18.19220542907715) + | > loss_duration: 1.8371785879135132 (1.8371785879135132) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4106831550598145 (3.4106831550598145) + | > loss_feat: nan (nan) + | > loss_mel: 19.17423439025879 (19.17423439025879) + | > loss_duration: 1.8886451721191406 (1.8886451721191406) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0845518112182617 (3.247617483139038) + | > loss_feat: nan (nan) + | > loss_mel: 18.936084747314453 (19.05515956878662) + | > loss_duration: 1.8146729469299316 (1.8516590595245361) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003644704818725586 (+0.00026917457580566406) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.247617483139038 (+0.08389604091644287) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.05515956878662 (+0.4703941345214844) + | > avg_loss_duration: 1.8516590595245361 (+0.005198001861572266) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 550/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:55:53)  + + --> TIME: 2025-04-19 07:55:59 -- STEP: 18/142 -- GLOBAL_STEP: 457400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.474132537841797 (18.311183081732857) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018671120584011928 + | > current_lr_1: 0.00018671120584011928 + | > step_time: 0.3122 (0.3038485580020481) + | > loader_time: 0.0052 (0.004753417438930935) + + + --> TIME: 2025-04-19 07:56:07 -- STEP: 43/142 -- GLOBAL_STEP: 457425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.150802612304688 (18.058127336723864) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018671120584011928 + | > current_lr_1: 0.00018671120584011928 + | > step_time: 0.3388 (0.31648773925248974) + | > loader_time: 0.0054 (0.0051018692726312685) + + + --> TIME: 2025-04-19 07:56:16 -- STEP: 68/142 -- GLOBAL_STEP: 457450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.051725387573242 (18.027418024399708) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018671120584011928 + | > current_lr_1: 0.00018671120584011928 + | > step_time: 0.3617 (0.32831589965259333) + | > loader_time: 0.006 (0.005441549946280088) + + + --> TIME: 2025-04-19 07:56:26 -- STEP: 93/142 -- GLOBAL_STEP: 457475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.363059997558594 (17.99586384270781) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018671120584011928 + | > current_lr_1: 0.00018671120584011928 + | > step_time: 0.3748 (0.33977566226836164) + | > loader_time: 0.0071 (0.0057193258757232334) + + + --> TIME: 2025-04-19 07:56:36 -- STEP: 118/142 -- GLOBAL_STEP: 457500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.49160385131836 (17.90768185308424) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018671120584011928 + | > current_lr_1: 0.00018671120584011928 + | > step_time: 0.4164 (0.35095461344314827) + | > loader_time: 0.0072 (0.006016727221214165) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8525726795196533 (2.8525726795196533) + | > loss_feat: nan (nan) + | > loss_mel: 18.756813049316406 (18.756813049316406) + | > loss_duration: 1.8190743923187256 (1.8190743923187256) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.353577136993408 (3.353577136993408) + | > loss_feat: nan (nan) + | > loss_mel: 15.981340408325195 (15.981340408325195) + | > loss_duration: 1.9053847789764404 (1.9053847789764404) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8839075565338135 (3.118742346763611) + | > loss_feat: nan (nan) + | > loss_mel: 19.137393951416016 (17.559367179870605) + | > loss_duration: 1.7889821529388428 (1.8471834659576416) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038650035858154297 (+0.00022029876708984375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.118742346763611 (-0.12887513637542725) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.559367179870605 (-1.4957923889160156) + | > avg_loss_duration: 1.8471834659576416 (-0.004475593566894531) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 551/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:56:48)  + + --> TIME: 2025-04-19 07:56:48 -- STEP: 1/142 -- GLOBAL_STEP: 457525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.335895538330078 (18.335895538330078) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.3013 (0.3012692928314209) + | > loader_time: 0.0041 (0.004114389419555664) + + + --> TIME: 2025-04-19 07:56:56 -- STEP: 26/142 -- GLOBAL_STEP: 457550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.785846710205078 (17.9386315712562) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.323 (0.3071290529691256) + | > loader_time: 0.0058 (0.0049189695945152856) + + + --> TIME: 2025-04-19 07:57:05 -- STEP: 51/142 -- GLOBAL_STEP: 457575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.569473266601562 (17.812725534626072) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.3414 (0.3195982446857527) + | > loader_time: 0.006 (0.0053699390560972915) + + + --> TIME: 2025-04-19 07:57:14 -- STEP: 76/142 -- GLOBAL_STEP: 457600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.34145164489746 (17.81337536008734) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.383 (0.3338981459015294) + | > loader_time: 0.0068 (0.005686123120157343) + + + --> TIME: 2025-04-19 07:57:24 -- STEP: 101/142 -- GLOBAL_STEP: 457625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.796873092651367 (17.911047312292716) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.4038 (0.34756292919121157) + | > loader_time: 0.0072 (0.005976681662077953) + + + --> TIME: 2025-04-19 07:57:35 -- STEP: 126/142 -- GLOBAL_STEP: 457650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.797836303710938 (17.885069506508955) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018668786693938925 + | > current_lr_1: 0.00018668786693938925 + | > step_time: 0.427 (0.36139308270953946) + | > loader_time: 0.0075 (0.006218912109496103) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8924400806427 (2.8924400806427) + | > loss_feat: nan (nan) + | > loss_mel: 17.603986740112305 (17.603986740112305) + | > loss_duration: 1.8274725675582886 (1.8274725675582886) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1471574306488037 (3.1471574306488037) + | > loss_feat: nan (nan) + | > loss_mel: 19.55913734436035 (19.55913734436035) + | > loss_duration: 1.9007230997085571 (1.9007230997085571) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.793999195098877 (2.9705783128738403) + | > loss_feat: nan (nan) + | > loss_mel: 16.66767692565918 (18.113407135009766) + | > loss_duration: 1.808166742324829 (1.8544449210166931) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035140514373779297 (-0.0003509521484375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9705783128738403 (-0.1481640338897705) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.113407135009766 (+0.5540399551391602) + | > avg_loss_duration: 1.8544449210166931 (+0.007261455059051514) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 552/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:57:43)  + + --> TIME: 2025-04-19 07:57:47 -- STEP: 9/142 -- GLOBAL_STEP: 457675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.342735290527344 (17.94713677300347) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.2979 (0.2988273832533095) + | > loader_time: 0.0045 (0.0045807626512315534) + + + --> TIME: 2025-04-19 07:57:55 -- STEP: 34/142 -- GLOBAL_STEP: 457700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.98749542236328 (18.052513515248023) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.3308 (0.31272523543413955) + | > loader_time: 0.0056 (0.005034075063817641) + + + --> TIME: 2025-04-19 07:58:04 -- STEP: 59/142 -- GLOBAL_STEP: 457725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.709089279174805 (17.93341200230487) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.3496 (0.3241113646555755) + | > loader_time: 0.0059 (0.005398079500360004) + + + --> TIME: 2025-04-19 07:58:13 -- STEP: 84/142 -- GLOBAL_STEP: 457750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.457286834716797 (17.850769065675294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.3788 (0.3365456007775805) + | > loader_time: 0.0084 (0.005719738347189768) + + + --> TIME: 2025-04-19 07:58:23 -- STEP: 109/142 -- GLOBAL_STEP: 457775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.463945388793945 (17.884409930727898) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.3918 (0.34804892539978016) + | > loader_time: 0.0069 (0.005980305715438424) + + + --> TIME: 2025-04-19 07:58:34 -- STEP: 134/142 -- GLOBAL_STEP: 457800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.630619049072266 (17.895319611278932) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866645309560218 + | > current_lr_1: 0.0001866645309560218 + | > step_time: 0.4244 (0.3615983920310859) + | > loader_time: 0.0072 (0.0062157253720867105) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.898378849029541 (2.898378849029541) + | > loss_feat: nan (nan) + | > loss_mel: 18.90203857421875 (18.90203857421875) + | > loss_duration: 1.8398953676223755 (1.8398953676223755) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2493720054626465 (3.2493720054626465) + | > loss_feat: nan (nan) + | > loss_mel: 18.621164321899414 (18.621164321899414) + | > loss_duration: 1.8980642557144165 (1.8980642557144165) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.862299680709839 (3.0558358430862427) + | > loss_feat: nan (nan) + | > loss_mel: 18.74599838256836 (18.683581352233887) + | > loss_duration: 1.7830431461334229 (1.8405537009239197) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003464221954345703 (-4.982948303222656e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0558358430862427 (+0.08525753021240234) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.683581352233887 (+0.5701742172241211) + | > avg_loss_duration: 1.8405537009239197 (-0.013891220092773438) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 553/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:58:38)  + + --> TIME: 2025-04-19 07:58:44 -- STEP: 17/142 -- GLOBAL_STEP: 457825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5086963176727295 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.156658172607422 (17.893830860362335) + | > loss_duration: 1.5546497106552124 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018664119788965231 + | > current_lr_1: 0.00018664119788965231 + | > step_time: 0.3273 (0.30623179323533) + | > loader_time: 0.0059 (0.0048489290125229775) + + + --> TIME: 2025-04-19 07:58:53 -- STEP: 42/142 -- GLOBAL_STEP: 457850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.009689331054688 (17.891103858039493) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018664119788965231 + | > current_lr_1: 0.00018664119788965231 + | > step_time: 0.3469 (0.32383469740549725) + | > loader_time: 0.0058 (0.005452133360363189) + + + --> TIME: 2025-04-19 07:59:02 -- STEP: 67/142 -- GLOBAL_STEP: 457875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.299896240234375 (17.90567060726792) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018664119788965231 + | > current_lr_1: 0.00018664119788965231 + | > step_time: 0.3684 (0.33724794103138483) + | > loader_time: 0.0066 (0.005743870094640931) + + + --> TIME: 2025-04-19 07:59:12 -- STEP: 92/142 -- GLOBAL_STEP: 457900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.12373924255371 (17.806792497634884) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018664119788965231 + | > current_lr_1: 0.00018664119788965231 + | > step_time: 0.3987 (0.3496532984401869) + | > loader_time: 0.007 (0.005995239900506061) + + + --> TIME: 2025-04-19 07:59:22 -- STEP: 117/142 -- GLOBAL_STEP: 457925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.115278244018555 (17.774195133111416) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018664119788965231 + | > current_lr_1: 0.00018664119788965231 + | > step_time: 0.4215 (0.3622202791719355) + | > loader_time: 0.007 (0.006252484443860176) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8350672721862793 (2.8350672721862793) + | > loss_feat: nan (nan) + | > loss_mel: 18.34636116027832 (18.34636116027832) + | > loss_duration: 1.827465534210205 (1.827465534210205) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.391508102416992 (3.391508102416992) + | > loss_feat: nan (nan) + | > loss_mel: 20.029083251953125 (20.029083251953125) + | > loss_duration: 1.894162654876709 (1.894162654876709) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8964157104492188 (3.1439619064331055) + | > loss_feat: nan (nan) + | > loss_mel: 18.17184829711914 (19.100465774536133) + | > loss_duration: 1.7930982112884521 (1.8436304330825806) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00338900089263916 (-7.522106170654297e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1439619064331055 (+0.08812606334686279) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.100465774536133 (+0.4168844223022461) + | > avg_loss_duration: 1.8436304330825806 (+0.0030767321586608887) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 554/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 07:59:35)  + + --> TIME: 2025-04-19 07:59:36 -- STEP: 0/142 -- GLOBAL_STEP: 457950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.827739715576172 (16.827739715576172) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.3268 (0.32682180404663086) + | > loader_time: 0.3347 (0.33466386795043945) + + + --> TIME: 2025-04-19 07:59:43 -- STEP: 25/142 -- GLOBAL_STEP: 457975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.940832138061523 (17.766201553344725) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.3204 (0.30743167877197264) + | > loader_time: 0.0054 (0.004802083969116211) + + + --> TIME: 2025-04-19 07:59:52 -- STEP: 50/142 -- GLOBAL_STEP: 458000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.67701530456543 (17.83268999099732) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.343 (0.3204591608047485) + | > loader_time: 0.0062 (0.005254344940185547) + + + --> TIME: 2025-04-19 08:00:01 -- STEP: 75/142 -- GLOBAL_STEP: 458025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.833595275878906 (17.860014457702643) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.3772 (0.33311133384704594) + | > loader_time: 0.0065 (0.0055504735310872395) + + + --> TIME: 2025-04-19 08:00:11 -- STEP: 100/142 -- GLOBAL_STEP: 458050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.2979679107666 (17.829723825454717) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.3918 (0.34430941820144656) + | > loader_time: 0.0066 (0.005792496204376222) + + + --> TIME: 2025-04-19 08:00:21 -- STEP: 125/142 -- GLOBAL_STEP: 458075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.86185646057129 (17.90034894561768) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001866178677399161 + | > current_lr_1: 0.0001866178677399161 + | > step_time: 0.413 (0.3563418045043945) + | > loader_time: 0.0072 (0.006034265518188478) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0488121509552 (3.0488121509552) + | > loss_feat: nan (nan) + | > loss_mel: 17.124338150024414 (17.124338150024414) + | > loss_duration: 1.8469395637512207 (1.8469395637512207) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3759043216705322 (3.3759043216705322) + | > loss_feat: nan (nan) + | > loss_mel: 19.55115509033203 (19.55115509033203) + | > loss_duration: 1.8911033868789673 (1.8911033868789673) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8853678703308105 (3.1306360960006714) + | > loss_feat: nan (nan) + | > loss_mel: 17.46109962463379 (18.50612735748291) + | > loss_duration: 1.7810425758361816 (1.8360729813575745) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032235383987426758 (-0.00016546249389648438) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1306360960006714 (-0.013325810432434082) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.50612735748291 (-0.5943384170532227) + | > avg_loss_duration: 1.8360729813575745 (-0.0075574517250061035) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 555/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:00:30)  + + --> TIME: 2025-04-19 08:00:33 -- STEP: 8/142 -- GLOBAL_STEP: 458100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.431692123413086 (17.60044491291046) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.2986 (0.3002028465270996) + | > loader_time: 0.0047 (0.004409193992614746) + + + --> TIME: 2025-04-19 08:00:41 -- STEP: 33/142 -- GLOBAL_STEP: 458125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.599321365356445 (18.070154507954918) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.3383 (0.31291705189329205) + | > loader_time: 0.0052 (0.005033644762906161) + + + --> TIME: 2025-04-19 08:00:50 -- STEP: 58/142 -- GLOBAL_STEP: 458150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.01978874206543 (18.114023816996617) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.3592 (0.3286879473719104) + | > loader_time: 0.0065 (0.005433053805910307) + + + --> TIME: 2025-04-19 08:01:00 -- STEP: 83/142 -- GLOBAL_STEP: 458175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.57291603088379 (17.96741590155177) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.3847 (0.34483553702572745) + | > loader_time: 0.0065 (0.005805127591971892) + + + --> TIME: 2025-04-19 08:01:10 -- STEP: 108/142 -- GLOBAL_STEP: 458200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.015335083007812 (17.957292071095218) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.4114 (0.3565050584298593) + | > loader_time: 0.007 (0.006098539740950973) + + + --> TIME: 2025-04-19 08:01:21 -- STEP: 133/142 -- GLOBAL_STEP: 458225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.10597038269043 (17.925531265430873) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001865945405064486 + | > current_lr_1: 0.0001865945405064486 + | > step_time: 0.4217 (0.36912331724525393) + | > loader_time: 0.0079 (0.006389037110751733) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0128657817840576 (3.0128657817840576) + | > loss_feat: nan (nan) + | > loss_mel: 18.562108993530273 (18.562108993530273) + | > loss_duration: 1.8137019872665405 (1.8137019872665405) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.226938009262085 (3.226938009262085) + | > loss_feat: nan (nan) + | > loss_mel: 18.45940589904785 (18.45940589904785) + | > loss_duration: 1.9221736192703247 (1.9221736192703247) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8755390644073486 (3.051238536834717) + | > loss_feat: nan (nan) + | > loss_mel: 19.393423080444336 (18.926414489746094) + | > loss_duration: 1.7921156883239746 (1.8571446537971497) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033611059188842773 (+0.00013756752014160156) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.051238536834717 (-0.07939755916595459) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.926414489746094 (+0.4202871322631836) + | > avg_loss_duration: 1.8571446537971497 (+0.021071672439575195) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 556/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:01:26)  + + --> TIME: 2025-04-19 08:01:32 -- STEP: 16/142 -- GLOBAL_STEP: 458250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.7957706451416 (17.88050782680512) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.3141 (0.3050047308206558) + | > loader_time: 0.0048 (0.004610136151313782) + + + --> TIME: 2025-04-19 08:01:40 -- STEP: 41/142 -- GLOBAL_STEP: 458275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.62505340576172 (17.827810333996283) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.3381 (0.318713508001188) + | > loader_time: 0.0058 (0.005132698431247618) + + + --> TIME: 2025-04-19 08:01:49 -- STEP: 66/142 -- GLOBAL_STEP: 458300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.6738338470459 (17.92336403239856) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.358 (0.3297142404498476) + | > loader_time: 0.0066 (0.005437890688578288) + + + --> TIME: 2025-04-19 08:01:59 -- STEP: 91/142 -- GLOBAL_STEP: 458325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.84218978881836 (17.835327525715243) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.3818 (0.34158001103243985) + | > loader_time: 0.0069 (0.005760444389594777) + + + --> TIME: 2025-04-19 08:02:09 -- STEP: 116/142 -- GLOBAL_STEP: 458350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.014209747314453 (17.81281705560355) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.4025 (0.35220240724497826) + | > loader_time: 0.0074 (0.0059990348487064735) + + + --> TIME: 2025-04-19 08:02:19 -- STEP: 141/142 -- GLOBAL_STEP: 458375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.046131134033203 (17.81847265933422) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018657121618888527 + | > current_lr_1: 0.00018657121618888527 + | > step_time: 0.1831 (0.3616212919248756) + | > loader_time: 0.0032 (0.00619313564706356) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.970374584197998 (2.970374584197998) + | > loss_feat: nan (nan) + | > loss_mel: 19.139440536499023 (19.139440536499023) + | > loss_duration: 1.8182976245880127 (1.8182976245880127) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2703235149383545 (3.2703235149383545) + | > loss_feat: nan (nan) + | > loss_mel: 18.021432876586914 (18.021432876586914) + | > loss_duration: 1.89656400680542 (1.89656400680542) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7933349609375 (3.0318292379379272) + | > loss_feat: nan (nan) + | > loss_mel: 17.235963821411133 (17.628698348999023) + | > loss_duration: 1.7973756790161133 (1.8469698429107666) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003462076187133789 (+0.00010097026824951172) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0318292379379272 (-0.01940929889678955) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.628698348999023 (-1.2977161407470703) + | > avg_loss_duration: 1.8469698429107666 (-0.010174810886383057) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 557/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:02:21)  + + --> TIME: 2025-04-19 08:02:29 -- STEP: 24/142 -- GLOBAL_STEP: 458400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.62566375732422 (17.996092240015667) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018654789478686164 + | > current_lr_1: 0.00018654789478686164 + | > step_time: 0.3171 (0.30734015504519147) + | > loader_time: 0.0053 (0.005024453004201253) + + + --> TIME: 2025-04-19 08:02:38 -- STEP: 49/142 -- GLOBAL_STEP: 458425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4405014514923096 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.02545738220215 (18.011318206787106) + | > loss_duration: 1.6092445850372314 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018654789478686164 + | > current_lr_1: 0.00018654789478686164 + | > step_time: 0.3438 (0.31955391533520766) + | > loader_time: 0.0057 (0.00531403872431541) + + + --> TIME: 2025-04-19 08:02:47 -- STEP: 74/142 -- GLOBAL_STEP: 458450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5399727821350098 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.994075775146484 (18.013616690764557) + | > loss_duration: 1.6576800346374512 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018654789478686164 + | > current_lr_1: 0.00018654789478686164 + | > step_time: 0.3614 (0.33179323737685756) + | > loader_time: 0.0064 (0.0055936059436282595) + + + --> TIME: 2025-04-19 08:02:56 -- STEP: 99/142 -- GLOBAL_STEP: 458475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5751709938049316 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.63313102722168 (18.05047790450279) + | > loss_duration: 1.692387342453003 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018654789478686164 + | > current_lr_1: 0.00018654789478686164 + | > step_time: 0.3769 (0.342470494183627) + | > loader_time: 0.0066 (0.005828746641525114) + + + --> TIME: 2025-04-19 08:03:07 -- STEP: 124/142 -- GLOBAL_STEP: 458500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.661029815673828 (18.013693801818352) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018654789478686164 + | > current_lr_1: 0.00018654789478686164 + | > step_time: 0.4071 (0.3547072179855839) + | > loader_time: 0.0072 (0.006116563274014381) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.885423421859741 (2.885423421859741) + | > loss_feat: nan (nan) + | > loss_mel: 19.17506217956543 (19.17506217956543) + | > loss_duration: 1.8494387865066528 (1.8494387865066528) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3052079677581787 (3.3052079677581787) + | > loss_feat: nan (nan) + | > loss_mel: 18.949018478393555 (18.949018478393555) + | > loss_duration: 1.9037957191467285 (1.9037957191467285) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9074535369873047 (3.1063307523727417) + | > loss_feat: nan (nan) + | > loss_mel: 19.077611923217773 (19.013315200805664) + | > loss_duration: 1.7905611991882324 (1.8471784591674805) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032622814178466797 (-0.00019979476928710938) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1063307523727417 (+0.07450151443481445) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.013315200805664 (+1.3846168518066406) + | > avg_loss_duration: 1.8471784591674805 (+0.0002086162567138672) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 558/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:03:16)  + + --> TIME: 2025-04-19 08:03:19 -- STEP: 7/142 -- GLOBAL_STEP: 458525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.248821258544922 (18.27530206952776) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.3021 (0.3008921146392822) + | > loader_time: 0.0041 (0.004386833735874721) + + + --> TIME: 2025-04-19 08:03:27 -- STEP: 32/142 -- GLOBAL_STEP: 458550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.52461051940918 (18.22172611951828) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.3333 (0.3138011693954468) + | > loader_time: 0.0054 (0.0049974024295806885) + + + --> TIME: 2025-04-19 08:03:36 -- STEP: 57/142 -- GLOBAL_STEP: 458575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5433571338653564 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.95607566833496 (18.071456976104205) + | > loss_duration: 1.688622236251831 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.3522 (0.32504373265985864) + | > loader_time: 0.0059 (0.005359118444877758) + + + --> TIME: 2025-04-19 08:03:45 -- STEP: 82/142 -- GLOBAL_STEP: 458600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.2375545501709 (18.13165622804223) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.3694 (0.3369484761866128) + | > loader_time: 0.0065 (0.005663505414637124) + + + --> TIME: 2025-04-19 08:03:55 -- STEP: 107/142 -- GLOBAL_STEP: 458625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.97262954711914 (18.060448022646323) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.3908 (0.3473198592105758) + | > loader_time: 0.0064 (0.0059356266093031265) + + + --> TIME: 2025-04-19 08:04:05 -- STEP: 132/142 -- GLOBAL_STEP: 458650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.179710388183594 (17.955495697079282) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018652457630001328 + | > current_lr_1: 0.00018652457630001328 + | > step_time: 0.4336 (0.3604808124628933) + | > loader_time: 0.0076 (0.006169526865988067) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9571523666381836 (2.9571523666381836) + | > loss_feat: nan (nan) + | > loss_mel: 18.5361328125 (18.5361328125) + | > loss_duration: 1.8321752548217773 (1.8321752548217773) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.348982334136963 (3.348982334136963) + | > loss_feat: nan (nan) + | > loss_mel: 17.165607452392578 (17.165607452392578) + | > loss_duration: 1.9084088802337646 (1.9084088802337646) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0132899284362793 (3.181136131286621) + | > loss_feat: nan (nan) + | > loss_mel: 19.212491989135742 (18.18904972076416) + | > loss_duration: 1.7898969650268555 (1.84915292263031) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032281875610351562 (-3.409385681152344e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.181136131286621 (+0.0748053789138794) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.18904972076416 (-0.8242654800415039) + | > avg_loss_duration: 1.84915292263031 (+0.00197446346282959) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 559/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:04:11)  + + --> TIME: 2025-04-19 08:04:16 -- STEP: 15/142 -- GLOBAL_STEP: 458675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.50383758544922 (17.512196032206216) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.321 (0.3100536028544108) + | > loader_time: 0.0053 (0.004727776845296224) + + + --> TIME: 2025-04-19 08:04:25 -- STEP: 40/142 -- GLOBAL_STEP: 458700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.1630859375 (17.902233910560614) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.3366 (0.3211063504219055) + | > loader_time: 0.0057 (0.005290031433105469) + + + --> TIME: 2025-04-19 08:04:34 -- STEP: 65/142 -- GLOBAL_STEP: 458725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.8878231048584 (17.9466462795551) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.3657 (0.3315259529994084) + | > loader_time: 0.0064 (0.005582438982450045) + + + --> TIME: 2025-04-19 08:04:43 -- STEP: 90/142 -- GLOBAL_STEP: 458750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.845474243164062 (17.849063407050238) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.3767 (0.3417237838109335) + | > loader_time: 0.0067 (0.005837853749593098) + + + --> TIME: 2025-04-19 08:04:53 -- STEP: 115/142 -- GLOBAL_STEP: 458775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.341419219970703 (17.893140461133857) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.4139 (0.35250335361646573) + | > loader_time: 0.0072 (0.006086824251257854) + + + --> TIME: 2025-04-19 08:05:04 -- STEP: 140/142 -- GLOBAL_STEP: 458800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64082145690918 (17.86134185109819) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018650126072797577 + | > current_lr_1: 0.00018650126072797577 + | > step_time: 0.4624 (0.36620704787118097) + | > loader_time: 0.0084 (0.006344728810446601) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.078772783279419 (3.078772783279419) + | > loss_feat: nan (nan) + | > loss_mel: 17.453636169433594 (17.453636169433594) + | > loss_duration: 1.8183467388153076 (1.8183467388153076) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.342024326324463 (3.342024326324463) + | > loss_feat: nan (nan) + | > loss_mel: 17.444381713867188 (17.444381713867188) + | > loss_duration: 1.9154956340789795 (1.9154956340789795) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8096230030059814 (3.075823664665222) + | > loss_feat: nan (nan) + | > loss_mel: 18.725379943847656 (18.084880828857422) + | > loss_duration: 1.774701476097107 (1.8450985550880432) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004787802696228027 (+0.001559615135192871) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.075823664665222 (-0.10531246662139893) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.084880828857422 (-0.10416889190673828) + | > avg_loss_duration: 1.8450985550880432 (-0.004054367542266846) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 560/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:05:07)  + + --> TIME: 2025-04-19 08:05:15 -- STEP: 23/142 -- GLOBAL_STEP: 458825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6456170082092285 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.13945960998535 (18.116383262302563) + | > loss_duration: 1.6192877292633057 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018647794807038476 + | > current_lr_1: 0.00018647794807038476 + | > step_time: 0.3216 (0.3077082530311916) + | > loader_time: 0.0057 (0.004678653634112814) + + + --> TIME: 2025-04-19 08:05:23 -- STEP: 48/142 -- GLOBAL_STEP: 458850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.140060424804688 (18.043881654739376) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018647794807038476 + | > current_lr_1: 0.00018647794807038476 + | > step_time: 0.3394 (0.32008202870686836) + | > loader_time: 0.0062 (0.005137676994005839) + + + --> TIME: 2025-04-19 08:05:32 -- STEP: 73/142 -- GLOBAL_STEP: 458875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.2454833984375 (17.918109292853366) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018647794807038476 + | > current_lr_1: 0.00018647794807038476 + | > step_time: 0.3771 (0.33274979787330095) + | > loader_time: 0.0065 (0.005441796289731378) + + + --> TIME: 2025-04-19 08:05:42 -- STEP: 98/142 -- GLOBAL_STEP: 458900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.074796676635742 (17.764135720778484) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018647794807038476 + | > current_lr_1: 0.00018647794807038476 + | > step_time: 0.4001 (0.3447797955298909) + | > loader_time: 0.0064 (0.005709652998009507) + + + --> TIME: 2025-04-19 08:05:52 -- STEP: 123/142 -- GLOBAL_STEP: 458925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.528518676757812 (17.749613808422552) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018647794807038476 + | > current_lr_1: 0.00018647794807038476 + | > step_time: 0.4144 (0.35671187416324757) + | > loader_time: 0.0073 (0.0059856845111381715) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7966372966766357 (2.7966372966766357) + | > loss_feat: nan (nan) + | > loss_mel: 17.178485870361328 (17.178485870361328) + | > loss_duration: 1.8341503143310547 (1.8341503143310547) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.305307388305664 (3.305307388305664) + | > loss_feat: nan (nan) + | > loss_mel: 17.996627807617188 (17.996627807617188) + | > loss_duration: 1.9194140434265137 (1.9194140434265137) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8872036933898926 (3.0962555408477783) + | > loss_feat: nan (nan) + | > loss_mel: 18.729148864746094 (18.36288833618164) + | > loss_duration: 1.7900893688201904 (1.854751706123352) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004011034965515137 (-0.0007767677307128906) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0962555408477783 (+0.020431876182556152) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.36288833618164 (+0.27800750732421875) + | > avg_loss_duration: 1.854751706123352 (+0.009653151035308838) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 561/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:06:02)  + + --> TIME: 2025-04-19 08:06:04 -- STEP: 6/142 -- GLOBAL_STEP: 458950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.87510108947754 (17.536935170491535) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.3046 (0.3001154661178589) + | > loader_time: 0.004 (0.004418849945068359) + + + --> TIME: 2025-04-19 08:06:12 -- STEP: 31/142 -- GLOBAL_STEP: 458975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.776464462280273 (17.739800483949722) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.3461 (0.31365023120757074) + | > loader_time: 0.0053 (0.0048761444707070625) + + + --> TIME: 2025-04-19 08:06:21 -- STEP: 56/142 -- GLOBAL_STEP: 459000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.05582046508789 (17.898212466921123) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.3476 (0.32456202592168537) + | > loader_time: 0.006 (0.005263081618717739) + + + --> TIME: 2025-04-19 08:06:30 -- STEP: 81/142 -- GLOBAL_STEP: 459025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.992626190185547 (17.76349380869925) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.3718 (0.3371878759360607) + | > loader_time: 0.0065 (0.00554889808466405) + + + --> TIME: 2025-04-19 08:06:40 -- STEP: 106/142 -- GLOBAL_STEP: 459050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.745235443115234 (17.867954380107392) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.387 (0.3481027585155559) + | > loader_time: 0.0067 (0.005811574324122015) + + + --> TIME: 2025-04-19 08:06:51 -- STEP: 131/142 -- GLOBAL_STEP: 459075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.665884017944336 (17.865894259387314) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018645463832687594 + | > current_lr_1: 0.00018645463832687594 + | > step_time: 0.4222 (0.35999780574827706) + | > loader_time: 0.0071 (0.00607555331164644) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.894655704498291 (2.894655704498291) + | > loss_feat: nan (nan) + | > loss_mel: 17.34903335571289 (17.34903335571289) + | > loss_duration: 1.8489634990692139 (1.8489634990692139) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.152327299118042 (3.152327299118042) + | > loss_feat: nan (nan) + | > loss_mel: 19.45453453063965 (19.45453453063965) + | > loss_duration: 1.9021097421646118 (1.9021097421646118) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.909348249435425 (3.0308377742767334) + | > loss_feat: nan (nan) + | > loss_mel: 17.241910934448242 (18.348222732543945) + | > loss_duration: 1.8020968437194824 (1.8521032929420471) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036460161209106445 (-0.0003650188446044922) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0308377742767334 (-0.06541776657104492) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.348222732543945 (-0.014665603637695312) + | > avg_loss_duration: 1.8521032929420471 (-0.0026484131813049316) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 562/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:06:57)  + + --> TIME: 2025-04-19 08:07:02 -- STEP: 14/142 -- GLOBAL_STEP: 459100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6035964488983154 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.115087509155273 (17.85814312526158) + | > loss_duration: 1.6254150867462158 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.3074 (0.3029670885631016) + | > loader_time: 0.0046 (0.004268118313380651) + + + --> TIME: 2025-04-19 08:07:10 -- STEP: 39/142 -- GLOBAL_STEP: 459125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.66962242126465 (17.864134886325935) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.3386 (0.31741755436628294) + | > loader_time: 0.0056 (0.004879071162297177) + + + --> TIME: 2025-04-19 08:07:19 -- STEP: 64/142 -- GLOBAL_STEP: 459150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.9321346282959 (17.93234737217427) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.3578 (0.3319339193403722) + | > loader_time: 0.0063 (0.005281709134578706) + + + --> TIME: 2025-04-19 08:07:29 -- STEP: 89/142 -- GLOBAL_STEP: 459175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.79689598083496 (18.000905561982915) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.3743 (0.34227094489536947) + | > loader_time: 0.0063 (0.005635858921522505) + + + --> TIME: 2025-04-19 08:07:39 -- STEP: 114/142 -- GLOBAL_STEP: 459200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.41769027709961 (18.009550730387367) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.394 (0.35247030592801293) + | > loader_time: 0.0071 (0.00593437019147371) + + + --> TIME: 2025-04-19 08:07:50 -- STEP: 139/142 -- GLOBAL_STEP: 459225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.169836044311523 (17.994774221516337) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018643133149708508 + | > current_lr_1: 0.00018643133149708508 + | > step_time: 0.4277 (0.3644130950351414) + | > loader_time: 0.0073 (0.006194848808453238) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.045073986053467 (3.045073986053467) + | > loss_feat: nan (nan) + | > loss_mel: 18.965248107910156 (18.965248107910156) + | > loss_duration: 1.8385891914367676 (1.8385891914367676) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2289090156555176 (3.2289090156555176) + | > loss_feat: nan (nan) + | > loss_mel: 18.97113609313965 (18.97113609313965) + | > loss_duration: 1.884952425956726 (1.884952425956726) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.099447727203369 (3.1641783714294434) + | > loss_feat: nan (nan) + | > loss_mel: 18.41651153564453 (18.69382381439209) + | > loss_duration: 1.800846815109253 (1.8428996205329895) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003358602523803711 (-0.0002874135971069336) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1641783714294434 (+0.13334059715270996) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.69382381439209 (+0.34560108184814453) + | > avg_loss_duration: 1.8428996205329895 (-0.009203672409057617) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 563/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:07:52)  + + --> TIME: 2025-04-19 08:08:00 -- STEP: 22/142 -- GLOBAL_STEP: 459250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.354129791259766 (17.96004364707254) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018640802758064794 + | > current_lr_1: 0.00018640802758064794 + | > step_time: 0.3192 (0.3057155934247104) + | > loader_time: 0.0053 (0.004599679600108753) + + + --> TIME: 2025-04-19 08:08:08 -- STEP: 47/142 -- GLOBAL_STEP: 459275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.939838409423828 (17.959720124589644) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018640802758064794 + | > current_lr_1: 0.00018640802758064794 + | > step_time: 0.3401 (0.3189476804530367) + | > loader_time: 0.0056 (0.0050702805214739854) + + + --> TIME: 2025-04-19 08:08:17 -- STEP: 72/142 -- GLOBAL_STEP: 459300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.23093605041504 (18.035089996125976) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018640802758064794 + | > current_lr_1: 0.00018640802758064794 + | > step_time: 0.3655 (0.3314588069915772) + | > loader_time: 0.0061 (0.005404363075892129) + + + --> TIME: 2025-04-19 08:08:27 -- STEP: 97/142 -- GLOBAL_STEP: 459325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.907196044921875 (18.018566564186333) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018640802758064794 + | > current_lr_1: 0.00018640802758064794 + | > step_time: 0.3847 (0.34219349536699134) + | > loader_time: 0.0067 (0.005644318983726894) + + + --> TIME: 2025-04-19 08:08:37 -- STEP: 122/142 -- GLOBAL_STEP: 459350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.744096755981445 (18.02121173358355) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018640802758064794 + | > current_lr_1: 0.00018640802758064794 + | > step_time: 0.4278 (0.3535162796739673) + | > loader_time: 0.0073 (0.005896923971957847) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9992969036102295 (2.9992969036102295) + | > loss_feat: nan (nan) + | > loss_mel: 16.47951316833496 (16.47951316833496) + | > loss_duration: 1.8348560333251953 (1.8348560333251953) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2711753845214844 (3.2711753845214844) + | > loss_feat: nan (nan) + | > loss_mel: 17.931352615356445 (17.931352615356445) + | > loss_duration: 1.9201542139053345 (1.9201542139053345) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.901794672012329 (3.0864850282669067) + | > loss_feat: nan (nan) + | > loss_mel: 16.81397819519043 (17.372665405273438) + | > loss_duration: 1.79585862159729 (1.8580064177513123) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035927295684814453 (+0.00023412704467773438) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0864850282669067 (-0.07769334316253662) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.372665405273438 (-1.3211584091186523) + | > avg_loss_duration: 1.8580064177513123 (+0.015106797218322754) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 564/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:08:47)  + + --> TIME: 2025-04-19 08:08:49 -- STEP: 5/142 -- GLOBAL_STEP: 459375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.65973663330078 (18.38000030517578) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.3002 (0.2974689483642578) + | > loader_time: 0.004 (0.004322862625122071) + + + --> TIME: 2025-04-19 08:08:57 -- STEP: 30/142 -- GLOBAL_STEP: 459400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.667301177978516 (17.886307334899907) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.3329 (0.31412563323974607) + | > loader_time: 0.0055 (0.004800875981648764) + + + --> TIME: 2025-04-19 08:09:06 -- STEP: 55/142 -- GLOBAL_STEP: 459425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.278278350830078 (18.083029903065082) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.3491 (0.3303543741052802) + | > loader_time: 0.0055 (0.005186380039561877) + + + --> TIME: 2025-04-19 08:09:15 -- STEP: 80/142 -- GLOBAL_STEP: 459450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6567904949188232 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.20819664001465 (17.9581288099289) + | > loss_duration: 1.6725821495056152 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.3832 (0.34052821099758157) + | > loader_time: 0.0063 (0.005466535687446594) + + + --> TIME: 2025-04-19 08:09:25 -- STEP: 105/142 -- GLOBAL_STEP: 459475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.84987449645996 (18.064413433983212) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.3911 (0.35124990826561336) + | > loader_time: 0.0066 (0.005830033620198567) + + + --> TIME: 2025-04-19 08:09:36 -- STEP: 130/142 -- GLOBAL_STEP: 459500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.419696807861328 (17.962985412891094) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018638472657720034 + | > current_lr_1: 0.00018638472657720034 + | > step_time: 0.4227 (0.36223816321446345) + | > loader_time: 0.0075 (0.00608683366041917) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0921578407287598 (3.0921578407287598) + | > loss_feat: nan (nan) + | > loss_mel: 16.020648956298828 (16.020648956298828) + | > loss_duration: 1.8362305164337158 (1.8362305164337158) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.269286632537842 (3.269286632537842) + | > loss_feat: nan (nan) + | > loss_mel: 18.751523971557617 (18.751523971557617) + | > loss_duration: 1.8928157091140747 (1.8928157091140747) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.852243185043335 (3.0607649087905884) + | > loss_feat: nan (nan) + | > loss_mel: 17.3732967376709 (18.062410354614258) + | > loss_duration: 1.7744622230529785 (1.8336389660835266) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004098773002624512 (+0.0005060434341430664) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0607649087905884 (-0.02572011947631836) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.062410354614258 (+0.6897449493408203) + | > avg_loss_duration: 1.8336389660835266 (-0.024367451667785645) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 565/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:09:42)  + + --> TIME: 2025-04-19 08:09:47 -- STEP: 13/142 -- GLOBAL_STEP: 459525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.309967041015625 (17.78431686988244) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.3052 (0.3038935477917011) + | > loader_time: 0.0051 (0.004647456682645357) + + + --> TIME: 2025-04-19 08:09:55 -- STEP: 38/142 -- GLOBAL_STEP: 459550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7144601345062256 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.271774291992188 (17.747194716804906) + | > loss_duration: 1.6318695545196533 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.3262 (0.31504676843944357) + | > loader_time: 0.0057 (0.005092376156857139) + + + --> TIME: 2025-04-19 08:10:04 -- STEP: 63/142 -- GLOBAL_STEP: 459575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.092695236206055 (17.84250687795972) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.3623 (0.3266931791154166) + | > loader_time: 0.0063 (0.005410667449708968) + + + --> TIME: 2025-04-19 08:10:13 -- STEP: 88/142 -- GLOBAL_STEP: 459600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.47531509399414 (17.709534124894574) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.3709 (0.33834072676571936) + | > loader_time: 0.0067 (0.005735773931850086) + + + --> TIME: 2025-04-19 08:10:23 -- STEP: 113/142 -- GLOBAL_STEP: 459625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.85120964050293 (17.734422194219267) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.4013 (0.34938623419905135) + | > loader_time: 0.007 (0.005968311191660113) + + + --> TIME: 2025-04-19 08:10:34 -- STEP: 138/142 -- GLOBAL_STEP: 459650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.50388526916504 (17.775975462319195) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018636142848637817 + | > current_lr_1: 0.00018636142848637817 + | > step_time: 0.4352 (0.3613303312356922) + | > loader_time: 0.0075 (0.006220335545747175) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.808027982711792 (2.808027982711792) + | > loss_feat: nan (nan) + | > loss_mel: 16.57691192626953 (16.57691192626953) + | > loss_duration: 1.8284716606140137 (1.8284716606140137) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2570414543151855 (3.2570414543151855) + | > loss_feat: nan (nan) + | > loss_mel: 18.007402420043945 (18.007402420043945) + | > loss_duration: 1.9203028678894043 (1.9203028678894043) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9291083812713623 (3.093074917793274) + | > loss_feat: nan (nan) + | > loss_mel: 18.941970825195312 (18.47468662261963) + | > loss_duration: 1.8048315048217773 (1.8625671863555908) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003641963005065918 (-0.00045680999755859375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.093074917793274 (+0.03231000900268555) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.47468662261963 (+0.4122762680053711) + | > avg_loss_duration: 1.8625671863555908 (+0.02892822027206421) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 566/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:10:37)  + + --> TIME: 2025-04-19 08:10:44 -- STEP: 21/142 -- GLOBAL_STEP: 459675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.66298484802246 (18.02873466128395) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018633813330781737 + | > current_lr_1: 0.00018633813330781737 + | > step_time: 0.3238 (0.30634041059584843) + | > loader_time: 0.0053 (0.004912126631963821) + + + --> TIME: 2025-04-19 08:10:53 -- STEP: 46/142 -- GLOBAL_STEP: 459700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.569345474243164 (18.076375671055008) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018633813330781737 + | > current_lr_1: 0.00018633813330781737 + | > step_time: 0.3514 (0.3237800338993902) + | > loader_time: 0.0062 (0.005341141120247219) + + + --> TIME: 2025-04-19 08:11:02 -- STEP: 71/142 -- GLOBAL_STEP: 459725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.49325942993164 (18.026864132411042) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018633813330781737 + | > current_lr_1: 0.00018633813330781737 + | > step_time: 0.3733 (0.33772712358286683) + | > loader_time: 0.0063 (0.005651960910206109) + + + --> TIME: 2025-04-19 08:11:12 -- STEP: 96/142 -- GLOBAL_STEP: 459750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5840508937835693 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.131364822387695 (17.980873336394623) + | > loss_duration: 1.6754810810089111 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018633813330781737 + | > current_lr_1: 0.00018633813330781737 + | > step_time: 0.3811 (0.3517991602420807) + | > loader_time: 0.0065 (0.005935470263163249) + + + --> TIME: 2025-04-19 08:11:23 -- STEP: 121/142 -- GLOBAL_STEP: 459775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.428869247436523 (17.884361479893197) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018633813330781737 + | > current_lr_1: 0.00018633813330781737 + | > step_time: 0.4619 (0.362179633999659) + | > loader_time: 0.008 (0.0061475974469145465) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.951978921890259 (2.951978921890259) + | > loss_feat: nan (nan) + | > loss_mel: 18.752044677734375 (18.752044677734375) + | > loss_duration: 1.8439853191375732 (1.8439853191375732) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2906570434570312 (3.2906570434570312) + | > loss_feat: nan (nan) + | > loss_mel: 16.85236930847168 (16.85236930847168) + | > loss_duration: 1.8887169361114502 (1.8887169361114502) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.922013282775879 (3.106335163116455) + | > loss_feat: nan (nan) + | > loss_mel: 17.03627586364746 (16.94432258605957) + | > loss_duration: 1.7837603092193604 (1.8362386226654053) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038031339645385742 (+0.00016117095947265625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.106335163116455 (+0.013260245323181152) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.94432258605957 (-1.5303640365600586) + | > avg_loss_duration: 1.8362386226654053 (-0.026328563690185547) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 567/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:11:33)  + + --> TIME: 2025-04-19 08:11:35 -- STEP: 4/142 -- GLOBAL_STEP: 459800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.383155822753906 (18.40431499481201) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.3037 (0.3028935194015503) + | > loader_time: 0.0052 (0.004503786563873291) + + + --> TIME: 2025-04-19 08:11:43 -- STEP: 29/142 -- GLOBAL_STEP: 459825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.77916145324707 (18.054341086025897) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.3239 (0.3109484294365192) + | > loader_time: 0.005 (0.005037759912425074) + + + --> TIME: 2025-04-19 08:11:52 -- STEP: 54/142 -- GLOBAL_STEP: 459850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5980629920959473 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.982507705688477 (17.919605908570464) + | > loss_duration: 1.612640380859375 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.3444 (0.32358588554241025) + | > loader_time: 0.0058 (0.005336982232552988) + + + --> TIME: 2025-04-19 08:12:01 -- STEP: 79/142 -- GLOBAL_STEP: 459875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.00189781188965 (17.950317467315287) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.3615 (0.33518033088008053) + | > loader_time: 0.0062 (0.005650303031824812) + + + --> TIME: 2025-04-19 08:12:11 -- STEP: 104/142 -- GLOBAL_STEP: 459900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.617229461669922 (18.01527046240293) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.3819 (0.34544096543238706) + | > loader_time: 0.007 (0.005889147520065308) + + + --> TIME: 2025-04-19 08:12:21 -- STEP: 129/142 -- GLOBAL_STEP: 459925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.42725944519043 (17.938022546989977) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018631484104115388 + | > current_lr_1: 0.00018631484104115388 + | > step_time: 0.4191 (0.3570317150086395) + | > loader_time: 0.0076 (0.006148356799931489) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0057873725891113 (3.0057873725891113) + | > loss_feat: nan (nan) + | > loss_mel: 19.516353607177734 (19.516353607177734) + | > loss_duration: 1.8490033149719238 (1.8490033149719238) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4419074058532715 (3.4419074058532715) + | > loss_feat: nan (nan) + | > loss_mel: 17.631549835205078 (17.631549835205078) + | > loss_duration: 1.8930985927581787 (1.8930985927581787) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.858081817626953 (3.1499946117401123) + | > loss_feat: nan (nan) + | > loss_mel: 17.599950790405273 (17.615750312805176) + | > loss_duration: 1.8023606538772583 (1.8477296233177185) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003736257553100586 (-6.687641143798828e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1499946117401123 (+0.04365944862365723) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.615750312805176 (+0.6714277267456055) + | > avg_loss_duration: 1.8477296233177185 (+0.011491000652313232) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 568/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:12:28)  + + --> TIME: 2025-04-19 08:12:32 -- STEP: 12/142 -- GLOBAL_STEP: 459950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.44129180908203 (18.102674166361492) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.3023 (0.30617902676264447) + | > loader_time: 0.0047 (0.004314303398132324) + + + --> TIME: 2025-04-19 08:12:41 -- STEP: 37/142 -- GLOBAL_STEP: 459975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.06608009338379 (18.100823041555046) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.3353 (0.3239971818150701) + | > loader_time: 0.0052 (0.00488717491562302) + + + --> TIME: 2025-04-19 08:12:50 -- STEP: 62/142 -- GLOBAL_STEP: 460000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.980955123901367 (18.051205958089522) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.3705 (0.3353535398360222) + | > loader_time: 0.0062 (0.005248808091686618) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_460000.pth + + --> TIME: 2025-04-19 08:13:01 -- STEP: 87/142 -- GLOBAL_STEP: 460025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.750782012939453 (18.06860964325653) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.3765 (0.345103617372184) + | > loader_time: 0.0067 (0.005591707668085208) + + + --> TIME: 2025-04-19 08:13:11 -- STEP: 112/142 -- GLOBAL_STEP: 460050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.875779151916504 (18.071481525897983) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.4015 (0.35408726973193033) + | > loader_time: 0.0073 (0.005957818457058499) + + + --> TIME: 2025-04-19 08:13:21 -- STEP: 137/142 -- GLOBAL_STEP: 460075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5497231483459473 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.34137535095215 (17.98548187478616) + | > loss_duration: 1.6563177108764648 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018629155168602374 + | > current_lr_1: 0.00018629155168602374 + | > step_time: 0.4376 (0.3652592993130649) + | > loader_time: 0.0078 (0.0062678382344489554) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.061276435852051 (3.061276435852051) + | > loss_feat: nan (nan) + | > loss_mel: 19.962621688842773 (19.962621688842773) + | > loss_duration: 1.8218066692352295 (1.8218066692352295) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5444581508636475 (3.5444581508636475) + | > loss_feat: nan (nan) + | > loss_mel: 17.465707778930664 (17.465707778930664) + | > loss_duration: 1.8973560333251953 (1.8973560333251953) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.81998872756958 (3.1822234392166138) + | > loss_feat: nan (nan) + | > loss_mel: 17.628002166748047 (17.546854972839355) + | > loss_duration: 1.7815475463867188 (1.839451789855957) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033255815505981445 (-0.0004106760025024414) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1822234392166138 (+0.032228827476501465) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.546854972839355 (-0.06889533996582031) + | > avg_loss_duration: 1.839451789855957 (-0.008277833461761475) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 569/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:13:25)  + + --> TIME: 2025-04-19 08:13:32 -- STEP: 20/142 -- GLOBAL_STEP: 460100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.94184112548828 (18.176248455047606) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018626826524206297 + | > current_lr_1: 0.00018626826524206297 + | > step_time: 0.3163 (0.3138959169387817) + | > loader_time: 0.0053 (0.005262911319732666) + + + --> TIME: 2025-04-19 08:13:41 -- STEP: 45/142 -- GLOBAL_STEP: 460125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.913589477539062 (17.972035959031846) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018626826524206297 + | > current_lr_1: 0.00018626826524206297 + | > step_time: 0.3458 (0.3225061787499323) + | > loader_time: 0.0061 (0.005505593617757162) + + + --> TIME: 2025-04-19 08:13:50 -- STEP: 70/142 -- GLOBAL_STEP: 460150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.65856170654297 (17.928311634063725) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018626826524206297 + | > current_lr_1: 0.00018626826524206297 + | > step_time: 0.364 (0.33285752023969395) + | > loader_time: 0.0066 (0.005800206320626395) + + + --> TIME: 2025-04-19 08:13:59 -- STEP: 95/142 -- GLOBAL_STEP: 460175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.32326889038086 (17.93657827879254) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018626826524206297 + | > current_lr_1: 0.00018626826524206297 + | > step_time: 0.3795 (0.3430957392642374) + | > loader_time: 0.0082 (0.006031430395025955) + + + --> TIME: 2025-04-19 08:14:09 -- STEP: 120/142 -- GLOBAL_STEP: 460200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.39557647705078 (17.919015304247555) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018626826524206297 + | > current_lr_1: 0.00018626826524206297 + | > step_time: 0.41 (0.35363011360168467) + | > loader_time: 0.0077 (0.006213972965876261) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.812168836593628 (2.812168836593628) + | > loss_feat: nan (nan) + | > loss_mel: 16.364788055419922 (16.364788055419922) + | > loss_duration: 1.8428359031677246 (1.8428359031677246) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.417565107345581 (3.417565107345581) + | > loss_feat: nan (nan) + | > loss_mel: 19.726530075073242 (19.726530075073242) + | > loss_duration: 1.8886024951934814 (1.8886024951934814) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8163466453552246 (3.116955876350403) + | > loss_feat: nan (nan) + | > loss_mel: 18.230066299438477 (18.97829818725586) + | > loss_duration: 1.7710168361663818 (1.8298096656799316) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038770437240600586 (+0.0005514621734619141) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.116955876350403 (-0.06526756286621094) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.97829818725586 (+1.431443214416504) + | > avg_loss_duration: 1.8298096656799316 (-0.00964212417602539) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 570/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:14:20)  + + --> TIME: 2025-04-19 08:14:22 -- STEP: 3/142 -- GLOBAL_STEP: 460225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.347070693969727 (18.03507423400879) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.296 (0.2977448304494222) + | > loader_time: 0.0039 (0.00442655881245931) + + + --> TIME: 2025-04-19 08:14:30 -- STEP: 28/142 -- GLOBAL_STEP: 460250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.563032627105713 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.826383590698242 (18.11135411262512) + | > loss_duration: 1.6438689231872559 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.3183 (0.31152279036385677) + | > loader_time: 0.0054 (0.005061115537370956) + + + --> TIME: 2025-04-19 08:14:38 -- STEP: 53/142 -- GLOBAL_STEP: 460275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.496938943862915 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.522178649902344 (17.977433096687754) + | > loss_duration: 1.6133092641830444 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.3427 (0.32271100889961674) + | > loader_time: 0.0058 (0.0054034646951927326) + + + --> TIME: 2025-04-19 08:14:48 -- STEP: 78/142 -- GLOBAL_STEP: 460300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6680855751037598 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.294267654418945 (17.975318578573376) + | > loss_duration: 1.6782742738723755 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.3813 (0.3365840697899843) + | > loader_time: 0.0065 (0.005682667096455892) + + + --> TIME: 2025-04-19 08:14:57 -- STEP: 103/142 -- GLOBAL_STEP: 460325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5828030109405518 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.3316593170166 (18.03027024315399) + | > loss_duration: 1.7119513750076294 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.3929 (0.3477501383105528) + | > loader_time: 0.0068 (0.005934745362661417) + + + --> TIME: 2025-04-19 08:15:08 -- STEP: 128/142 -- GLOBAL_STEP: 460350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.796707153320312 (17.937345631420612) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001862449817089077 + | > current_lr_1: 0.0001862449817089077 + | > step_time: 0.4118 (0.359213437885046) + | > loader_time: 0.0073 (0.006198817864060402) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7966532707214355 (2.7966532707214355) + | > loss_feat: nan (nan) + | > loss_mel: 20.52130126953125 (20.52130126953125) + | > loss_duration: 1.8472241163253784 (1.8472241163253784) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3708691596984863 (3.3708691596984863) + | > loss_feat: nan (nan) + | > loss_mel: 18.429712295532227 (18.429712295532227) + | > loss_duration: 1.8871859312057495 (1.8871859312057495) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8201165199279785 (3.0954928398132324) + | > loss_feat: nan (nan) + | > loss_mel: 19.198505401611328 (18.814108848571777) + | > loss_duration: 1.793811559677124 (1.8404987454414368) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004015326499938965 (+0.00013828277587890625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0954928398132324 (-0.02146303653717041) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.814108848571777 (-0.16418933868408203) + | > avg_loss_duration: 1.8404987454414368 (+0.010689079761505127) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 571/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:15:15)  + + --> TIME: 2025-04-19 08:15:19 -- STEP: 11/142 -- GLOBAL_STEP: 460375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.442962646484375 (17.805621927434746) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.3055 (0.3005081090060147) + | > loader_time: 0.0048 (0.00483393669128418) + + + --> TIME: 2025-04-19 08:15:28 -- STEP: 36/142 -- GLOBAL_STEP: 460400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.24856948852539 (17.90938817130194) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.3453 (0.3183961576885647) + | > loader_time: 0.0058 (0.005447096294826931) + + + --> TIME: 2025-04-19 08:15:37 -- STEP: 61/142 -- GLOBAL_STEP: 460425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.851394653320312 (18.068988893852854) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.3771 (0.3331159333713719) + | > loader_time: 0.0064 (0.005695006886466607) + + + --> TIME: 2025-04-19 08:15:46 -- STEP: 86/142 -- GLOBAL_STEP: 460450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6845030784606934 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.715396881103516 (17.98077789572782) + | > loss_duration: 1.6968361139297485 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.3857 (0.34702075082202294) + | > loader_time: 0.0072 (0.006008406018101893) + + + --> TIME: 2025-04-19 08:15:57 -- STEP: 111/142 -- GLOBAL_STEP: 460475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.417884826660156 (17.957839192570855) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.4137 (0.3588044557485495) + | > loader_time: 0.0088 (0.006280417914863106) + + + --> TIME: 2025-04-19 08:16:07 -- STEP: 136/142 -- GLOBAL_STEP: 460500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.730213165283203 (17.94285806487587) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018622170108619407 + | > current_lr_1: 0.00018622170108619407 + | > step_time: 0.4339 (0.3711619832936455) + | > loader_time: 0.0079 (0.006526235271902646) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0810298919677734 (3.0810298919677734) + | > loss_feat: nan (nan) + | > loss_mel: 16.908353805541992 (16.908353805541992) + | > loss_duration: 1.8062763214111328 (1.8062763214111328) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3225367069244385 (3.3225367069244385) + | > loss_feat: nan (nan) + | > loss_mel: 17.15094757080078 (17.15094757080078) + | > loss_duration: 1.903283715248108 (1.903283715248108) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.731057643890381 (3.0267971754074097) + | > loss_feat: nan (nan) + | > loss_mel: 15.826502799987793 (16.488725185394287) + | > loss_duration: 1.801463007926941 (1.8523733615875244) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031480789184570312 (-0.0008672475814819336) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0267971754074097 (-0.06869566440582275) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.488725185394287 (-2.3253836631774902) + | > avg_loss_duration: 1.8523733615875244 (+0.011874616146087646) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 572/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:16:12)  + + --> TIME: 2025-04-19 08:16:18 -- STEP: 19/142 -- GLOBAL_STEP: 460525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.546405553817749 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.899188995361328 (18.017264617116826) + | > loss_duration: 1.602445363998413 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861984233735583 + | > current_lr_1: 0.0001861984233735583 + | > step_time: 0.3124 (0.30584133298773514) + | > loader_time: 0.0058 (0.0047935937580309415) + + + --> TIME: 2025-04-19 08:16:27 -- STEP: 44/142 -- GLOBAL_STEP: 460550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.014368057250977 (17.822005662051115) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861984233735583 + | > current_lr_1: 0.0001861984233735583 + | > step_time: 0.3336 (0.3184722607786006) + | > loader_time: 0.0056 (0.005287571386857467) + + + --> TIME: 2025-04-19 08:16:36 -- STEP: 69/142 -- GLOBAL_STEP: 460575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.55681800842285 (17.781085111092825) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861984233735583 + | > current_lr_1: 0.0001861984233735583 + | > step_time: 0.3673 (0.3299565177032914) + | > loader_time: 0.0061 (0.00553680848384249) + + + --> TIME: 2025-04-19 08:16:45 -- STEP: 94/142 -- GLOBAL_STEP: 460600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.934524536132812 (17.775017971688126) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861984233735583 + | > current_lr_1: 0.0001861984233735583 + | > step_time: 0.3756 (0.3413554531462649) + | > loader_time: 0.0066 (0.005783743046699686) + + + --> TIME: 2025-04-19 08:16:55 -- STEP: 119/142 -- GLOBAL_STEP: 460625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.18014144897461 (17.74010088463791) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861984233735583 + | > current_lr_1: 0.0001861984233735583 + | > step_time: 0.4014 (0.3521870965717219) + | > loader_time: 0.0073 (0.006068221661223083) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1555771827697754 (3.1555771827697754) + | > loss_feat: nan (nan) + | > loss_mel: 15.635856628417969 (15.635856628417969) + | > loss_duration: 1.8210762739181519 (1.8210762739181519) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2518718242645264 (3.2518718242645264) + | > loss_feat: nan (nan) + | > loss_mel: 19.031753540039062 (19.031753540039062) + | > loss_duration: 1.92023766040802 (1.92023766040802) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7489659786224365 (3.0004189014434814) + | > loss_feat: nan (nan) + | > loss_mel: 19.0394344329834 (19.03559398651123) + | > loss_duration: 1.7785178422927856 (1.8493777513504028) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003464937210083008 (+0.00031685829162597656) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0004189014434814 (-0.026378273963928223) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.03559398651123 (+2.5468688011169434) + | > avg_loss_duration: 1.8493777513504028 (-0.002995610237121582) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 573/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:17:07)  + + --> TIME: 2025-04-19 08:17:08 -- STEP: 2/142 -- GLOBAL_STEP: 460650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.632524013519287 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.24120330810547 (17.99858283996582) + | > loss_duration: 1.6494650840759277 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.2945 (0.2977031469345093) + | > loader_time: 0.0045 (0.004293084144592285) + + + --> TIME: 2025-04-19 08:17:16 -- STEP: 27/142 -- GLOBAL_STEP: 460675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.543380737304688 (17.84076153790509) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.3394 (0.3146090860720034) + | > loader_time: 0.0056 (0.005159952022411206) + + + --> TIME: 2025-04-19 08:17:25 -- STEP: 52/142 -- GLOBAL_STEP: 460700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.70577049255371 (17.94077990605281) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.3505 (0.3297482866507311) + | > loader_time: 0.0063 (0.005553896610553448) + + + --> TIME: 2025-04-19 08:17:34 -- STEP: 77/142 -- GLOBAL_STEP: 460725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.96993064880371 (17.942150239820606) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.3756 (0.3429525270090475) + | > loader_time: 0.0064 (0.005873856606421533) + + + --> TIME: 2025-04-19 08:17:44 -- STEP: 102/142 -- GLOBAL_STEP: 460750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.575528860092163 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.331432342529297 (17.902462716196105) + | > loss_duration: 1.6689561605453491 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.4136 (0.3557397244023341) + | > loader_time: 0.0073 (0.00619947442821428) + + + --> TIME: 2025-04-19 08:17:55 -- STEP: 127/142 -- GLOBAL_STEP: 460775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.079933166503906 (17.843265248095896) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001861751485706366 + | > current_lr_1: 0.0001861751485706366 + | > step_time: 0.4147 (0.3668723669577771) + | > loader_time: 0.0072 (0.006396526426780882) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.980084180831909 (2.980084180831909) + | > loss_feat: nan (nan) + | > loss_mel: 17.16697120666504 (17.16697120666504) + | > loss_duration: 1.8235993385314941 (1.8235993385314941) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3029699325561523 (3.3029699325561523) + | > loss_feat: nan (nan) + | > loss_mel: 19.176734924316406 (19.176734924316406) + | > loss_duration: 1.871829867362976 (1.871829867362976) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.747697353363037 (3.0253336429595947) + | > loss_feat: nan (nan) + | > loss_mel: 16.875701904296875 (18.02621841430664) + | > loss_duration: 1.8085510730743408 (1.8401904702186584) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00370633602142334 (+0.00024139881134033203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0253336429595947 (+0.02491474151611328) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.02621841430664 (-1.0093755722045898) + | > avg_loss_duration: 1.8401904702186584 (-0.009187281131744385) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 574/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:18:03)  + + --> TIME: 2025-04-19 08:18:06 -- STEP: 10/142 -- GLOBAL_STEP: 460800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.50732421875 (17.37500982284546) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.308 (0.299868106842041) + | > loader_time: 0.005 (0.004715633392333984) + + + --> TIME: 2025-04-19 08:18:15 -- STEP: 35/142 -- GLOBAL_STEP: 460825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.357872009277344 (17.790180233546664) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.3296 (0.31293456213814863) + | > loader_time: 0.0055 (0.005167021070207869) + + + --> TIME: 2025-04-19 08:18:23 -- STEP: 60/142 -- GLOBAL_STEP: 460850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6009154319763184 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.79212760925293 (17.843158292770383) + | > loss_duration: 1.6323177814483643 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.3541 (0.3245798627535501) + | > loader_time: 0.0068 (0.0055229425430297855) + + + --> TIME: 2025-04-19 08:18:33 -- STEP: 85/142 -- GLOBAL_STEP: 460875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.52402663230896 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.951534271240234 (17.909412148419538) + | > loss_duration: 1.6723284721374512 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.3697 (0.33667566636029406) + | > loader_time: 0.0067 (0.005855327494004192) + + + --> TIME: 2025-04-19 08:18:42 -- STEP: 110/142 -- GLOBAL_STEP: 460900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.890399932861328 (17.903224563598624) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.4048 (0.34730861186981193) + | > loader_time: 0.0077 (0.0061101458289406505) + + + --> TIME: 2025-04-19 08:18:53 -- STEP: 135/142 -- GLOBAL_STEP: 460925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.58056640625 (17.903432902583358) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018615187667706527 + | > current_lr_1: 0.00018615187667706527 + | > step_time: 0.4311 (0.35930577207494657) + | > loader_time: 0.0076 (0.006355464016949688) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9154529571533203 (2.9154529571533203) + | > loss_feat: nan (nan) + | > loss_mel: 18.922658920288086 (18.922658920288086) + | > loss_duration: 1.841719388961792 (1.841719388961792) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3492395877838135 (3.3492395877838135) + | > loss_feat: nan (nan) + | > loss_mel: 18.52393341064453 (18.52393341064453) + | > loss_duration: 1.9078550338745117 (1.9078550338745117) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6080071926116943 (2.978623390197754) + | > loss_feat: nan (nan) + | > loss_mel: 16.657638549804688 (17.59078598022461) + | > loss_duration: 1.8031857013702393 (1.8555203676223755) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033658742904663086 (-0.00034046173095703125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.978623390197754 (-0.04671025276184082) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.59078598022461 (-0.43543243408203125) + | > avg_loss_duration: 1.8555203676223755 (+0.015329897403717041) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 575/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:18:57)  + + --> TIME: 2025-04-19 08:19:04 -- STEP: 18/142 -- GLOBAL_STEP: 460950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.211421966552734 (17.9044771194458) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018612860769248062 + | > current_lr_1: 0.00018612860769248062 + | > step_time: 0.3147 (0.3178610536787245) + | > loader_time: 0.0051 (0.004886150360107422) + + + --> TIME: 2025-04-19 08:19:12 -- STEP: 43/142 -- GLOBAL_STEP: 460975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.349536895751953 (18.146468983140107) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018612860769248062 + | > current_lr_1: 0.00018612860769248062 + | > step_time: 0.3375 (0.3230957485908686) + | > loader_time: 0.0059 (0.005202614983846975) + + + --> TIME: 2025-04-19 08:19:21 -- STEP: 68/142 -- GLOBAL_STEP: 461000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.496063232421875 (18.169694395626294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018612860769248062 + | > current_lr_1: 0.00018612860769248062 + | > step_time: 0.3628 (0.3333571097430061) + | > loader_time: 0.0061 (0.0055242601562948785) + + + --> TIME: 2025-04-19 08:19:31 -- STEP: 93/142 -- GLOBAL_STEP: 461025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.58371353149414 (18.11970800994545) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018612860769248062 + | > current_lr_1: 0.00018612860769248062 + | > step_time: 0.3748 (0.3438276347293648) + | > loader_time: 0.0075 (0.0058107965735978975) + + + --> TIME: 2025-04-19 08:19:41 -- STEP: 118/142 -- GLOBAL_STEP: 461050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.101320266723633 (18.073901701781715) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018612860769248062 + | > current_lr_1: 0.00018612860769248062 + | > step_time: 0.4173 (0.3540765713837187) + | > loader_time: 0.0073 (0.006040470074799101) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0916261672973633 (3.0916261672973633) + | > loss_feat: nan (nan) + | > loss_mel: 18.100418090820312 (18.100418090820312) + | > loss_duration: 1.8454067707061768 (1.8454067707061768) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4008982181549072 (3.4008982181549072) + | > loss_feat: nan (nan) + | > loss_mel: 19.172897338867188 (19.172897338867188) + | > loss_duration: 1.8939374685287476 (1.8939374685287476) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8873002529144287 (3.144099235534668) + | > loss_feat: nan (nan) + | > loss_mel: 17.657522201538086 (18.415209770202637) + | > loss_duration: 1.8013134002685547 (1.8476254343986511) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003501415252685547 (+0.00013554096221923828) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.144099235534668 (+0.16547584533691406) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.415209770202637 (+0.8244237899780273) + | > avg_loss_duration: 1.8476254343986511 (-0.007894933223724365) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 576/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:19:53)  + + --> TIME: 2025-04-19 08:19:54 -- STEP: 1/142 -- GLOBAL_STEP: 461075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.991384506225586 (17.991384506225586) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.3107 (0.31066203117370605) + | > loader_time: 0.0042 (0.0041675567626953125) + + + --> TIME: 2025-04-19 08:20:02 -- STEP: 26/142 -- GLOBAL_STEP: 461100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.08588409423828 (17.91229464457585) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.3236 (0.3114989262360793) + | > loader_time: 0.0056 (0.005048990249633789) + + + --> TIME: 2025-04-19 08:20:10 -- STEP: 51/142 -- GLOBAL_STEP: 461125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.420238494873047 (17.967377344767254) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.3436 (0.32258076293795723) + | > loader_time: 0.0064 (0.0054497765559776154) + + + --> TIME: 2025-04-19 08:20:19 -- STEP: 76/142 -- GLOBAL_STEP: 461150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.877172470092773 (17.907115333958675) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.3829 (0.3347356099831431) + | > loader_time: 0.0061 (0.005721851399070338) + + + --> TIME: 2025-04-19 08:20:29 -- STEP: 101/142 -- GLOBAL_STEP: 461175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.320695877075195 (17.953092896112125) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.3892 (0.3449688075792673) + | > loader_time: 0.0066 (0.005974403702386535) + + + --> TIME: 2025-04-19 08:20:39 -- STEP: 126/142 -- GLOBAL_STEP: 461200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.85538101196289 (17.918789795466836) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018610534161651906 + | > current_lr_1: 0.00018610534161651906 + | > step_time: 0.4123 (0.3563034723675441) + | > loader_time: 0.0071 (0.006213568505786714) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.92868971824646 (2.92868971824646) + | > loss_feat: nan (nan) + | > loss_mel: 20.055986404418945 (20.055986404418945) + | > loss_duration: 1.8333039283752441 (1.8333039283752441) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4073076248168945 (3.4073076248168945) + | > loss_feat: nan (nan) + | > loss_mel: 16.94591522216797 (16.94591522216797) + | > loss_duration: 1.9229118824005127 (1.9229118824005127) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9740216732025146 (3.1906646490097046) + | > loss_feat: nan (nan) + | > loss_mel: 17.94938850402832 (17.447651863098145) + | > loss_duration: 1.7993783950805664 (1.8611451387405396) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003590822219848633 (+8.940696716308594e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1906646490097046 (+0.04656541347503662) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.447651863098145 (-0.9675579071044922) + | > avg_loss_duration: 1.8611451387405396 (+0.013519704341888428) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 577/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:20:48)  + + --> TIME: 2025-04-19 08:20:51 -- STEP: 9/142 -- GLOBAL_STEP: 461225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.913345336914062 (17.979691823323567) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.2984 (0.29887157016330296) + | > loader_time: 0.005 (0.004444864061143663) + + + --> TIME: 2025-04-19 08:20:59 -- STEP: 34/142 -- GLOBAL_STEP: 461250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.70743727684021 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.82402229309082 (18.137733291177188) + | > loss_duration: 1.6408207416534424 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.3286 (0.3131023364908555) + | > loader_time: 0.0055 (0.0051970972734339095) + + + --> TIME: 2025-04-19 08:21:08 -- STEP: 59/142 -- GLOBAL_STEP: 461275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5584378242492676 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.159799575805664 (18.015780691373138) + | > loss_duration: 1.6503326892852783 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.3532 (0.3251949286056777) + | > loader_time: 0.0064 (0.005587969796132233) + + + --> TIME: 2025-04-19 08:21:17 -- STEP: 84/142 -- GLOBAL_STEP: 461300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.70890736579895 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.625503540039062 (17.97054081871395) + | > loss_duration: 1.7066043615341187 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.3686 (0.3370073040326436) + | > loader_time: 0.0064 (0.005885090146745954) + + + --> TIME: 2025-04-19 08:21:27 -- STEP: 109/142 -- GLOBAL_STEP: 461325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.734407424926758 (17.944098411350065) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.3895 (0.3474117856506908) + | > loader_time: 0.007 (0.0061197674602543545) + + + --> TIME: 2025-04-19 08:21:38 -- STEP: 134/142 -- GLOBAL_STEP: 461350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.257646560668945 (17.892076826807273) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018608207844881698 + | > current_lr_1: 0.00018608207844881698 + | > step_time: 0.4143 (0.3590860206689408) + | > loader_time: 0.0077 (0.0063425160166042955) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.006625175476074 (3.006625175476074) + | > loss_feat: nan (nan) + | > loss_mel: 18.642932891845703 (18.642932891845703) + | > loss_duration: 1.817246437072754 (1.817246437072754) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4228196144104004 (3.4228196144104004) + | > loss_feat: nan (nan) + | > loss_mel: 19.38211441040039 (19.38211441040039) + | > loss_duration: 1.9062628746032715 (1.9062628746032715) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.668285846710205 (3.0455527305603027) + | > loss_feat: nan (nan) + | > loss_mel: 19.69267463684082 (19.537394523620605) + | > loss_duration: 1.7866582870483398 (1.8464605808258057) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003765106201171875 (+0.0001742839813232422) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0455527305603027 (-0.14511191844940186) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.537394523620605 (+2.089742660522461) + | > avg_loss_duration: 1.8464605808258057 (-0.014684557914733887) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 578/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:21:42)  + + --> TIME: 2025-04-19 08:21:48 -- STEP: 17/142 -- GLOBAL_STEP: 461375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.201255798339844 (17.890821849598606) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018605881818901086 + | > current_lr_1: 0.00018605881818901086 + | > step_time: 0.317 (0.3067793425391702) + | > loader_time: 0.0053 (0.004972542033475988) + + + --> TIME: 2025-04-19 08:21:57 -- STEP: 42/142 -- GLOBAL_STEP: 461400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.644756317138672 (17.92089705240159) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018605881818901086 + | > current_lr_1: 0.00018605881818901086 + | > step_time: 0.351 (0.32298400288536433) + | > loader_time: 0.0061 (0.005371139163062686) + + + --> TIME: 2025-04-19 08:22:06 -- STEP: 67/142 -- GLOBAL_STEP: 461425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.158714294433594 (17.90782359108997) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018605881818901086 + | > current_lr_1: 0.00018605881818901086 + | > step_time: 0.3681 (0.3390799671856325) + | > loader_time: 0.0064 (0.00578157581500153) + + + --> TIME: 2025-04-19 08:22:16 -- STEP: 92/142 -- GLOBAL_STEP: 461450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.45870018005371 (17.8603745750759) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018605881818901086 + | > current_lr_1: 0.00018605881818901086 + | > step_time: 0.4053 (0.35115077184594196) + | > loader_time: 0.0071 (0.006061144497083581) + + + --> TIME: 2025-04-19 08:22:27 -- STEP: 117/142 -- GLOBAL_STEP: 461475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.098316192626953 (17.935424331925887) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018605881818901086 + | > current_lr_1: 0.00018605881818901086 + | > step_time: 0.4101 (0.36430459348564476) + | > loader_time: 0.0066 (0.00633840071849334) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.023897647857666 (3.023897647857666) + | > loss_feat: nan (nan) + | > loss_mel: 17.543304443359375 (17.543304443359375) + | > loss_duration: 1.844158411026001 (1.844158411026001) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3118629455566406 (3.3118629455566406) + | > loss_feat: nan (nan) + | > loss_mel: 16.476865768432617 (16.476865768432617) + | > loss_duration: 1.9071109294891357 (1.9071109294891357) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.726752758026123 (3.019307851791382) + | > loss_feat: nan (nan) + | > loss_mel: 18.812175750732422 (17.64452075958252) + | > loss_duration: 1.7862595319747925 (1.8466852307319641) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003923296928405762 (+0.00015819072723388672) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.019307851791382 (-0.0262448787689209) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.64452075958252 (-1.892873764038086) + | > avg_loss_duration: 1.8466852307319641 (+0.00022464990615844727) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 579/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:22:39)  + + --> TIME: 2025-04-19 08:22:39 -- STEP: 0/142 -- GLOBAL_STEP: 461500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.037994384765625 (17.037994384765625) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.3283 (0.3283107280731201) + | > loader_time: 0.3239 (0.32386255264282227) + + + --> TIME: 2025-04-19 08:22:47 -- STEP: 25/142 -- GLOBAL_STEP: 461525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.659362554550171 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.22541046142578 (18.094177780151366) + | > loss_duration: 1.6615214347839355 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.3192 (0.3074414539337158) + | > loader_time: 0.0061 (0.005170688629150392) + + + --> TIME: 2025-04-19 08:22:56 -- STEP: 50/142 -- GLOBAL_STEP: 461550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.068099975585938 (18.183528633117668) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.3437 (0.3196297168731689) + | > loader_time: 0.0063 (0.0055342626571655285) + + + --> TIME: 2025-04-19 08:23:05 -- STEP: 75/142 -- GLOBAL_STEP: 461575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.195756912231445 (18.118712234497064) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.3719 (0.33201902071634926) + | > loader_time: 0.0063 (0.0057969506581624355) + + + --> TIME: 2025-04-19 08:23:15 -- STEP: 100/142 -- GLOBAL_STEP: 461600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.963199615478516 (18.025608701705927) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.3882 (0.3427951550483704) + | > loader_time: 0.0071 (0.006041553020477296) + + + --> TIME: 2025-04-19 08:23:25 -- STEP: 125/142 -- GLOBAL_STEP: 461625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64617919921875 (18.031955055236818) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018603556083673722 + | > current_lr_1: 0.00018603556083673722 + | > step_time: 0.4176 (0.35466483879089356) + | > loader_time: 0.0081 (0.006249513626098634) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.986436605453491 (2.986436605453491) + | > loss_feat: nan (nan) + | > loss_mel: 18.164167404174805 (18.164167404174805) + | > loss_duration: 1.8375787734985352 (1.8375787734985352) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3965811729431152 (3.3965811729431152) + | > loss_feat: nan (nan) + | > loss_mel: 18.77259635925293 (18.77259635925293) + | > loss_duration: 1.8901236057281494 (1.8901236057281494) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0073037147521973 (3.2019424438476562) + | > loss_feat: nan (nan) + | > loss_mel: 19.6337947845459 (19.203195571899414) + | > loss_duration: 1.7782177925109863 (1.8341706991195679) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003651261329650879 (-0.0002720355987548828) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2019424438476562 (+0.18263459205627441) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.203195571899414 (+1.5586748123168945) + | > avg_loss_duration: 1.8341706991195679 (-0.01251453161239624) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 580/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:23:33)  + + --> TIME: 2025-04-19 08:23:37 -- STEP: 8/142 -- GLOBAL_STEP: 461650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.97962760925293 (17.43346631526947) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.2994 (0.29958516359329224) + | > loader_time: 0.005 (0.004554450511932373) + + + --> TIME: 2025-04-19 08:23:45 -- STEP: 33/142 -- GLOBAL_STEP: 461675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5817813873291 (17.813080498666473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.3391 (0.31615406094175397) + | > loader_time: 0.0059 (0.00535392038749926) + + + --> TIME: 2025-04-19 08:23:54 -- STEP: 58/142 -- GLOBAL_STEP: 461700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.42542266845703 (17.85834123348368) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.359 (0.33029621222923533) + | > loader_time: 0.0063 (0.0056978505233238495) + + + --> TIME: 2025-04-19 08:24:03 -- STEP: 83/142 -- GLOBAL_STEP: 461725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4007034301757812 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.922170639038086 (17.849475113742322) + | > loss_duration: 1.690739631652832 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.3867 (0.3452141514743665) + | > loader_time: 0.0065 (0.006029180733554336) + + + --> TIME: 2025-04-19 08:24:14 -- STEP: 108/142 -- GLOBAL_STEP: 461750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.540611267089844 (17.818231149956038) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.41 (0.35763750694416185) + | > loader_time: 0.0074 (0.006343541321931063) + + + --> TIME: 2025-04-19 08:24:24 -- STEP: 133/142 -- GLOBAL_STEP: 461775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.44413185119629 (17.812941450821732) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018601230639163263 + | > current_lr_1: 0.00018601230639163263 + | > step_time: 0.4209 (0.3702778188805831) + | > loader_time: 0.0078 (0.0065858848112866405) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.03633713722229 (3.03633713722229) + | > loss_feat: nan (nan) + | > loss_mel: 19.080617904663086 (19.080617904663086) + | > loss_duration: 1.8101576566696167 (1.8101576566696167) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4671266078948975 (3.4671266078948975) + | > loss_feat: nan (nan) + | > loss_mel: 17.292861938476562 (17.292861938476562) + | > loss_duration: 1.8706536293029785 (1.8706536293029785) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7541935443878174 (3.1106600761413574) + | > loss_feat: nan (nan) + | > loss_mel: 18.236936569213867 (17.764899253845215) + | > loss_duration: 1.7965214252471924 (1.8335875272750854) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033349990844726562 (-0.00031626224517822266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1106600761413574 (-0.09128236770629883) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.764899253845215 (-1.4382963180541992) + | > avg_loss_duration: 1.8335875272750854 (-0.0005831718444824219) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 581/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:24:30)  + + --> TIME: 2025-04-19 08:24:36 -- STEP: 16/142 -- GLOBAL_STEP: 461800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.71925163269043 (17.821426391601562) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.3125 (0.303404301404953) + | > loader_time: 0.0053 (0.004871010780334473) + + + --> TIME: 2025-04-19 08:24:44 -- STEP: 41/142 -- GLOBAL_STEP: 461825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.535691499710083 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.719167709350586 (17.935737726165026) + | > loss_duration: 1.5953718423843384 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.3373 (0.31610744173933814) + | > loader_time: 0.0055 (0.005246429908566357) + + + --> TIME: 2025-04-19 08:24:53 -- STEP: 66/142 -- GLOBAL_STEP: 461850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.725811004638672 (17.94169501102331) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.3569 (0.32785452857162) + | > loader_time: 0.0062 (0.005518692912477435) + + + --> TIME: 2025-04-19 08:25:02 -- STEP: 91/142 -- GLOBAL_STEP: 461875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.244125366210938 (17.946017254839866) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.3758 (0.3396321207612424) + | > loader_time: 0.0061 (0.00577757646749308) + + + --> TIME: 2025-04-19 08:25:12 -- STEP: 116/142 -- GLOBAL_STEP: 461900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.468875885009766 (17.917252729678967) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.4033 (0.3504452376530087) + | > loader_time: 0.0073 (0.00602663590990264) + + + --> TIME: 2025-04-19 08:25:23 -- STEP: 141/142 -- GLOBAL_STEP: 461925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.128700256347656 (17.86832665382547) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018598905485333366 + | > current_lr_1: 0.00018598905485333366 + | > step_time: 0.1794 (0.3609158400948165) + | > loader_time: 0.0028 (0.006257527263451977) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.027865171432495 (3.027865171432495) + | > loss_feat: nan (nan) + | > loss_mel: 18.45552635192871 (18.45552635192871) + | > loss_duration: 1.8106926679611206 (1.8106926679611206) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5469045639038086 (3.5469045639038086) + | > loss_feat: nan (nan) + | > loss_mel: 18.603343963623047 (18.603343963623047) + | > loss_duration: 1.8980662822723389 (1.8980662822723389) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.898736000061035 (3.222820281982422) + | > loss_feat: nan (nan) + | > loss_mel: 18.128137588500977 (18.36574077606201) + | > loss_duration: 1.7888240814208984 (1.8434451818466187) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031921863555908203 (-0.00014281272888183594) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.222820281982422 (+0.11216020584106445) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.36574077606201 (+0.6008415222167969) + | > avg_loss_duration: 1.8434451818466187 (+0.009857654571533203) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 582/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:25:25)  + + --> TIME: 2025-04-19 08:25:33 -- STEP: 24/142 -- GLOBAL_STEP: 461950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.77531623840332 (17.96209196249644) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185965806221477 + | > current_lr_1: 0.000185965806221477 + | > step_time: 0.3269 (0.30924685796101886) + | > loader_time: 0.0056 (0.0052318274974823) + + + --> TIME: 2025-04-19 08:25:42 -- STEP: 49/142 -- GLOBAL_STEP: 461975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.455246925354004 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.42449188232422 (17.889893337171902) + | > loss_duration: 1.6204032897949219 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185965806221477 + | > current_lr_1: 0.000185965806221477 + | > step_time: 0.3537 (0.3253072816498426) + | > loader_time: 0.0063 (0.005633534217367367) + + + --> TIME: 2025-04-19 08:25:51 -- STEP: 74/142 -- GLOBAL_STEP: 462000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6348202228546143 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.181140899658203 (17.96546579051662) + | > loss_duration: 1.6526633501052856 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185965806221477 + | > current_lr_1: 0.000185965806221477 + | > step_time: 0.362 (0.34086888867455567) + | > loader_time: 0.0058 (0.005936120007489179) + + + --> TIME: 2025-04-19 08:26:01 -- STEP: 99/142 -- GLOBAL_STEP: 462025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.600790500640869 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.244522094726562 (18.038141568501796) + | > loss_duration: 1.6871883869171143 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185965806221477 + | > current_lr_1: 0.000185965806221477 + | > step_time: 0.3769 (0.3494460943973426) + | > loader_time: 0.0072 (0.006095524990197384) + + + --> TIME: 2025-04-19 08:26:11 -- STEP: 124/142 -- GLOBAL_STEP: 462050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.14432716369629 (17.99750439582333) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185965806221477 + | > current_lr_1: 0.000185965806221477 + | > step_time: 0.4175 (0.3600103335995828) + | > loader_time: 0.0075 (0.006344345308119251) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7987756729125977 (2.7987756729125977) + | > loss_feat: nan (nan) + | > loss_mel: 19.32085609436035 (19.32085609436035) + | > loss_duration: 1.8379909992218018 (1.8379909992218018) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4173967838287354 (3.4173967838287354) + | > loss_feat: nan (nan) + | > loss_mel: 18.914878845214844 (18.914878845214844) + | > loss_duration: 1.8898687362670898 (1.8898687362670898) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.743851900100708 (3.0806243419647217) + | > loss_feat: nan (nan) + | > loss_mel: 17.376480102539062 (18.145679473876953) + | > loss_duration: 1.7820913791656494 (1.8359800577163696) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037109851837158203 (+0.000518798828125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0806243419647217 (-0.1421959400177002) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.145679473876953 (-0.2200613021850586) + | > avg_loss_duration: 1.8359800577163696 (-0.0074651241302490234) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 583/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:26:20)  + + --> TIME: 2025-04-19 08:26:23 -- STEP: 7/142 -- GLOBAL_STEP: 462075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.816137313842773 (18.390785217285156) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.3145 (0.303544282913208) + | > loader_time: 0.0038 (0.003998688289097377) + + + --> TIME: 2025-04-19 08:26:31 -- STEP: 32/142 -- GLOBAL_STEP: 462100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.131834030151367 (18.11901205778122) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.3309 (0.3136006891727447) + | > loader_time: 0.0056 (0.0048990026116371155) + + + --> TIME: 2025-04-19 08:26:40 -- STEP: 57/142 -- GLOBAL_STEP: 462125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.455188751220703 (18.086642432631105) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.3497 (0.32493112798322704) + | > loader_time: 0.0057 (0.005322025533308062) + + + --> TIME: 2025-04-19 08:26:49 -- STEP: 82/142 -- GLOBAL_STEP: 462150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.531824111938477 (18.070943564903445) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.369 (0.3365233903977929) + | > loader_time: 0.0064 (0.005573173848594106) + + + --> TIME: 2025-04-19 08:26:59 -- STEP: 107/142 -- GLOBAL_STEP: 462175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.205066680908203 (17.99757791679596) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.3896 (0.34676799818734144) + | > loader_time: 0.0066 (0.005809743827748521) + + + --> TIME: 2025-04-19 08:27:09 -- STEP: 132/142 -- GLOBAL_STEP: 462200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.949909210205078 (17.96008149060336) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001859425604956993 + | > current_lr_1: 0.0001859425604956993 + | > step_time: 0.4223 (0.3585504311503786) + | > loader_time: 0.0074 (0.00608300259619048) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8303139209747314 (2.8303139209747314) + | > loss_feat: nan (nan) + | > loss_mel: 19.298660278320312 (19.298660278320312) + | > loss_duration: 1.8294146060943604 (1.8294146060943604) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3371684551239014 (3.3371684551239014) + | > loss_feat: nan (nan) + | > loss_mel: 18.791608810424805 (18.791608810424805) + | > loss_duration: 1.9130182266235352 (1.9130182266235352) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.999260425567627 (3.168214440345764) + | > loss_feat: nan (nan) + | > loss_mel: 18.943937301635742 (18.867773056030273) + | > loss_duration: 1.778958797454834 (1.8459885120391846) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003901958465576172 (+0.00019097328186035156) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.168214440345764 (+0.08759009838104248) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.867773056030273 (+0.7220935821533203) + | > avg_loss_duration: 1.8459885120391846 (+0.010008454322814941) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 584/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:27:15)  + + --> TIME: 2025-04-19 08:27:20 -- STEP: 15/142 -- GLOBAL_STEP: 462225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.887693405151367 (17.70330721537272) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.307 (0.3011236826578776) + | > loader_time: 0.0057 (0.004699850082397461) + + + --> TIME: 2025-04-19 08:27:29 -- STEP: 40/142 -- GLOBAL_STEP: 462250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.50193977355957 (17.921155834197997) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.343 (0.3190234303474426) + | > loader_time: 0.0059 (0.005168229341506957) + + + --> TIME: 2025-04-19 08:27:38 -- STEP: 65/142 -- GLOBAL_STEP: 462275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.680625915527344 (17.828790899423453) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.3793 (0.333441737981943) + | > loader_time: 0.0066 (0.005521018688495342) + + + --> TIME: 2025-04-19 08:27:48 -- STEP: 90/142 -- GLOBAL_STEP: 462300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.823577880859375 (17.931257014804423) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.391 (0.3465335342619154) + | > loader_time: 0.0071 (0.005808313687642415) + + + --> TIME: 2025-04-19 08:27:58 -- STEP: 115/142 -- GLOBAL_STEP: 462325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.38359832763672 (17.8978398696236) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.4156 (0.35878690429355786) + | > loader_time: 0.0076 (0.006066635380620542) + + + --> TIME: 2025-04-19 08:28:09 -- STEP: 140/142 -- GLOBAL_STEP: 462350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.687894821166992 (17.89832260949271) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018591931767563734 + | > current_lr_1: 0.00018591931767563734 + | > step_time: 0.4344 (0.3710425479071481) + | > loader_time: 0.007 (0.006279194355010986) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0482523441314697 (3.0482523441314697) + | > loss_feat: nan (nan) + | > loss_mel: 17.960901260375977 (17.960901260375977) + | > loss_duration: 1.8432961702346802 (1.8432961702346802) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.297947406768799 (3.297947406768799) + | > loss_feat: nan (nan) + | > loss_mel: 18.29320526123047 (18.29320526123047) + | > loss_duration: 1.905693531036377 (1.905693531036377) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.961500406265259 (3.129723906517029) + | > loss_feat: nan (nan) + | > loss_mel: 18.739105224609375 (18.516155242919922) + | > loss_duration: 1.8175342082977295 (1.8616138696670532) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035219192504882812 (-0.0003800392150878906) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.129723906517029 (-0.03849053382873535) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.516155242919922 (-0.35161781311035156) + | > avg_loss_duration: 1.8616138696670532 (+0.015625357627868652) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 585/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:28:11)  + + --> TIME: 2025-04-19 08:28:19 -- STEP: 23/142 -- GLOBAL_STEP: 462375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.626573085784912 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.786439895629883 (17.591255602629285) + | > loss_duration: 1.6264400482177734 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018589607776092786 + | > current_lr_1: 0.00018589607776092786 + | > step_time: 0.3217 (0.3072617157645847) + | > loader_time: 0.0072 (0.004970042601875637) + + + --> TIME: 2025-04-19 08:28:28 -- STEP: 48/142 -- GLOBAL_STEP: 462400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.739402770996094 (17.902917504310604) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018589607776092786 + | > current_lr_1: 0.00018589607776092786 + | > step_time: 0.3361 (0.3217639525731404) + | > loader_time: 0.0059 (0.005429074168205261) + + + --> TIME: 2025-04-19 08:28:37 -- STEP: 73/142 -- GLOBAL_STEP: 462425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.83955192565918 (17.91140807164858) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018589607776092786 + | > current_lr_1: 0.00018589607776092786 + | > step_time: 0.368 (0.33299403974454694) + | > loader_time: 0.0063 (0.0056822822518544655) + + + --> TIME: 2025-04-19 08:28:46 -- STEP: 98/142 -- GLOBAL_STEP: 462450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.743074417114258 (17.88310011065736) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018589607776092786 + | > current_lr_1: 0.00018589607776092786 + | > step_time: 0.3845 (0.34347195284707205) + | > loader_time: 0.0068 (0.005948013188887616) + + + --> TIME: 2025-04-19 08:28:56 -- STEP: 123/142 -- GLOBAL_STEP: 462475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.86247444152832 (17.89066832627707) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018589607776092786 + | > current_lr_1: 0.00018589607776092786 + | > step_time: 0.414 (0.35478297094019445) + | > loader_time: 0.0072 (0.006223308361642729) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8030898571014404 (2.8030898571014404) + | > loss_feat: nan (nan) + | > loss_mel: 19.40399932861328 (19.40399932861328) + | > loss_duration: 1.8342750072479248 (1.8342750072479248) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3453543186187744 (3.3453543186187744) + | > loss_feat: nan (nan) + | > loss_mel: 19.016843795776367 (19.016843795776367) + | > loss_duration: 1.9100077152252197 (1.9100077152252197) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8594963550567627 (3.1024253368377686) + | > loss_feat: nan (nan) + | > loss_mel: 18.607269287109375 (18.81205654144287) + | > loss_duration: 1.7852444648742676 (1.8476260900497437) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003538846969604492 (+1.6927719116210938e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1024253368377686 (-0.027298569679260254) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.81205654144287 (+0.2959012985229492) + | > avg_loss_duration: 1.8476260900497437 (-0.01398777961730957) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 586/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:29:06)  + + --> TIME: 2025-04-19 08:29:08 -- STEP: 6/142 -- GLOBAL_STEP: 462500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.96088218688965 (17.853450775146484) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.3056 (0.29909642537434894) + | > loader_time: 0.0043 (0.004437764485677083) + + + --> TIME: 2025-04-19 08:29:17 -- STEP: 31/142 -- GLOBAL_STEP: 462525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.955965042114258 (17.98420407695155) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.3359 (0.3164695847419) + | > loader_time: 0.0059 (0.005111648190406061) + + + --> TIME: 2025-04-19 08:29:25 -- STEP: 56/142 -- GLOBAL_STEP: 462550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.664798736572266 (18.01359035287584) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.3579 (0.33022731968334734) + | > loader_time: 0.0059 (0.005506174904959542) + + + --> TIME: 2025-04-19 08:29:35 -- STEP: 81/142 -- GLOBAL_STEP: 462575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.076684951782227 (17.866493213323892) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.3786 (0.3450877430998249) + | > loader_time: 0.0067 (0.0058584713641508124) + + + --> TIME: 2025-04-19 08:29:45 -- STEP: 106/142 -- GLOBAL_STEP: 462600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.519201278686523 (17.850600521519493) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.3993 (0.35684954670240293) + | > loader_time: 0.0075 (0.006128542828110029) + + + --> TIME: 2025-04-19 08:29:56 -- STEP: 131/142 -- GLOBAL_STEP: 462625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.978206634521484 (17.77369106816881) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018587284075120773 + | > current_lr_1: 0.00018587284075120773 + | > step_time: 0.4354 (0.3692835760480575) + | > loader_time: 0.0076 (0.006399067303606572) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8743536472320557 (2.8743536472320557) + | > loss_feat: nan (nan) + | > loss_mel: 19.109037399291992 (19.109037399291992) + | > loss_duration: 1.8484009504318237 (1.8484009504318237) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2003278732299805 (3.2003278732299805) + | > loss_feat: nan (nan) + | > loss_mel: 19.62904930114746 (19.62904930114746) + | > loss_duration: 1.9184273481369019 (1.9184273481369019) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0138401985168457 (3.107084035873413) + | > loss_feat: nan (nan) + | > loss_mel: 18.183942794799805 (18.906496047973633) + | > loss_duration: 1.7964884042739868 (1.8574578762054443) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003438711166381836 (-0.00010013580322265625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.107084035873413 (+0.004658699035644531) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.906496047973633 (+0.09443950653076172) + | > avg_loss_duration: 1.8574578762054443 (+0.009831786155700684) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 587/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:30:02)  + + --> TIME: 2025-04-19 08:30:08 -- STEP: 14/142 -- GLOBAL_STEP: 462650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5495288372039795 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.799638748168945 (18.21484511239188) + | > loss_duration: 1.6377553939819336 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.3061 (0.30500144617898123) + | > loader_time: 0.0048 (0.004660027367728097) + + + --> TIME: 2025-04-19 08:30:16 -- STEP: 39/142 -- GLOBAL_STEP: 462675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.7993221282959 (18.184172116793118) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.3367 (0.315916684957651) + | > loader_time: 0.0061 (0.005110410543588491) + + + --> TIME: 2025-04-19 08:30:25 -- STEP: 64/142 -- GLOBAL_STEP: 462700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.23050308227539 (18.074741467833523) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.3551 (0.32760683447122574) + | > loader_time: 0.0059 (0.005424749106168747) + + + --> TIME: 2025-04-19 08:30:34 -- STEP: 89/142 -- GLOBAL_STEP: 462725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5988712310791 (17.943104293908974) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.3754 (0.34040007966288033) + | > loader_time: 0.0089 (0.005769317069750154) + + + --> TIME: 2025-04-19 08:30:44 -- STEP: 114/142 -- GLOBAL_STEP: 462750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.23296546936035 (17.854617955391873) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.4059 (0.35074062514723403) + | > loader_time: 0.0073 (0.006040642136021664) + + + --> TIME: 2025-04-19 08:30:55 -- STEP: 139/142 -- GLOBAL_STEP: 462775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.665647506713867 (17.875004665457098) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018584960664611383 + | > current_lr_1: 0.00018584960664611383 + | > step_time: 0.4272 (0.3627043782378273) + | > loader_time: 0.0074 (0.006318010014595745) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0155885219573975 (3.0155885219573975) + | > loss_feat: nan (nan) + | > loss_mel: 16.374561309814453 (16.374561309814453) + | > loss_duration: 1.831091284751892 (1.831091284751892) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.280907154083252 (3.280907154083252) + | > loss_feat: nan (nan) + | > loss_mel: 17.896635055541992 (17.896635055541992) + | > loss_duration: 1.9171476364135742 (1.9171476364135742) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7858810424804688 (3.0333940982818604) + | > loss_feat: nan (nan) + | > loss_mel: 17.806655883789062 (17.851645469665527) + | > loss_duration: 1.8019499778747559 (1.859548807144165) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036773681640625 (+0.00023865699768066406) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0333940982818604 (-0.07368993759155273) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.851645469665527 (-1.0548505783081055) + | > avg_loss_duration: 1.859548807144165 (+0.002090930938720703) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 588/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:30:57)  + + --> TIME: 2025-04-19 08:31:05 -- STEP: 22/142 -- GLOBAL_STEP: 462800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.18444061279297 (17.952115665782582) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018582637544528306 + | > current_lr_1: 0.00018582637544528306 + | > step_time: 0.3298 (0.3087805076078935) + | > loader_time: 0.0058 (0.004902774637395684) + + + --> TIME: 2025-04-19 08:31:14 -- STEP: 47/142 -- GLOBAL_STEP: 462825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.605496883392334 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.155975341796875 (17.96714206452065) + | > loss_duration: 1.5875831842422485 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018582637544528306 + | > current_lr_1: 0.00018582637544528306 + | > step_time: 0.3479 (0.3247024911515256) + | > loader_time: 0.006 (0.005398613341311191) + + + --> TIME: 2025-04-19 08:31:23 -- STEP: 72/142 -- GLOBAL_STEP: 462850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.677655220031738 (17.904153386751812) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018582637544528306 + | > current_lr_1: 0.00018582637544528306 + | > step_time: 0.3797 (0.33854097790188264) + | > loader_time: 0.0066 (0.005740784936481052) + + + --> TIME: 2025-04-19 08:31:33 -- STEP: 97/142 -- GLOBAL_STEP: 462875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.67475700378418 (17.901831577733628) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018582637544528306 + | > current_lr_1: 0.00018582637544528306 + | > step_time: 0.3977 (0.3508757463435537) + | > loader_time: 0.0068 (0.006050463804264658) + + + --> TIME: 2025-04-19 08:31:43 -- STEP: 122/142 -- GLOBAL_STEP: 462900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.1510066986084 (17.890675966856918) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018582637544528306 + | > current_lr_1: 0.00018582637544528306 + | > step_time: 0.426 (0.3642956350670486) + | > loader_time: 0.0071 (0.006317781620338315) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1000161170959473 (3.1000161170959473) + | > loss_feat: nan (nan) + | > loss_mel: 16.880807876586914 (16.880807876586914) + | > loss_duration: 1.859168529510498 (1.859168529510498) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.328904390335083 (3.328904390335083) + | > loss_feat: nan (nan) + | > loss_mel: 17.481433868408203 (17.481433868408203) + | > loss_duration: 1.9290026426315308 (1.9290026426315308) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.754890203475952 (3.0418972969055176) + | > loss_feat: nan (nan) + | > loss_mel: 17.63655662536621 (17.558995246887207) + | > loss_duration: 1.7949144840240479 (1.8619585633277893) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032395124435424805 (-0.00043785572052001953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0418972969055176 (+0.008503198623657227) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.558995246887207 (-0.2926502227783203) + | > avg_loss_duration: 1.8619585633277893 (+0.0024097561836242676) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 589/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:31:53)  + + --> TIME: 2025-04-19 08:31:56 -- STEP: 5/142 -- GLOBAL_STEP: 462925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.475334405899048 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.719308853149414 (18.197505950927734) + | > loss_duration: 1.611739993095398 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.3038 (0.30062131881713866) + | > loader_time: 0.0043 (0.0042610645294189455) + + + --> TIME: 2025-04-19 08:32:04 -- STEP: 30/142 -- GLOBAL_STEP: 462950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.1433048248291 (18.08024482727051) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.3257 (0.3124270757039388) + | > loader_time: 0.0058 (0.005038905143737795) + + + --> TIME: 2025-04-19 08:32:12 -- STEP: 55/142 -- GLOBAL_STEP: 462975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.75102996826172 (17.955712838606406) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.3477 (0.3246479901400479) + | > loader_time: 0.0055 (0.005398763309825552) + + + --> TIME: 2025-04-19 08:32:22 -- STEP: 80/142 -- GLOBAL_STEP: 463000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.348068237304688 (17.957767248153687) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.3832 (0.33627418875694276) + | > loader_time: 0.0065 (0.005692416429519653) + + + --> TIME: 2025-04-19 08:32:31 -- STEP: 105/142 -- GLOBAL_STEP: 463025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.260265350341797 (17.949728929428815) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.3972 (0.34666474206106995) + | > loader_time: 0.0065 (0.0059412615639822825) + + + --> TIME: 2025-04-19 08:32:42 -- STEP: 130/142 -- GLOBAL_STEP: 463050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.944643020629883 (17.969239242260258) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001858031471483524 + | > current_lr_1: 0.0001858031471483524 + | > step_time: 0.4221 (0.35884291208707364) + | > loader_time: 0.0075 (0.006232166290283203) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.008308172225952 (3.008308172225952) + | > loss_feat: nan (nan) + | > loss_mel: 19.775991439819336 (19.775991439819336) + | > loss_duration: 1.836376428604126 (1.836376428604126) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5889155864715576 (3.5889155864715576) + | > loss_feat: nan (nan) + | > loss_mel: 18.110363006591797 (18.110363006591797) + | > loss_duration: 1.8986272811889648 (1.8986272811889648) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9122660160064697 (3.2505908012390137) + | > loss_feat: nan (nan) + | > loss_mel: 17.017810821533203 (17.5640869140625) + | > loss_duration: 1.7855944633483887 (1.8421108722686768) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003806591033935547 (+0.0005670785903930664) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2505908012390137 (+0.2086935043334961) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.5640869140625 (+0.005091667175292969) + | > avg_loss_duration: 1.8421108722686768 (-0.01984769105911255) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 590/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:32:49)  + + --> TIME: 2025-04-19 08:32:53 -- STEP: 13/142 -- GLOBAL_STEP: 463075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.11190414428711 (17.129033528841457) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.3039 (0.3002749222975511) + | > loader_time: 0.0049 (0.0043656275822566105) + + + --> TIME: 2025-04-19 08:33:01 -- STEP: 38/142 -- GLOBAL_STEP: 463100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.139230728149414 (17.706970541100752) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.3369 (0.3177855265767951) + | > loader_time: 0.0056 (0.004983343576130114) + + + --> TIME: 2025-04-19 08:33:11 -- STEP: 63/142 -- GLOBAL_STEP: 463125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.727511167526245 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.524198532104492 (17.82150993649922) + | > loss_duration: 1.65501070022583 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.373 (0.33225906462896443) + | > loader_time: 0.0064 (0.0054464832184806695) + + + --> TIME: 2025-04-19 08:33:20 -- STEP: 88/142 -- GLOBAL_STEP: 463150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.338886260986328 (17.877465107224218) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.3833 (0.34574073824015533) + | > loader_time: 0.0067 (0.0057562129064039754) + + + --> TIME: 2025-04-19 08:33:30 -- STEP: 113/142 -- GLOBAL_STEP: 463175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.745372772216797 (17.87906115033987) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.4006 (0.3570183568296181) + | > loader_time: 0.0072 (0.005993018108131611) + + + --> TIME: 2025-04-19 08:33:41 -- STEP: 138/142 -- GLOBAL_STEP: 463200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7099955081939697 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.307613372802734 (17.827434850775692) + | > loss_duration: 1.7327525615692139 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018577992175495886 + | > current_lr_1: 0.00018577992175495886 + | > step_time: 0.4318 (0.3677457951117254) + | > loader_time: 0.0075 (0.006232954453730928) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.975783348083496 (2.975783348083496) + | > loss_feat: nan (nan) + | > loss_mel: 18.790205001831055 (18.790205001831055) + | > loss_duration: 1.8496332168579102 (1.8496332168579102) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.362304449081421 (3.362304449081421) + | > loss_feat: nan (nan) + | > loss_mel: 20.185468673706055 (20.185468673706055) + | > loss_duration: 1.912309169769287 (1.912309169769287) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8529934883117676 (3.1076489686965942) + | > loss_feat: nan (nan) + | > loss_mel: 17.1351375579834 (18.660303115844727) + | > loss_duration: 1.80170738697052 (1.8570082783699036) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003662705421447754 (-0.00014388561248779297) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1076489686965942 (-0.14294183254241943) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.660303115844727 (+1.0962162017822266) + | > avg_loss_duration: 1.8570082783699036 (+0.014897406101226807) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 591/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:33:44)  + + --> TIME: 2025-04-19 08:33:51 -- STEP: 21/142 -- GLOBAL_STEP: 463225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.170167922973633 (17.896897815522692) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018575669926473948 + | > current_lr_1: 0.00018575669926473948 + | > step_time: 0.3158 (0.3056078751881917) + | > loader_time: 0.0052 (0.004780735288347516) + + + --> TIME: 2025-04-19 08:34:00 -- STEP: 46/142 -- GLOBAL_STEP: 463250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.919788360595703 (17.93271682573401) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018575669926473948 + | > current_lr_1: 0.00018575669926473948 + | > step_time: 0.3418 (0.3197834958200868) + | > loader_time: 0.0056 (0.005239248275756837) + + + --> TIME: 2025-04-19 08:34:09 -- STEP: 71/142 -- GLOBAL_STEP: 463275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.50839614868164 (17.93003513443638) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018575669926473948 + | > current_lr_1: 0.00018575669926473948 + | > step_time: 0.361 (0.33103879404739583) + | > loader_time: 0.0063 (0.005507966162453235) + + + --> TIME: 2025-04-19 08:34:18 -- STEP: 96/142 -- GLOBAL_STEP: 463300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6804637908935547 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.097070693969727 (17.9790696799755) + | > loss_duration: 1.6651941537857056 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018575669926473948 + | > current_lr_1: 0.00018575669926473948 + | > step_time: 0.3797 (0.34190919498602534) + | > loader_time: 0.0063 (0.005746364593505857) + + + --> TIME: 2025-04-19 08:34:29 -- STEP: 121/142 -- GLOBAL_STEP: 463325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.80855941772461 (18.0267950247142) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018575669926473948 + | > current_lr_1: 0.00018575669926473948 + | > step_time: 0.4197 (0.3531296844324789) + | > loader_time: 0.0076 (0.006002055711982662) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7659425735473633 (2.7659425735473633) + | > loss_feat: nan (nan) + | > loss_mel: 18.770736694335938 (18.770736694335938) + | > loss_duration: 1.8259589672088623 (1.8259589672088623) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.169654369354248 (3.169654369354248) + | > loss_feat: nan (nan) + | > loss_mel: 18.215578079223633 (18.215578079223633) + | > loss_duration: 1.8877569437026978 (1.8877569437026978) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.802699089050293 (2.9861767292022705) + | > loss_feat: nan (nan) + | > loss_mel: 18.42439842224121 (18.319988250732422) + | > loss_duration: 1.7906866073608398 (1.8392217755317688) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003687739372253418 (+2.5033950805664062e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9861767292022705 (-0.12147223949432373) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.319988250732422 (-0.3403148651123047) + | > avg_loss_duration: 1.8392217755317688 (-0.017786502838134766) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 592/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:34:39)  + + --> TIME: 2025-04-19 08:34:41 -- STEP: 4/142 -- GLOBAL_STEP: 463350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.48151206970215 (18.76148796081543) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.3005 (0.30032283067703247) + | > loader_time: 0.0055 (0.0045874714851379395) + + + --> TIME: 2025-04-19 08:34:49 -- STEP: 29/142 -- GLOBAL_STEP: 463375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.563003540039062 (17.848050742313777) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.3313 (0.3186250061824404) + | > loader_time: 0.0061 (0.005195560126469053) + + + --> TIME: 2025-04-19 08:34:58 -- STEP: 54/142 -- GLOBAL_STEP: 463400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.588136672973633 (17.934671967117875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.3558 (0.3318184790787875) + | > loader_time: 0.0063 (0.0055634357311107495) + + + --> TIME: 2025-04-19 08:35:08 -- STEP: 79/142 -- GLOBAL_STEP: 463425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.740856170654297 (17.975018452994433) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.3614 (0.3474665201162991) + | > loader_time: 0.0067 (0.005897359002994586) + + + --> TIME: 2025-04-19 08:35:17 -- STEP: 104/142 -- GLOBAL_STEP: 463450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.394853591918945 (17.986033173707817) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.3863 (0.3552153614851145) + | > loader_time: 0.007 (0.006099673417898325) + + + --> TIME: 2025-04-19 08:35:28 -- STEP: 129/142 -- GLOBAL_STEP: 463475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.064462661743164 (17.90708067990089) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018573347967733138 + | > current_lr_1: 0.00018573347967733138 + | > step_time: 0.4222 (0.3651784630708917) + | > loader_time: 0.0073 (0.0063273278317710225) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0531017780303955 (3.0531017780303955) + | > loss_feat: nan (nan) + | > loss_mel: 19.485794067382812 (19.485794067382812) + | > loss_duration: 1.8315129280090332 (1.8315129280090332) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.159421443939209 (3.159421443939209) + | > loss_feat: nan (nan) + | > loss_mel: 19.361539840698242 (19.361539840698242) + | > loss_duration: 1.900089979171753 (1.900089979171753) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9095802307128906 (3.03450083732605) + | > loss_feat: nan (nan) + | > loss_mel: 19.07887840270996 (19.2202091217041) + | > loss_duration: 1.8071787357330322 (1.8536343574523926) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031446218490600586 (-0.0005431175231933594) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.03450083732605 (+0.0483241081237793) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.2202091217041 (+0.9002208709716797) + | > avg_loss_duration: 1.8536343574523926 (+0.01441258192062378) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 593/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:35:35)  + + --> TIME: 2025-04-19 08:35:39 -- STEP: 12/142 -- GLOBAL_STEP: 463500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.248031616210938 (18.37284771601359) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.3051 (0.3022580146789551) + | > loader_time: 0.0052 (0.004536668459574382) + + + --> TIME: 2025-04-19 08:35:47 -- STEP: 37/142 -- GLOBAL_STEP: 463525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.3753719329834 (18.08775973964382) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.3276 (0.31445001267098094) + | > loader_time: 0.0055 (0.005038796244440852) + + + --> TIME: 2025-04-19 08:35:56 -- STEP: 62/142 -- GLOBAL_STEP: 463550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.498754501342773 (18.11867092501732) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.357 (0.32697619161298197) + | > loader_time: 0.0066 (0.005350835861698274) + + + --> TIME: 2025-04-19 08:36:06 -- STEP: 87/142 -- GLOBAL_STEP: 463575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.108856201171875 (18.039312987492) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.3757 (0.33886351256534974) + | > loader_time: 0.0063 (0.005658182604559536) + + + --> TIME: 2025-04-19 08:36:16 -- STEP: 112/142 -- GLOBAL_STEP: 463600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.961532592773438 (17.947285848004473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.4004 (0.3495410063437054) + | > loader_time: 0.0068 (0.00592360964843205) + + + --> TIME: 2025-04-19 08:36:26 -- STEP: 137/142 -- GLOBAL_STEP: 463625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.013750076293945 (17.869342038231167) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001857102629923717 + | > current_lr_1: 0.0001857102629923717 + | > step_time: 0.4334 (0.3613699143820436) + | > loader_time: 0.0077 (0.006183197898586301) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.860759973526001 (2.860759973526001) + | > loss_feat: nan (nan) + | > loss_mel: 17.426532745361328 (17.426532745361328) + | > loss_duration: 1.8504266738891602 (1.8504266738891602) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.311765193939209 (3.311765193939209) + | > loss_feat: nan (nan) + | > loss_mel: 19.35302734375 (19.35302734375) + | > loss_duration: 1.9349281787872314 (1.9349281787872314) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.983266592025757 (3.147515892982483) + | > loss_feat: nan (nan) + | > loss_mel: 19.591459274291992 (19.472243309020996) + | > loss_duration: 1.802161455154419 (1.8685448169708252) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035734176635742188 (+0.00042879581451416016) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.147515892982483 (+0.1130150556564331) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.472243309020996 (+0.25203418731689453) + | > avg_loss_duration: 1.8685448169708252 (+0.014910459518432617) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 594/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:36:30)  + + --> TIME: 2025-04-19 08:36:37 -- STEP: 20/142 -- GLOBAL_STEP: 463650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5910162925720215 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.935613632202148 (17.898951530456543) + | > loss_duration: 1.6515971422195435 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018568704920949764 + | > current_lr_1: 0.00018568704920949764 + | > step_time: 0.3165 (0.3078766822814941) + | > loader_time: 0.0052 (0.004781937599182129) + + + --> TIME: 2025-04-19 08:36:45 -- STEP: 45/142 -- GLOBAL_STEP: 463675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.19983673095703 (17.779737684461807) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018568704920949764 + | > current_lr_1: 0.00018568704920949764 + | > step_time: 0.3539 (0.3243998845418295) + | > loader_time: 0.0063 (0.005398564868503146) + + + --> TIME: 2025-04-19 08:36:55 -- STEP: 70/142 -- GLOBAL_STEP: 463700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.87104034423828 (17.831557205745163) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018568704920949764 + | > current_lr_1: 0.00018568704920949764 + | > step_time: 0.3648 (0.34211997645241876) + | > loader_time: 0.0065 (0.005726872171674454) + + + --> TIME: 2025-04-19 08:37:04 -- STEP: 95/142 -- GLOBAL_STEP: 463725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.614990234375 (17.850022024857378) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018568704920949764 + | > current_lr_1: 0.00018568704920949764 + | > step_time: 0.3871 (0.35043733747381905) + | > loader_time: 0.0068 (0.005985445725290398) + + + --> TIME: 2025-04-19 08:37:15 -- STEP: 120/142 -- GLOBAL_STEP: 463750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.740821838378906 (17.823898569742845) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018568704920949764 + | > current_lr_1: 0.00018568704920949764 + | > step_time: 0.4148 (0.3608927528063456) + | > loader_time: 0.0076 (0.006238935391108194) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.007436990737915 (3.007436990737915) + | > loss_feat: nan (nan) + | > loss_mel: 18.545961380004883 (18.545961380004883) + | > loss_duration: 1.8383387327194214 (1.8383387327194214) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.187134027481079 (3.187134027481079) + | > loss_feat: nan (nan) + | > loss_mel: 16.82577133178711 (16.82577133178711) + | > loss_duration: 1.8756299018859863 (1.8756299018859863) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8400990962982178 (3.0136165618896484) + | > loss_feat: nan (nan) + | > loss_mel: 16.42064094543457 (16.62320613861084) + | > loss_duration: 1.7804794311523438 (1.828054666519165) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003591299057006836 (+1.7881393432617188e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0136165618896484 (-0.13389933109283447) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.62320613861084 (-2.8490371704101562) + | > avg_loss_duration: 1.828054666519165 (-0.040490150451660156) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 595/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:37:26)  + + --> TIME: 2025-04-19 08:37:27 -- STEP: 3/142 -- GLOBAL_STEP: 463775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6406197547912598 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.164819717407227 (17.114866256713867) + | > loss_duration: 1.5975635051727295 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.2965 (0.2992209593454997) + | > loader_time: 0.0041 (0.004016002019246419) + + + --> TIME: 2025-04-19 08:37:35 -- STEP: 28/142 -- GLOBAL_STEP: 463800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.80675506591797 (17.765001910073416) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.3197 (0.310469993523189) + | > loader_time: 0.0056 (0.005031509058816093) + + + --> TIME: 2025-04-19 08:37:44 -- STEP: 53/142 -- GLOBAL_STEP: 463825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.24518394470215 (17.779115245027366) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.3415 (0.3223516266300993) + | > loader_time: 0.006 (0.005372479276837043) + + + --> TIME: 2025-04-19 08:37:53 -- STEP: 78/142 -- GLOBAL_STEP: 463850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.317134857177734 (17.81502674787473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.3675 (0.3357943846629216) + | > loader_time: 0.0063 (0.005703507325588128) + + + --> TIME: 2025-04-19 08:38:03 -- STEP: 103/142 -- GLOBAL_STEP: 463875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7614541053771973 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.6381778717041 (17.88524896195791) + | > loss_duration: 1.6827625036239624 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.3865 (0.3464002377778581) + | > loader_time: 0.0068 (0.005951913815100216) + + + --> TIME: 2025-04-19 08:38:13 -- STEP: 128/142 -- GLOBAL_STEP: 463900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.647825241088867 (17.83446779847145) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018566383832834644 + | > current_lr_1: 0.00018566383832834644 + | > step_time: 0.4356 (0.358066413551569) + | > loader_time: 0.0076 (0.006187649443745613) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.879439353942871 (2.879439353942871) + | > loss_feat: nan (nan) + | > loss_mel: 18.734989166259766 (18.734989166259766) + | > loss_duration: 1.8272749185562134 (1.8272749185562134) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3201653957366943 (3.3201653957366943) + | > loss_feat: nan (nan) + | > loss_mel: 15.967402458190918 (15.967402458190918) + | > loss_duration: 1.8958660364151 (1.8958660364151) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9190502166748047 (3.1196078062057495) + | > loss_feat: nan (nan) + | > loss_mel: 17.980741500854492 (16.974071979522705) + | > loss_duration: 1.7851579189300537 (1.840511977672577) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034052133560180664 (-0.00018608570098876953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1196078062057495 (+0.10599124431610107) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.974071979522705 (+0.35086584091186523) + | > avg_loss_duration: 1.840511977672577 (+0.012457311153411865) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 596/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:38:20)  + + --> TIME: 2025-04-19 08:38:25 -- STEP: 11/142 -- GLOBAL_STEP: 463925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.574103593826294 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.168745040893555 (18.347190683538262) + | > loss_duration: 1.5748984813690186 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.3076 (0.30437996170737525) + | > loader_time: 0.0045 (0.004713426936756481) + + + --> TIME: 2025-04-19 08:38:33 -- STEP: 36/142 -- GLOBAL_STEP: 463950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.254247665405273 (18.06954860687256) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.3367 (0.3184685839547051) + | > loader_time: 0.0058 (0.00530259476767646) + + + --> TIME: 2025-04-19 08:38:42 -- STEP: 61/142 -- GLOBAL_STEP: 463975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.816146850585938 (18.050266000091046) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.369 (0.3321370453131003) + | > loader_time: 0.0064 (0.0056636138040511335) + + + --> TIME: 2025-04-19 08:38:52 -- STEP: 86/142 -- GLOBAL_STEP: 464000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.412597179412842 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.49141502380371 (18.005829156831247) + | > loss_duration: 1.6905255317687988 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.3818 (0.34538183378618814) + | > loader_time: 0.0065 (0.005879912265511446) + + + --> TIME: 2025-04-19 08:39:01 -- STEP: 111/142 -- GLOBAL_STEP: 464025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.108930587768555 (17.993945723181376) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.4007 (0.3550247286891077) + | > loader_time: 0.0073 (0.00613409119683343) + + + --> TIME: 2025-04-19 08:39:12 -- STEP: 136/142 -- GLOBAL_STEP: 464050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.763486862182617 (17.95675957202912) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018564063034855538 + | > current_lr_1: 0.00018564063034855538 + | > step_time: 0.4255 (0.36587547729997066) + | > loader_time: 0.0073 (0.006384139551835901) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.037757158279419 (3.037757158279419) + | > loss_feat: nan (nan) + | > loss_mel: 17.12032127380371 (17.12032127380371) + | > loss_duration: 1.8427151441574097 (1.8427151441574097) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.251664400100708 (3.251664400100708) + | > loss_feat: nan (nan) + | > loss_mel: 18.537395477294922 (18.537395477294922) + | > loss_duration: 1.9058995246887207 (1.9058995246887207) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.781050205230713 (3.0163573026657104) + | > loss_feat: nan (nan) + | > loss_mel: 19.317852020263672 (18.927623748779297) + | > loss_duration: 1.7726740837097168 (1.8392868041992188) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037276744842529297 (+0.0003224611282348633) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0163573026657104 (-0.10325050354003906) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.927623748779297 (+1.9535517692565918) + | > avg_loss_duration: 1.8392868041992188 (-0.0012251734733581543) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 597/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:39:16)  + + --> TIME: 2025-04-19 08:39:23 -- STEP: 19/142 -- GLOBAL_STEP: 464075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.956071853637695 (18.106725592362253) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001856174252697618 + | > current_lr_1: 0.0001856174252697618 + | > step_time: 0.3311 (0.3074133647115607) + | > loader_time: 0.0051 (0.004814022465756065) + + + --> TIME: 2025-04-19 08:39:31 -- STEP: 44/142 -- GLOBAL_STEP: 464100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.390522003173828 (17.974882494319566) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001856174252697618 + | > current_lr_1: 0.0001856174252697618 + | > step_time: 0.3386 (0.31861994483254175) + | > loader_time: 0.0054 (0.005281237038699065) + + + --> TIME: 2025-04-19 08:39:40 -- STEP: 69/142 -- GLOBAL_STEP: 464125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.76144790649414 (18.044128708217453) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001856174252697618 + | > current_lr_1: 0.0001856174252697618 + | > step_time: 0.3701 (0.33015879340793775) + | > loader_time: 0.0061 (0.0055904733961907) + + + --> TIME: 2025-04-19 08:39:50 -- STEP: 94/142 -- GLOBAL_STEP: 464150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.399877548217773 (17.9585387351665) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001856174252697618 + | > current_lr_1: 0.0001856174252697618 + | > step_time: 0.3722 (0.341214915539356) + | > loader_time: 0.0061 (0.005871141210515449) + + + --> TIME: 2025-04-19 08:40:00 -- STEP: 119/142 -- GLOBAL_STEP: 464175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.89419174194336 (17.871055474802226) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001856174252697618 + | > current_lr_1: 0.0001856174252697618 + | > step_time: 0.4004 (0.3522068833102699) + | > loader_time: 0.0071 (0.006097605248459247) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.925318956375122 (2.925318956375122) + | > loss_feat: nan (nan) + | > loss_mel: 17.89994239807129 (17.89994239807129) + | > loss_duration: 1.8272662162780762 (1.8272662162780762) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.160949945449829 (3.160949945449829) + | > loss_feat: nan (nan) + | > loss_mel: 17.1361141204834 (17.1361141204834) + | > loss_duration: 1.904496431350708 (1.904496431350708) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.841204881668091 (3.00107741355896) + | > loss_feat: nan (nan) + | > loss_mel: 18.474214553833008 (17.805164337158203) + | > loss_duration: 1.7926771640777588 (1.8485867977142334) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036774873733520508 (-5.0187110900878906e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.00107741355896 (-0.015279889106750488) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.805164337158203 (-1.1224594116210938) + | > avg_loss_duration: 1.8485867977142334 (+0.009299993515014648) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 598/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:40:11)  + + --> TIME: 2025-04-19 08:40:12 -- STEP: 2/142 -- GLOBAL_STEP: 464200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.560352325439453 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.891162872314453 (17.00559711456299) + | > loss_duration: 1.6672358512878418 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.2958 (0.2985495328903198) + | > loader_time: 0.0044 (0.004163026809692383) + + + --> TIME: 2025-04-19 08:40:20 -- STEP: 27/142 -- GLOBAL_STEP: 464225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.432775497436523 (17.62148111837882) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.3346 (0.3115369567164668) + | > loader_time: 0.0052 (0.005133990888242369) + + + --> TIME: 2025-04-19 08:40:29 -- STEP: 52/142 -- GLOBAL_STEP: 464250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.88578224182129 (17.81128272643456) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.3499 (0.327376888348506) + | > loader_time: 0.0064 (0.005447236391214224) + + + --> TIME: 2025-04-19 08:40:38 -- STEP: 77/142 -- GLOBAL_STEP: 464275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.320724487304688 (17.869433142922144) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.3767 (0.34167232141866305) + | > loader_time: 0.0074 (0.005780461546662566) + + + --> TIME: 2025-04-19 08:40:48 -- STEP: 102/142 -- GLOBAL_STEP: 464300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.931194305419922 (17.867452640159463) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.4087 (0.35358345508575423) + | > loader_time: 0.0066 (0.006097104035171808) + + + --> TIME: 2025-04-19 08:40:59 -- STEP: 127/142 -- GLOBAL_STEP: 464325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.544673919677734 (17.91084242993453) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018559422309160308 + | > current_lr_1: 0.00018559422309160308 + | > step_time: 0.4429 (0.3675513887029932) + | > loader_time: 0.0076 (0.006422754347793699) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9547908306121826 (2.9547908306121826) + | > loss_feat: nan (nan) + | > loss_mel: 17.144479751586914 (17.144479751586914) + | > loss_duration: 1.8351515531539917 (1.8351515531539917) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.125854969024658 (3.125854969024658) + | > loss_feat: nan (nan) + | > loss_mel: 18.71497344970703 (18.71497344970703) + | > loss_duration: 1.9256267547607422 (1.9256267547607422) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9673829078674316 (3.046618938446045) + | > loss_feat: nan (nan) + | > loss_mel: 17.3956298828125 (18.055301666259766) + | > loss_duration: 1.8187415599822998 (1.872184157371521) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00371551513671875 (+3.802776336669922e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.046618938446045 (+0.04554152488708496) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.055301666259766 (+0.2501373291015625) + | > avg_loss_duration: 1.872184157371521 (+0.023597359657287598) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 599/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:41:07)  + + --> TIME: 2025-04-19 08:41:11 -- STEP: 10/142 -- GLOBAL_STEP: 464350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.57917594909668 (18.377535247802733) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.3079 (0.3000095129013062) + | > loader_time: 0.0048 (0.004606223106384278) + + + --> TIME: 2025-04-19 08:41:19 -- STEP: 35/142 -- GLOBAL_STEP: 464375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.494476318359375 (18.089316177368165) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.3305 (0.31295948709760396) + | > loader_time: 0.0056 (0.005074637276785715) + + + --> TIME: 2025-04-19 08:41:28 -- STEP: 60/142 -- GLOBAL_STEP: 464400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.177120208740234 (18.070033597946168) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.3515 (0.32512272198994957) + | > loader_time: 0.0063 (0.0054551760355631505) + + + --> TIME: 2025-04-19 08:41:37 -- STEP: 85/142 -- GLOBAL_STEP: 464425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.667973041534424 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.241310119628906 (18.01182975769043) + | > loss_duration: 1.6631312370300293 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.3704 (0.33706729271832636) + | > loader_time: 0.0069 (0.005752235300400679) + + + --> TIME: 2025-04-19 08:41:47 -- STEP: 110/142 -- GLOBAL_STEP: 464450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.721611261367798 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.198116302490234 (18.04763624017889) + | > loss_duration: 1.6643345355987549 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.4057 (0.34765143177726054) + | > loader_time: 0.0069 (0.006014988639137963) + + + --> TIME: 2025-04-19 08:41:58 -- STEP: 135/142 -- GLOBAL_STEP: 464475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.88703155517578 (17.98220794112594) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855710238137166 + | > current_lr_1: 0.0001855710238137166 + | > step_time: 0.4297 (0.3597478478043168) + | > loader_time: 0.0076 (0.00630005024097584) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8477914333343506 (2.8477914333343506) + | > loss_feat: nan (nan) + | > loss_mel: 18.707399368286133 (18.707399368286133) + | > loss_duration: 1.8182934522628784 (1.8182934522628784) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4865922927856445 (3.4865922927856445) + | > loss_feat: nan (nan) + | > loss_mel: 18.353620529174805 (18.353620529174805) + | > loss_duration: 1.88770592212677 (1.88770592212677) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.663383722305298 (3.074988007545471) + | > loss_feat: nan (nan) + | > loss_mel: 19.081928253173828 (18.717774391174316) + | > loss_duration: 1.780444622039795 (1.8340752720832825) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003913283348083496 (+0.0001977682113647461) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.074988007545471 (+0.02836906909942627) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.717774391174316 (+0.6624727249145508) + | > avg_loss_duration: 1.8340752720832825 (-0.038108885288238525) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 600/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:42:02)  + + --> TIME: 2025-04-19 08:42:08 -- STEP: 18/142 -- GLOBAL_STEP: 464500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.850933074951172 (18.168876859876843) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855478274357399 + | > current_lr_1: 0.0001855478274357399 + | > step_time: 0.3113 (0.3053401576148139) + | > loader_time: 0.0051 (0.004859301779005263) + + + --> TIME: 2025-04-19 08:42:17 -- STEP: 43/142 -- GLOBAL_STEP: 464525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.892871856689453 (18.042190862256422) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855478274357399 + | > current_lr_1: 0.0001855478274357399 + | > step_time: 0.3508 (0.3233663093212039) + | > loader_time: 0.0058 (0.005374387253162475) + + + --> TIME: 2025-04-19 08:42:26 -- STEP: 68/142 -- GLOBAL_STEP: 464550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.342111587524414 (18.05451339833876) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855478274357399 + | > current_lr_1: 0.0001855478274357399 + | > step_time: 0.3757 (0.33716156202204095) + | > loader_time: 0.0062 (0.005735348252689138) + + + --> TIME: 2025-04-19 08:42:36 -- STEP: 93/142 -- GLOBAL_STEP: 464575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.82122230529785 (18.046721591744372) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855478274357399 + | > current_lr_1: 0.0001855478274357399 + | > step_time: 0.3886 (0.3500215571413757) + | > loader_time: 0.0066 (0.006035356111423943) + + + --> TIME: 2025-04-19 08:42:46 -- STEP: 118/142 -- GLOBAL_STEP: 464600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.667294502258301 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.627872467041016 (18.026949914835267) + | > loss_duration: 1.6403236389160156 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001855478274357399 + | > current_lr_1: 0.0001855478274357399 + | > step_time: 0.428 (0.36197632652218054) + | > loader_time: 0.0075 (0.006283802501225876) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.845334529876709 (2.845334529876709) + | > loss_feat: nan (nan) + | > loss_mel: 18.970792770385742 (18.970792770385742) + | > loss_duration: 1.8402765989303589 (1.8402765989303589) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2511961460113525 (3.2511961460113525) + | > loss_feat: nan (nan) + | > loss_mel: 17.65591812133789 (17.65591812133789) + | > loss_duration: 1.9142038822174072 (1.9142038822174072) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.966562271118164 (3.1088792085647583) + | > loss_feat: nan (nan) + | > loss_mel: 18.327035903930664 (17.991477012634277) + | > loss_duration: 1.8033604621887207 (1.858782172203064) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033167600631713867 (-0.0005965232849121094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1088792085647583 (+0.03389120101928711) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.991477012634277 (-0.7262973785400391) + | > avg_loss_duration: 1.858782172203064 (+0.024706900119781494) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 601/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:42:58)  + + --> TIME: 2025-04-19 08:42:59 -- STEP: 1/142 -- GLOBAL_STEP: 464625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.303504943847656 (18.303504943847656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.3255 (0.32552170753479004) + | > loader_time: 0.0039 (0.0038726329803466797) + + + --> TIME: 2025-04-19 08:43:08 -- STEP: 26/142 -- GLOBAL_STEP: 464650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.63511061668396 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.25570297241211 (17.800077071556682) + | > loss_duration: 1.661372423171997 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.3263 (0.3160050648909349) + | > loader_time: 0.0055 (0.005316926882817196) + + + --> TIME: 2025-04-19 08:43:16 -- STEP: 51/142 -- GLOBAL_STEP: 464675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.719715118408203 (17.947572034948013) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.3442 (0.3250579039255778) + | > loader_time: 0.0061 (0.005541591083302219) + + + --> TIME: 2025-04-19 08:43:25 -- STEP: 76/142 -- GLOBAL_STEP: 464700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.958545684814453 (17.87357307735242) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.3688 (0.3358279027436909) + | > loader_time: 0.0064 (0.005777911136024877) + + + --> TIME: 2025-04-19 08:43:35 -- STEP: 101/142 -- GLOBAL_STEP: 464725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.299715042114258 (17.891088051371057) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.3892 (0.34617018935704) + | > loader_time: 0.0066 (0.006018190100641534) + + + --> TIME: 2025-04-19 08:43:45 -- STEP: 126/142 -- GLOBAL_STEP: 464750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.11672592163086 (17.901917624095134) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018552463395731043 + | > current_lr_1: 0.00018552463395731043 + | > step_time: 0.4132 (0.35719463560316306) + | > loader_time: 0.0073 (0.006265326151772151) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9318509101867676 (2.9318509101867676) + | > loss_feat: nan (nan) + | > loss_mel: 18.66722297668457 (18.66722297668457) + | > loss_duration: 1.8197435140609741 (1.8197435140609741) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.215479612350464 (3.215479612350464) + | > loss_feat: nan (nan) + | > loss_mel: 18.60540199279785 (18.60540199279785) + | > loss_duration: 1.9035065174102783 (1.9035065174102783) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.820289373397827 (3.0178844928741455) + | > loss_feat: nan (nan) + | > loss_mel: 19.54775047302246 (19.076576232910156) + | > loss_duration: 1.793921709060669 (1.8487141132354736) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0040847063064575195 (+0.0007679462432861328) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0178844928741455 (-0.09099471569061279) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.076576232910156 (+1.085099220275879) + | > avg_loss_duration: 1.8487141132354736 (-0.010068058967590332) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 602/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:43:53)  + + --> TIME: 2025-04-19 08:43:57 -- STEP: 9/142 -- GLOBAL_STEP: 464775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.167789459228516 (17.490605460272896) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.3003 (0.3037836816575792) + | > loader_time: 0.0044 (0.004363854726155599) + + + --> TIME: 2025-04-19 08:44:05 -- STEP: 34/142 -- GLOBAL_STEP: 464800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5707499980926514 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.411548614501953 (17.93865736793069) + | > loss_duration: 1.647396206855774 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.3381 (0.3175819621366613) + | > loader_time: 0.0056 (0.00515918170704561) + + + --> TIME: 2025-04-19 08:44:14 -- STEP: 59/142 -- GLOBAL_STEP: 464825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.473939895629883 (17.84845788600082) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.3619 (0.33103631714643056) + | > loader_time: 0.0066 (0.005597591400146484) + + + --> TIME: 2025-04-19 08:44:24 -- STEP: 84/142 -- GLOBAL_STEP: 464850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.196630477905273 (17.843859547660468) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.3824 (0.3449947606949579) + | > loader_time: 0.0067 (0.005917177313850037) + + + --> TIME: 2025-04-19 08:44:34 -- STEP: 109/142 -- GLOBAL_STEP: 464875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.732925415039062 (17.80200712396464) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.4038 (0.3568223069567199) + | > loader_time: 0.0066 (0.00617265045096021) + + + --> TIME: 2025-04-19 08:44:45 -- STEP: 134/142 -- GLOBAL_STEP: 464900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.657588958740234 (17.80841517804274) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018550144337806576 + | > current_lr_1: 0.00018550144337806576 + | > step_time: 0.4269 (0.3695088197935873) + | > loader_time: 0.0073 (0.006439591521647438) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9412190914154053 (2.9412190914154053) + | > loss_feat: nan (nan) + | > loss_mel: 17.66120147705078 (17.66120147705078) + | > loss_duration: 1.841063380241394 (1.841063380241394) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.492770195007324 (3.492770195007324) + | > loss_feat: nan (nan) + | > loss_mel: 18.8840389251709 (18.8840389251709) + | > loss_duration: 1.8948571681976318 (1.8948571681976318) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.979271650314331 (3.2360209226608276) + | > loss_feat: nan (nan) + | > loss_mel: 17.692176818847656 (18.288107872009277) + | > loss_duration: 1.7692238092422485 (1.8320404887199402) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032215118408203125 (-0.000863194465637207) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2360209226608276 (+0.21813642978668213) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.288107872009277 (-0.7884683609008789) + | > avg_loss_duration: 1.8320404887199402 (-0.016673624515533447) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 603/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:44:50)  + + --> TIME: 2025-04-19 08:44:56 -- STEP: 17/142 -- GLOBAL_STEP: 464925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.005062103271484 (17.967881314894733) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854782556976435 + | > current_lr_1: 0.0001854782556976435 + | > step_time: 0.3112 (0.30413631831898413) + | > loader_time: 0.0052 (0.004847596673404469) + + + --> TIME: 2025-04-19 08:45:04 -- STEP: 42/142 -- GLOBAL_STEP: 464950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.003307342529297 (18.035311903272348) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854782556976435 + | > current_lr_1: 0.0001854782556976435 + | > step_time: 0.338 (0.31645328090304425) + | > loader_time: 0.0059 (0.005202117420378187) + + + --> TIME: 2025-04-19 08:45:13 -- STEP: 67/142 -- GLOBAL_STEP: 464975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.141613006591797 (18.03916300588579) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854782556976435 + | > current_lr_1: 0.0001854782556976435 + | > step_time: 0.3561 (0.329853292721421) + | > loader_time: 0.0064 (0.00550545863251188) + + + --> TIME: 2025-04-19 08:45:23 -- STEP: 92/142 -- GLOBAL_STEP: 465000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.72410011291504 (17.932740646859873) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854782556976435 + | > current_lr_1: 0.0001854782556976435 + | > step_time: 0.3852 (0.3409636124320652) + | > loader_time: 0.0069 (0.005755364894866946) + + + --> TIME: 2025-04-19 08:45:33 -- STEP: 117/142 -- GLOBAL_STEP: 465025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.440147399902344 (17.93256607218686) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854782556976435 + | > current_lr_1: 0.0001854782556976435 + | > step_time: 0.4087 (0.35159182956076074) + | > loader_time: 0.0069 (0.006037924024793839) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8682897090911865 (2.8682897090911865) + | > loss_feat: nan (nan) + | > loss_mel: 17.359241485595703 (17.359241485595703) + | > loss_duration: 1.8506062030792236 (1.8506062030792236) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.307912588119507 (3.307912588119507) + | > loss_feat: nan (nan) + | > loss_mel: 17.83103370666504 (17.83103370666504) + | > loss_duration: 1.9011197090148926 (1.9011197090148926) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.759582757949829 (3.033747673034668) + | > loss_feat: nan (nan) + | > loss_mel: 16.66691017150879 (17.248971939086914) + | > loss_duration: 1.769821047782898 (1.8354703783988953) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00336456298828125 (+0.0001430511474609375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.033747673034668 (-0.20227324962615967) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.248971939086914 (-1.0391359329223633) + | > avg_loss_duration: 1.8354703783988953 (+0.003429889678955078) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 604/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:45:45)  + + --> TIME: 2025-04-19 08:45:45 -- STEP: 0/142 -- GLOBAL_STEP: 465050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.541275024414062 (16.541275024414062) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.3242 (0.324216365814209) + | > loader_time: 0.3068 (0.3068277835845947) + + + --> TIME: 2025-04-19 08:45:53 -- STEP: 25/142 -- GLOBAL_STEP: 465075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6683907508850098 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.218889236450195 (18.01560874938965) + | > loss_duration: 1.658104658126831 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.3166 (0.309949083328247) + | > loader_time: 0.006 (0.005129327774047851) + + + --> TIME: 2025-04-19 08:46:02 -- STEP: 50/142 -- GLOBAL_STEP: 465100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6081910133361816 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.574649810791016 (18.048628501892093) + | > loss_duration: 1.6664063930511475 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.3436 (0.3214514970779419) + | > loader_time: 0.0061 (0.005543298721313476) + + + --> TIME: 2025-04-19 08:46:11 -- STEP: 75/142 -- GLOBAL_STEP: 465125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.82672119140625 (17.927965545654295) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.3727 (0.33401873270670573) + | > loader_time: 0.0067 (0.005855979919433594) + + + --> TIME: 2025-04-19 08:46:21 -- STEP: 100/142 -- GLOBAL_STEP: 465150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.582359313964844 (17.955337524414062) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.4025 (0.34563256502151485) + | > loader_time: 0.0068 (0.006119053363800051) + + + --> TIME: 2025-04-19 08:46:31 -- STEP: 125/142 -- GLOBAL_STEP: 465175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.327138900756836 (17.9591329498291) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854550709156813 + | > current_lr_1: 0.0001854550709156813 + | > step_time: 0.4283 (0.35973929786682124) + | > loader_time: 0.0074 (0.006417339324951174) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9722862243652344 (2.9722862243652344) + | > loss_feat: nan (nan) + | > loss_mel: 18.43196678161621 (18.43196678161621) + | > loss_duration: 1.8216660022735596 (1.8216660022735596) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4173316955566406 (3.4173316955566406) + | > loss_feat: nan (nan) + | > loss_mel: 15.643034934997559 (15.643034934997559) + | > loss_duration: 1.890688180923462 (1.890688180923462) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8274834156036377 (3.122407555580139) + | > loss_feat: nan (nan) + | > loss_mel: 16.648313522338867 (16.145674228668213) + | > loss_duration: 1.8033777475357056 (1.8470329642295837) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003979682922363281 (+0.0006151199340820312) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.122407555580139 (+0.08865988254547119) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.145674228668213 (-1.1032977104187012) + | > avg_loss_duration: 1.8470329642295837 (+0.011562585830688477) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 605/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:46:40)  + + --> TIME: 2025-04-19 08:46:44 -- STEP: 8/142 -- GLOBAL_STEP: 465200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.613006591796875 (18.322324991226196) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.2997 (0.29921239614486694) + | > loader_time: 0.0047 (0.004566818475723267) + + + --> TIME: 2025-04-19 08:46:52 -- STEP: 33/142 -- GLOBAL_STEP: 465225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.431082248687744 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.455129623413086 (18.232632492527816) + | > loss_duration: 1.6239862442016602 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.3306 (0.31287816799048224) + | > loader_time: 0.0054 (0.005178278142755682) + + + --> TIME: 2025-04-19 08:47:00 -- STEP: 58/142 -- GLOBAL_STEP: 465250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.71900177001953 (18.07664636085774) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.3485 (0.32474609490098627) + | > loader_time: 0.0058 (0.005435840836886702) + + + --> TIME: 2025-04-19 08:47:10 -- STEP: 83/142 -- GLOBAL_STEP: 465275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5442123413085938 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.613412857055664 (17.95460465442704) + | > loss_duration: 1.7252795696258545 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.3723 (0.33681079278509307) + | > loader_time: 0.0063 (0.005672239395509283) + + + --> TIME: 2025-04-19 08:47:20 -- STEP: 108/142 -- GLOBAL_STEP: 465300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.540771484375 (17.977626968313146) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.3976 (0.34790115886264367) + | > loader_time: 0.0065 (0.005982524818844265) + + + --> TIME: 2025-04-19 08:47:30 -- STEP: 133/142 -- GLOBAL_STEP: 465325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.91077423095703 (17.908755876067886) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001854318890318168 + | > current_lr_1: 0.0001854318890318168 + | > step_time: 0.4075 (0.35936796754822686) + | > loader_time: 0.0071 (0.006231686226407388) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8965864181518555 (2.8965864181518555) + | > loss_feat: nan (nan) + | > loss_mel: 18.589059829711914 (18.589059829711914) + | > loss_duration: 1.8146837949752808 (1.8146837949752808) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4534690380096436 (3.4534690380096436) + | > loss_feat: nan (nan) + | > loss_mel: 17.80495262145996 (17.80495262145996) + | > loss_duration: 1.9021334648132324 (1.9021334648132324) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7336082458496094 (3.0935386419296265) + | > loss_feat: nan (nan) + | > loss_mel: 17.865768432617188 (17.835360527038574) + | > loss_duration: 1.8008198738098145 (1.8514766693115234) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037775039672851562 (-0.000202178955078125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0935386419296265 (-0.028868913650512695) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.835360527038574 (+1.6896862983703613) + | > avg_loss_duration: 1.8514766693115234 (+0.004443705081939697) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 606/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:47:35)  + + --> TIME: 2025-04-19 08:47:41 -- STEP: 16/142 -- GLOBAL_STEP: 465350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.064109802246094 (17.92535984516144) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.3146 (0.30468977987766266) + | > loader_time: 0.0058 (0.004941031336784363) + + + --> TIME: 2025-04-19 08:47:49 -- STEP: 41/142 -- GLOBAL_STEP: 465375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.971281051635742 (17.978859087316003) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.3361 (0.3164573064664515) + | > loader_time: 0.0058 (0.005374344383797993) + + + --> TIME: 2025-04-19 08:47:58 -- STEP: 66/142 -- GLOBAL_STEP: 465400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5488991737365723 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.628164291381836 (18.04060331980388) + | > loss_duration: 1.6165496110916138 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.3577 (0.32808955149217073) + | > loader_time: 0.007 (0.005727908828041769) + + + --> TIME: 2025-04-19 08:48:08 -- STEP: 91/142 -- GLOBAL_STEP: 465425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.85686683654785 (17.9172218134115) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.3754 (0.3392308413327394) + | > loader_time: 0.0065 (0.005983355281117196) + + + --> TIME: 2025-04-19 08:48:17 -- STEP: 116/142 -- GLOBAL_STEP: 465450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.858190536499023 (17.937027906549396) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.4029 (0.34989706195633974) + | > loader_time: 0.007 (0.006198102030260807) + + + --> TIME: 2025-04-19 08:48:28 -- STEP: 141/142 -- GLOBAL_STEP: 465475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.320159673690796 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.720199584960938 (17.906798173349806) + | > loss_duration: 1.554774284362793 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018540871004568783 + | > current_lr_1: 0.00018540871004568783 + | > step_time: 0.1797 (0.35952190135387657) + | > loader_time: 0.0029 (0.006383714946449224) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.164452075958252 (3.164452075958252) + | > loss_feat: nan (nan) + | > loss_mel: 17.49432945251465 (17.49432945251465) + | > loss_duration: 1.8506438732147217 (1.8506438732147217) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.472764253616333 (3.472764253616333) + | > loss_feat: nan (nan) + | > loss_mel: 19.393815994262695 (19.393815994262695) + | > loss_duration: 1.8945553302764893 (1.8945553302764893) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.893965482711792 (3.1833648681640625) + | > loss_feat: nan (nan) + | > loss_mel: 18.79384422302246 (19.093830108642578) + | > loss_duration: 1.7894105911254883 (1.8419829607009888) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003515481948852539 (-0.0002620220184326172) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1833648681640625 (+0.08982622623443604) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.093830108642578 (+1.258469581604004) + | > avg_loss_duration: 1.8419829607009888 (-0.009493708610534668) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 607/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:48:30)  + + --> TIME: 2025-04-19 08:48:38 -- STEP: 24/142 -- GLOBAL_STEP: 465500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.837778091430664 (17.925796985626224) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001853855339569321 + | > current_lr_1: 0.0001853855339569321 + | > step_time: 0.3199 (0.31635265549023944) + | > loader_time: 0.0052 (0.0049457550048828125) + + + --> TIME: 2025-04-19 08:48:47 -- STEP: 49/142 -- GLOBAL_STEP: 465525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6218955516815186 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.70818328857422 (17.94370841979981) + | > loss_duration: 1.618727445602417 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001853855339569321 + | > current_lr_1: 0.0001853855339569321 + | > step_time: 0.3458 (0.32496960795655544) + | > loader_time: 0.0061 (0.005427102653347716) + + + --> TIME: 2025-04-19 08:48:56 -- STEP: 74/142 -- GLOBAL_STEP: 465550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5848519802093506 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.116130828857422 (17.849512680156817) + | > loss_duration: 1.6445906162261963 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001853855339569321 + | > current_lr_1: 0.0001853855339569321 + | > step_time: 0.3708 (0.3354115260613931) + | > loader_time: 0.0065 (0.005713620701351682) + + + --> TIME: 2025-04-19 08:49:06 -- STEP: 99/142 -- GLOBAL_STEP: 465575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6497249603271484 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.949277877807617 (17.857379229381834) + | > loss_duration: 1.7050191164016724 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001853855339569321 + | > current_lr_1: 0.0001853855339569321 + | > step_time: 0.378 (0.3453477055135399) + | > loader_time: 0.007 (0.005986148660833184) + + + --> TIME: 2025-04-19 08:49:16 -- STEP: 124/142 -- GLOBAL_STEP: 465600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.812484741210938 (17.85002580765755) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001853855339569321 + | > current_lr_1: 0.0001853855339569321 + | > step_time: 0.4191 (0.3565806727255544) + | > loader_time: 0.007 (0.006199579085073162) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.024613857269287 (3.024613857269287) + | > loss_feat: nan (nan) + | > loss_mel: 19.90565299987793 (19.90565299987793) + | > loss_duration: 1.8331429958343506 (1.8331429958343506) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.538081645965576 (3.538081645965576) + | > loss_feat: nan (nan) + | > loss_mel: 16.895299911499023 (16.895299911499023) + | > loss_duration: 1.8913977146148682 (1.8913977146148682) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8108017444610596 (3.174441695213318) + | > loss_feat: nan (nan) + | > loss_mel: 18.259294509887695 (17.57729721069336) + | > loss_duration: 1.8096339702606201 (1.8505158424377441) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035213232040405273 (+5.841255187988281e-06) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.174441695213318 (-0.008923172950744629) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.57729721069336 (-1.5165328979492188) + | > avg_loss_duration: 1.8505158424377441 (+0.008532881736755371) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 608/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:49:25)  + + --> TIME: 2025-04-19 08:49:28 -- STEP: 7/142 -- GLOBAL_STEP: 465625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.661514282226562 (17.95778042929513) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.2989 (0.299523081098284) + | > loader_time: 0.0041 (0.004290955407278878) + + + --> TIME: 2025-04-19 08:49:36 -- STEP: 32/142 -- GLOBAL_STEP: 465650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.930679321289062 (18.145517557859428) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.3285 (0.3117242828011513) + | > loader_time: 0.0053 (0.005186632275581359) + + + --> TIME: 2025-04-19 08:49:45 -- STEP: 57/142 -- GLOBAL_STEP: 465675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.77603530883789 (18.01558333949039) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.3477 (0.3232597468192117) + | > loader_time: 0.0061 (0.005574565184743781) + + + --> TIME: 2025-04-19 08:49:54 -- STEP: 82/142 -- GLOBAL_STEP: 465700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.462207794189453 (17.954078778987974) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.3694 (0.3362462026316946) + | > loader_time: 0.0066 (0.005859767518392424) + + + --> TIME: 2025-04-19 08:50:04 -- STEP: 107/142 -- GLOBAL_STEP: 465725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.268495559692383 (17.92267894744872) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.3938 (0.34753276477350253) + | > loader_time: 0.0071 (0.006119220056266428) + + + --> TIME: 2025-04-19 08:50:15 -- STEP: 132/142 -- GLOBAL_STEP: 465750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.820062637329102 (17.8990714261026) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018536236076518748 + | > current_lr_1: 0.00018536236076518748 + | > step_time: 0.4237 (0.3609856367111206) + | > loader_time: 0.0076 (0.0063722711620908794) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.105287790298462 (3.105287790298462) + | > loss_feat: nan (nan) + | > loss_mel: 17.31941032409668 (17.31941032409668) + | > loss_duration: 1.8195183277130127 (1.8195183277130127) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2346928119659424 (3.2346928119659424) + | > loss_feat: nan (nan) + | > loss_mel: 15.79654312133789 (15.79654312133789) + | > loss_duration: 1.9165518283843994 (1.9165518283843994) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8823142051696777 (3.05850350856781) + | > loss_feat: nan (nan) + | > loss_mel: 16.001272201538086 (15.898907661437988) + | > loss_duration: 1.7977521419525146 (1.857151985168457) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003171086311340332 (-0.0003502368927001953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.05850350856781 (-0.11593818664550781) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 15.898907661437988 (-1.678389549255371) + | > avg_loss_duration: 1.857151985168457 (+0.006636142730712891) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 609/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:50:20)  + + --> TIME: 2025-04-19 08:50:25 -- STEP: 15/142 -- GLOBAL_STEP: 465775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.450942993164062 (17.879988861083984) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.3079 (0.30218234062194826) + | > loader_time: 0.0055 (0.00487828254699707) + + + --> TIME: 2025-04-19 08:50:34 -- STEP: 40/142 -- GLOBAL_STEP: 465800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.728900909423828 (17.921564865112302) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.3454 (0.3161098599433899) + | > loader_time: 0.0055 (0.005318105220794678) + + + --> TIME: 2025-04-19 08:50:43 -- STEP: 65/142 -- GLOBAL_STEP: 465825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.852731704711914 (17.967430261465218) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.3647 (0.32899780640235315) + | > loader_time: 0.0061 (0.005609123523418719) + + + --> TIME: 2025-04-19 08:50:52 -- STEP: 90/142 -- GLOBAL_STEP: 465850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.042539596557617 (17.875393528408473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.3762 (0.34016911983489984) + | > loader_time: 0.0071 (0.005881362491183808) + + + --> TIME: 2025-04-19 08:51:02 -- STEP: 115/142 -- GLOBAL_STEP: 465875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.213607788085938 (17.93009345842445) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.3995 (0.3507230385490085) + | > loader_time: 0.0093 (0.006184380987416141) + + + --> TIME: 2025-04-19 08:51:13 -- STEP: 140/142 -- GLOBAL_STEP: 465900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.98544692993164 (17.893565838677553) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018533919047009182 + | > current_lr_1: 0.00018533919047009182 + | > step_time: 0.4316 (0.3628828542573111) + | > loader_time: 0.0077 (0.006419045584542409) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9462337493896484 (2.9462337493896484) + | > loss_feat: nan (nan) + | > loss_mel: 17.35836410522461 (17.35836410522461) + | > loss_duration: 1.8487173318862915 (1.8487173318862915) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.339808225631714 (3.339808225631714) + | > loss_feat: nan (nan) + | > loss_mel: 19.034061431884766 (19.034061431884766) + | > loss_duration: 1.9060213565826416 (1.9060213565826416) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.765143871307373 (3.0524760484695435) + | > loss_feat: nan (nan) + | > loss_mel: 16.33951759338379 (17.686789512634277) + | > loss_duration: 1.7999658584594727 (1.8529936075210571) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036821365356445312 (+0.0005110502243041992) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0524760484695435 (-0.0060274600982666016) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.686789512634277 (+1.787881851196289) + | > avg_loss_duration: 1.8529936075210571 (-0.004158377647399902) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 610/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:51:15)  + + --> TIME: 2025-04-19 08:51:23 -- STEP: 23/142 -- GLOBAL_STEP: 465925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4661130905151367 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.852689743041992 (17.99683645497198) + | > loss_duration: 1.6282927989959717 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018531602307128304 + | > current_lr_1: 0.00018531602307128304 + | > step_time: 0.3382 (0.31177113367163617) + | > loader_time: 0.0059 (0.005041879156361456) + + + --> TIME: 2025-04-19 08:51:32 -- STEP: 48/142 -- GLOBAL_STEP: 465950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.3228816986084 (18.031350692113246) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018531602307128304 + | > current_lr_1: 0.00018531602307128304 + | > step_time: 0.3468 (0.32735636333624524) + | > loader_time: 0.0067 (0.005527744690577189) + + + --> TIME: 2025-04-19 08:51:41 -- STEP: 73/142 -- GLOBAL_STEP: 465975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.88189697265625 (18.00994640507111) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018531602307128304 + | > current_lr_1: 0.00018531602307128304 + | > step_time: 0.3799 (0.3413748218588633) + | > loader_time: 0.0068 (0.0058417908132892766) + + + --> TIME: 2025-04-19 08:51:51 -- STEP: 98/142 -- GLOBAL_STEP: 466000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.124961853027344 (18.0000310236094) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018531602307128304 + | > current_lr_1: 0.00018531602307128304 + | > step_time: 0.3844 (0.3546217096095181) + | > loader_time: 0.0062 (0.006116721094871054) + + + --> TIME: 2025-04-19 08:52:02 -- STEP: 123/142 -- GLOBAL_STEP: 466025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.519321918487549 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.67247200012207 (18.012198657524298) + | > loss_duration: 1.6462383270263672 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018531602307128304 + | > current_lr_1: 0.00018531602307128304 + | > step_time: 0.4138 (0.3638697504028071) + | > loader_time: 0.0081 (0.006366514578098204) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.057112216949463 (3.057112216949463) + | > loss_feat: nan (nan) + | > loss_mel: 17.3905086517334 (17.3905086517334) + | > loss_duration: 1.8202908039093018 (1.8202908039093018) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2019548416137695 (3.2019548416137695) + | > loss_feat: nan (nan) + | > loss_mel: 18.79169273376465 (18.79169273376465) + | > loss_duration: 1.901787519454956 (1.901787519454956) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8735222816467285 (3.037738561630249) + | > loss_feat: nan (nan) + | > loss_mel: 19.03880500793457 (18.91524887084961) + | > loss_duration: 1.8012288808822632 (1.8515082001686096) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004057049751281738 (+0.00037491321563720703) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.037738561630249 (-0.014737486839294434) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.91524887084961 (+1.228459358215332) + | > avg_loss_duration: 1.8515082001686096 (-0.0014854073524475098) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 611/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:52:11)  + + --> TIME: 2025-04-19 08:52:14 -- STEP: 6/142 -- GLOBAL_STEP: 466050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.60658860206604 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.416278839111328 (18.09582455952962) + | > loss_duration: 1.6540368795394897 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.3027 (0.3040281931559245) + | > loader_time: 0.0042 (0.004460453987121582) + + + --> TIME: 2025-04-19 08:52:22 -- STEP: 31/142 -- GLOBAL_STEP: 466075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.168733596801758 (18.032966183077903) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.3242 (0.31221969666019567) + | > loader_time: 0.0052 (0.005162492875129946) + + + --> TIME: 2025-04-19 08:52:30 -- STEP: 56/142 -- GLOBAL_STEP: 466100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.138179779052734 (18.163049646786284) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.3492 (0.32353668553488596) + | > loader_time: 0.006 (0.00546497106552124) + + + --> TIME: 2025-04-19 08:52:40 -- STEP: 81/142 -- GLOBAL_STEP: 466125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.74462127685547 (18.066284874339168) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.3679 (0.33596594539689423) + | > loader_time: 0.0068 (0.005789156313295719) + + + --> TIME: 2025-04-19 08:52:49 -- STEP: 106/142 -- GLOBAL_STEP: 466150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.393198013305664 (17.99758820263845) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.3857 (0.3467062599254104) + | > loader_time: 0.0095 (0.006059302473967931) + + + --> TIME: 2025-04-19 08:53:00 -- STEP: 131/142 -- GLOBAL_STEP: 466175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.9367733001709 (17.917559514518913) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018529285856839912 + | > current_lr_1: 0.00018529285856839912 + | > step_time: 0.4238 (0.35857319649849223) + | > loader_time: 0.0077 (0.0063170094526451065) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0130608081817627 (3.0130608081817627) + | > loss_feat: nan (nan) + | > loss_mel: 17.682859420776367 (17.682859420776367) + | > loss_duration: 1.838900089263916 (1.838900089263916) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4474852085113525 (3.4474852085113525) + | > loss_feat: nan (nan) + | > loss_mel: 18.04966926574707 (18.04966926574707) + | > loss_duration: 1.9008736610412598 (1.9008736610412598) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9184539318084717 (3.182969570159912) + | > loss_feat: nan (nan) + | > loss_mel: 20.029882431030273 (19.039775848388672) + | > loss_duration: 1.7636030912399292 (1.8322383761405945) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034902095794677734 (-0.0005668401718139648) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.182969570159912 (+0.14523100852966309) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.039775848388672 (+0.1245269775390625) + | > avg_loss_duration: 1.8322383761405945 (-0.019269824028015137) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 612/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:53:06)  + + --> TIME: 2025-04-19 08:53:11 -- STEP: 14/142 -- GLOBAL_STEP: 466200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.517072916030884 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.36246109008789 (17.841566562652588) + | > loss_duration: 1.631112813949585 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.3075 (0.30296410833086285) + | > loader_time: 0.0046 (0.004812989916120257) + + + --> TIME: 2025-04-19 08:53:19 -- STEP: 39/142 -- GLOBAL_STEP: 466225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.262130737304688 (17.81459072308663) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.3378 (0.31973981857299805) + | > loader_time: 0.006 (0.005356537990081003) + + + --> TIME: 2025-04-19 08:53:28 -- STEP: 64/142 -- GLOBAL_STEP: 466250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.507322311401367 (17.799465522170077) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.3682 (0.3339183181524277) + | > loader_time: 0.0065 (0.005715884268283844) + + + --> TIME: 2025-04-19 08:53:38 -- STEP: 89/142 -- GLOBAL_STEP: 466275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.609302520751953 (17.849719904781736) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.3876 (0.34866208172916024) + | > loader_time: 0.0071 (0.0060659472861986485) + + + --> TIME: 2025-04-19 08:53:49 -- STEP: 114/142 -- GLOBAL_STEP: 466300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.741491317749023 (17.8853029033594) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.4094 (0.36058767636617023) + | > loader_time: 0.0074 (0.006373382451241477) + + + --> TIME: 2025-04-19 08:54:00 -- STEP: 139/142 -- GLOBAL_STEP: 466325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.823286056518555 (17.840098737812728) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018526969696107805 + | > current_lr_1: 0.00018526969696107805 + | > step_time: 0.4393 (0.3731398308019844) + | > loader_time: 0.0078 (0.006663425363225045) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.017594575881958 (3.017594575881958) + | > loss_feat: nan (nan) + | > loss_mel: 19.877700805664062 (19.877700805664062) + | > loss_duration: 1.8288609981536865 (1.8288609981536865) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4554996490478516 (3.4554996490478516) + | > loss_feat: nan (nan) + | > loss_mel: 16.631305694580078 (16.631305694580078) + | > loss_duration: 1.9294967651367188 (1.9294967651367188) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8825740814208984 (3.169036865234375) + | > loss_feat: nan (nan) + | > loss_mel: 19.668094635009766 (18.149700164794922) + | > loss_duration: 1.7949210405349731 (1.862208902835846) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037959814071655273 (+0.0003057718276977539) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.169036865234375 (-0.01393270492553711) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.149700164794922 (-0.89007568359375) + | > avg_loss_duration: 1.862208902835846 (+0.029970526695251465) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 613/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:54:02)  + + --> TIME: 2025-04-19 08:54:10 -- STEP: 22/142 -- GLOBAL_STEP: 466350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.581645965576172 (17.821561033075508) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001852465382489579 + | > current_lr_1: 0.0001852465382489579 + | > step_time: 0.3259 (0.3075389645316384) + | > loader_time: 0.0051 (0.004960461096330123) + + + --> TIME: 2025-04-19 08:54:19 -- STEP: 47/142 -- GLOBAL_STEP: 466375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5226147174835205 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.68307876586914 (17.979433222019928) + | > loss_duration: 1.5855507850646973 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001852465382489579 + | > current_lr_1: 0.0001852465382489579 + | > step_time: 0.3391 (0.320992611824198) + | > loader_time: 0.0059 (0.0052770807387981) + + + --> TIME: 2025-04-19 08:54:28 -- STEP: 72/142 -- GLOBAL_STEP: 466400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.638370513916016 (18.035696400536438) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001852465382489579 + | > current_lr_1: 0.0001852465382489579 + | > step_time: 0.3721 (0.33266813556353253) + | > loader_time: 0.0066 (0.005592925681008233) + + + --> TIME: 2025-04-19 08:54:37 -- STEP: 97/142 -- GLOBAL_STEP: 466425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.17466163635254 (18.08551288880024) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001852465382489579 + | > current_lr_1: 0.0001852465382489579 + | > step_time: 0.3845 (0.3431132577129246) + | > loader_time: 0.007 (0.0058627669344243315) + + + --> TIME: 2025-04-19 08:54:47 -- STEP: 122/142 -- GLOBAL_STEP: 466450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.072994232177734 (17.96242508341054) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001852465382489579 + | > current_lr_1: 0.0001852465382489579 + | > step_time: 0.4259 (0.3542608608965014) + | > loader_time: 0.0072 (0.006095350765791096) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.104379892349243 (3.104379892349243) + | > loss_feat: nan (nan) + | > loss_mel: 18.484867095947266 (18.484867095947266) + | > loss_duration: 1.8249855041503906 (1.8249855041503906) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.308349132537842 (3.308349132537842) + | > loss_feat: nan (nan) + | > loss_mel: 18.969337463378906 (18.969337463378906) + | > loss_duration: 1.874884843826294 (1.874884843826294) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8890838623046875 (3.0987164974212646) + | > loss_feat: nan (nan) + | > loss_mel: 17.77742576599121 (18.37338161468506) + | > loss_duration: 1.810758352279663 (1.8428215980529785) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036383867263793945 (-0.0001575946807861328) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0987164974212646 (-0.07032036781311035) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.37338161468506 (+0.22368144989013672) + | > avg_loss_duration: 1.8428215980529785 (-0.01938730478286743) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 614/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:54:57)  + + --> TIME: 2025-04-19 08:54:59 -- STEP: 5/142 -- GLOBAL_STEP: 466475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.218181610107422 (18.20483283996582) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.2984 (0.29665322303771974) + | > loader_time: 0.0045 (0.004656648635864258) + + + --> TIME: 2025-04-19 08:55:08 -- STEP: 30/142 -- GLOBAL_STEP: 466500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.368803024291992 (17.79586086273193) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.3338 (0.3142199754714966) + | > loader_time: 0.0064 (0.005201252301534017) + + + --> TIME: 2025-04-19 08:55:17 -- STEP: 55/142 -- GLOBAL_STEP: 466525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.96071434020996 (17.918256378173822) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.3596 (0.33011799292130906) + | > loader_time: 0.0063 (0.005587070638483221) + + + --> TIME: 2025-04-19 08:55:26 -- STEP: 80/142 -- GLOBAL_STEP: 466550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.583998918533325 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.820125579833984 (17.877028799057) + | > loss_duration: 1.6596534252166748 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.3984 (0.3439725607633591) + | > loader_time: 0.0069 (0.005937933921813966) + + + --> TIME: 2025-04-19 08:55:36 -- STEP: 105/142 -- GLOBAL_STEP: 466575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7334539890289307 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.85988712310791 (17.85828927357991) + | > loss_duration: 1.695428490638733 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.4181 (0.3557684966496059) + | > loader_time: 0.0072 (0.0062128566560291105) + + + --> TIME: 2025-04-19 08:55:47 -- STEP: 130/142 -- GLOBAL_STEP: 466600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.831387519836426 (17.836759053743794) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018522338243167678 + | > current_lr_1: 0.00018522338243167678 + | > step_time: 0.4341 (0.36870913688953105) + | > loader_time: 0.0079 (0.006450205582838792) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.805544853210449 (2.805544853210449) + | > loss_feat: nan (nan) + | > loss_mel: 18.28961944580078 (18.28961944580078) + | > loss_duration: 1.8468427658081055 (1.8468427658081055) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.324410915374756 (3.324410915374756) + | > loss_feat: nan (nan) + | > loss_mel: 19.07256317138672 (19.07256317138672) + | > loss_duration: 1.8813942670822144 (1.8813942670822144) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.796281337738037 (3.0603461265563965) + | > loss_feat: nan (nan) + | > loss_mel: 17.366140365600586 (18.219351768493652) + | > loss_duration: 1.8014588356018066 (1.8414265513420105) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036287307739257812 (-9.655952453613281e-06) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0603461265563965 (-0.038370370864868164) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.219351768493652 (-0.15402984619140625) + | > avg_loss_duration: 1.8414265513420105 (-0.0013950467109680176) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 615/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:55:54)  + + --> TIME: 2025-04-19 08:55:58 -- STEP: 13/142 -- GLOBAL_STEP: 466625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.04684829711914 (18.12568737910344) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.3055 (0.3014846031482403) + | > loader_time: 0.005 (0.004663687485914964) + + + --> TIME: 2025-04-19 08:56:07 -- STEP: 38/142 -- GLOBAL_STEP: 466650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.398452758789062 (18.094103662591234) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.3272 (0.3172827770835475) + | > loader_time: 0.0054 (0.005091736191197445) + + + --> TIME: 2025-04-19 08:56:16 -- STEP: 63/142 -- GLOBAL_STEP: 466675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.74236297607422 (17.967766534714475) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.3631 (0.32816698816087514) + | > loader_time: 0.0062 (0.005462196138170029) + + + --> TIME: 2025-04-19 08:56:25 -- STEP: 88/142 -- GLOBAL_STEP: 466700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.62500762939453 (18.0351083278656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.3722 (0.33930442279035394) + | > loader_time: 0.0061 (0.00576434081250971) + + + --> TIME: 2025-04-19 08:56:35 -- STEP: 113/142 -- GLOBAL_STEP: 466725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.727360725402832 (17.95690723014089) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.4022 (0.3500180117851865) + | > loader_time: 0.007 (0.006007523663276065) + + + --> TIME: 2025-04-19 08:56:46 -- STEP: 138/142 -- GLOBAL_STEP: 466750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.950822830200195 (17.897066973257758) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018520022950887282 + | > current_lr_1: 0.00018520022950887282 + | > step_time: 0.4259 (0.3619241645370704) + | > loader_time: 0.0072 (0.006245082703189573) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7602694034576416 (2.7602694034576416) + | > loss_feat: nan (nan) + | > loss_mel: 16.675853729248047 (16.675853729248047) + | > loss_duration: 1.839034080505371 (1.839034080505371) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2715792655944824 (3.2715792655944824) + | > loss_feat: nan (nan) + | > loss_mel: 18.977853775024414 (18.977853775024414) + | > loss_duration: 1.9227678775787354 (1.9227678775787354) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9107441902160645 (3.0911617279052734) + | > loss_feat: nan (nan) + | > loss_mel: 19.530803680419922 (19.254328727722168) + | > loss_duration: 1.8071712255477905 (1.864969551563263) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032181739807128906 (-0.0004105567932128906) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0911617279052734 (+0.030815601348876953) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.254328727722168 (+1.0349769592285156) + | > avg_loss_duration: 1.864969551563263 (+0.02354300022125244) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 616/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:56:49)  + + --> TIME: 2025-04-19 08:56:56 -- STEP: 21/142 -- GLOBAL_STEP: 466775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.007213592529297 (17.849628176007947) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001851770794801842 + | > current_lr_1: 0.0001851770794801842 + | > step_time: 0.3167 (0.3054952848525274) + | > loader_time: 0.0058 (0.004977600915091378) + + + --> TIME: 2025-04-19 08:57:04 -- STEP: 46/142 -- GLOBAL_STEP: 466800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.956226348876953 (17.826295728268825) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001851770794801842 + | > current_lr_1: 0.0001851770794801842 + | > step_time: 0.341 (0.31768359826958703) + | > loader_time: 0.0061 (0.0053870677947998056) + + + --> TIME: 2025-04-19 08:57:13 -- STEP: 71/142 -- GLOBAL_STEP: 466825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.071475982666016 (17.91491734142035) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001851770794801842 + | > current_lr_1: 0.0001851770794801842 + | > step_time: 0.3636 (0.3300363379465023) + | > loader_time: 0.0067 (0.005730538301064934) + + + --> TIME: 2025-04-19 08:57:23 -- STEP: 96/142 -- GLOBAL_STEP: 466850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.456866979598999 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.832265853881836 (17.859056353569038) + | > loss_duration: 1.6825687885284424 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001851770794801842 + | > current_lr_1: 0.0001851770794801842 + | > step_time: 0.3865 (0.3417527874310811) + | > loader_time: 0.007 (0.005967246989409129) + + + --> TIME: 2025-04-19 08:57:33 -- STEP: 121/142 -- GLOBAL_STEP: 466875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.126035690307617 (17.833072331326075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001851770794801842 + | > current_lr_1: 0.0001851770794801842 + | > step_time: 0.4182 (0.35364032579847604) + | > loader_time: 0.0078 (0.006250639592320466) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8609817028045654 (2.8609817028045654) + | > loss_feat: nan (nan) + | > loss_mel: 17.36790657043457 (17.36790657043457) + | > loss_duration: 1.8114097118377686 (1.8114097118377686) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5270590782165527 (3.5270590782165527) + | > loss_feat: nan (nan) + | > loss_mel: 17.60041618347168 (17.60041618347168) + | > loss_duration: 1.8958537578582764 (1.8958537578582764) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.982032060623169 (3.254545569419861) + | > loss_feat: nan (nan) + | > loss_mel: 19.32638168334961 (18.463398933410645) + | > loss_duration: 1.784620761871338 (1.8402372598648071) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00334930419921875 (+0.00013113021850585938) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.254545569419861 (+0.1633838415145874) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.463398933410645 (-0.7909297943115234) + | > avg_loss_duration: 1.8402372598648071 (-0.02473229169845581) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 617/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:57:44)  + + --> TIME: 2025-04-19 08:57:46 -- STEP: 4/142 -- GLOBAL_STEP: 466900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.891876220703125 (18.103612422943115) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.3049 (0.3164283037185669) + | > loader_time: 0.0064 (0.004923760890960693) + + + --> TIME: 2025-04-19 08:57:54 -- STEP: 29/142 -- GLOBAL_STEP: 466925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.268796920776367 (17.897085847525762) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.3206 (0.31262032739047346) + | > loader_time: 0.0054 (0.004970706742385338) + + + --> TIME: 2025-04-19 08:58:02 -- STEP: 54/142 -- GLOBAL_STEP: 466950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.384037017822266 (17.892625632109468) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.3436 (0.32346977127922905) + | > loader_time: 0.0054 (0.005371614738746926) + + + --> TIME: 2025-04-19 08:58:12 -- STEP: 79/142 -- GLOBAL_STEP: 466975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.10286521911621 (17.971537481380416) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.3649 (0.3355074290987812) + | > loader_time: 0.0065 (0.005656861051728454) + + + --> TIME: 2025-04-19 08:58:22 -- STEP: 104/142 -- GLOBAL_STEP: 467000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.146100997924805 (17.971641430487992) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.3887 (0.34677961239447963) + | > loader_time: 0.0073 (0.005935139380968534) + + + --> TIME: 2025-04-19 08:58:32 -- STEP: 129/142 -- GLOBAL_STEP: 467025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.635101318359375 (17.93189944777377) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018515393234524918 + | > current_lr_1: 0.00018515393234524918 + | > step_time: 0.4307 (0.35988826714744865) + | > loader_time: 0.0075 (0.006243772284929143) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.983776330947876 (2.983776330947876) + | > loss_feat: nan (nan) + | > loss_mel: 17.824487686157227 (17.824487686157227) + | > loss_duration: 1.7977752685546875 (1.7977752685546875) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.473783493041992 (3.473783493041992) + | > loss_feat: nan (nan) + | > loss_mel: 18.197309494018555 (18.197309494018555) + | > loss_duration: 1.8939136266708374 (1.8939136266708374) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6556601524353027 (3.0647218227386475) + | > loss_feat: nan (nan) + | > loss_mel: 12.571796417236328 (15.384552955627441) + | > loss_duration: 1.8271410465240479 (1.8605273365974426) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00326693058013916 (-8.237361907958984e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0647218227386475 (-0.18982374668121338) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 15.384552955627441 (-3.078845977783203) + | > avg_loss_duration: 1.8605273365974426 (+0.020290076732635498) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 618/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:58:39)  + + --> TIME: 2025-04-19 08:58:44 -- STEP: 12/142 -- GLOBAL_STEP: 467050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.271793365478516 (17.97393226623535) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.3045 (0.3027492165565491) + | > loader_time: 0.005 (0.004688739776611328) + + + --> TIME: 2025-04-19 08:58:52 -- STEP: 37/142 -- GLOBAL_STEP: 467075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.18975830078125 (17.951170586250925) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.3365 (0.3185045654709275) + | > loader_time: 0.006 (0.005305419097075593) + + + --> TIME: 2025-04-19 08:59:01 -- STEP: 62/142 -- GLOBAL_STEP: 467100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.999984741210938 (17.898590641637007) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.3691 (0.3322916030883789) + | > loader_time: 0.0067 (0.00569408170638546) + + + --> TIME: 2025-04-19 08:59:11 -- STEP: 87/142 -- GLOBAL_STEP: 467125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.563499450683594 (17.87673593937666) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.3894 (0.34576879424610363) + | > loader_time: 0.0071 (0.006006819078291972) + + + --> TIME: 2025-04-19 08:59:21 -- STEP: 112/142 -- GLOBAL_STEP: 467150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.065135955810547 (17.898425689765382) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.4166 (0.3582205218928201) + | > loader_time: 0.0069 (0.006291376692908153) + + + --> TIME: 2025-04-19 08:59:32 -- STEP: 137/142 -- GLOBAL_STEP: 467175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 14.915736198425293 (17.85617921648234) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000185130788103706 + | > current_lr_1: 0.000185130788103706 + | > step_time: 0.4418 (0.3707960991963853) + | > loader_time: 0.0075 (0.006544945013784146) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9721148014068604 (2.9721148014068604) + | > loss_feat: nan (nan) + | > loss_mel: 19.585214614868164 (19.585214614868164) + | > loss_duration: 1.8315367698669434 (1.8315367698669434) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.244878053665161 (3.244878053665161) + | > loss_feat: nan (nan) + | > loss_mel: 17.251977920532227 (17.251977920532227) + | > loss_duration: 1.884987711906433 (1.884987711906433) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.991954803466797 (3.118416428565979) + | > loss_feat: nan (nan) + | > loss_mel: 17.827831268310547 (17.539904594421387) + | > loss_duration: 1.7928125858306885 (1.8389001488685608) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003519296646118164 (+0.0002523660659790039) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.118416428565979 (+0.05369460582733154) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.539904594421387 (+2.1553516387939453) + | > avg_loss_duration: 1.8389001488685608 (-0.021627187728881836) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 619/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 08:59:36)  + + --> TIME: 2025-04-19 08:59:43 -- STEP: 20/142 -- GLOBAL_STEP: 467200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.34891128540039 (18.004410839080812) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018510764675519302 + | > current_lr_1: 0.00018510764675519302 + | > step_time: 0.3166 (0.30875109434127807) + | > loader_time: 0.0051 (0.004779362678527832) + + + --> TIME: 2025-04-19 08:59:51 -- STEP: 45/142 -- GLOBAL_STEP: 467225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.457124710083008 (17.992384952969022) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018510764675519302 + | > current_lr_1: 0.00018510764675519302 + | > step_time: 0.3508 (0.322092056274414) + | > loader_time: 0.0065 (0.005345386928982205) + + + --> TIME: 2025-04-19 09:00:00 -- STEP: 70/142 -- GLOBAL_STEP: 467250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.678804397583008 (17.912317480359757) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018510764675519302 + | > current_lr_1: 0.00018510764675519302 + | > step_time: 0.3643 (0.3331027065004621) + | > loader_time: 0.0062 (0.005607805933271135) + + + --> TIME: 2025-04-19 09:00:10 -- STEP: 95/142 -- GLOBAL_STEP: 467275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.2507266998291 (17.93136135904413) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018510764675519302 + | > current_lr_1: 0.00018510764675519302 + | > step_time: 0.3803 (0.3434449497022125) + | > loader_time: 0.0066 (0.005853560096339178) + + + --> TIME: 2025-04-19 09:00:20 -- STEP: 120/142 -- GLOBAL_STEP: 467300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.685901641845703 (17.88716790676117) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018510764675519302 + | > current_lr_1: 0.00018510764675519302 + | > step_time: 0.4081 (0.35410883426666245) + | > loader_time: 0.0073 (0.00612453619639079) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.93076753616333 (2.93076753616333) + | > loss_feat: nan (nan) + | > loss_mel: 18.385982513427734 (18.385982513427734) + | > loss_duration: 1.840638518333435 (1.840638518333435) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.242793321609497 (3.242793321609497) + | > loss_feat: nan (nan) + | > loss_mel: 15.8956298828125 (15.8956298828125) + | > loss_duration: 1.9106194972991943 (1.9106194972991943) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.843714475631714 (3.0432538986206055) + | > loss_feat: nan (nan) + | > loss_mel: 17.2286376953125 (16.5621337890625) + | > loss_duration: 1.7831087112426758 (1.846864104270935) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033434629440307617 (-0.00017583370208740234) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0432538986206055 (-0.07516252994537354) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.5621337890625 (-0.9777708053588867) + | > avg_loss_duration: 1.846864104270935 (+0.007963955402374268) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 620/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:00:31)  + + --> TIME: 2025-04-19 09:00:32 -- STEP: 3/142 -- GLOBAL_STEP: 467325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.99492835998535 (17.84162966410319) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.2968 (0.2983388106028239) + | > loader_time: 0.0048 (0.004453102747599284) + + + --> TIME: 2025-04-19 09:00:40 -- STEP: 28/142 -- GLOBAL_STEP: 467350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.27587127685547 (17.73895985739572) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.3197 (0.309429407119751) + | > loader_time: 0.0067 (0.005243667534419468) + + + --> TIME: 2025-04-19 09:00:49 -- STEP: 53/142 -- GLOBAL_STEP: 467375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.94329261779785 (17.80822329251272) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.3487 (0.32148876730001197) + | > loader_time: 0.0067 (0.005578378461441903) + + + --> TIME: 2025-04-19 09:00:58 -- STEP: 78/142 -- GLOBAL_STEP: 467400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.902753829956055 (17.88085432541677) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.3709 (0.3339442748289842) + | > loader_time: 0.007 (0.005884097172663761) + + + --> TIME: 2025-04-19 09:01:08 -- STEP: 103/142 -- GLOBAL_STEP: 467425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6983935832977295 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64885902404785 (17.91895058085617) + | > loss_duration: 1.6921491622924805 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.3887 (0.34447600077656865) + | > loader_time: 0.0071 (0.006111672781046154) + + + --> TIME: 2025-04-19 09:01:18 -- STEP: 128/142 -- GLOBAL_STEP: 467450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.046127319335938 (17.90156012773513) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018508450829934862 + | > current_lr_1: 0.00018508450829934862 + | > step_time: 0.4051 (0.3555565234273672) + | > loader_time: 0.0073 (0.006300227716565132) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.836627244949341 (2.836627244949341) + | > loss_feat: nan (nan) + | > loss_mel: 18.17633056640625 (18.17633056640625) + | > loss_duration: 1.848203420639038 (1.848203420639038) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2817952632904053 (3.2817952632904053) + | > loss_feat: nan (nan) + | > loss_mel: 19.768028259277344 (19.768028259277344) + | > loss_duration: 1.9139704704284668 (1.9139704704284668) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.923215866088867 (3.1025055646896362) + | > loss_feat: nan (nan) + | > loss_mel: 18.819955825805664 (19.293992042541504) + | > loss_duration: 1.7953590154647827 (1.8546647429466248) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037038326263427734 (+0.0003603696823120117) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1025055646896362 (+0.05925166606903076) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.293992042541504 (+2.731858253479004) + | > avg_loss_duration: 1.8546647429466248 (+0.007800638675689697) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 621/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:01:25)  + + --> TIME: 2025-04-19 09:01:29 -- STEP: 11/142 -- GLOBAL_STEP: 467475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.04039764404297 (18.12710918079723) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.308 (0.32094634662974963) + | > loader_time: 0.0049 (0.004656574942848899) + + + --> TIME: 2025-04-19 09:01:38 -- STEP: 36/142 -- GLOBAL_STEP: 467500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.921039581298828 (18.273020214504665) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.3292 (0.3214586046006944) + | > loader_time: 0.0054 (0.005190458562639026) + + + --> TIME: 2025-04-19 09:01:46 -- STEP: 61/142 -- GLOBAL_STEP: 467525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.481740951538086 (18.080860982175736) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.3594 (0.33029151353679714) + | > loader_time: 0.0062 (0.00546796986314117) + + + --> TIME: 2025-04-19 09:01:56 -- STEP: 86/142 -- GLOBAL_STEP: 467550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.526683807373047 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.592496871948242 (18.023131880649306) + | > loss_duration: 1.6630373001098633 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.3805 (0.3420691601065701) + | > loader_time: 0.0062 (0.005773283714471862) + + + --> TIME: 2025-04-19 09:02:06 -- STEP: 111/142 -- GLOBAL_STEP: 467575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.159351348876953 (17.97836798590583) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.4007 (0.3527818821571968) + | > loader_time: 0.0078 (0.006069204828760645) + + + --> TIME: 2025-04-19 09:02:16 -- STEP: 136/142 -- GLOBAL_STEP: 467600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5142879486084 (17.904884674969843) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850613727358112 + | > current_lr_1: 0.0001850613727358112 + | > step_time: 0.4259 (0.36408684884800624) + | > loader_time: 0.0076 (0.006337768891278435) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8709983825683594 (2.8709983825683594) + | > loss_feat: nan (nan) + | > loss_mel: 15.712984085083008 (15.712984085083008) + | > loss_duration: 1.8237087726593018 (1.8237087726593018) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3829166889190674 (3.3829166889190674) + | > loss_feat: nan (nan) + | > loss_mel: 18.2049617767334 (18.2049617767334) + | > loss_duration: 1.9104946851730347 (1.9104946851730347) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8842878341674805 (3.133602261543274) + | > loss_feat: nan (nan) + | > loss_mel: 18.35932731628418 (18.28214454650879) + | > loss_duration: 1.7826372385025024 (1.8465659618377686) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032078027725219727 (-0.0004960298538208008) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.133602261543274 (+0.031096696853637695) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.28214454650879 (-1.0118474960327148) + | > avg_loss_duration: 1.8465659618377686 (-0.008098781108856201) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 622/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:02:20)  + + --> TIME: 2025-04-19 09:02:27 -- STEP: 19/142 -- GLOBAL_STEP: 467625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.0710506439209 (17.785802038092363) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850382400642192 + | > current_lr_1: 0.0001850382400642192 + | > step_time: 0.3123 (0.3087754249572754) + | > loader_time: 0.005 (0.0048533489829615546) + + + --> TIME: 2025-04-19 09:02:35 -- STEP: 44/142 -- GLOBAL_STEP: 467650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.446795701980591 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.998706817626953 (17.740567770871248) + | > loss_duration: 1.633199691772461 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850382400642192 + | > current_lr_1: 0.0001850382400642192 + | > step_time: 0.3342 (0.3194660869511691) + | > loader_time: 0.0061 (0.005296512083573775) + + + --> TIME: 2025-04-19 09:02:44 -- STEP: 69/142 -- GLOBAL_STEP: 467675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6416289806365967 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.810909271240234 (17.871387011762977) + | > loss_duration: 1.7002527713775635 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850382400642192 + | > current_lr_1: 0.0001850382400642192 + | > step_time: 0.3712 (0.33094316980113153) + | > loader_time: 0.0066 (0.005635068036507869) + + + --> TIME: 2025-04-19 09:02:54 -- STEP: 94/142 -- GLOBAL_STEP: 467700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.478240966796875 (17.899632017663183) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850382400642192 + | > current_lr_1: 0.0001850382400642192 + | > step_time: 0.3729 (0.34190979156088314) + | > loader_time: 0.0065 (0.0059102748302703205) + + + --> TIME: 2025-04-19 09:03:04 -- STEP: 119/142 -- GLOBAL_STEP: 467725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.36442756652832 (17.833934255006934) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001850382400642192 + | > current_lr_1: 0.0001850382400642192 + | > step_time: 0.4133 (0.3528051276166899) + | > loader_time: 0.0074 (0.006165290079196962) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8767168521881104 (2.8767168521881104) + | > loss_feat: nan (nan) + | > loss_mel: 16.893308639526367 (16.893308639526367) + | > loss_duration: 1.805527925491333 (1.805527925491333) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.451230764389038 (3.451230764389038) + | > loss_feat: nan (nan) + | > loss_mel: 19.929306030273438 (19.929306030273438) + | > loss_duration: 1.892250418663025 (1.892250418663025) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9405455589294434 (3.1958881616592407) + | > loss_feat: nan (nan) + | > loss_mel: 17.224910736083984 (18.57710838317871) + | > loss_duration: 1.7940740585327148 (1.8431622385978699) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003571629524230957 (+0.0003638267517089844) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1958881616592407 (+0.0622859001159668) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.57710838317871 (+0.2949638366699219) + | > avg_loss_duration: 1.8431622385978699 (-0.0034037232398986816) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 623/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:03:16)  + + --> TIME: 2025-04-19 09:03:17 -- STEP: 2/142 -- GLOBAL_STEP: 467750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6806130409240723 (2.6813766956329346) + | > loss_feat: nan (nan) + | > loss_mel: 16.415843963623047 (17.133495330810547) + | > loss_duration: 1.6636008024215698 (1.6325680017471313) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.2971 (0.299152135848999) + | > loader_time: 0.0039 (0.0039827823638916016) + + + --> TIME: 2025-04-19 09:03:25 -- STEP: 27/142 -- GLOBAL_STEP: 467775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.224695205688477 (18.01942429719148) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.3268 (0.3094412662364818) + | > loader_time: 0.0054 (0.005031082365247939) + + + --> TIME: 2025-04-19 09:03:33 -- STEP: 52/142 -- GLOBAL_STEP: 467800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.790802001953125 (18.046373569048367) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.3416 (0.32154545417198765) + | > loader_time: 0.0058 (0.005410510760087235) + + + --> TIME: 2025-04-19 09:03:43 -- STEP: 77/142 -- GLOBAL_STEP: 467825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.06636619567871 (18.00341555360075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.3706 (0.3345505850655692) + | > loader_time: 0.0068 (0.005711434723494891) + + + --> TIME: 2025-04-19 09:03:52 -- STEP: 102/142 -- GLOBAL_STEP: 467850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.006929397583008 (17.904601480446594) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.3989 (0.34655135986851704) + | > loader_time: 0.0074 (0.0059765437070061185) + + + --> TIME: 2025-04-19 09:04:03 -- STEP: 127/142 -- GLOBAL_STEP: 467875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.46296501159668 (17.95316411566546) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018501511028421117 + | > current_lr_1: 0.00018501511028421117 + | > step_time: 0.423 (0.3593815217806598) + | > loader_time: 0.0073 (0.006287972758135458) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9288532733917236 (2.9288532733917236) + | > loss_feat: nan (nan) + | > loss_mel: 14.681713104248047 (14.681713104248047) + | > loss_duration: 1.8329720497131348 (1.8329720497131348) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4292478561401367 (3.4292478561401367) + | > loss_feat: nan (nan) + | > loss_mel: 19.327098846435547 (19.327098846435547) + | > loss_duration: 1.8804361820220947 (1.8804361820220947) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0551202297210693 (3.242184042930603) + | > loss_feat: nan (nan) + | > loss_mel: 16.222341537475586 (17.774720191955566) + | > loss_duration: 1.7917038202285767 (1.8360700011253357) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003434896469116211 (-0.0001367330551147461) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.242184042930603 (+0.046295881271362305) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.774720191955566 (-0.8023881912231445) + | > avg_loss_duration: 1.8360700011253357 (-0.00709223747253418) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 624/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:04:11)  + + --> TIME: 2025-04-19 09:04:15 -- STEP: 10/142 -- GLOBAL_STEP: 467900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.05228042602539 (17.884441566467284) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.3091 (0.30025155544281007) + | > loader_time: 0.0052 (0.004616951942443848) + + + --> TIME: 2025-04-19 09:04:23 -- STEP: 35/142 -- GLOBAL_STEP: 467925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.205219268798828 (17.674964523315435) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.3509 (0.3154351098196847) + | > loader_time: 0.0053 (0.005067184993198939) + + + --> TIME: 2025-04-19 09:04:32 -- STEP: 60/142 -- GLOBAL_STEP: 467950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5286917686462402 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.62552261352539 (17.840612506866464) + | > loss_duration: 1.6545891761779785 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.3488 (0.32623723745346084) + | > loader_time: 0.006 (0.0053637981414794925) + + + --> TIME: 2025-04-19 09:04:41 -- STEP: 85/142 -- GLOBAL_STEP: 467975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.634660243988037 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.649349212646484 (17.814902978784897) + | > loss_duration: 1.6589329242706299 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.376 (0.33880469378303096) + | > loader_time: 0.0059 (0.005695458019480985) + + + --> TIME: 2025-04-19 09:04:51 -- STEP: 110/142 -- GLOBAL_STEP: 468000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.723095417022705 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.155969619750977 (17.84138091694225) + | > loss_duration: 1.6697957515716553 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.4088 (0.3503026723861694) + | > loader_time: 0.0071 (0.005963279984214089) + + + --> TIME: 2025-04-19 09:05:02 -- STEP: 135/142 -- GLOBAL_STEP: 468025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5963058471679688 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.784475326538086 (17.75338183508979) + | > loss_duration: 1.6407053470611572 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018499198339542563 + | > current_lr_1: 0.00018499198339542563 + | > step_time: 0.4289 (0.36258662011888293) + | > loader_time: 0.0077 (0.006227597483882198) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.868605613708496 (2.868605613708496) + | > loss_feat: nan (nan) + | > loss_mel: 18.14653205871582 (18.14653205871582) + | > loss_duration: 1.8280014991760254 (1.8280014991760254) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5368640422821045 (3.5368640422821045) + | > loss_feat: nan (nan) + | > loss_mel: 15.67402172088623 (15.67402172088623) + | > loss_duration: 1.8789234161376953 (1.8789234161376953) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.91131329536438 (3.224088668823242) + | > loss_feat: nan (nan) + | > loss_mel: 18.839046478271484 (17.256534099578857) + | > loss_duration: 1.7857584953308105 (1.832340955734253) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034486055374145508 (+1.3709068298339844e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.224088668823242 (-0.01809537410736084) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.256534099578857 (-0.518186092376709) + | > avg_loss_duration: 1.832340955734253 (-0.0037290453910827637) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 625/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:05:06)  + + --> TIME: 2025-04-19 09:05:12 -- STEP: 18/142 -- GLOBAL_STEP: 468050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.554962158203125 (18.030863814883762) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849688593975012 + | > current_lr_1: 0.0001849688593975012 + | > step_time: 0.3127 (0.3051581250296699) + | > loader_time: 0.0046 (0.00456930531395806) + + + --> TIME: 2025-04-19 09:05:21 -- STEP: 43/142 -- GLOBAL_STEP: 468075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.763004302978516 (17.932503434114683) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849688593975012 + | > current_lr_1: 0.0001849688593975012 + | > step_time: 0.3372 (0.31779579783594886) + | > loader_time: 0.0053 (0.005006657090297965) + + + --> TIME: 2025-04-19 09:05:30 -- STEP: 68/142 -- GLOBAL_STEP: 468100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.190189361572266 (17.962529350729554) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849688593975012 + | > current_lr_1: 0.0001849688593975012 + | > step_time: 0.3632 (0.3293594823164098) + | > loader_time: 0.0059 (0.0053219935473273776) + + + --> TIME: 2025-04-19 09:05:39 -- STEP: 93/142 -- GLOBAL_STEP: 468125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.709514617919922 (17.997805697943583) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849688593975012 + | > current_lr_1: 0.0001849688593975012 + | > step_time: 0.3739 (0.3407530502606463) + | > loader_time: 0.0075 (0.005612893771099791) + + + --> TIME: 2025-04-19 09:05:49 -- STEP: 118/142 -- GLOBAL_STEP: 468150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.634129047393799 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.404443740844727 (18.027734530174154) + | > loss_duration: 1.6273059844970703 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849688593975012 + | > current_lr_1: 0.0001849688593975012 + | > step_time: 0.4158 (0.3517453347222279) + | > loader_time: 0.0074 (0.0058743771860155026) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.82057523727417 (2.82057523727417) + | > loss_feat: nan (nan) + | > loss_mel: 17.46988868713379 (17.46988868713379) + | > loss_duration: 1.8243200778961182 (1.8243200778961182) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2811098098754883 (3.2811098098754883) + | > loss_feat: nan (nan) + | > loss_mel: 18.257152557373047 (18.257152557373047) + | > loss_duration: 1.9046149253845215 (1.9046149253845215) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7085213661193848 (2.9948155879974365) + | > loss_feat: nan (nan) + | > loss_mel: 16.867650985717773 (17.56240177154541) + | > loss_duration: 1.7891110181808472 (1.8468629717826843) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036565065383911133 (+0.0002079010009765625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9948155879974365 (-0.22927308082580566) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.56240177154541 (+0.30586767196655273) + | > avg_loss_duration: 1.8468629717826843 (+0.014522016048431396) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 626/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:06:01)  + + --> TIME: 2025-04-19 09:06:02 -- STEP: 1/142 -- GLOBAL_STEP: 468175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.92804718017578 (17.92804718017578) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.3043 (0.3042871952056885) + | > loader_time: 0.0037 (0.0037457942962646484) + + + --> TIME: 2025-04-19 09:06:10 -- STEP: 26/142 -- GLOBAL_STEP: 468200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.792898178100586 (17.771872447087215) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.3413 (0.31944475724146915) + | > loader_time: 0.0053 (0.004948625197777381) + + + --> TIME: 2025-04-19 09:06:19 -- STEP: 51/142 -- GLOBAL_STEP: 468225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.689241409301758 (17.806337057375444) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.3528 (0.33175576901903353) + | > loader_time: 0.0056 (0.0053466535082050395) + + + --> TIME: 2025-04-19 09:06:28 -- STEP: 76/142 -- GLOBAL_STEP: 468250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.536561965942383 (17.743688859437647) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.3805 (0.3442729993870385) + | > loader_time: 0.0064 (0.005685294929303621) + + + --> TIME: 2025-04-19 09:06:38 -- STEP: 101/142 -- GLOBAL_STEP: 468275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.02747917175293 (17.809130262620386) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.4045 (0.355351745492161) + | > loader_time: 0.0069 (0.005947660691667312) + + + --> TIME: 2025-04-19 09:06:49 -- STEP: 126/142 -- GLOBAL_STEP: 468300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.098966598510742 (17.752946096753316) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001849457382900765 + | > current_lr_1: 0.0001849457382900765 + | > step_time: 0.4262 (0.36738074204278376) + | > loader_time: 0.0077 (0.006214508934626504) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8921961784362793 (2.8921961784362793) + | > loss_feat: nan (nan) + | > loss_mel: 19.068716049194336 (19.068716049194336) + | > loss_duration: 1.8204004764556885 (1.8204004764556885) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2811636924743652 (3.2811636924743652) + | > loss_feat: nan (nan) + | > loss_mel: 18.518962860107422 (18.518962860107422) + | > loss_duration: 1.8876876831054688 (1.8876876831054688) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.948129177093506 (3.1146464347839355) + | > loss_feat: nan (nan) + | > loss_mel: 14.959734916687012 (16.739348888397217) + | > loss_duration: 1.793135643005371 (1.84041166305542) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003570556640625 (-8.594989776611328e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1146464347839355 (+0.11983084678649902) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.739348888397217 (-0.8230528831481934) + | > avg_loss_duration: 1.84041166305542 (-0.006451308727264404) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 627/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:06:57)  + + --> TIME: 2025-04-19 09:07:01 -- STEP: 9/142 -- GLOBAL_STEP: 468325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.935869216918945 (18.359933429294163) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.2995 (0.29882404539320206) + | > loader_time: 0.0045 (0.0042909251319037545) + + + --> TIME: 2025-04-19 09:07:09 -- STEP: 34/142 -- GLOBAL_STEP: 468350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.595594644546509 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.37200164794922 (18.010958587422092) + | > loss_duration: 1.652512550354004 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.3343 (0.31388310123892393) + | > loader_time: 0.0057 (0.004927277565002441) + + + --> TIME: 2025-04-19 09:07:18 -- STEP: 59/142 -- GLOBAL_STEP: 468375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.913761138916016 (17.954457347675906) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.3492 (0.32508130800926077) + | > loader_time: 0.0061 (0.005292278225139036) + + + --> TIME: 2025-04-19 09:07:27 -- STEP: 84/142 -- GLOBAL_STEP: 468400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.385473251342773 (17.95252781822568) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.3699 (0.33705084096817745) + | > loader_time: 0.006 (0.00557839586621239) + + + --> TIME: 2025-04-19 09:07:37 -- STEP: 109/142 -- GLOBAL_STEP: 468425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.2900447845459 (17.96238229909074) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.392 (0.34753364816718146) + | > loader_time: 0.0067 (0.00581730615108385) + + + --> TIME: 2025-04-19 09:07:47 -- STEP: 134/142 -- GLOBAL_STEP: 468450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.41939353942871 (17.953498591237988) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018492262007279025 + | > current_lr_1: 0.00018492262007279025 + | > step_time: 0.4142 (0.35958080861105846) + | > loader_time: 0.007 (0.006077004902398408) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0453619956970215 (3.0453619956970215) + | > loss_feat: nan (nan) + | > loss_mel: 17.93490219116211 (17.93490219116211) + | > loss_duration: 1.8415530920028687 (1.8415530920028687) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4460370540618896 (3.4460370540618896) + | > loss_feat: nan (nan) + | > loss_mel: 18.13441276550293 (18.13441276550293) + | > loss_duration: 1.8880157470703125 (1.8880157470703125) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.942578077316284 (3.194307565689087) + | > loss_feat: nan (nan) + | > loss_mel: 19.264162063598633 (18.69928741455078) + | > loss_duration: 1.8111730813980103 (1.8495944142341614) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038946866989135742 (+0.0003241300582885742) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.194307565689087 (+0.07966113090515137) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.69928741455078 (+1.9599385261535645) + | > avg_loss_duration: 1.8495944142341614 (+0.009182751178741455) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 628/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:07:52)  + + --> TIME: 2025-04-19 09:07:58 -- STEP: 17/142 -- GLOBAL_STEP: 468475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.028549194335938 (18.120131548713236) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018489950474528114 + | > current_lr_1: 0.00018489950474528114 + | > step_time: 0.3104 (0.302685863831464) + | > loader_time: 0.005 (0.004749185898724724) + + + --> TIME: 2025-04-19 09:08:07 -- STEP: 42/142 -- GLOBAL_STEP: 468500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.940837860107422 (18.05252502078102) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018489950474528114 + | > current_lr_1: 0.00018489950474528114 + | > step_time: 0.3459 (0.31903470697857095) + | > loader_time: 0.0054 (0.0052091677983601885) + + + --> TIME: 2025-04-19 09:08:16 -- STEP: 67/142 -- GLOBAL_STEP: 468525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.897441864013672 (17.95172968907143) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018489950474528114 + | > current_lr_1: 0.00018489950474528114 + | > step_time: 0.3691 (0.33572222937398893) + | > loader_time: 0.0065 (0.005656986094232815) + + + --> TIME: 2025-04-19 09:08:26 -- STEP: 92/142 -- GLOBAL_STEP: 468550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.311819076538086 (17.93506034560826) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018489950474528114 + | > current_lr_1: 0.00018489950474528114 + | > step_time: 0.3991 (0.34860843420028687) + | > loader_time: 0.0069 (0.005955499151478643) + + + --> TIME: 2025-04-19 09:08:36 -- STEP: 117/142 -- GLOBAL_STEP: 468575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64208984375 (17.90430694156224) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018489950474528114 + | > current_lr_1: 0.00018489950474528114 + | > step_time: 0.4223 (0.3605975411896013) + | > loader_time: 0.0068 (0.006229891736283261) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.855114221572876 (2.855114221572876) + | > loss_feat: nan (nan) + | > loss_mel: 17.67631721496582 (17.67631721496582) + | > loss_duration: 1.8424357175827026 (1.8424357175827026) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.432270050048828 (3.432270050048828) + | > loss_feat: nan (nan) + | > loss_mel: 18.39937400817871 (18.39937400817871) + | > loss_duration: 1.8780068159103394 (1.8780068159103394) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.796316146850586 (3.114293098449707) + | > loss_feat: nan (nan) + | > loss_mel: 17.776334762573242 (18.087854385375977) + | > loss_duration: 1.8116464614868164 (1.8448266386985779) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037658214569091797 (-0.00012886524200439453) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.114293098449707 (-0.08001446723937988) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.087854385375977 (-0.6114330291748047) + | > avg_loss_duration: 1.8448266386985779 (-0.004767775535583496) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 629/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:08:49)  + + --> TIME: 2025-04-19 09:08:49 -- STEP: 0/142 -- GLOBAL_STEP: 468600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.444461822509766 (17.444461822509766) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.3376 (0.3376283645629883) + | > loader_time: 0.3402 (0.3402104377746582) + + + --> TIME: 2025-04-19 09:08:57 -- STEP: 25/142 -- GLOBAL_STEP: 468625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.305654525756836 (17.933062591552734) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.3192 (0.30871532440185545) + | > loader_time: 0.0054 (0.004866819381713868) + + + --> TIME: 2025-04-19 09:09:06 -- STEP: 50/142 -- GLOBAL_STEP: 468650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.820898056030273 (17.92935297012329) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.3442 (0.32068148136138924) + | > loader_time: 0.0064 (0.005375423431396484) + + + --> TIME: 2025-04-19 09:09:15 -- STEP: 75/142 -- GLOBAL_STEP: 468675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.348743438720703 (17.915405972798666) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.3728 (0.3325581804911296) + | > loader_time: 0.0064 (0.005615342458089193) + + + --> TIME: 2025-04-19 09:09:24 -- STEP: 100/142 -- GLOBAL_STEP: 468700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.37286949157715 (17.885542497634887) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.3872 (0.34307420730590826) + | > loader_time: 0.0071 (0.005856766700744629) + + + --> TIME: 2025-04-19 09:09:35 -- STEP: 125/142 -- GLOBAL_STEP: 468725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.440603256225586 (17.857014686584474) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018487639230718798 + | > current_lr_1: 0.00018487639230718798 + | > step_time: 0.411 (0.3546871662139893) + | > loader_time: 0.0076 (0.006097904205322266) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.940169095993042 (2.940169095993042) + | > loss_feat: nan (nan) + | > loss_mel: 16.998451232910156 (16.998451232910156) + | > loss_duration: 1.8135331869125366 (1.8135331869125366) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1630442142486572 (3.1630442142486572) + | > loss_feat: nan (nan) + | > loss_mel: 17.6914005279541 (17.6914005279541) + | > loss_duration: 1.912677526473999 (1.912677526473999) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9431538581848145 (3.053099036216736) + | > loss_feat: nan (nan) + | > loss_mel: 17.441082000732422 (17.56624126434326) + | > loss_duration: 1.780279517173767 (1.846478521823883) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036693811416625977 (-9.644031524658203e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.053099036216736 (-0.06119406223297119) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.56624126434326 (-0.5216131210327148) + | > avg_loss_duration: 1.846478521823883 (+0.0016518831253051758) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 630/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:09:43)  + + --> TIME: 2025-04-19 09:09:46 -- STEP: 8/142 -- GLOBAL_STEP: 468750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.31171417236328 (17.840205907821655) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.2979 (0.30042076110839844) + | > loader_time: 0.0049 (0.004383862018585205) + + + --> TIME: 2025-04-19 09:09:55 -- STEP: 33/142 -- GLOBAL_STEP: 468775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.600116729736328 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.791234970092773 (18.07903301354611) + | > loss_duration: 1.6246323585510254 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.3394 (0.31677922335537995) + | > loader_time: 0.0057 (0.005270033171682647) + + + --> TIME: 2025-04-19 09:10:04 -- STEP: 58/142 -- GLOBAL_STEP: 468800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.12861442565918 (17.99749806831623) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.3587 (0.3305548593915742) + | > loader_time: 0.0061 (0.005587512049181707) + + + --> TIME: 2025-04-19 09:10:13 -- STEP: 83/142 -- GLOBAL_STEP: 468825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.547276496887207 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.053537368774414 (17.955500579742058) + | > loss_duration: 1.705317735671997 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.3844 (0.3445336761244808) + | > loader_time: 0.0068 (0.005873740437519118) + + + --> TIME: 2025-04-19 09:10:23 -- STEP: 108/142 -- GLOBAL_STEP: 468850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.170927047729492 (17.88753917482163) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.4094 (0.3570238638807226) + | > loader_time: 0.0071 (0.006135788228776719) + + + --> TIME: 2025-04-19 09:10:34 -- STEP: 133/142 -- GLOBAL_STEP: 468875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.562028884887695 (17.834508279212418) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018485328275814956 + | > current_lr_1: 0.00018485328275814956 + | > step_time: 0.4243 (0.36952525511720125) + | > loader_time: 0.0079 (0.006381111933772724) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0305392742156982 (3.0305392742156982) + | > loss_feat: nan (nan) + | > loss_mel: 17.961292266845703 (17.961292266845703) + | > loss_duration: 1.8303601741790771 (1.8303601741790771) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3282909393310547 (3.3282909393310547) + | > loss_feat: nan (nan) + | > loss_mel: 17.70175552368164 (17.70175552368164) + | > loss_duration: 1.9169379472732544 (1.9169379472732544) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.66191029548645 (2.9951006174087524) + | > loss_feat: nan (nan) + | > loss_mel: 19.789026260375977 (18.74539089202881) + | > loss_duration: 1.8248646259307861 (1.8709012866020203) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036290884017944336 (-4.029273986816406e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9951006174087524 (-0.0579984188079834) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.74539089202881 (+1.1791496276855469) + | > avg_loss_duration: 1.8709012866020203 (+0.024422764778137207) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 631/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:10:40)  + + --> TIME: 2025-04-19 09:10:45 -- STEP: 16/142 -- GLOBAL_STEP: 468900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.19678497314453 (17.845069408416748) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.3144 (0.30335743725299835) + | > loader_time: 0.005 (0.004642322659492493) + + + --> TIME: 2025-04-19 09:10:54 -- STEP: 41/142 -- GLOBAL_STEP: 468925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5494093894958496 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.20416831970215 (17.883746728664487) + | > loss_duration: 1.5958590507507324 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.3364 (0.31591696273989794) + | > loader_time: 0.0059 (0.005111880418730945) + + + --> TIME: 2025-04-19 09:11:03 -- STEP: 66/142 -- GLOBAL_STEP: 468950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.9453182220459 (17.92699227188572) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.3578 (0.3280366803660537) + | > loader_time: 0.0058 (0.0054504654624245395) + + + --> TIME: 2025-04-19 09:11:12 -- STEP: 91/142 -- GLOBAL_STEP: 468975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.907150268554688 (17.91538743658379) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.376 (0.33944606781005854) + | > loader_time: 0.0066 (0.0057143903040623905) + + + --> TIME: 2025-04-19 09:11:22 -- STEP: 116/142 -- GLOBAL_STEP: 469000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.195283889770508 (17.916457324192432) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.4046 (0.3501946165643889) + | > loader_time: 0.0083 (0.005984931156553069) + + + --> TIME: 2025-04-19 09:11:33 -- STEP: 141/142 -- GLOBAL_STEP: 469025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.21553897857666 (17.829295970023946) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001848301760978048 + | > current_lr_1: 0.0001848301760978048 + | > step_time: 0.1799 (0.36083277404731035) + | > loader_time: 0.0026 (0.006194391994611589) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.92232608795166 (2.92232608795166) + | > loss_feat: nan (nan) + | > loss_mel: 19.782180786132812 (19.782180786132812) + | > loss_duration: 1.8363163471221924 (1.8363163471221924) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4018640518188477 (3.4018640518188477) + | > loss_feat: nan (nan) + | > loss_mel: 18.46724510192871 (18.46724510192871) + | > loss_duration: 1.9122872352600098 (1.9122872352600098) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8879330158233643 (3.144898533821106) + | > loss_feat: nan (nan) + | > loss_mel: 19.594104766845703 (19.030674934387207) + | > loss_duration: 1.7909537553787231 (1.8516204953193665) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003442525863647461 (-0.00018656253814697266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.144898533821106 (+0.14979791641235352) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.030674934387207 (+0.28528404235839844) + | > avg_loss_duration: 1.8516204953193665 (-0.01928079128265381) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 632/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:11:35)  + + --> TIME: 2025-04-19 09:11:43 -- STEP: 24/142 -- GLOBAL_STEP: 469050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.82265281677246 (17.96859304110209) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018480707232579257 + | > current_lr_1: 0.00018480707232579257 + | > step_time: 0.3172 (0.30753100911776227) + | > loader_time: 0.0052 (0.0049833257993062334) + + + --> TIME: 2025-04-19 09:11:51 -- STEP: 49/142 -- GLOBAL_STEP: 469075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.530623197555542 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.124267578125 (18.022349493844164) + | > loss_duration: 1.6010420322418213 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018480707232579257 + | > current_lr_1: 0.00018480707232579257 + | > step_time: 0.3453 (0.3191728737889503) + | > loader_time: 0.0057 (0.00531001480258241) + + + --> TIME: 2025-04-19 09:12:00 -- STEP: 74/142 -- GLOBAL_STEP: 469100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.138072967529297 (17.98919048824826) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018480707232579257 + | > current_lr_1: 0.00018480707232579257 + | > step_time: 0.3638 (0.332632886396872) + | > loader_time: 0.007 (0.0056556044398127355) + + + --> TIME: 2025-04-19 09:12:10 -- STEP: 99/142 -- GLOBAL_STEP: 469125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.667703628540039 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.152626037597656 (17.864357842339402) + | > loss_duration: 1.6883864402770996 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018480707232579257 + | > current_lr_1: 0.00018480707232579257 + | > step_time: 0.3915 (0.34627340297506315) + | > loader_time: 0.0072 (0.00595923385234794) + + + --> TIME: 2025-04-19 09:12:21 -- STEP: 124/142 -- GLOBAL_STEP: 469150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.862564086914062 (17.886246512013095) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018480707232579257 + | > current_lr_1: 0.00018480707232579257 + | > step_time: 0.4347 (0.36047978170456424) + | > loader_time: 0.0069 (0.006249258595128211) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.03218936920166 (3.03218936920166) + | > loss_feat: nan (nan) + | > loss_mel: 19.32665252685547 (19.32665252685547) + | > loss_duration: 1.8417892456054688 (1.8417892456054688) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.281398057937622 (3.281398057937622) + | > loss_feat: nan (nan) + | > loss_mel: 19.25982666015625 (19.25982666015625) + | > loss_duration: 1.9000385999679565 (1.9000385999679565) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.856682538986206 (3.069040298461914) + | > loss_feat: nan (nan) + | > loss_mel: 20.081064224243164 (19.670445442199707) + | > loss_duration: 1.7759231328964233 (1.83798086643219) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037795305252075195 (+0.0003370046615600586) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.069040298461914 (-0.0758582353591919) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.670445442199707 (+0.6397705078125) + | > avg_loss_duration: 1.83798086643219 (-0.013639628887176514) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 633/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:12:31)  + + --> TIME: 2025-04-19 09:12:33 -- STEP: 7/142 -- GLOBAL_STEP: 469175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.28693389892578 (18.196499143327987) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.3001 (0.29952846254621235) + | > loader_time: 0.004 (0.004384143011910575) + + + --> TIME: 2025-04-19 09:12:42 -- STEP: 32/142 -- GLOBAL_STEP: 469200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.872373580932617 (17.721574872732162) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.3568 (0.3138253241777419) + | > loader_time: 0.006 (0.005080588161945343) + + + --> TIME: 2025-04-19 09:12:50 -- STEP: 57/142 -- GLOBAL_STEP: 469225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.909273147583008 (17.849358759428323) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.351 (0.3260407615126224) + | > loader_time: 0.0062 (0.005521937420493676) + + + --> TIME: 2025-04-19 09:13:00 -- STEP: 82/142 -- GLOBAL_STEP: 469250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.442699432373047 (17.927685121210608) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.3692 (0.3376120416129508) + | > loader_time: 0.0065 (0.00582601965927496) + + + --> TIME: 2025-04-19 09:13:09 -- STEP: 107/142 -- GLOBAL_STEP: 469275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.81435203552246 (17.900115333984942) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.3921 (0.34778472864739246) + | > loader_time: 0.0071 (0.006069608937914125) + + + --> TIME: 2025-04-19 09:13:20 -- STEP: 132/142 -- GLOBAL_STEP: 469300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.467308044433594 (17.882905309850514) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018478397144175182 + | > current_lr_1: 0.00018478397144175182 + | > step_time: 0.4179 (0.35925687262506195) + | > loader_time: 0.0074 (0.006299333138899368) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9768831729888916 (2.9768831729888916) + | > loss_feat: nan (nan) + | > loss_mel: 19.435806274414062 (19.435806274414062) + | > loss_duration: 1.8307676315307617 (1.8307676315307617) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3751471042633057 (3.3751471042633057) + | > loss_feat: nan (nan) + | > loss_mel: 18.16864776611328 (18.16864776611328) + | > loss_duration: 1.8943179845809937 (1.8943179845809937) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.932631254196167 (3.1538891792297363) + | > loss_feat: nan (nan) + | > loss_mel: 16.619762420654297 (17.39420509338379) + | > loss_duration: 1.8079768419265747 (1.8511474132537842) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003733992576599121 (-4.553794860839844e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1538891792297363 (+0.08484888076782227) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.39420509338379 (-2.276240348815918) + | > avg_loss_duration: 1.8511474132537842 (+0.013166546821594238) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 634/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:13:26)  + + --> TIME: 2025-04-19 09:13:31 -- STEP: 15/142 -- GLOBAL_STEP: 469325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.43526840209961 (17.996073277791343) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.3093 (0.30375185012817385) + | > loader_time: 0.0052 (0.00486901601155599) + + + --> TIME: 2025-04-19 09:13:39 -- STEP: 40/142 -- GLOBAL_STEP: 469350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.616111755371094 (17.989043045043946) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.3355 (0.3163651406764984) + | > loader_time: 0.0061 (0.005266082286834717) + + + --> TIME: 2025-04-19 09:13:48 -- STEP: 65/142 -- GLOBAL_STEP: 469375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.074554443359375 (18.081171123798075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.3682 (0.3281048077803392) + | > loader_time: 0.0067 (0.005623373618492714) + + + --> TIME: 2025-04-19 09:13:57 -- STEP: 90/142 -- GLOBAL_STEP: 469400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.216142654418945 (18.09950471454197) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.3768 (0.3393276267581516) + | > loader_time: 0.0076 (0.0059118562274509) + + + --> TIME: 2025-04-19 09:14:07 -- STEP: 115/142 -- GLOBAL_STEP: 469425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.031335830688477 (18.04627109195875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.3977 (0.34994735510452935) + | > loader_time: 0.0076 (0.006118745389192001) + + + --> TIME: 2025-04-19 09:14:18 -- STEP: 140/142 -- GLOBAL_STEP: 469450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.394784927368164 (17.99893202781677) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847608734453216 + | > current_lr_1: 0.0001847608734453216 + | > step_time: 0.4303 (0.3613198637962342) + | > loader_time: 0.0075 (0.006337491103581019) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.863455057144165 (2.863455057144165) + | > loss_feat: nan (nan) + | > loss_mel: 19.136333465576172 (19.136333465576172) + | > loss_duration: 1.8474756479263306 (1.8474756479263306) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2365522384643555 (3.2365522384643555) + | > loss_feat: nan (nan) + | > loss_mel: 18.143278121948242 (18.143278121948242) + | > loss_duration: 1.912527322769165 (1.912527322769165) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8799352645874023 (3.058243751525879) + | > loss_feat: nan (nan) + | > loss_mel: 18.790128707885742 (18.466703414916992) + | > loss_duration: 1.8045989274978638 (1.8585631251335144) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034835338592529297 (-0.0002504587173461914) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.058243751525879 (-0.09564542770385742) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.466703414916992 (+1.0724983215332031) + | > avg_loss_duration: 1.8585631251335144 (+0.007415711879730225) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 635/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:14:20)  + + --> TIME: 2025-04-19 09:14:28 -- STEP: 23/142 -- GLOBAL_STEP: 469475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.96326446533203 (17.859734452289082) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847377783361409 + | > current_lr_1: 0.0001847377783361409 + | > step_time: 0.3205 (0.309068254802538) + | > loader_time: 0.006 (0.005020856857299805) + + + --> TIME: 2025-04-19 09:14:37 -- STEP: 48/142 -- GLOBAL_STEP: 469500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.11451530456543 (18.13805045684178) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847377783361409 + | > current_lr_1: 0.0001847377783361409 + | > step_time: 0.3374 (0.32435692846775044) + | > loader_time: 0.0053 (0.005378941694895427) + + + --> TIME: 2025-04-19 09:14:46 -- STEP: 73/142 -- GLOBAL_STEP: 469525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.519306182861328 (18.085580499204866) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847377783361409 + | > current_lr_1: 0.0001847377783361409 + | > step_time: 0.3697 (0.3349832639302292) + | > loader_time: 0.0067 (0.005654570174543825) + + + --> TIME: 2025-04-19 09:14:56 -- STEP: 98/142 -- GLOBAL_STEP: 469550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.26343536376953 (18.06670602487057) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847377783361409 + | > current_lr_1: 0.0001847377783361409 + | > step_time: 0.3842 (0.3453333596793971) + | > loader_time: 0.007 (0.005971509583142339) + + + --> TIME: 2025-04-19 09:15:06 -- STEP: 123/142 -- GLOBAL_STEP: 469575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.533623218536377 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.007003784179688 (18.033794736474498) + | > loss_duration: 1.6474733352661133 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847377783361409 + | > current_lr_1: 0.0001847377783361409 + | > step_time: 0.4157 (0.3566009959554283) + | > loader_time: 0.0077 (0.006211959249605008) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.94120192527771 (2.94120192527771) + | > loss_feat: nan (nan) + | > loss_mel: 17.475107192993164 (17.475107192993164) + | > loss_duration: 1.8256103992462158 (1.8256103992462158) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.416376829147339 (3.416376829147339) + | > loss_feat: nan (nan) + | > loss_mel: 18.059667587280273 (18.059667587280273) + | > loss_duration: 1.8888959884643555 (1.8888959884643555) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8307042121887207 (3.12354052066803) + | > loss_feat: nan (nan) + | > loss_mel: 14.589189529418945 (16.32442855834961) + | > loss_duration: 1.7968682050704956 (1.8428820967674255) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00339353084564209 (-9.000301361083984e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.12354052066803 (+0.06529676914215088) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.32442855834961 (-2.142274856567383) + | > avg_loss_duration: 1.8428820967674255 (-0.015681028366088867) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 636/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:15:15)  + + --> TIME: 2025-04-19 09:15:18 -- STEP: 6/142 -- GLOBAL_STEP: 469600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.091466903686523 (17.848687171936035) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.3032 (0.29883189996083576) + | > loader_time: 0.005 (0.004626154899597168) + + + --> TIME: 2025-04-19 09:15:26 -- STEP: 31/142 -- GLOBAL_STEP: 469625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.12289047241211 (17.84586254242928) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.3334 (0.3137838840484619) + | > loader_time: 0.006 (0.005247300670992943) + + + --> TIME: 2025-04-19 09:15:35 -- STEP: 56/142 -- GLOBAL_STEP: 469650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.16522789001465 (17.77087533473968) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.3596 (0.32842552236148287) + | > loader_time: 0.0062 (0.005601968084062848) + + + --> TIME: 2025-04-19 09:15:45 -- STEP: 81/142 -- GLOBAL_STEP: 469675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.0262508392334 (17.780376975918987) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.3781 (0.3426979559439199) + | > loader_time: 0.0066 (0.005903273452947168) + + + --> TIME: 2025-04-19 09:15:55 -- STEP: 106/142 -- GLOBAL_STEP: 469700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.926399230957031 (17.72083937447025) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.3983 (0.3551587990994723) + | > loader_time: 0.0071 (0.006162620940298404) + + + --> TIME: 2025-04-19 09:16:05 -- STEP: 131/142 -- GLOBAL_STEP: 469725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.88229751586914 (17.750688450937037) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001847146861138489 + | > current_lr_1: 0.0001847146861138489 + | > step_time: 0.4346 (0.3678245653632942) + | > loader_time: 0.0074 (0.006401680808030922) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.046097993850708 (3.046097993850708) + | > loss_feat: nan (nan) + | > loss_mel: 19.710865020751953 (19.710865020751953) + | > loss_duration: 1.8267550468444824 (1.8267550468444824) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5745065212249756 (3.5745065212249756) + | > loss_feat: nan (nan) + | > loss_mel: 18.90041160583496 (18.90041160583496) + | > loss_duration: 1.9166171550750732 (1.9166171550750732) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9428563117980957 (3.2586814165115356) + | > loss_feat: nan (nan) + | > loss_mel: 16.195114135742188 (17.547762870788574) + | > loss_duration: 1.7933290004730225 (1.8549730777740479) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004144906997680664 (+0.0007513761520385742) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2586814165115356 (+0.13514089584350586) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.547762870788574 (+1.2233343124389648) + | > avg_loss_duration: 1.8549730777740479 (+0.012090981006622314) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 637/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:16:12)  + + --> TIME: 2025-04-19 09:16:17 -- STEP: 14/142 -- GLOBAL_STEP: 469750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4702823162078857 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.922842025756836 (18.18559237888881) + | > loss_duration: 1.6165587902069092 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.3076 (0.30223558630262104) + | > loader_time: 0.0048 (0.004675728934151786) + + + --> TIME: 2025-04-19 09:16:25 -- STEP: 39/142 -- GLOBAL_STEP: 469775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5210185050964355 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.045724868774414 (18.007698890490417) + | > loss_duration: 1.613827109336853 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.3339 (0.3151558668185503) + | > loader_time: 0.0059 (0.005147719994569436) + + + --> TIME: 2025-04-19 09:16:34 -- STEP: 64/142 -- GLOBAL_STEP: 469800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.511760711669922 (17.9471613764763) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.3746 (0.3276243954896927) + | > loader_time: 0.0058 (0.005421902984380724) + + + --> TIME: 2025-04-19 09:16:43 -- STEP: 89/142 -- GLOBAL_STEP: 469825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.196697235107422 (17.937179426128953) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.3732 (0.34026852618442494) + | > loader_time: 0.0065 (0.0057040841391917) + + + --> TIME: 2025-04-19 09:16:53 -- STEP: 114/142 -- GLOBAL_STEP: 469850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.662357330322266 (17.897551829354818) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.3933 (0.35056853294372564) + | > loader_time: 0.008 (0.005988899030183489) + + + --> TIME: 2025-04-19 09:17:04 -- STEP: 139/142 -- GLOBAL_STEP: 469875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.928508758544922 (17.83939379410779) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018469159677808464 + | > current_lr_1: 0.00018469159677808464 + | > step_time: 0.4281 (0.3624806455571017) + | > loader_time: 0.0073 (0.0062586609408152175) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9300920963287354 (2.9300920963287354) + | > loss_feat: nan (nan) + | > loss_mel: 18.747344970703125 (18.747344970703125) + | > loss_duration: 1.8489058017730713 (1.8489058017730713) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5678679943084717 (3.5678679943084717) + | > loss_feat: nan (nan) + | > loss_mel: 19.262685775756836 (19.262685775756836) + | > loss_duration: 1.8973004817962646 (1.8973004817962646) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.780775785446167 (3.1743218898773193) + | > loss_feat: nan (nan) + | > loss_mel: 15.761590003967285 (17.51213788986206) + | > loss_duration: 1.7789698839187622 (1.8381351828575134) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032988786697387695 (-0.0008460283279418945) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1743218898773193 (-0.08435952663421631) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.51213788986206 (-0.03562498092651367) + | > avg_loss_duration: 1.8381351828575134 (-0.016837894916534424) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 638/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:17:07)  + + --> TIME: 2025-04-19 09:17:14 -- STEP: 22/142 -- GLOBAL_STEP: 469900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.00791358947754 (18.025199630043723) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018466851032848737 + | > current_lr_1: 0.00018466851032848737 + | > step_time: 0.3189 (0.3064213015816428) + | > loader_time: 0.0058 (0.0049952810460870915) + + + --> TIME: 2025-04-19 09:17:23 -- STEP: 47/142 -- GLOBAL_STEP: 469925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.108448028564453 (17.947170622805338) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018466851032848737 + | > current_lr_1: 0.00018466851032848737 + | > step_time: 0.337 (0.31849607508233246) + | > loader_time: 0.0053 (0.0054296381930087475) + + + --> TIME: 2025-04-19 09:17:32 -- STEP: 72/142 -- GLOBAL_STEP: 469950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.490978240966797 (17.966026730007602) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018466851032848737 + | > current_lr_1: 0.00018466851032848737 + | > step_time: 0.3675 (0.3310657938321431) + | > loader_time: 0.0062 (0.005733946959177653) + + + --> TIME: 2025-04-19 09:17:41 -- STEP: 97/142 -- GLOBAL_STEP: 469975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.22665786743164 (17.95312792984481) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018466851032848737 + | > current_lr_1: 0.00018466851032848737 + | > step_time: 0.3871 (0.34260851329134895) + | > loader_time: 0.007 (0.006019393193353085) + + + --> TIME: 2025-04-19 09:17:52 -- STEP: 122/142 -- GLOBAL_STEP: 470000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.375160217285156 (17.92234812408197) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018466851032848737 + | > current_lr_1: 0.00018466851032848737 + | > step_time: 0.4271 (0.354656407090484) + | > loader_time: 0.0077 (0.006263277569755181) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_470000.pth + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.977463722229004 (2.977463722229004) + | > loss_feat: nan (nan) + | > loss_mel: 20.1755313873291 (20.1755313873291) + | > loss_duration: 1.850712776184082 (1.850712776184082) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3184447288513184 (3.3184447288513184) + | > loss_feat: nan (nan) + | > loss_mel: 15.493541717529297 (15.493541717529297) + | > loss_duration: 1.9280184507369995 (1.9280184507369995) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.786311388015747 (3.0523780584335327) + | > loss_feat: nan (nan) + | > loss_mel: 18.870065689086914 (17.181803703308105) + | > loss_duration: 1.7851132154464722 (1.8565658330917358) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003642439842224121 (+0.00034356117248535156) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0523780584335327 (-0.12194383144378662) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.181803703308105 (-0.3303341865539551) + | > avg_loss_duration: 1.8565658330917358 (+0.018430650234222412) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 639/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:18:03)  + + --> TIME: 2025-04-19 09:18:05 -- STEP: 5/142 -- GLOBAL_STEP: 470025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5995728969573975 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.499160766601562 (17.822576904296874) + | > loss_duration: 1.6598073244094849 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.3019 (0.3065951824188232) + | > loader_time: 0.0043 (0.004427576065063476) + + + --> TIME: 2025-04-19 09:18:13 -- STEP: 30/142 -- GLOBAL_STEP: 470050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.847885131835938 (17.701384671529134) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.3253 (0.3129501740137736) + | > loader_time: 0.0062 (0.005139183998107911) + + + --> TIME: 2025-04-19 09:18:22 -- STEP: 55/142 -- GLOBAL_STEP: 470075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.468841552734375 (17.962741782448504) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.3467 (0.32374835881319913) + | > loader_time: 0.0063 (0.00550946755842729) + + + --> TIME: 2025-04-19 09:18:31 -- STEP: 80/142 -- GLOBAL_STEP: 470100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.655242443084717 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.682476043701172 (17.929992973804467) + | > loss_duration: 1.6502822637557983 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.3854 (0.3364346563816071) + | > loader_time: 0.0066 (0.005783277750015259) + + + --> TIME: 2025-04-19 09:18:41 -- STEP: 105/142 -- GLOBAL_STEP: 470125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.7917537689209 (17.92258694058373) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.3969 (0.3476720469338553) + | > loader_time: 0.0088 (0.006078622454688662) + + + --> TIME: 2025-04-19 09:18:52 -- STEP: 130/142 -- GLOBAL_STEP: 470150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64126205444336 (17.94050730191745) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846454267646963 + | > current_lr_1: 0.0001846454267646963 + | > step_time: 0.424 (0.3601872425812941) + | > loader_time: 0.0079 (0.006363995258624737) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0380702018737793 (3.0380702018737793) + | > loss_feat: nan (nan) + | > loss_mel: 18.18009376525879 (18.18009376525879) + | > loss_duration: 1.8445329666137695 (1.8445329666137695) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4133858680725098 (3.4133858680725098) + | > loss_feat: nan (nan) + | > loss_mel: 17.908851623535156 (17.908851623535156) + | > loss_duration: 1.9191924333572388 (1.9191924333572388) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.865307092666626 (3.139346480369568) + | > loss_feat: nan (nan) + | > loss_mel: 19.50919532775879 (18.709023475646973) + | > loss_duration: 1.7864255905151367 (1.8528090119361877) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003945112228393555 (+0.0003026723861694336) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.139346480369568 (+0.08696842193603516) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.709023475646973 (+1.5272197723388672) + | > avg_loss_duration: 1.8528090119361877 (-0.0037568211555480957) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 640/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:18:58)  + + --> TIME: 2025-04-19 09:19:03 -- STEP: 13/142 -- GLOBAL_STEP: 470175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.21149444580078 (17.900170986468975) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.3071 (0.3009372674501859) + | > loader_time: 0.0048 (0.0044397757603571964) + + + --> TIME: 2025-04-19 09:19:11 -- STEP: 38/142 -- GLOBAL_STEP: 470200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.67249870300293 (17.901040830110244) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.3366 (0.31864122967971) + | > loader_time: 0.0054 (0.004898021095677426) + + + --> TIME: 2025-04-19 09:19:21 -- STEP: 63/142 -- GLOBAL_STEP: 470225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.613422393798828 (18.024592989966976) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.3648 (0.334616430222042) + | > loader_time: 0.0061 (0.005242480172051324) + + + --> TIME: 2025-04-19 09:19:30 -- STEP: 88/142 -- GLOBAL_STEP: 470250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.53271484375 (17.956438703970473) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.3742 (0.34431155432354305) + | > loader_time: 0.0066 (0.005500094457106156) + + + --> TIME: 2025-04-19 09:19:40 -- STEP: 113/142 -- GLOBAL_STEP: 470275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.484352111816406 (17.91063134860148) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.4013 (0.35397431491750525) + | > loader_time: 0.0068 (0.0057836844857815095) + + + --> TIME: 2025-04-19 09:19:51 -- STEP: 138/142 -- GLOBAL_STEP: 470300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.902921199798584 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.563369750976562 (17.9300887481026) + | > loss_duration: 1.750436782836914 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001846223460863507 + | > current_lr_1: 0.0001846223460863507 + | > step_time: 0.4259 (0.3656462638274483) + | > loader_time: 0.0074 (0.006041741025620612) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.740701913833618 (2.740701913833618) + | > loss_feat: nan (nan) + | > loss_mel: 17.276628494262695 (17.276628494262695) + | > loss_duration: 1.824776530265808 (1.824776530265808) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5212948322296143 (3.5212948322296143) + | > loss_feat: nan (nan) + | > loss_mel: 17.933696746826172 (17.933696746826172) + | > loss_duration: 1.8972141742706299 (1.8972141742706299) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6936874389648438 (3.107491135597229) + | > loss_feat: nan (nan) + | > loss_mel: 18.09720802307129 (18.01545238494873) + | > loss_duration: 1.787196159362793 (1.8422051668167114) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033028125762939453 (-0.0006422996520996094) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.107491135597229 (-0.03185534477233887) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.01545238494873 (-0.6935710906982422) + | > avg_loss_duration: 1.8422051668167114 (-0.010603845119476318) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 641/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:19:54)  + + --> TIME: 2025-04-19 09:20:01 -- STEP: 21/142 -- GLOBAL_STEP: 470325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.135196685791016 (17.738786833626882) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845992682930899 + | > current_lr_1: 0.0001845992682930899 + | > step_time: 0.3196 (0.3096891130719866) + | > loader_time: 0.0056 (0.004996810640607562) + + + --> TIME: 2025-04-19 09:20:09 -- STEP: 46/142 -- GLOBAL_STEP: 470350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.179115295410156 (17.776471863622245) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845992682930899 + | > current_lr_1: 0.0001845992682930899 + | > step_time: 0.3411 (0.32017139248226) + | > loader_time: 0.0055 (0.005290487538213315) + + + --> TIME: 2025-04-19 09:20:18 -- STEP: 71/142 -- GLOBAL_STEP: 470375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.32130241394043 (17.81274289144596) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845992682930899 + | > current_lr_1: 0.0001845992682930899 + | > step_time: 0.363 (0.33190822937119174) + | > loader_time: 0.0061 (0.005512650583831357) + + + --> TIME: 2025-04-19 09:20:28 -- STEP: 96/142 -- GLOBAL_STEP: 470400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5178186893463135 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.10950469970703 (17.86665071050325) + | > loss_duration: 1.669189214706421 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845992682930899 + | > current_lr_1: 0.0001845992682930899 + | > step_time: 0.3802 (0.3423454736669858) + | > loader_time: 0.0065 (0.005727355678876241) + + + --> TIME: 2025-04-19 09:20:38 -- STEP: 121/142 -- GLOBAL_STEP: 470425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.697057723999023 (17.79016957401244) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845992682930899 + | > current_lr_1: 0.0001845992682930899 + | > step_time: 0.4155 (0.3533649562804167) + | > loader_time: 0.0074 (0.00594428945178828) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9429197311401367 (2.9429197311401367) + | > loss_feat: nan (nan) + | > loss_mel: 16.383869171142578 (16.383869171142578) + | > loss_duration: 1.841237187385559 (1.841237187385559) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.504941463470459 (3.504941463470459) + | > loss_feat: nan (nan) + | > loss_mel: 18.395126342773438 (18.395126342773438) + | > loss_duration: 1.915450930595398 (1.915450930595398) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9034392833709717 (3.2041903734207153) + | > loss_feat: nan (nan) + | > loss_mel: 15.874506950378418 (17.134816646575928) + | > loss_duration: 1.8061519861221313 (1.8608014583587646) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033042430877685547 (+1.430511474609375e-06) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2041903734207153 (+0.09669923782348633) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.134816646575928 (-0.8806357383728027) + | > avg_loss_duration: 1.8608014583587646 (+0.018596291542053223) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 642/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:20:49)  + + --> TIME: 2025-04-19 09:20:51 -- STEP: 4/142 -- GLOBAL_STEP: 470450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.73253631591797 (18.530098915100098) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.3046 (0.30707764625549316) + | > loader_time: 0.0043 (0.004425764083862305) + + + --> TIME: 2025-04-19 09:20:59 -- STEP: 29/142 -- GLOBAL_STEP: 470475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.55392837524414 (18.066170528017242) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.3295 (0.31518611414679165) + | > loader_time: 0.0059 (0.005192403135628536) + + + --> TIME: 2025-04-19 09:21:08 -- STEP: 54/142 -- GLOBAL_STEP: 470500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5416219234466553 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.41558265686035 (17.983822434036817) + | > loss_duration: 1.6335413455963135 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.3539 (0.3290150916134871) + | > loader_time: 0.0064 (0.0055190545541268805) + + + --> TIME: 2025-04-19 09:21:17 -- STEP: 79/142 -- GLOBAL_STEP: 470525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.954296112060547 (17.980284787431547) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.3706 (0.3429217398921146) + | > loader_time: 0.0062 (0.005840280387974993) + + + --> TIME: 2025-04-19 09:21:27 -- STEP: 104/142 -- GLOBAL_STEP: 470550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.135648727416992 (18.031860415752117) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.3967 (0.3548864217904898) + | > loader_time: 0.0084 (0.006105060760791485) + + + --> TIME: 2025-04-19 09:21:38 -- STEP: 129/142 -- GLOBAL_STEP: 470575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.241283416748047 (17.996056815450512) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018457619338455326 + | > current_lr_1: 0.00018457619338455326 + | > step_time: 0.4206 (0.3663938174876132) + | > loader_time: 0.0082 (0.006338182345841282) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9969053268432617 (2.9969053268432617) + | > loss_feat: nan (nan) + | > loss_mel: 18.190473556518555 (18.190473556518555) + | > loss_duration: 1.8340117931365967 (1.8340117931365967) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1310040950775146 (3.1310040950775146) + | > loss_feat: nan (nan) + | > loss_mel: 18.92179298400879 (18.92179298400879) + | > loss_duration: 1.909332513809204 (1.909332513809204) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8268861770629883 (2.9789451360702515) + | > loss_feat: nan (nan) + | > loss_mel: 16.640897750854492 (17.78134536743164) + | > loss_duration: 1.7945953607559204 (1.8519639372825623) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034297704696655273 (+0.00012552738189697266) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9789451360702515 (-0.22524523735046387) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.78134536743164 (+0.6465287208557129) + | > avg_loss_duration: 1.8519639372825623 (-0.008837521076202393) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 643/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:21:45)  + + --> TIME: 2025-04-19 09:21:49 -- STEP: 12/142 -- GLOBAL_STEP: 470600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.994121551513672 (18.03112840652466) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.3068 (0.3060055573781331) + | > loader_time: 0.005 (0.004617532094319661) + + + --> TIME: 2025-04-19 09:21:57 -- STEP: 37/142 -- GLOBAL_STEP: 470625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.22698974609375 (17.947436822427292) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.3277 (0.31639295655327876) + | > loader_time: 0.0056 (0.005088303540204023) + + + --> TIME: 2025-04-19 09:22:06 -- STEP: 62/142 -- GLOBAL_STEP: 470650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.64350700378418 (17.937322185885517) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.3564 (0.3270450099822014) + | > loader_time: 0.006 (0.005402684211730958) + + + --> TIME: 2025-04-19 09:22:16 -- STEP: 87/142 -- GLOBAL_STEP: 470675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.897275924682617 (17.967575478827825) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.3738 (0.33847650988348593) + | > loader_time: 0.0063 (0.005657445425274727) + + + --> TIME: 2025-04-19 09:22:25 -- STEP: 112/142 -- GLOBAL_STEP: 470700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.179210662841797 (17.93169835635593) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.4011 (0.34884422166006906) + | > loader_time: 0.0071 (0.005873597093990871) + + + --> TIME: 2025-04-19 09:22:36 -- STEP: 137/142 -- GLOBAL_STEP: 470725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5969347953796387 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.54842758178711 (17.92204822066926) + | > loss_duration: 1.6731234788894653 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845531213603802 + | > current_lr_1: 0.0001845531213603802 + | > step_time: 0.4288 (0.3607417614790645) + | > loader_time: 0.0073 (0.006157493939364914) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8695764541625977 (2.8695764541625977) + | > loss_feat: nan (nan) + | > loss_mel: 19.93553352355957 (19.93553352355957) + | > loss_duration: 1.8371553421020508 (1.8371553421020508) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.452191114425659 (3.452191114425659) + | > loss_feat: nan (nan) + | > loss_mel: 17.854759216308594 (17.854759216308594) + | > loss_duration: 1.895841121673584 (1.895841121673584) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7368483543395996 (3.0945197343826294) + | > loss_feat: nan (nan) + | > loss_mel: 17.15985679626465 (17.50730800628662) + | > loss_duration: 1.7731201648712158 (1.8344806432724) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004229068756103516 (+0.0007992982864379883) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0945197343826294 (+0.11557459831237793) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.50730800628662 (-0.27403736114501953) + | > avg_loss_duration: 1.8344806432724 (-0.017483294010162354) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 644/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:22:40)  + + --> TIME: 2025-04-19 09:22:46 -- STEP: 20/142 -- GLOBAL_STEP: 470750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.634263515472412 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.85942268371582 (17.66589946746826) + | > loss_duration: 1.6887283325195312 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018453005222021014 + | > current_lr_1: 0.00018453005222021014 + | > step_time: 0.3169 (0.30665829181671145) + | > loader_time: 0.0053 (0.004989457130432129) + + + --> TIME: 2025-04-19 09:22:55 -- STEP: 45/142 -- GLOBAL_STEP: 470775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.484865188598633 (17.763404104444717) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018453005222021014 + | > current_lr_1: 0.00018453005222021014 + | > step_time: 0.3532 (0.32570794423421223) + | > loader_time: 0.0064 (0.005440012613932291) + + + --> TIME: 2025-04-19 09:23:04 -- STEP: 70/142 -- GLOBAL_STEP: 470800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.974770545959473 (17.890921810695104) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018453005222021014 + | > current_lr_1: 0.00018453005222021014 + | > step_time: 0.3749 (0.3392207247870309) + | > loader_time: 0.0063 (0.0057320560727800625) + + + --> TIME: 2025-04-19 09:23:14 -- STEP: 95/142 -- GLOBAL_STEP: 470825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.35122299194336 (17.852389978107656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018453005222021014 + | > current_lr_1: 0.00018453005222021014 + | > step_time: 0.3929 (0.3512720860933002) + | > loader_time: 0.0069 (0.006078624725341796) + + + --> TIME: 2025-04-19 09:23:25 -- STEP: 120/142 -- GLOBAL_STEP: 470850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.39654541015625 (17.908361999193826) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018453005222021014 + | > current_lr_1: 0.00018453005222021014 + | > step_time: 0.4246 (0.36336562832196545) + | > loader_time: 0.0073 (0.006313931941986083) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0429298877716064 (3.0429298877716064) + | > loss_feat: nan (nan) + | > loss_mel: 17.97992515563965 (17.97992515563965) + | > loss_duration: 1.841947078704834 (1.841947078704834) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3830056190490723 (3.3830056190490723) + | > loss_feat: nan (nan) + | > loss_mel: 17.389312744140625 (17.389312744140625) + | > loss_duration: 1.881758451461792 (1.881758451461792) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.702683925628662 (3.042844772338867) + | > loss_feat: nan (nan) + | > loss_mel: 18.10904884338379 (17.749180793762207) + | > loss_duration: 1.7965712547302246 (1.8391648530960083) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031527280807495117 (-0.001076340675354004) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.042844772338867 (-0.05167496204376221) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.749180793762207 (+0.24187278747558594) + | > avg_loss_duration: 1.8391648530960083 (+0.0046842098236083984) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 645/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:23:36)  + + --> TIME: 2025-04-19 09:23:38 -- STEP: 3/142 -- GLOBAL_STEP: 470875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.203466415405273 (18.326545079549152) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.2964 (0.29756005605061847) + | > loader_time: 0.0044 (0.004285971323649089) + + + --> TIME: 2025-04-19 09:23:46 -- STEP: 28/142 -- GLOBAL_STEP: 470900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.50564193725586 (17.8921422958374) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.3195 (0.30984361682619366) + | > loader_time: 0.0049 (0.004797288349696568) + + + --> TIME: 2025-04-19 09:23:54 -- STEP: 53/142 -- GLOBAL_STEP: 470925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4178123474121094 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.318349838256836 (17.952977450388772) + | > loss_duration: 1.6078836917877197 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.3425 (0.32197942373887556) + | > loader_time: 0.0056 (0.005118122640645729) + + + --> TIME: 2025-04-19 09:24:03 -- STEP: 78/142 -- GLOBAL_STEP: 470950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.4778995513916 (18.000540562165085) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.3671 (0.33424084614484745) + | > loader_time: 0.0064 (0.005402723948160808) + + + --> TIME: 2025-04-19 09:24:13 -- STEP: 103/142 -- GLOBAL_STEP: 470975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.751096725463867 (17.970608220517065) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.3917 (0.3448768865714953) + | > loader_time: 0.0064 (0.005669584552061212) + + + --> TIME: 2025-04-19 09:24:23 -- STEP: 128/142 -- GLOBAL_STEP: 471000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.505374908447266 (17.96008925139904) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001845069859636826 + | > current_lr_1: 0.0001845069859636826 + | > step_time: 0.4117 (0.3564776089042425) + | > loader_time: 0.007 (0.0059089232236146936) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9313247203826904 (2.9313247203826904) + | > loss_feat: nan (nan) + | > loss_mel: 19.618330001831055 (19.618330001831055) + | > loss_duration: 1.8466427326202393 (1.8466427326202393) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2240984439849854 (3.2240984439849854) + | > loss_feat: nan (nan) + | > loss_mel: 17.364511489868164 (17.364511489868164) + | > loss_duration: 1.9035181999206543 (1.9035181999206543) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8391356468200684 (3.031617045402527) + | > loss_feat: nan (nan) + | > loss_mel: 19.658742904663086 (18.511627197265625) + | > loss_duration: 1.801879644393921 (1.8526989221572876) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034837722778320312 (+0.00033104419708251953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.031617045402527 (-0.011227726936340332) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.511627197265625 (+0.762446403503418) + | > avg_loss_duration: 1.8526989221572876 (+0.013534069061279297) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 646/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:24:31)  + + --> TIME: 2025-04-19 09:24:35 -- STEP: 11/142 -- GLOBAL_STEP: 471025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.443988800048828 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.01214599609375 (18.132498827847567) + | > loss_duration: 1.6085126399993896 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.3078 (0.3010419498790394) + | > loader_time: 0.0051 (0.004487492821433328) + + + --> TIME: 2025-04-19 09:24:43 -- STEP: 36/142 -- GLOBAL_STEP: 471050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.374814987182617 (17.901303715176052) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.3401 (0.3171893027093675) + | > loader_time: 0.0057 (0.005028247833251953) + + + --> TIME: 2025-04-19 09:24:52 -- STEP: 61/142 -- GLOBAL_STEP: 471075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.051944732666016 (18.011141292384416) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.3674 (0.3313940431250901) + | > loader_time: 0.0057 (0.005430936813354492) + + + --> TIME: 2025-04-19 09:25:02 -- STEP: 86/142 -- GLOBAL_STEP: 471100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.462852478027344 (18.07415505342706) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.3858 (0.34804375504338453) + | > loader_time: 0.0066 (0.005751490592956543) + + + --> TIME: 2025-04-19 09:25:12 -- STEP: 111/142 -- GLOBAL_STEP: 471125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.872724533081055 (18.096706063897766) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.3998 (0.36054929956659526) + | > loader_time: 0.0071 (0.006063287322585647) + + + --> TIME: 2025-04-19 09:25:23 -- STEP: 136/142 -- GLOBAL_STEP: 471150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.289351463317871 (18.02887677445132) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018448392259043714 + | > current_lr_1: 0.00018448392259043714 + | > step_time: 0.4274 (0.37051861075793985) + | > loader_time: 0.0074 (0.006306124084136065) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8711133003234863 (2.8711133003234863) + | > loss_feat: nan (nan) + | > loss_mel: 17.95767593383789 (17.95767593383789) + | > loss_duration: 1.832214593887329 (1.832214593887329) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3597662448883057 (3.3597662448883057) + | > loss_feat: nan (nan) + | > loss_mel: 19.111177444458008 (19.111177444458008) + | > loss_duration: 1.9029648303985596 (1.9029648303985596) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.865079402923584 (3.112422823905945) + | > loss_feat: nan (nan) + | > loss_mel: 19.234935760498047 (19.173056602478027) + | > loss_duration: 1.7989134788513184 (1.850939154624939) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032595396041870117 (-0.00022423267364501953) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.112422823905945 (+0.08080577850341797) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.173056602478027 (+0.6614294052124023) + | > avg_loss_duration: 1.850939154624939 (-0.0017597675323486328) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 647/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:25:27)  + + --> TIME: 2025-04-19 09:25:34 -- STEP: 19/142 -- GLOBAL_STEP: 471175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.99230194091797 (17.939940301995527) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018446086210011334 + | > current_lr_1: 0.00018446086210011334 + | > step_time: 0.3123 (0.3053050668616044) + | > loader_time: 0.005 (0.004723975532933285) + + + --> TIME: 2025-04-19 09:25:42 -- STEP: 44/142 -- GLOBAL_STEP: 471200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.8172607421875 (18.074838616631247) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018446086210011334 + | > current_lr_1: 0.00018446086210011334 + | > step_time: 0.3344 (0.31749988144094293) + | > loader_time: 0.006 (0.00521110946481878) + + + --> TIME: 2025-04-19 09:25:51 -- STEP: 69/142 -- GLOBAL_STEP: 471225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.184078216552734 (18.007500012715656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018446086210011334 + | > current_lr_1: 0.00018446086210011334 + | > step_time: 0.3861 (0.32985605018726294) + | > loader_time: 0.0066 (0.0054865021636520605) + + + --> TIME: 2025-04-19 09:26:00 -- STEP: 94/142 -- GLOBAL_STEP: 471250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.30586051940918 (17.968287153446937) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018446086210011334 + | > current_lr_1: 0.00018446086210011334 + | > step_time: 0.3709 (0.34129751235880745) + | > loader_time: 0.0066 (0.005790565876250571) + + + --> TIME: 2025-04-19 09:26:11 -- STEP: 119/142 -- GLOBAL_STEP: 471275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.52720069885254 (17.9315721287447) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018446086210011334 + | > current_lr_1: 0.00018446086210011334 + | > step_time: 0.401 (0.3521028947429496) + | > loader_time: 0.0072 (0.006029159081082384) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0201168060302734 (3.0201168060302734) + | > loss_feat: nan (nan) + | > loss_mel: 18.359609603881836 (18.359609603881836) + | > loss_duration: 1.8100579977035522 (1.8100579977035522) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.336926221847534 (3.336926221847534) + | > loss_feat: nan (nan) + | > loss_mel: 17.15084457397461 (17.15084457397461) + | > loss_duration: 1.894631266593933 (1.894631266593933) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.749971389770508 (3.043448805809021) + | > loss_feat: nan (nan) + | > loss_mel: 17.313508987426758 (17.232176780700684) + | > loss_duration: 1.8103160858154297 (1.8524736762046814) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033921003341674805 (+0.00013256072998046875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.043448805809021 (-0.06897401809692383) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.232176780700684 (-1.9408798217773438) + | > avg_loss_duration: 1.8524736762046814 (+0.0015345215797424316) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 648/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:26:22)  + + --> TIME: 2025-04-19 09:26:23 -- STEP: 2/142 -- GLOBAL_STEP: 471300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3970112800598145 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.197534561157227 (17.944669723510742) + | > loss_duration: 1.6382355690002441 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.494 (0.39784741401672363) + | > loader_time: 0.004 (0.004011869430541992) + + + --> TIME: 2025-04-19 09:26:31 -- STEP: 27/142 -- GLOBAL_STEP: 471325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.916231155395508 (17.758143424987797) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.3255 (0.3185841065865976) + | > loader_time: 0.0057 (0.004873408211602105) + + + --> TIME: 2025-04-19 09:26:40 -- STEP: 52/142 -- GLOBAL_STEP: 471350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.181806564331055 (17.806887975105873) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.3414 (0.327218981889578) + | > loader_time: 0.0058 (0.00531710569675152) + + + --> TIME: 2025-04-19 09:26:49 -- STEP: 77/142 -- GLOBAL_STEP: 471375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.987340927124023 (17.91284106613754) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.3678 (0.3382748628591562) + | > loader_time: 0.0065 (0.005625449217759168) + + + --> TIME: 2025-04-19 09:26:59 -- STEP: 102/142 -- GLOBAL_STEP: 471400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5937788486480713 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.56744956970215 (17.98473820966834) + | > loss_duration: 1.6595993041992188 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.4035 (0.34857607121561096) + | > loader_time: 0.0064 (0.005902108024148379) + + + --> TIME: 2025-04-19 09:27:09 -- STEP: 127/142 -- GLOBAL_STEP: 471425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.747051239013672 (18.001102026991962) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018443780449235083 + | > current_lr_1: 0.00018443780449235083 + | > step_time: 0.4152 (0.3604085839639499) + | > loader_time: 0.0074 (0.0061734116922213325) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.805039405822754 (2.805039405822754) + | > loss_feat: nan (nan) + | > loss_mel: 18.334928512573242 (18.334928512573242) + | > loss_duration: 1.8364304304122925 (1.8364304304122925) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4446613788604736 (3.4446613788604736) + | > loss_feat: nan (nan) + | > loss_mel: 18.71830177307129 (18.71830177307129) + | > loss_duration: 1.9046846628189087 (1.9046846628189087) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9746525287628174 (3.2096569538116455) + | > loss_feat: nan (nan) + | > loss_mel: 19.3398380279541 (19.029069900512695) + | > loss_duration: 1.786445140838623 (1.8455649018287659) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037908554077148438 (+0.0003987550735473633) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2096569538116455 (+0.1662081480026245) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.029069900512695 (+1.7968931198120117) + | > avg_loss_duration: 1.8455649018287659 (-0.006908774375915527) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 649/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:27:17)  + + --> TIME: 2025-04-19 09:27:21 -- STEP: 10/142 -- GLOBAL_STEP: 471450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.543487548828125 (17.690140342712404) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.3106 (0.30035440921783446) + | > loader_time: 0.0051 (0.004695630073547364) + + + --> TIME: 2025-04-19 09:27:29 -- STEP: 35/142 -- GLOBAL_STEP: 471475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.660362243652344 (17.814715031215123) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.3416 (0.3137365681784494) + | > loader_time: 0.0061 (0.005191074098859515) + + + --> TIME: 2025-04-19 09:27:38 -- STEP: 60/142 -- GLOBAL_STEP: 471500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5297818183898926 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.354291915893555 (17.925282621383666) + | > loss_duration: 1.6377805471420288 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.3503 (0.3248992323875427) + | > loader_time: 0.0064 (0.005523022015889486) + + + --> TIME: 2025-04-19 09:27:47 -- STEP: 85/142 -- GLOBAL_STEP: 471525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5867958068847656 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.74070167541504 (17.938480859644276) + | > loss_duration: 1.6522060632705688 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.3716 (0.3370636238771326) + | > loader_time: 0.0067 (0.005788671269136314) + + + --> TIME: 2025-04-19 09:27:57 -- STEP: 110/142 -- GLOBAL_STEP: 471550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.31927490234375 (17.932260738719602) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.4069 (0.34767190759832206) + | > loader_time: 0.0075 (0.0060649698430841606) + + + --> TIME: 2025-04-19 09:28:08 -- STEP: 135/142 -- GLOBAL_STEP: 471575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.954072952270508 (17.899961139537677) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018441474976678928 + | > current_lr_1: 0.00018441474976678928 + | > step_time: 0.4295 (0.35981443723042805) + | > loader_time: 0.0076 (0.00631182105452926) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9632935523986816 (2.9632935523986816) + | > loss_feat: nan (nan) + | > loss_mel: 19.21877098083496 (19.21877098083496) + | > loss_duration: 1.844122290611267 (1.844122290611267) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.475015640258789 (3.475015640258789) + | > loss_feat: nan (nan) + | > loss_mel: 17.974531173706055 (17.974531173706055) + | > loss_duration: 1.905448317527771 (1.905448317527771) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0351295471191406 (3.255072593688965) + | > loss_feat: nan (nan) + | > loss_mel: 18.044178009033203 (18.00935459136963) + | > loss_duration: 1.7761168479919434 (1.8407825827598572) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00402987003326416 (+0.0002390146255493164) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.255072593688965 (+0.045415639877319336) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.00935459136963 (-1.0197153091430664) + | > avg_loss_duration: 1.8407825827598572 (-0.004782319068908691) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 650/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:28:12)  + + --> TIME: 2025-04-19 09:28:18 -- STEP: 18/142 -- GLOBAL_STEP: 471600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.0101318359375 (18.020452393425835) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001843916979230684 + | > current_lr_1: 0.0001843916979230684 + | > step_time: 0.313 (0.3048022720548842) + | > loader_time: 0.0047 (0.004887368943956163) + + + --> TIME: 2025-04-19 09:28:27 -- STEP: 43/142 -- GLOBAL_STEP: 471625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.437856912612915 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.657930374145508 (17.859517563221065) + | > loss_duration: 1.6211833953857422 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001843916979230684 + | > current_lr_1: 0.0001843916979230684 + | > step_time: 0.3381 (0.31748466713483936) + | > loader_time: 0.0055 (0.005213576693867528) + + + --> TIME: 2025-04-19 09:28:36 -- STEP: 68/142 -- GLOBAL_STEP: 471650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.999584197998047 (17.832885994630693) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001843916979230684 + | > current_lr_1: 0.0001843916979230684 + | > step_time: 0.3626 (0.3294848975013283) + | > loader_time: 0.0057 (0.005510786000420065) + + + --> TIME: 2025-04-19 09:28:45 -- STEP: 93/142 -- GLOBAL_STEP: 471675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5275943279266357 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.325244903564453 (17.8894704388034) + | > loss_duration: 1.6592766046524048 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001843916979230684 + | > current_lr_1: 0.0001843916979230684 + | > step_time: 0.3786 (0.3418177481620542) + | > loader_time: 0.0066 (0.005760149289202948) + + + --> TIME: 2025-04-19 09:28:55 -- STEP: 118/142 -- GLOBAL_STEP: 471700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.489699125289917 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.2286376953125 (17.895279084221794) + | > loss_duration: 1.6442559957504272 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001843916979230684 + | > current_lr_1: 0.0001843916979230684 + | > step_time: 0.4195 (0.3538870831667366) + | > loader_time: 0.008 (0.005978529736147089) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.99356746673584 (2.99356746673584) + | > loss_feat: nan (nan) + | > loss_mel: 20.01413345336914 (20.01413345336914) + | > loss_duration: 1.843597412109375 (1.843597412109375) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3907172679901123 (3.3907172679901123) + | > loss_feat: nan (nan) + | > loss_mel: 16.945009231567383 (16.945009231567383) + | > loss_duration: 1.9112757444381714 (1.9112757444381714) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.971928596496582 (3.181322932243347) + | > loss_feat: nan (nan) + | > loss_mel: 17.562503814697266 (17.253756523132324) + | > loss_duration: 1.810197114944458 (1.8607364296913147) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00348508358001709 (-0.0005447864532470703) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.181322932243347 (-0.07374966144561768) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.253756523132324 (-0.7555980682373047) + | > avg_loss_duration: 1.8607364296913147 (+0.01995384693145752) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 651/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:29:07)  + + --> TIME: 2025-04-19 09:29:08 -- STEP: 1/142 -- GLOBAL_STEP: 471725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.55368995666504 (16.55368995666504) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.3026 (0.3026466369628906) + | > loader_time: 0.004 (0.0040318965911865234) + + + --> TIME: 2025-04-19 09:29:16 -- STEP: 26/142 -- GLOBAL_STEP: 471750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5870401859283447 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.46756362915039 (17.795049190521237) + | > loss_duration: 1.6323853731155396 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.3225 (0.3139573702445397) + | > loader_time: 0.0054 (0.005087109712453989) + + + --> TIME: 2025-04-19 09:29:25 -- STEP: 51/142 -- GLOBAL_STEP: 471775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.571369171142578 (18.001570552003155) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.3431 (0.3240140419380337) + | > loader_time: 0.0062 (0.00546115052466299) + + + --> TIME: 2025-04-19 09:29:34 -- STEP: 76/142 -- GLOBAL_STEP: 471800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.561279296875 (17.972084861052668) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.3742 (0.33614635781237945) + | > loader_time: 0.0067 (0.005786779679750142) + + + --> TIME: 2025-04-19 09:29:44 -- STEP: 101/142 -- GLOBAL_STEP: 471825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.312646865844727 (17.88793081340223) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.3944 (0.34694465080110154) + | > loader_time: 0.0064 (0.00606587853762183) + + + --> TIME: 2025-04-19 09:29:54 -- STEP: 126/142 -- GLOBAL_STEP: 471850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.922006607055664 (17.904613441891133) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018436864896082802 + | > current_lr_1: 0.00018436864896082802 + | > step_time: 0.4172 (0.3589080307218764) + | > loader_time: 0.0077 (0.006289924894060408) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9894378185272217 (2.9894378185272217) + | > loss_feat: nan (nan) + | > loss_mel: 19.168025970458984 (19.168025970458984) + | > loss_duration: 1.8403868675231934 (1.8403868675231934) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4674394130706787 (3.4674394130706787) + | > loss_feat: nan (nan) + | > loss_mel: 18.868640899658203 (18.868640899658203) + | > loss_duration: 1.9027884006500244 (1.9027884006500244) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0694210529327393 (3.268430233001709) + | > loss_feat: nan (nan) + | > loss_mel: 19.043704986572266 (18.956172943115234) + | > loss_duration: 1.7862039804458618 (1.8444961905479431) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033904314041137695 (-9.465217590332031e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.268430233001709 (+0.08710730075836182) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.956172943115234 (+1.7024164199829102) + | > avg_loss_duration: 1.8444961905479431 (-0.016240239143371582) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 652/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:30:03)  + + --> TIME: 2025-04-19 09:30:06 -- STEP: 9/142 -- GLOBAL_STEP: 471875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.3377628326416 (18.336686028374565) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.3011 (0.29966266949971515) + | > loader_time: 0.0046 (0.004491249720255534) + + + --> TIME: 2025-04-19 09:30:14 -- STEP: 34/142 -- GLOBAL_STEP: 471900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6232831478118896 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.1984806060791 (18.05006975286147) + | > loss_duration: 1.6458678245544434 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.33 (0.3130672468858606) + | > loader_time: 0.0054 (0.0050204641678754025) + + + --> TIME: 2025-04-19 09:30:23 -- STEP: 59/142 -- GLOBAL_STEP: 471925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.536399841308594 (17.995878122620663) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.3555 (0.32590328636815985) + | > loader_time: 0.0057 (0.00532264628652799) + + + --> TIME: 2025-04-19 09:30:32 -- STEP: 84/142 -- GLOBAL_STEP: 471950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.620603084564209 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.763896942138672 (17.956174680164885) + | > loss_duration: 1.6943403482437134 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.3704 (0.337669517312731) + | > loader_time: 0.0068 (0.005614519119262697) + + + --> TIME: 2025-04-19 09:30:42 -- STEP: 109/142 -- GLOBAL_STEP: 471975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.563690185546875 (17.983950387447255) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.39 (0.3480442620198661) + | > loader_time: 0.0069 (0.005872571140254312) + + + --> TIME: 2025-04-19 09:30:53 -- STEP: 134/142 -- GLOBAL_STEP: 472000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.763317108154297 (17.9328461832075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018434560287970792 + | > current_lr_1: 0.00018434560287970792 + | > step_time: 0.417 (0.3599798768313963) + | > loader_time: 0.0076 (0.006153578188881947) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9554877281188965 (2.9554877281188965) + | > loss_feat: nan (nan) + | > loss_mel: 16.708202362060547 (16.708202362060547) + | > loss_duration: 1.8258728981018066 (1.8258728981018066) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4681432247161865 (3.4681432247161865) + | > loss_feat: nan (nan) + | > loss_mel: 17.40192222595215 (17.40192222595215) + | > loss_duration: 1.887239694595337 (1.887239694595337) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.734679698944092 (3.101411461830139) + | > loss_feat: nan (nan) + | > loss_mel: 16.88966941833496 (17.145795822143555) + | > loss_duration: 1.8212296962738037 (1.8542346954345703) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003355264663696289 (-3.516674041748047e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.101411461830139 (-0.16701877117156982) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.145795822143555 (-1.8103771209716797) + | > avg_loss_duration: 1.8542346954345703 (+0.009738504886627197) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 653/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:30:57)  + + --> TIME: 2025-04-19 09:31:03 -- STEP: 17/142 -- GLOBAL_STEP: 472025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.984480857849121 (17.892579976250143) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018432255967934795 + | > current_lr_1: 0.00018432255967934795 + | > step_time: 0.31 (0.30334124845616955) + | > loader_time: 0.0053 (0.0047910634209127985) + + + --> TIME: 2025-04-19 09:31:12 -- STEP: 42/142 -- GLOBAL_STEP: 472050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.118661880493164 (17.92846504847208) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018432255967934795 + | > current_lr_1: 0.00018432255967934795 + | > step_time: 0.3368 (0.31645718074980234) + | > loader_time: 0.0057 (0.005233980360485259) + + + --> TIME: 2025-04-19 09:31:21 -- STEP: 67/142 -- GLOBAL_STEP: 472075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.4393253326416 (18.000635502943357) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018432255967934795 + | > current_lr_1: 0.00018432255967934795 + | > step_time: 0.3577 (0.3301862994236733) + | > loader_time: 0.0061 (0.005683859782432443) + + + --> TIME: 2025-04-19 09:31:30 -- STEP: 92/142 -- GLOBAL_STEP: 472100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.917895317077637 (17.898589393366944) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018432255967934795 + | > current_lr_1: 0.00018432255967934795 + | > step_time: 0.3848 (0.3412225583325261) + | > loader_time: 0.0068 (0.005890195784361468) + + + --> TIME: 2025-04-19 09:31:40 -- STEP: 117/142 -- GLOBAL_STEP: 472125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.824798583984375 (17.89450059385382) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018432255967934795 + | > current_lr_1: 0.00018432255967934795 + | > step_time: 0.4079 (0.3517030650733883) + | > loader_time: 0.0074 (0.006147851291884726) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9135096073150635 (2.9135096073150635) + | > loss_feat: nan (nan) + | > loss_mel: 19.075822830200195 (19.075822830200195) + | > loss_duration: 1.823133945465088 (1.823133945465088) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.26269268989563 (3.26269268989563) + | > loss_feat: nan (nan) + | > loss_mel: 18.72276496887207 (18.72276496887207) + | > loss_duration: 1.914937973022461 (1.914937973022461) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.95717453956604 (3.109933614730835) + | > loss_feat: nan (nan) + | > loss_mel: 18.023977279663086 (18.373371124267578) + | > loss_duration: 1.7845057249069214 (1.8497218489646912) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032559633255004883 (-9.930133819580078e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.109933614730835 (+0.0085221529006958) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.373371124267578 (+1.2275753021240234) + | > avg_loss_duration: 1.8497218489646912 (-0.00451284646987915) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 654/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:31:52)  + + --> TIME: 2025-04-19 09:31:53 -- STEP: 0/142 -- GLOBAL_STEP: 472150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.775304794311523 (18.775304794311523) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.3177 (0.3177163600921631) + | > loader_time: 0.3101 (0.31014227867126465) + + + --> TIME: 2025-04-19 09:32:01 -- STEP: 25/142 -- GLOBAL_STEP: 472175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.433229684829712 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.06247329711914 (18.129845809936526) + | > loss_duration: 1.6773648262023926 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.321 (0.3174459648132324) + | > loader_time: 0.0058 (0.0050916290283203125) + + + --> TIME: 2025-04-19 09:32:09 -- STEP: 50/142 -- GLOBAL_STEP: 472200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.712669372558594 (18.061205711364742) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.3443 (0.3253553581237793) + | > loader_time: 0.0061 (0.0053587532043457035) + + + --> TIME: 2025-04-19 09:32:19 -- STEP: 75/142 -- GLOBAL_STEP: 472225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.97301483154297 (17.96024084726969) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.372 (0.33652988433837894) + | > loader_time: 0.0068 (0.005644928614298503) + + + --> TIME: 2025-04-19 09:32:28 -- STEP: 100/142 -- GLOBAL_STEP: 472250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.0745906829834 (18.036547002792357) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.394 (0.34632485866546636) + | > loader_time: 0.0066 (0.0059041428565979005) + + + --> TIME: 2025-04-19 09:32:39 -- STEP: 125/142 -- GLOBAL_STEP: 472275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.915668487548828 (18.025249885559077) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018429951935938803 + | > current_lr_1: 0.00018429951935938803 + | > step_time: 0.4139 (0.35760565185546883) + | > loader_time: 0.0082 (0.006160671234130859) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.979020357131958 (2.979020357131958) + | > loss_feat: nan (nan) + | > loss_mel: 16.01342010498047 (16.01342010498047) + | > loss_duration: 1.843140959739685 (1.843140959739685) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3706750869750977 (3.3706750869750977) + | > loss_feat: nan (nan) + | > loss_mel: 18.720199584960938 (18.720199584960938) + | > loss_duration: 1.890284776687622 (1.890284776687622) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.772068500518799 (3.0713717937469482) + | > loss_feat: nan (nan) + | > loss_mel: 17.820707321166992 (18.270453453063965) + | > loss_duration: 1.789955496788025 (1.8401201367378235) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038660764694213867 (+0.0006101131439208984) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0713717937469482 (-0.03856182098388672) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.270453453063965 (-0.10291767120361328) + | > avg_loss_duration: 1.8401201367378235 (-0.009601712226867676) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 655/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:32:47)  + + --> TIME: 2025-04-19 09:32:50 -- STEP: 8/142 -- GLOBAL_STEP: 472300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.124799728393555 (17.665971755981445) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.2992 (0.29996800422668457) + | > loader_time: 0.0048 (0.004540205001831055) + + + --> TIME: 2025-04-19 09:32:58 -- STEP: 33/142 -- GLOBAL_STEP: 472325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.148929595947266 (17.779545032616813) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.3319 (0.31201353217616223) + | > loader_time: 0.0054 (0.005198724342114997) + + + --> TIME: 2025-04-19 09:33:07 -- STEP: 58/142 -- GLOBAL_STEP: 472350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.904621124267578 (17.905350306938438) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.3488 (0.3237032890319825) + | > loader_time: 0.0062 (0.0054871789340315185) + + + --> TIME: 2025-04-19 09:33:16 -- STEP: 83/142 -- GLOBAL_STEP: 472375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.74956703186035 (17.920242573841506) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.3728 (0.3359416364187218) + | > loader_time: 0.0063 (0.005723737808595221) + + + --> TIME: 2025-04-19 09:33:26 -- STEP: 108/142 -- GLOBAL_STEP: 472400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.89532470703125 (17.941917534227724) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.3943 (0.3479217644091006) + | > loader_time: 0.0072 (0.006027592553032769) + + + --> TIME: 2025-04-19 09:33:37 -- STEP: 133/142 -- GLOBAL_STEP: 472425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.356048583984375 (17.971781673287985) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001842764819194681 + | > current_lr_1: 0.0001842764819194681 + | > step_time: 0.4111 (0.35968663280171564) + | > loader_time: 0.0078 (0.006279049063087406) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0463569164276123 (3.0463569164276123) + | > loss_feat: nan (nan) + | > loss_mel: 16.416784286499023 (16.416784286499023) + | > loss_duration: 1.835436224937439 (1.835436224937439) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2967307567596436 (3.2967307567596436) + | > loss_feat: nan (nan) + | > loss_mel: 19.302000045776367 (19.302000045776367) + | > loss_duration: 1.89798104763031 (1.89798104763031) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9192593097686768 (3.10799503326416) + | > loss_feat: nan (nan) + | > loss_mel: 16.309974670410156 (17.80598735809326) + | > loss_duration: 1.7615392208099365 (1.8297601342201233) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003401041030883789 (-0.00046503543853759766) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.10799503326416 (+0.036623239517211914) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.80598735809326 (-0.4644660949707031) + | > avg_loss_duration: 1.8297601342201233 (-0.010360002517700195) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 656/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:33:42)  + + --> TIME: 2025-04-19 09:33:48 -- STEP: 16/142 -- GLOBAL_STEP: 472450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.21610450744629 (18.028496026992798) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.3156 (0.303242564201355) + | > loader_time: 0.005 (0.004994571208953857) + + + --> TIME: 2025-04-19 09:33:56 -- STEP: 41/142 -- GLOBAL_STEP: 472475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5006701946258545 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.980022430419922 (18.07258671086009) + | > loss_duration: 1.6059927940368652 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.346 (0.32056839291642353) + | > loader_time: 0.0059 (0.005375530661606207) + + + --> TIME: 2025-04-19 09:34:06 -- STEP: 66/142 -- GLOBAL_STEP: 472500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.874378204345703 (17.93683914704757) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.3592 (0.33720585794159863) + | > loader_time: 0.0065 (0.0057371638037941675) + + + --> TIME: 2025-04-19 09:34:15 -- STEP: 91/142 -- GLOBAL_STEP: 472525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.871318817138672 (17.951332207564466) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.3855 (0.34629025826087373) + | > loader_time: 0.0063 (0.005972961803059002) + + + --> TIME: 2025-04-19 09:34:25 -- STEP: 116/142 -- GLOBAL_STEP: 472550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.93346405029297 (17.8707188326737) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.4062 (0.35590431608002776) + | > loader_time: 0.0068 (0.006222626258587015) + + + --> TIME: 2025-04-19 09:34:36 -- STEP: 141/142 -- GLOBAL_STEP: 472575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.252727508544922 (17.840830924663138) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018425344735922817 + | > current_lr_1: 0.00018425344735922817 + | > step_time: 0.1837 (0.36544797099228454) + | > loader_time: 0.0029 (0.0064327497008844475) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0318663120269775 (3.0318663120269775) + | > loss_feat: nan (nan) + | > loss_mel: 17.867921829223633 (17.867921829223633) + | > loss_duration: 1.8079521656036377 (1.8079521656036377) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2989590167999268 (3.2989590167999268) + | > loss_feat: nan (nan) + | > loss_mel: 17.883541107177734 (17.883541107177734) + | > loss_duration: 1.8917415142059326 (1.8917415142059326) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.090380907058716 (3.1946699619293213) + | > loss_feat: nan (nan) + | > loss_mel: 18.81489372253418 (18.349217414855957) + | > loss_duration: 1.7882342338562012 (1.839987874031067) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003612518310546875 (+0.00021147727966308594) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1946699619293213 (+0.08667492866516113) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.349217414855957 (+0.5432300567626953) + | > avg_loss_duration: 1.839987874031067 (+0.010227739810943604) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 657/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:34:38)  + + --> TIME: 2025-04-19 09:34:46 -- STEP: 24/142 -- GLOBAL_STEP: 472600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.201017379760742 (17.848999460538227) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018423041567830825 + | > current_lr_1: 0.00018423041567830825 + | > step_time: 0.3183 (0.31081643700599676) + | > loader_time: 0.0057 (0.005020747582117716) + + + --> TIME: 2025-04-19 09:34:54 -- STEP: 49/142 -- GLOBAL_STEP: 472625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4698376655578613 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.72214126586914 (17.97445723475242) + | > loss_duration: 1.6078218221664429 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018423041567830825 + | > current_lr_1: 0.00018423041567830825 + | > step_time: 0.344 (0.3215487538551798) + | > loader_time: 0.006 (0.00545576153969278) + + + --> TIME: 2025-04-19 09:35:03 -- STEP: 74/142 -- GLOBAL_STEP: 472650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6170380115509033 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.38359832763672 (18.02363396979667) + | > loss_duration: 1.6350656747817993 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018423041567830825 + | > current_lr_1: 0.00018423041567830825 + | > step_time: 0.3619 (0.33307420240866176) + | > loader_time: 0.0065 (0.005718308526116448) + + + --> TIME: 2025-04-19 09:35:13 -- STEP: 99/142 -- GLOBAL_STEP: 472675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.693490982055664 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.75977897644043 (17.994186834855512) + | > loss_duration: 1.7028865814208984 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018423041567830825 + | > current_lr_1: 0.00018423041567830825 + | > step_time: 0.38 (0.3435124267231334) + | > loader_time: 0.007 (0.0059518621425436) + + + --> TIME: 2025-04-19 09:35:23 -- STEP: 124/142 -- GLOBAL_STEP: 472700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.4976806640625 (17.927275403853383) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018423041567830825 + | > current_lr_1: 0.00018423041567830825 + | > step_time: 0.4151 (0.3549728124372421) + | > loader_time: 0.0071 (0.006175220012664795) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0437843799591064 (3.0437843799591064) + | > loss_feat: nan (nan) + | > loss_mel: 18.499801635742188 (18.499801635742188) + | > loss_duration: 1.8247891664505005 (1.8247891664505005) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.55662202835083 (3.55662202835083) + | > loss_feat: nan (nan) + | > loss_mel: 19.720962524414062 (19.720962524414062) + | > loss_duration: 1.8913666009902954 (1.8913666009902954) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.796933650970459 (3.1767778396606445) + | > loss_feat: nan (nan) + | > loss_mel: 19.515785217285156 (19.61837387084961) + | > loss_duration: 1.8031773567199707 (1.847271978855133) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003872513771057129 (+0.0002599954605102539) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1767778396606445 (-0.017892122268676758) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.61837387084961 (+1.2691564559936523) + | > avg_loss_duration: 1.847271978855133 (+0.007284104824066162) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 658/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:35:33)  + + --> TIME: 2025-04-19 09:35:35 -- STEP: 7/142 -- GLOBAL_STEP: 472725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.855493545532227 (17.893314634050643) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.2978 (0.29905257906232563) + | > loader_time: 0.0045 (0.004343645913260323) + + + --> TIME: 2025-04-19 09:35:44 -- STEP: 32/142 -- GLOBAL_STEP: 472750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.31824493408203 (17.98722642660141) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.3372 (0.3152908831834793) + | > loader_time: 0.006 (0.005238376557826996) + + + --> TIME: 2025-04-19 09:35:53 -- STEP: 57/142 -- GLOBAL_STEP: 472775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6816816329956055 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.337011337280273 (17.9934556358739) + | > loss_duration: 1.6742464303970337 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.3598 (0.3296607209925066) + | > loader_time: 0.0064 (0.005698936027393007) + + + --> TIME: 2025-04-19 09:36:02 -- STEP: 82/142 -- GLOBAL_STEP: 472800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.417194366455078 (17.97876452236641) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.3796 (0.34379616307049266) + | > loader_time: 0.0072 (0.006020208684409539) + + + --> TIME: 2025-04-19 09:36:12 -- STEP: 107/142 -- GLOBAL_STEP: 472825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.679576873779297 (18.017098070305096) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.4053 (0.35552020384886557) + | > loader_time: 0.0075 (0.006308250338117654) + + + --> TIME: 2025-04-19 09:36:23 -- STEP: 132/142 -- GLOBAL_STEP: 472850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.570018768310547 (17.925435593633953) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018420738687634846 + | > current_lr_1: 0.00018420738687634846 + | > step_time: 0.4232 (0.3679120685115005) + | > loader_time: 0.0082 (0.006538770415566185) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.133983612060547 (3.133983612060547) + | > loss_feat: nan (nan) + | > loss_mel: 20.087684631347656 (20.087684631347656) + | > loss_duration: 1.8331756591796875 (1.8331756591796875) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3786070346832275 (3.3786070346832275) + | > loss_feat: nan (nan) + | > loss_mel: 19.31716537475586 (19.31716537475586) + | > loss_duration: 1.8966095447540283 (1.8966095447540283) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.950343370437622 (3.164475202560425) + | > loss_feat: nan (nan) + | > loss_mel: 19.575746536254883 (19.44645595550537) + | > loss_duration: 1.7831358909606934 (1.8398727178573608) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033686161041259766 (-0.0005038976669311523) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.164475202560425 (-0.012302637100219727) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.44645595550537 (-0.17191791534423828) + | > avg_loss_duration: 1.8398727178573608 (-0.007399260997772217) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 659/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:36:29)  + + --> TIME: 2025-04-19 09:36:34 -- STEP: 15/142 -- GLOBAL_STEP: 472875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.12588119506836 (17.899511528015136) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.3102 (0.30247850418090827) + | > loader_time: 0.0054 (0.004704475402832031) + + + --> TIME: 2025-04-19 09:36:42 -- STEP: 40/142 -- GLOBAL_STEP: 472900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.7584228515625 (18.0132497549057) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.3366 (0.3155852437019349) + | > loader_time: 0.0069 (0.005235791206359863) + + + --> TIME: 2025-04-19 09:36:51 -- STEP: 65/142 -- GLOBAL_STEP: 472925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.327255249023438 (17.891298836928147) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.3679 (0.32769644810603227) + | > loader_time: 0.0064 (0.005616690562321589) + + + --> TIME: 2025-04-19 09:37:01 -- STEP: 90/142 -- GLOBAL_STEP: 472950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.44963836669922 (17.851673709021675) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.3774 (0.3392792648739285) + | > loader_time: 0.0069 (0.005876225895351834) + + + --> TIME: 2025-04-19 09:37:11 -- STEP: 115/142 -- GLOBAL_STEP: 472975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.765310287475586 (17.812580324255894) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.3984 (0.34993647907091224) + | > loader_time: 0.0076 (0.006131916460783585) + + + --> TIME: 2025-04-19 09:37:21 -- STEP: 140/142 -- GLOBAL_STEP: 473000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.334177017211914 (17.78813914571489) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841843609529889 + | > current_lr_1: 0.0001841843609529889 + | > step_time: 0.4317 (0.3622085520199367) + | > loader_time: 0.0073 (0.00635915994644165) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9177215099334717 (2.9177215099334717) + | > loss_feat: nan (nan) + | > loss_mel: 18.074363708496094 (18.074363708496094) + | > loss_duration: 1.8454128503799438 (1.8454128503799438) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4271528720855713 (3.4271528720855713) + | > loss_feat: nan (nan) + | > loss_mel: 18.936573028564453 (18.936573028564453) + | > loss_duration: 1.908806324005127 (1.908806324005127) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.822500228881836 (3.1248265504837036) + | > loss_feat: nan (nan) + | > loss_mel: 18.22337532043457 (18.57997417449951) + | > loss_duration: 1.7804944515228271 (1.844650387763977) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033702850341796875 (+1.6689300537109375e-06) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1248265504837036 (-0.03964865207672119) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.57997417449951 (-0.8664817810058594) + | > avg_loss_duration: 1.844650387763977 (+0.004777669906616211) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 660/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:37:24)  + + --> TIME: 2025-04-19 09:37:31 -- STEP: 23/142 -- GLOBAL_STEP: 473025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.32316017150879 (17.910380487856656) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018416133790786978 + | > current_lr_1: 0.00018416133790786978 + | > step_time: 0.3262 (0.309139697448067) + | > loader_time: 0.0053 (0.005119541416997495) + + + --> TIME: 2025-04-19 09:37:40 -- STEP: 48/142 -- GLOBAL_STEP: 473050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.280372619628906 (17.903457085291546) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018416133790786978 + | > current_lr_1: 0.00018416133790786978 + | > step_time: 0.3473 (0.32817301650842023) + | > loader_time: 0.0067 (0.00564005970954895) + + + --> TIME: 2025-04-19 09:37:50 -- STEP: 73/142 -- GLOBAL_STEP: 473075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.493072509765625 (17.768149715580357) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018416133790786978 + | > current_lr_1: 0.00018416133790786978 + | > step_time: 0.3804 (0.3420794108142591) + | > loader_time: 0.0065 (0.0059125194810841185) + + + --> TIME: 2025-04-19 09:38:00 -- STEP: 98/142 -- GLOBAL_STEP: 473100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.020841598510742 (17.750255321969792) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018416133790786978 + | > current_lr_1: 0.00018416133790786978 + | > step_time: 0.3973 (0.3535489817054904) + | > loader_time: 0.0074 (0.0061907111381997865) + + + --> TIME: 2025-04-19 09:38:10 -- STEP: 123/142 -- GLOBAL_STEP: 473125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7748591899871826 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.75934410095215 (17.740545621732387) + | > loss_duration: 1.6570887565612793 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018416133790786978 + | > current_lr_1: 0.00018416133790786978 + | > step_time: 0.4243 (0.3656193802996379) + | > loader_time: 0.0076 (0.006457065179096005) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8623225688934326 (2.8623225688934326) + | > loss_feat: nan (nan) + | > loss_mel: 14.966109275817871 (14.966109275817871) + | > loss_duration: 1.8479976654052734 (1.8479976654052734) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5131657123565674 (3.5131657123565674) + | > loss_feat: nan (nan) + | > loss_mel: 16.60335350036621 (16.60335350036621) + | > loss_duration: 1.9025788307189941 (1.9025788307189941) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.795395851135254 (3.1542807817459106) + | > loss_feat: nan (nan) + | > loss_mel: 18.756820678710938 (17.680087089538574) + | > loss_duration: 1.799828290939331 (1.8512035608291626) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033055543899536133 (-6.473064422607422e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1542807817459106 (+0.02945423126220703) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.680087089538574 (-0.8998870849609375) + | > avg_loss_duration: 1.8512035608291626 (+0.006553173065185547) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 661/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:38:20)  + + --> TIME: 2025-04-19 09:38:23 -- STEP: 6/142 -- GLOBAL_STEP: 473150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.587313175201416 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.846712112426758 (17.751060485839844) + | > loss_duration: 1.6453065872192383 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.3059 (0.29913143316904706) + | > loader_time: 0.0041 (0.004241029421488444) + + + --> TIME: 2025-04-19 09:38:31 -- STEP: 31/142 -- GLOBAL_STEP: 473175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.39543914794922 (17.94720717399351) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.3243 (0.3112660069619456) + | > loader_time: 0.0054 (0.00506921737424789) + + + --> TIME: 2025-04-19 09:38:39 -- STEP: 56/142 -- GLOBAL_STEP: 473200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.832868576049805 (17.973594018391204) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.3476 (0.32298412067549576) + | > loader_time: 0.0059 (0.0053807369300297335) + + + --> TIME: 2025-04-19 09:38:49 -- STEP: 81/142 -- GLOBAL_STEP: 473225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.198030471801758 (17.977417533780322) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.3687 (0.337304053483186) + | > loader_time: 0.0066 (0.005633195241292317) + + + --> TIME: 2025-04-19 09:38:58 -- STEP: 106/142 -- GLOBAL_STEP: 473250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.695411682128906 (17.9267881591365) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.3841 (0.34727677534211354) + | > loader_time: 0.0067 (0.005887951491014013) + + + --> TIME: 2025-04-19 09:39:09 -- STEP: 131/142 -- GLOBAL_STEP: 473275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.208927154541016 (17.836767931930886) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018413831774063128 + | > current_lr_1: 0.00018413831774063128 + | > step_time: 0.425 (0.3588877434038934) + | > loader_time: 0.0073 (0.006138342937440363) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.981232166290283 (2.981232166290283) + | > loss_feat: nan (nan) + | > loss_mel: 20.016368865966797 (20.016368865966797) + | > loss_duration: 1.8239941596984863 (1.8239941596984863) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4043431282043457 (3.4043431282043457) + | > loss_feat: nan (nan) + | > loss_mel: 18.982322692871094 (18.982322692871094) + | > loss_duration: 1.8934556245803833 (1.8934556245803833) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.124582052230835 (3.2644625902175903) + | > loss_feat: nan (nan) + | > loss_mel: 17.65937614440918 (18.320849418640137) + | > loss_duration: 1.814083218574524 (1.8537694215774536) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036062002182006836 (+0.0003006458282470703) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2644625902175903 (+0.11018180847167969) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.320849418640137 (+0.6407623291015625) + | > avg_loss_duration: 1.8537694215774536 (+0.0025658607482910156) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 662/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:39:15)  + + --> TIME: 2025-04-19 09:39:20 -- STEP: 14/142 -- GLOBAL_STEP: 473300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.488527774810791 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.622243881225586 (18.25527667999268) + | > loss_duration: 1.605531096458435 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.3055 (0.3073366028921945) + | > loader_time: 0.0049 (0.004818609782627651) + + + --> TIME: 2025-04-19 09:39:28 -- STEP: 39/142 -- GLOBAL_STEP: 473325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.72684097290039 (18.20094803052071) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.3399 (0.31911881764729827) + | > loader_time: 0.0058 (0.005340936856392102) + + + --> TIME: 2025-04-19 09:39:38 -- STEP: 64/142 -- GLOBAL_STEP: 473350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.18055534362793 (18.063540548086166) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.3696 (0.3332447558641434) + | > loader_time: 0.0063 (0.00572541356086731) + + + --> TIME: 2025-04-19 09:39:47 -- STEP: 89/142 -- GLOBAL_STEP: 473375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.76355743408203 (17.99428618356083) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.4024 (0.34870651062954683) + | > loader_time: 0.0078 (0.0060017617900719805) + + + --> TIME: 2025-04-19 09:39:58 -- STEP: 114/142 -- GLOBAL_STEP: 473400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.86060905456543 (18.002612833391154) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.4089 (0.3606773761280796) + | > loader_time: 0.0077 (0.006271889335230777) + + + --> TIME: 2025-04-19 09:40:09 -- STEP: 139/142 -- GLOBAL_STEP: 473425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.168855667114258 (18.036548641945817) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001841153004509137 + | > current_lr_1: 0.0001841153004509137 + | > step_time: 0.4362 (0.3730160949899138) + | > loader_time: 0.0073 (0.006509926679323047) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9783775806427 (2.9783775806427) + | > loss_feat: nan (nan) + | > loss_mel: 18.23049545288086 (18.23049545288086) + | > loss_duration: 1.8175758123397827 (1.8175758123397827) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2967822551727295 (3.2967822551727295) + | > loss_feat: nan (nan) + | > loss_mel: 17.040040969848633 (17.040040969848633) + | > loss_duration: 1.9085536003112793 (1.9085536003112793) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9514009952545166 (3.124091625213623) + | > loss_feat: nan (nan) + | > loss_mel: 18.000003814697266 (17.52002239227295) + | > loss_duration: 1.7834398746490479 (1.8459967374801636) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003628969192504883 (+2.276897430419922e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.124091625213623 (-0.14037096500396729) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.52002239227295 (-0.8008270263671875) + | > avg_loss_duration: 1.8459967374801636 (-0.007772684097290039) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 663/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:40:12)  + + --> TIME: 2025-04-19 09:40:19 -- STEP: 22/142 -- GLOBAL_STEP: 473450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.985963821411133 (18.17933281985196) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018409228603835735 + | > current_lr_1: 0.00018409228603835735 + | > step_time: 0.3182 (0.30612023310227826) + | > loader_time: 0.0053 (0.004862200130115856) + + + --> TIME: 2025-04-19 09:40:28 -- STEP: 47/142 -- GLOBAL_STEP: 473475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.40127182006836 (18.13442339795702) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018409228603835735 + | > current_lr_1: 0.00018409228603835735 + | > step_time: 0.3386 (0.3184865890665257) + | > loader_time: 0.0061 (0.005279667834018139) + + + --> TIME: 2025-04-19 09:40:37 -- STEP: 72/142 -- GLOBAL_STEP: 473500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.114126205444336 (18.11283889081743) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018409228603835735 + | > current_lr_1: 0.00018409228603835735 + | > step_time: 0.3668 (0.33098436064190334) + | > loader_time: 0.0064 (0.005590498447418213) + + + --> TIME: 2025-04-19 09:40:46 -- STEP: 97/142 -- GLOBAL_STEP: 473525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.120141983032227 (18.090771773426805) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018409228603835735 + | > current_lr_1: 0.00018409228603835735 + | > step_time: 0.3876 (0.34210621450365203) + | > loader_time: 0.0064 (0.005858244355191891) + + + --> TIME: 2025-04-19 09:40:57 -- STEP: 122/142 -- GLOBAL_STEP: 473550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.409135818481445 (18.092458748426594) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018409228603835735 + | > current_lr_1: 0.00018409228603835735 + | > step_time: 0.4401 (0.3542068024150661) + | > loader_time: 0.0072 (0.0061365541864614025) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.823666572570801 (2.823666572570801) + | > loss_feat: nan (nan) + | > loss_mel: 17.261247634887695 (17.261247634887695) + | > loss_duration: 1.824510097503662 (1.824510097503662) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.461704969406128 (3.461704969406128) + | > loss_feat: nan (nan) + | > loss_mel: 19.63654327392578 (19.63654327392578) + | > loss_duration: 1.896816611289978 (1.896816611289978) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.76291823387146 (3.112311601638794) + | > loss_feat: nan (nan) + | > loss_mel: 19.365201950073242 (19.50087261199951) + | > loss_duration: 1.7916909456253052 (1.8442537784576416) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034122467041015625 (-0.0002167224884033203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.112311601638794 (-0.011780023574829102) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.50087261199951 (+1.9808502197265625) + | > avg_loss_duration: 1.8442537784576416 (-0.0017429590225219727) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 664/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:41:07)  + + --> TIME: 2025-04-19 09:41:09 -- STEP: 5/142 -- GLOBAL_STEP: 473575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.12992286682129 (17.98399200439453) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.2992 (0.29754981994628904) + | > loader_time: 0.0054 (0.004845762252807617) + + + --> TIME: 2025-04-19 09:41:17 -- STEP: 30/142 -- GLOBAL_STEP: 473600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.05066680908203 (17.85833307902019) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.3248 (0.3099605639775594) + | > loader_time: 0.0061 (0.005182615915934245) + + + --> TIME: 2025-04-19 09:41:25 -- STEP: 55/142 -- GLOBAL_STEP: 473625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.938467025756836 (17.95405686118387) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.3555 (0.321833068674261) + | > loader_time: 0.0059 (0.005585362694480203) + + + --> TIME: 2025-04-19 09:41:35 -- STEP: 80/142 -- GLOBAL_STEP: 473650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.671055316925049 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.514097213745117 (17.98807703256608) + | > loss_duration: 1.6847680807113647 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.3891 (0.3366538107395173) + | > loader_time: 0.0072 (0.005881088972091674) + + + --> TIME: 2025-04-19 09:41:45 -- STEP: 105/142 -- GLOBAL_STEP: 473675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.580411672592163 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.333641052246094 (17.92928442273822) + | > loss_duration: 1.683359146118164 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.4073 (0.35003104891095843) + | > loader_time: 0.007 (0.006150138945806592) + + + --> TIME: 2025-04-19 09:41:56 -- STEP: 130/142 -- GLOBAL_STEP: 473700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.523984909057617 (17.935328652308545) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018406927450260254 + | > current_lr_1: 0.00018406927450260254 + | > step_time: 0.4385 (0.36632167742802546) + | > loader_time: 0.0074 (0.0064159154891967754) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.799394369125366 (2.799394369125366) + | > loss_feat: nan (nan) + | > loss_mel: 16.484655380249023 (16.484655380249023) + | > loss_duration: 1.833420753479004 (1.833420753479004) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4091315269470215 (3.4091315269470215) + | > loss_feat: nan (nan) + | > loss_mel: 19.043733596801758 (19.043733596801758) + | > loss_duration: 1.9036656618118286 (1.9036656618118286) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9014406204223633 (3.1552860736846924) + | > loss_feat: nan (nan) + | > loss_mel: 18.565841674804688 (18.804787635803223) + | > loss_duration: 1.8048384189605713 (1.8542520403862) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032695531845092773 (-0.00014269351959228516) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1552860736846924 (+0.04297447204589844) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.804787635803223 (-0.6960849761962891) + | > avg_loss_duration: 1.8542520403862 (+0.00999826192855835) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 665/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:42:03)  + + --> TIME: 2025-04-19 09:42:07 -- STEP: 13/142 -- GLOBAL_STEP: 473725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 14.984688758850098 (17.52703233865591) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.3048 (0.3079576859107384) + | > loader_time: 0.0048 (0.0043438214522141675) + + + --> TIME: 2025-04-19 09:42:16 -- STEP: 38/142 -- GLOBAL_STEP: 473750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.549407482147217 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.530986785888672 (17.871327475497594) + | > loss_duration: 1.6354204416275024 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.3283 (0.31684945759020355) + | > loader_time: 0.0057 (0.004955561537491648) + + + --> TIME: 2025-04-19 09:42:24 -- STEP: 63/142 -- GLOBAL_STEP: 473775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.3436279296875 (17.933208601815352) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.3628 (0.32781451467483763) + | > loader_time: 0.0059 (0.005313301843310159) + + + --> TIME: 2025-04-19 09:42:34 -- STEP: 88/142 -- GLOBAL_STEP: 473800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.1712589263916 (17.965346347201955) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.3707 (0.33899661356752564) + | > loader_time: 0.0068 (0.0055754699490287085) + + + --> TIME: 2025-04-19 09:42:44 -- STEP: 113/142 -- GLOBAL_STEP: 473825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.110071182250977 (17.99137258951642) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.401 (0.3495321737981475) + | > loader_time: 0.007 (0.005846135384213608) + + + --> TIME: 2025-04-19 09:42:54 -- STEP: 138/142 -- GLOBAL_STEP: 473850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.66829490661621 (17.97821584646252) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001840462658432897 + | > current_lr_1: 0.0001840462658432897 + | > step_time: 0.4255 (0.3615085664002792) + | > loader_time: 0.0073 (0.00610947090646495) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7950644493103027 (2.7950644493103027) + | > loss_feat: nan (nan) + | > loss_mel: 17.496850967407227 (17.496850967407227) + | > loss_duration: 1.8245431184768677 (1.8245431184768677) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2824134826660156 (3.2824134826660156) + | > loss_feat: nan (nan) + | > loss_mel: 16.420867919921875 (16.420867919921875) + | > loss_duration: 1.8863625526428223 (1.8863625526428223) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9012086391448975 (3.0918110609054565) + | > loss_feat: nan (nan) + | > loss_mel: 18.302982330322266 (17.36192512512207) + | > loss_duration: 1.80363130569458 (1.8449969291687012) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031894445419311523 (-8.0108642578125e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0918110609054565 (-0.06347501277923584) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.36192512512207 (-1.4428625106811523) + | > avg_loss_duration: 1.8449969291687012 (-0.00925511121749878) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 666/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:42:57)  + + --> TIME: 2025-04-19 09:43:05 -- STEP: 21/142 -- GLOBAL_STEP: 473875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.599897384643555 (17.880005836486816) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018402326006005927 + | > current_lr_1: 0.00018402326006005927 + | > step_time: 0.3245 (0.30610368365333196) + | > loader_time: 0.0051 (0.004652829397292365) + + + --> TIME: 2025-04-19 09:43:13 -- STEP: 46/142 -- GLOBAL_STEP: 473900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.35946273803711 (17.83834874111673) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018402326006005927 + | > current_lr_1: 0.00018402326006005927 + | > step_time: 0.351 (0.32296939518140705) + | > loader_time: 0.0057 (0.0051308766655299995) + + + --> TIME: 2025-04-19 09:43:23 -- STEP: 71/142 -- GLOBAL_STEP: 473925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.499401092529297 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.500701904296875 (17.828639769218338) + | > loss_duration: 1.6664167642593384 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018402326006005927 + | > current_lr_1: 0.00018402326006005927 + | > step_time: 0.371 (0.33708913897124815) + | > loader_time: 0.0064 (0.005474459957069076) + + + --> TIME: 2025-04-19 09:43:32 -- STEP: 96/142 -- GLOBAL_STEP: 473950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5927841663360596 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.31050682067871 (17.836120148499813) + | > loss_duration: 1.6814031600952148 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018402326006005927 + | > current_lr_1: 0.00018402326006005927 + | > step_time: 0.3911 (0.34946262588103627) + | > loader_time: 0.0069 (0.00577118992805481) + + + --> TIME: 2025-04-19 09:43:43 -- STEP: 121/142 -- GLOBAL_STEP: 473975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.67113494873047 (17.879063157010677) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018402326006005927 + | > current_lr_1: 0.00018402326006005927 + | > step_time: 0.4162 (0.3611204387727849) + | > loader_time: 0.0073 (0.006008692024167904) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.106045961380005 (3.106045961380005) + | > loss_feat: nan (nan) + | > loss_mel: 16.232553482055664 (16.232553482055664) + | > loss_duration: 1.8289985656738281 (1.8289985656738281) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.33958101272583 (3.33958101272583) + | > loss_feat: nan (nan) + | > loss_mel: 18.04833221435547 (18.04833221435547) + | > loss_duration: 1.9120056629180908 (1.9120056629180908) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8789238929748535 (3.109252452850342) + | > loss_feat: nan (nan) + | > loss_mel: 18.612300872802734 (18.3303165435791) + | > loss_duration: 1.769653081893921 (1.8408293724060059) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003756403923034668 (+0.0005669593811035156) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.109252452850342 (+0.017441391944885254) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.3303165435791 (+0.9683914184570312) + | > avg_loss_duration: 1.8408293724060059 (-0.0041675567626953125) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 667/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:43:53)  + + --> TIME: 2025-04-19 09:43:55 -- STEP: 4/142 -- GLOBAL_STEP: 474000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5656795501709 (17.59217929840088) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.2993 (0.29917681217193604) + | > loader_time: 0.0055 (0.004498779773712158) + + + --> TIME: 2025-04-19 09:44:03 -- STEP: 29/142 -- GLOBAL_STEP: 474025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.836301803588867 (18.06043684071508) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.3214 (0.3157212734222412) + | > loader_time: 0.0058 (0.005261232113016063) + + + --> TIME: 2025-04-19 09:44:12 -- STEP: 54/142 -- GLOBAL_STEP: 474050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.251874923706055 (18.00350099139743) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.3453 (0.3249728326444273) + | > loader_time: 0.0058 (0.005513376659817165) + + + --> TIME: 2025-04-19 09:44:21 -- STEP: 79/142 -- GLOBAL_STEP: 474075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.46587371826172 (17.979287606251397) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.3594 (0.33661806734302374) + | > loader_time: 0.0066 (0.005789590787283981) + + + --> TIME: 2025-04-19 09:44:31 -- STEP: 104/142 -- GLOBAL_STEP: 474100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.47160530090332 (17.99917267836057) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.3839 (0.3469318632896129) + | > loader_time: 0.007 (0.006008427876692552) + + + --> TIME: 2025-04-19 09:44:41 -- STEP: 129/142 -- GLOBAL_STEP: 474125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.647127151489258 (17.945267987805735) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018400025715255175 + | > current_lr_1: 0.00018400025715255175 + | > step_time: 0.4279 (0.3583397144495054) + | > loader_time: 0.0076 (0.006243519080701724) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.972301721572876 (2.972301721572876) + | > loss_feat: nan (nan) + | > loss_mel: 17.75981330871582 (17.75981330871582) + | > loss_duration: 1.835881233215332 (1.835881233215332) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.355844736099243 (3.355844736099243) + | > loss_feat: nan (nan) + | > loss_mel: 17.413061141967773 (17.413061141967773) + | > loss_duration: 1.9301953315734863 (1.9301953315734863) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8982534408569336 (3.1270490884780884) + | > loss_feat: nan (nan) + | > loss_mel: 15.89413070678711 (16.65359592437744) + | > loss_duration: 1.8109477758407593 (1.8705715537071228) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00328218936920166 (-0.0004742145538330078) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1270490884780884 (+0.017796635627746582) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.65359592437744 (-1.6767206192016602) + | > avg_loss_duration: 1.8705715537071228 (+0.029742181301116943) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 668/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:44:48)  + + --> TIME: 2025-04-19 09:44:52 -- STEP: 12/142 -- GLOBAL_STEP: 474150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.876270294189453 (18.115228017171223) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.3031 (0.3020153045654297) + | > loader_time: 0.0052 (0.0047095417976379395) + + + --> TIME: 2025-04-19 09:45:01 -- STEP: 37/142 -- GLOBAL_STEP: 474175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.312021255493164 (18.096550400192672) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.3382 (0.3183756261258512) + | > loader_time: 0.0055 (0.005235620447107263) + + + --> TIME: 2025-04-19 09:45:10 -- STEP: 62/142 -- GLOBAL_STEP: 474200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.804683685302734 (17.943036910026294) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.3692 (0.3325890225748862) + | > loader_time: 0.0064 (0.005609343128819619) + + + --> TIME: 2025-04-19 09:45:20 -- STEP: 87/142 -- GLOBAL_STEP: 474225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.239845275878906 (17.928733255671347) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.3885 (0.346673280343242) + | > loader_time: 0.0063 (0.0058839348540909) + + + --> TIME: 2025-04-19 09:45:30 -- STEP: 112/142 -- GLOBAL_STEP: 474250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.278032302856445 (17.8871029785701) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.415 (0.35848461517265856) + | > loader_time: 0.0071 (0.006175611700330463) + + + --> TIME: 2025-04-19 09:45:41 -- STEP: 137/142 -- GLOBAL_STEP: 474275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.76932716369629 (17.896122048371012) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018397725712040766 + | > current_lr_1: 0.00018397725712040766 + | > step_time: 0.4418 (0.3710506588873202) + | > loader_time: 0.0074 (0.006431010517760786) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9503839015960693 (2.9503839015960693) + | > loss_feat: nan (nan) + | > loss_mel: 18.73316192626953 (18.73316192626953) + | > loss_duration: 1.835542917251587 (1.835542917251587) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.408909320831299 (3.408909320831299) + | > loss_feat: nan (nan) + | > loss_mel: 18.847612380981445 (18.847612380981445) + | > loss_duration: 1.8900258541107178 (1.8900258541107178) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.996260166168213 (3.202584743499756) + | > loss_feat: nan (nan) + | > loss_mel: 19.618249893188477 (19.23293113708496) + | > loss_duration: 1.8032910823822021 (1.84665846824646) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036153793334960938 (+0.0003331899642944336) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.202584743499756 (+0.07553565502166748) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.23293113708496 (+2.5793352127075195) + | > avg_loss_duration: 1.84665846824646 (-0.023913085460662842) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 669/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:45:44)  + + --> TIME: 2025-04-19 09:45:51 -- STEP: 20/142 -- GLOBAL_STEP: 474300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4909794330596924 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.86153793334961 (18.22750949859619) + | > loss_duration: 1.6607903242111206 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839542599632676 + | > current_lr_1: 0.0001839542599632676 + | > step_time: 0.3172 (0.30605244636535645) + | > loader_time: 0.0059 (0.004754257202148437) + + + --> TIME: 2025-04-19 09:46:00 -- STEP: 45/142 -- GLOBAL_STEP: 474325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.91609764099121 (18.0754515753852) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839542599632676 + | > current_lr_1: 0.0001839542599632676 + | > step_time: 0.3432 (0.3191533194647894) + | > loader_time: 0.0058 (0.005193736818101672) + + + --> TIME: 2025-04-19 09:46:09 -- STEP: 70/142 -- GLOBAL_STEP: 474350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.560298919677734 (18.041067477634975) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839542599632676 + | > current_lr_1: 0.0001839542599632676 + | > step_time: 0.3639 (0.33332899638584684) + | > loader_time: 0.0063 (0.005590329851422992) + + + --> TIME: 2025-04-19 09:46:19 -- STEP: 95/142 -- GLOBAL_STEP: 474375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.285511016845703 (18.07333179273104) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839542599632676 + | > current_lr_1: 0.0001839542599632676 + | > step_time: 0.3793 (0.3440855804242587) + | > loader_time: 0.0067 (0.005852538660952921) + + + --> TIME: 2025-04-19 09:46:29 -- STEP: 120/142 -- GLOBAL_STEP: 474400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.915124893188477 (18.03889055252075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839542599632676 + | > current_lr_1: 0.0001839542599632676 + | > step_time: 0.4096 (0.35478889743487046) + | > loader_time: 0.0072 (0.0061190088589986175) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9633266925811768 (2.9633266925811768) + | > loss_feat: nan (nan) + | > loss_mel: 18.102643966674805 (18.102643966674805) + | > loss_duration: 1.8186999559402466 (1.8186999559402466) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.336303949356079 (3.336303949356079) + | > loss_feat: nan (nan) + | > loss_mel: 17.171661376953125 (17.171661376953125) + | > loss_duration: 1.9032222032546997 (1.9032222032546997) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.95125675201416 (3.1437803506851196) + | > loss_feat: nan (nan) + | > loss_mel: 19.916547775268555 (18.54410457611084) + | > loss_duration: 1.8096065521240234 (1.8564143776893616) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003178119659423828 (-0.0004372596740722656) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1437803506851196 (-0.05880439281463623) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.54410457611084 (-0.6888265609741211) + | > avg_loss_duration: 1.8564143776893616 (+0.009755909442901611) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 670/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:46:40)  + + --> TIME: 2025-04-19 09:46:41 -- STEP: 3/142 -- GLOBAL_STEP: 474425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.524877548217773 (18.03577995300293) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.3002 (0.30460818608601886) + | > loader_time: 0.0037 (0.003989458084106445) + + + --> TIME: 2025-04-19 09:46:49 -- STEP: 28/142 -- GLOBAL_STEP: 474450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6676061153411865 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.719545364379883 (18.061967032296316) + | > loss_duration: 1.6602497100830078 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.3263 (0.3141384720802307) + | > loader_time: 0.0053 (0.004790655204227993) + + + --> TIME: 2025-04-19 09:46:58 -- STEP: 53/142 -- GLOBAL_STEP: 474475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.359844207763672 (18.016959820153584) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.3528 (0.32788128672905686) + | > loader_time: 0.0057 (0.005171321473031674) + + + --> TIME: 2025-04-19 09:47:07 -- STEP: 78/142 -- GLOBAL_STEP: 474500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.502487182617188 (17.989895013662483) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.4026 (0.34270734664721364) + | > loader_time: 0.0061 (0.005535730948814978) + + + --> TIME: 2025-04-19 09:47:18 -- STEP: 103/142 -- GLOBAL_STEP: 474525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.54163360595703 (17.992438955214414) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.4014 (0.3553161922010403) + | > loader_time: 0.0066 (0.005795967231676415) + + + --> TIME: 2025-04-19 09:47:28 -- STEP: 128/142 -- GLOBAL_STEP: 474550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.59927749633789 (17.91221884638071) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839312656807722 + | > current_lr_1: 0.0001839312656807722 + | > step_time: 0.4163 (0.3678500521928072) + | > loader_time: 0.0072 (0.0060421936213970176) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.989579916000366 (2.989579916000366) + | > loss_feat: nan (nan) + | > loss_mel: 17.959993362426758 (17.959993362426758) + | > loss_duration: 1.8455909490585327 (1.8455909490585327) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.31003999710083 (3.31003999710083) + | > loss_feat: nan (nan) + | > loss_mel: 16.12734031677246 (16.12734031677246) + | > loss_duration: 1.8812282085418701 (1.8812282085418701) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0887107849121094 (3.1993753910064697) + | > loss_feat: nan (nan) + | > loss_mel: 18.894128799438477 (17.51073455810547) + | > loss_duration: 1.781287431716919 (1.8312578201293945) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034842491149902344 (+0.00030612945556640625) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1993753910064697 (+0.0555950403213501) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.51073455810547 (-1.033370018005371) + | > avg_loss_duration: 1.8312578201293945 (-0.02515655755996704) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 671/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:47:36)  + + --> TIME: 2025-04-19 09:47:40 -- STEP: 11/142 -- GLOBAL_STEP: 474575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.665409088134766 (17.62933349609375) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.3051 (0.300781401720914) + | > loader_time: 0.005 (0.004535198211669922) + + + --> TIME: 2025-04-19 09:47:48 -- STEP: 36/142 -- GLOBAL_STEP: 474600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.36628532409668 (17.90167077382405) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.3297 (0.31366172101762557) + | > loader_time: 0.0058 (0.005045122570461697) + + + --> TIME: 2025-04-19 09:47:57 -- STEP: 61/142 -- GLOBAL_STEP: 474625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.321449279785156 (17.983959354338094) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.3587 (0.32520544333536117) + | > loader_time: 0.0061 (0.005335717904763142) + + + --> TIME: 2025-04-19 09:48:06 -- STEP: 86/142 -- GLOBAL_STEP: 474650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5828869342803955 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.058229446411133 (17.998474320700005) + | > loss_duration: 1.6871697902679443 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.3735 (0.3375107798465463) + | > loader_time: 0.0065 (0.005612628404484236) + + + --> TIME: 2025-04-19 09:48:16 -- STEP: 111/142 -- GLOBAL_STEP: 474675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.904359817504883 (17.929285238455016) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.4005 (0.34948331171327884) + | > loader_time: 0.0074 (0.005946730708216759) + + + --> TIME: 2025-04-19 09:48:27 -- STEP: 136/142 -- GLOBAL_STEP: 474700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.757503509521484 (17.89131283760072) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001839082742725621 + | > current_lr_1: 0.0001839082742725621 + | > step_time: 0.4242 (0.3613055699011859) + | > loader_time: 0.0079 (0.006210169371436622) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8220055103302 (2.8220055103302) + | > loss_feat: nan (nan) + | > loss_mel: 18.060518264770508 (18.060518264770508) + | > loss_duration: 1.859626293182373 (1.859626293182373) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.356761932373047 (3.356761932373047) + | > loss_feat: nan (nan) + | > loss_mel: 19.194950103759766 (19.194950103759766) + | > loss_duration: 1.8908195495605469 (1.8908195495605469) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8967702388763428 (3.126766085624695) + | > loss_feat: nan (nan) + | > loss_mel: 18.390676498413086 (18.792813301086426) + | > loss_duration: 1.800679087638855 (1.845749318599701) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035262107849121094 (+4.1961669921875e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.126766085624695 (-0.0726093053817749) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.792813301086426 (+1.282078742980957) + | > avg_loss_duration: 1.845749318599701 (+0.014491498470306396) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 672/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:48:31)  + + --> TIME: 2025-04-19 09:48:37 -- STEP: 19/142 -- GLOBAL_STEP: 474725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.315093994140625 (17.74146200481214) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018388528573827801 + | > current_lr_1: 0.00018388528573827801 + | > step_time: 0.3117 (0.3038303224663985) + | > loader_time: 0.0052 (0.004979836313348068) + + + --> TIME: 2025-04-19 09:48:46 -- STEP: 44/142 -- GLOBAL_STEP: 474750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.48813247680664 (17.921834382143885) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018388528573827801 + | > current_lr_1: 0.00018388528573827801 + | > step_time: 0.3426 (0.32194012403488154) + | > loader_time: 0.0055 (0.005322028290141712) + + + --> TIME: 2025-04-19 09:48:55 -- STEP: 69/142 -- GLOBAL_STEP: 474775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.713815689086914 (17.937431459841513) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018388528573827801 + | > current_lr_1: 0.00018388528573827801 + | > step_time: 0.3946 (0.3369959022687829) + | > loader_time: 0.0082 (0.005726216495900914) + + + --> TIME: 2025-04-19 09:49:05 -- STEP: 94/142 -- GLOBAL_STEP: 474800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.32414436340332 (17.98625807051963) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018388528573827801 + | > current_lr_1: 0.00018388528573827801 + | > step_time: 0.3851 (0.3492977543080107) + | > loader_time: 0.0065 (0.006013844875579185) + + + --> TIME: 2025-04-19 09:49:15 -- STEP: 119/142 -- GLOBAL_STEP: 474825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.114259719848633 (18.0238451196366) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018388528573827801 + | > current_lr_1: 0.00018388528573827801 + | > step_time: 0.4111 (0.3614061239386807) + | > loader_time: 0.0078 (0.006277348814892168) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.114881992340088 (3.114881992340088) + | > loss_feat: nan (nan) + | > loss_mel: 16.994577407836914 (16.994577407836914) + | > loss_duration: 1.8436522483825684 (1.8436522483825684) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.331679582595825 (3.331679582595825) + | > loss_feat: nan (nan) + | > loss_mel: 18.348678588867188 (18.348678588867188) + | > loss_duration: 1.8930809497833252 (1.8930809497833252) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.645049571990967 (2.988364577293396) + | > loss_feat: nan (nan) + | > loss_mel: 18.951305389404297 (18.649991989135742) + | > loss_duration: 1.795759677886963 (1.844420313835144) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003209829330444336 (-0.00031638145446777344) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.988364577293396 (-0.13840150833129883) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.649991989135742 (-0.1428213119506836) + | > avg_loss_duration: 1.844420313835144 (-0.0013290047645568848) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 673/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:49:27)  + + --> TIME: 2025-04-19 09:49:28 -- STEP: 2/142 -- GLOBAL_STEP: 474850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6400372982025146 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.063756942749023 (17.666885375976562) + | > loss_duration: 1.6487336158752441 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.2947 (0.2996920347213745) + | > loader_time: 0.004 (0.004053592681884766) + + + --> TIME: 2025-04-19 09:49:36 -- STEP: 27/142 -- GLOBAL_STEP: 474875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.928308486938477 (17.997833216631854) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.3237 (0.3092070950402154) + | > loader_time: 0.0055 (0.004969826451054325) + + + --> TIME: 2025-04-19 09:49:45 -- STEP: 52/142 -- GLOBAL_STEP: 474900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.310331344604492 (17.785541259325466) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.3419 (0.32200786700615514) + | > loader_time: 0.0057 (0.005341442731710581) + + + --> TIME: 2025-04-19 09:49:54 -- STEP: 77/142 -- GLOBAL_STEP: 474925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.16469383239746 (17.832473878736614) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.37 (0.3349738306813426) + | > loader_time: 0.0061 (0.005638339302756569) + + + --> TIME: 2025-04-19 09:50:04 -- STEP: 102/142 -- GLOBAL_STEP: 474950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5981085300445557 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.884899139404297 (17.858968846938193) + | > loss_duration: 1.657118558883667 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.3903 (0.3452253178054211) + | > loader_time: 0.0065 (0.005855448105755974) + + + --> TIME: 2025-04-19 09:50:14 -- STEP: 127/142 -- GLOBAL_STEP: 474975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.077882766723633 (17.827933521721306) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018386230007756072 + | > current_lr_1: 0.00018386230007756072 + | > step_time: 0.4371 (0.357028337914174) + | > loader_time: 0.0083 (0.006134196529238242) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.897289991378784 (2.897289991378784) + | > loss_feat: nan (nan) + | > loss_mel: 19.24786376953125 (19.24786376953125) + | > loss_duration: 1.8182368278503418 (1.8182368278503418) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4337282180786133 (3.4337282180786133) + | > loss_feat: nan (nan) + | > loss_mel: 19.477659225463867 (19.477659225463867) + | > loss_duration: 1.90287184715271 (1.90287184715271) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.783568859100342 (3.1086485385894775) + | > loss_feat: nan (nan) + | > loss_mel: 17.652517318725586 (18.565088272094727) + | > loss_duration: 1.7839100360870361 (1.843390941619873) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004027605056762695 (+0.0008177757263183594) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1086485385894775 (+0.12028396129608154) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.565088272094727 (-0.08490371704101562) + | > avg_loss_duration: 1.843390941619873 (-0.001029372215270996) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 674/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:50:22)  + + --> TIME: 2025-04-19 09:50:26 -- STEP: 10/142 -- GLOBAL_STEP: 475000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.470165252685547 (18.288851928710937) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.3092 (0.30631959438323975) + | > loader_time: 0.0051 (0.0045575380325317385) + + + --> TIME: 2025-04-19 09:50:34 -- STEP: 35/142 -- GLOBAL_STEP: 475025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.17911148071289 (17.959923035757882) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.3395 (0.3187779358455113) + | > loader_time: 0.0058 (0.005152695519583566) + + + --> TIME: 2025-04-19 09:50:43 -- STEP: 60/142 -- GLOBAL_STEP: 475050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.282512664794922 (17.91052015622456) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.3596 (0.33201486269632985) + | > loader_time: 0.0067 (0.005574961503346761) + + + --> TIME: 2025-04-19 09:50:53 -- STEP: 85/142 -- GLOBAL_STEP: 475075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.580366373062134 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.793014526367188 (17.896166722914742) + | > loss_duration: 1.646914005279541 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.3829 (0.3456462579614976) + | > loader_time: 0.0067 (0.005908783744363223) + + + --> TIME: 2025-04-19 09:51:03 -- STEP: 110/142 -- GLOBAL_STEP: 475100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.758992910385132 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.36307144165039 (17.844597244262676) + | > loss_duration: 1.6693898439407349 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.4399 (0.357622131434354) + | > loader_time: 0.0078 (0.006223628737709738) + + + --> TIME: 2025-04-19 09:51:14 -- STEP: 135/142 -- GLOBAL_STEP: 475125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.554722547531128 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.550132751464844 (17.78779545536746) + | > loss_duration: 1.6324291229248047 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018383931729005102 + | > current_lr_1: 0.00018383931729005102 + | > step_time: 0.433 (0.37004132624025704) + | > loader_time: 0.0077 (0.006465253123530634) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.874417304992676 (2.874417304992676) + | > loss_feat: nan (nan) + | > loss_mel: 15.97124195098877 (15.97124195098877) + | > loss_duration: 1.8397302627563477 (1.8397302627563477) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2729485034942627 (3.2729485034942627) + | > loss_feat: nan (nan) + | > loss_mel: 17.063879013061523 (17.063879013061523) + | > loss_duration: 1.910007119178772 (1.910007119178772) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.845963478088379 (3.059455990791321) + | > loss_feat: nan (nan) + | > loss_mel: 17.9874267578125 (17.52565288543701) + | > loss_duration: 1.793276071548462 (1.851641595363617) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003576040267944336 (-0.0004515647888183594) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.059455990791321 (-0.04919254779815674) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.52565288543701 (-1.0394353866577148) + | > avg_loss_duration: 1.851641595363617 (+0.008250653743743896) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 675/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:51:18)  + + --> TIME: 2025-04-19 09:51:25 -- STEP: 18/142 -- GLOBAL_STEP: 475150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.168800354003906 (17.611496024661594) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018381633737538975 + | > current_lr_1: 0.00018381633737538975 + | > step_time: 0.313 (0.30466344621446395) + | > loader_time: 0.0049 (0.004794756571451823) + + + --> TIME: 2025-04-19 09:51:33 -- STEP: 43/142 -- GLOBAL_STEP: 475175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4856550693511963 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.92396354675293 (17.804234127665676) + | > loss_duration: 1.61674165725708 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018381633737538975 + | > current_lr_1: 0.00018381633737538975 + | > step_time: 0.3373 (0.31795059802920317) + | > loader_time: 0.0057 (0.005175523979719295) + + + --> TIME: 2025-04-19 09:51:42 -- STEP: 68/142 -- GLOBAL_STEP: 475200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.898090362548828 (17.792277013554298) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018381633737538975 + | > current_lr_1: 0.00018381633737538975 + | > step_time: 0.3666 (0.32956273415509385) + | > loader_time: 0.0062 (0.005471948315115536) + + + --> TIME: 2025-04-19 09:51:52 -- STEP: 93/142 -- GLOBAL_STEP: 475225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.155107498168945 (17.761008283143404) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018381633737538975 + | > current_lr_1: 0.00018381633737538975 + | > step_time: 0.3738 (0.3411183536693615) + | > loader_time: 0.0062 (0.00573952736393098) + + + --> TIME: 2025-04-19 09:52:02 -- STEP: 118/142 -- GLOBAL_STEP: 475250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5960817337036133 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.731985092163086 (17.80308505236093) + | > loss_duration: 1.6248399019241333 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018381633737538975 + | > current_lr_1: 0.00018381633737538975 + | > step_time: 0.4161 (0.35202608068110586) + | > loader_time: 0.0072 (0.006018349679849917) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7955496311187744 (2.7955496311187744) + | > loss_feat: nan (nan) + | > loss_mel: 13.014913558959961 (13.014913558959961) + | > loss_duration: 1.8515411615371704 (1.8515411615371704) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3652024269104004 (3.3652024269104004) + | > loss_feat: nan (nan) + | > loss_mel: 19.007335662841797 (19.007335662841797) + | > loss_duration: 1.8886582851409912 (1.8886582851409912) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8623123168945312 (3.113757371902466) + | > loss_feat: nan (nan) + | > loss_mel: 17.83524513244629 (18.421290397644043) + | > loss_duration: 1.8025908470153809 (1.845624566078186) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034584999084472656 (-0.00011754035949707031) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.113757371902466 (+0.05430138111114502) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.421290397644043 (+0.8956375122070312) + | > avg_loss_duration: 1.845624566078186 (-0.006017029285430908) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 676/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:52:13)  + + --> TIME: 2025-04-19 09:52:14 -- STEP: 1/142 -- GLOBAL_STEP: 475275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.94734001159668 (18.94734001159668) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.3032 (0.30324220657348633) + | > loader_time: 0.0041 (0.004080533981323242) + + + --> TIME: 2025-04-19 09:52:22 -- STEP: 26/142 -- GLOBAL_STEP: 475300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5299272537231445 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.130126953125 (17.84439068574172) + | > loss_duration: 1.6196317672729492 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.3301 (0.3105970254311195) + | > loader_time: 0.0055 (0.004955007479741023) + + + --> TIME: 2025-04-19 09:52:31 -- STEP: 51/142 -- GLOBAL_STEP: 475325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.47126579284668 (17.667461114771232) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.3564 (0.329227045470593) + | > loader_time: 0.0058 (0.005478513007070504) + + + --> TIME: 2025-04-19 09:52:41 -- STEP: 76/142 -- GLOBAL_STEP: 475350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.528453826904297 (17.733524987572114) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.3788 (0.342603159578223) + | > loader_time: 0.0068 (0.005743309071189479) + + + --> TIME: 2025-04-19 09:52:51 -- STEP: 101/142 -- GLOBAL_STEP: 475375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.118335723876953 (17.82676133070842) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.389 (0.3553139006737436) + | > loader_time: 0.0073 (0.006043998321684282) + + + --> TIME: 2025-04-19 09:53:01 -- STEP: 126/142 -- GLOBAL_STEP: 475400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.75770378112793 (17.88583934874761) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018379336033321782 + | > current_lr_1: 0.00018379336033321782 + | > step_time: 0.4141 (0.3647612787428357) + | > loader_time: 0.0074 (0.006314782869248164) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1328306198120117 (3.1328306198120117) + | > loss_feat: nan (nan) + | > loss_mel: 19.394052505493164 (19.394052505493164) + | > loss_duration: 1.8435851335525513 (1.8435851335525513) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.6270318031311035 (3.6270318031311035) + | > loss_feat: nan (nan) + | > loss_mel: 18.113882064819336 (18.113882064819336) + | > loss_duration: 1.9018583297729492 (1.9018583297729492) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8807690143585205 (3.253900408744812) + | > loss_feat: nan (nan) + | > loss_mel: 17.52639389038086 (17.820137977600098) + | > loss_duration: 1.8055429458618164 (1.8537006378173828) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0039463043212890625 (+0.0004878044128417969) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.253900408744812 (+0.1401430368423462) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.820137977600098 (-0.6011524200439453) + | > avg_loss_duration: 1.8537006378173828 (+0.008076071739196777) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 677/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:53:09)  + + --> TIME: 2025-04-19 09:53:13 -- STEP: 9/142 -- GLOBAL_STEP: 475425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.76072883605957 (18.32622506883409) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.2978 (0.29982222451104057) + | > loader_time: 0.0048 (0.004629029168023003) + + + --> TIME: 2025-04-19 09:53:21 -- STEP: 34/142 -- GLOBAL_STEP: 475450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5496017932891846 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.42867660522461 (18.09553617589614) + | > loss_duration: 1.634648323059082 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.329 (0.31295303737415986) + | > loader_time: 0.0056 (0.0051726172952090985) + + + --> TIME: 2025-04-19 09:53:29 -- STEP: 59/142 -- GLOBAL_STEP: 475475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.713342666625977 (17.96196087336137) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.3496 (0.3245086387052375) + | > loader_time: 0.0062 (0.005515761294607388) + + + --> TIME: 2025-04-19 09:53:39 -- STEP: 84/142 -- GLOBAL_STEP: 475500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.862661361694336 (17.842275505974186) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.369 (0.33653119064512715) + | > loader_time: 0.0063 (0.005774773302532376) + + + --> TIME: 2025-04-19 09:53:49 -- STEP: 109/142 -- GLOBAL_STEP: 475525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.557907104492188 (17.7749018712875) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.3876 (0.3469680143058847) + | > loader_time: 0.0074 (0.0059933137456211465) + + + --> TIME: 2025-04-19 09:53:59 -- STEP: 134/142 -- GLOBAL_STEP: 475550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.118427276611328 (17.772012055809828) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018377038616317615 + | > current_lr_1: 0.00018377038616317615 + | > step_time: 0.4193 (0.3590141730522042) + | > loader_time: 0.0076 (0.006245394251239832) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9027085304260254 (2.9027085304260254) + | > loss_feat: nan (nan) + | > loss_mel: 18.146268844604492 (18.146268844604492) + | > loss_duration: 1.820238471031189 (1.820238471031189) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.304759979248047 (3.304759979248047) + | > loss_feat: nan (nan) + | > loss_mel: 18.683637619018555 (18.683637619018555) + | > loss_duration: 1.91005277633667 (1.91005277633667) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7942464351654053 (3.049503207206726) + | > loss_feat: nan (nan) + | > loss_mel: 17.128948211669922 (17.90629291534424) + | > loss_duration: 1.8161334991455078 (1.8630931377410889) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003209710121154785 (-0.0007365942001342773) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.049503207206726 (-0.20439720153808594) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.90629291534424 (+0.08615493774414062) + | > avg_loss_duration: 1.8630931377410889 (+0.009392499923706055) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 678/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:54:04)  + + --> TIME: 2025-04-19 09:54:10 -- STEP: 17/142 -- GLOBAL_STEP: 475575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.511157751083374 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.454540252685547 (17.93868580986472) + | > loss_duration: 1.5730592012405396 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018374741486490575 + | > current_lr_1: 0.00018374741486490575 + | > step_time: 0.3169 (0.3059617491329418) + | > loader_time: 0.0051 (0.004671068752513212) + + + --> TIME: 2025-04-19 09:54:18 -- STEP: 42/142 -- GLOBAL_STEP: 475600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.291065216064453 (18.013481548854287) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018374741486490575 + | > current_lr_1: 0.00018374741486490575 + | > step_time: 0.346 (0.3219990616752988) + | > loader_time: 0.0061 (0.005350743021283831) + + + --> TIME: 2025-04-19 09:54:28 -- STEP: 67/142 -- GLOBAL_STEP: 475625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.652854919433594 (17.88456361685226) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018374741486490575 + | > current_lr_1: 0.00018374741486490575 + | > step_time: 0.3683 (0.33570870356773264) + | > loader_time: 0.0066 (0.00576011814288239) + + + --> TIME: 2025-04-19 09:54:38 -- STEP: 92/142 -- GLOBAL_STEP: 475650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.08830451965332 (17.897460989330124) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018374741486490575 + | > current_lr_1: 0.00018374741486490575 + | > step_time: 0.3878 (0.35184536550355994) + | > loader_time: 0.0063 (0.006043447100597879) + + + --> TIME: 2025-04-19 09:54:48 -- STEP: 117/142 -- GLOBAL_STEP: 475675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.07065200805664 (17.83970568730281) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018374741486490575 + | > current_lr_1: 0.00018374741486490575 + | > step_time: 0.4109 (0.3602582124563364) + | > loader_time: 0.0072 (0.006281119126539964) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.73785662651062 (2.73785662651062) + | > loss_feat: nan (nan) + | > loss_mel: 17.215425491333008 (17.215425491333008) + | > loss_duration: 1.8288121223449707 (1.8288121223449707) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3995180130004883 (3.3995180130004883) + | > loss_feat: nan (nan) + | > loss_mel: 18.573524475097656 (18.573524475097656) + | > loss_duration: 1.8889826536178589 (1.8889826536178589) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9737842082977295 (3.186651110649109) + | > loss_feat: nan (nan) + | > loss_mel: 18.19057846069336 (18.382051467895508) + | > loss_duration: 1.7937731742858887 (1.8413779139518738) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035001039505004883 (+0.0002903938293457031) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.186651110649109 (+0.1371479034423828) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.382051467895508 (+0.47575855255126953) + | > avg_loss_duration: 1.8413779139518738 (-0.021715223789215088) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 679/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:55:00)  + + --> TIME: 2025-04-19 09:55:00 -- STEP: 0/142 -- GLOBAL_STEP: 475700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.505992889404297 (16.505992889404297) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.3263 (0.3262596130371094) + | > loader_time: 0.3235 (0.3235156536102295) + + + --> TIME: 2025-04-19 09:55:08 -- STEP: 25/142 -- GLOBAL_STEP: 475725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.29339027404785 (18.12831703186035) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.3186 (0.30958553314208986) + | > loader_time: 0.0056 (0.00522028923034668) + + + --> TIME: 2025-04-19 09:55:17 -- STEP: 50/142 -- GLOBAL_STEP: 475750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.53268814086914 (18.164304161071776) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.3452 (0.3215891742706299) + | > loader_time: 0.006 (0.005598802566528319) + + + --> TIME: 2025-04-19 09:55:26 -- STEP: 75/142 -- GLOBAL_STEP: 475775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.416168212890625 (18.11768913269043) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.3733 (0.3334498691558838) + | > loader_time: 0.0069 (0.005843369166056315) + + + --> TIME: 2025-04-19 09:55:36 -- STEP: 100/142 -- GLOBAL_STEP: 475800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.560428619384766 (18.10975786209106) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.388 (0.34390841245651244) + | > loader_time: 0.0076 (0.006095256805419922) + + + --> TIME: 2025-04-19 09:55:46 -- STEP: 125/142 -- GLOBAL_STEP: 475825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.786483764648438 (18.028908325195314) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018372444643804763 + | > current_lr_1: 0.00018372444643804763 + | > step_time: 0.4115 (0.3552848434448242) + | > loader_time: 0.0074 (0.0063257083892822264) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0079619884490967 (3.0079619884490967) + | > loss_feat: nan (nan) + | > loss_mel: 19.562870025634766 (19.562870025634766) + | > loss_duration: 1.8489298820495605 (1.8489298820495605) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.518012046813965 (3.518012046813965) + | > loss_feat: nan (nan) + | > loss_mel: 19.552961349487305 (19.552961349487305) + | > loss_duration: 1.923022747039795 (1.923022747039795) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7053496837615967 (3.1116808652877808) + | > loss_feat: nan (nan) + | > loss_mel: 19.558317184448242 (19.555639266967773) + | > loss_duration: 1.7823021411895752 (1.852662444114685) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003492593765258789 (-7.510185241699219e-06) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1116808652877808 (-0.07497024536132812) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.555639266967773 (+1.1735877990722656) + | > avg_loss_duration: 1.852662444114685 (+0.01128453016281128) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 680/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:55:55)  + + --> TIME: 2025-04-19 09:55:58 -- STEP: 8/142 -- GLOBAL_STEP: 475850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.283018112182617 (17.436156153678894) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.3015 (0.32442227005958557) + | > loader_time: 0.0046 (0.00411030650138855) + + + --> TIME: 2025-04-19 09:56:06 -- STEP: 33/142 -- GLOBAL_STEP: 475875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.755220413208008 (17.807958805199824) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.331 (0.3190405585549094) + | > loader_time: 0.0053 (0.004864295323689778) + + + --> TIME: 2025-04-19 09:56:15 -- STEP: 58/142 -- GLOBAL_STEP: 475900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.294954299926758 (17.891895639485327) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.3497 (0.32822036332097543) + | > loader_time: 0.0059 (0.005214822703394395) + + + --> TIME: 2025-04-19 09:56:24 -- STEP: 83/142 -- GLOBAL_STEP: 475925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.654996156692505 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.679367065429688 (17.894337860934712) + | > loss_duration: 1.6996681690216064 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.3715 (0.33934181282319203) + | > loader_time: 0.0064 (0.005571971456688569) + + + --> TIME: 2025-04-19 09:56:34 -- STEP: 108/142 -- GLOBAL_STEP: 475950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.424680709838867 (17.9440444222203) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.3965 (0.34965241396868657) + | > loader_time: 0.007 (0.005841369982118959) + + + --> TIME: 2025-04-19 09:56:45 -- STEP: 133/142 -- GLOBAL_STEP: 475975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.3347225189209 (17.936603581994994) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018370148088224286 + | > current_lr_1: 0.00018370148088224286 + | > step_time: 0.4113 (0.3617921843564599) + | > loader_time: 0.0071 (0.0061548419464799685) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.956117630004883 (2.956117630004883) + | > loss_feat: nan (nan) + | > loss_mel: 18.452173233032227 (18.452173233032227) + | > loss_duration: 1.8309557437896729 (1.8309557437896729) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.253206253051758 (3.253206253051758) + | > loss_feat: nan (nan) + | > loss_mel: 16.960479736328125 (16.960479736328125) + | > loss_duration: 1.8940918445587158 (1.8940918445587158) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.774669885635376 (3.013938069343567) + | > loss_feat: nan (nan) + | > loss_mel: 17.987075805664062 (17.473777770996094) + | > loss_duration: 1.7795032262802124 (1.8367975354194641) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0031044483184814453 (-0.00038814544677734375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.013938069343567 (-0.09774279594421387) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.473777770996094 (-2.0818614959716797) + | > avg_loss_duration: 1.8367975354194641 (-0.015864908695220947) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 681/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:56:50)  + + --> TIME: 2025-04-19 09:56:55 -- STEP: 16/142 -- GLOBAL_STEP: 476000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.056949615478516 (17.716510713100433) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.3126 (0.3034629821777344) + | > loader_time: 0.0051 (0.004469260573387146) + + + --> TIME: 2025-04-19 09:57:04 -- STEP: 41/142 -- GLOBAL_STEP: 476025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3504719734191895 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.185434341430664 (17.95505498095257) + | > loss_duration: 1.611457109451294 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.3368 (0.3164618422345417) + | > loader_time: 0.0054 (0.004982134191001334) + + + --> TIME: 2025-04-19 09:57:13 -- STEP: 66/142 -- GLOBAL_STEP: 476050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.444904327392578 (17.980622768402103) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.3561 (0.3282293045159542) + | > loader_time: 0.0059 (0.005338928916237571) + + + --> TIME: 2025-04-19 09:57:22 -- STEP: 91/142 -- GLOBAL_STEP: 476075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.157625198364258 (17.90478219042768) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.3781 (0.3398112255138355) + | > loader_time: 0.0063 (0.005620558183271806) + + + --> TIME: 2025-04-19 09:57:32 -- STEP: 116/142 -- GLOBAL_STEP: 476100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.478239059448242 (17.89906119478159) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.4029 (0.3508028881303195) + | > loader_time: 0.0081 (0.005875632680695632) + + + --> TIME: 2025-04-19 09:57:43 -- STEP: 141/142 -- GLOBAL_STEP: 476125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.030065536499023 (17.92112535111447) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018367851819713258 + | > current_lr_1: 0.00018367851819713258 + | > step_time: 0.1794 (0.3615201517199793) + | > loader_time: 0.0025 (0.0060699476417920275) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8789849281311035 (2.8789849281311035) + | > loss_feat: nan (nan) + | > loss_mel: 19.014118194580078 (19.014118194580078) + | > loss_duration: 1.8138787746429443 (1.8138787746429443) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.588520050048828 (3.588520050048828) + | > loss_feat: nan (nan) + | > loss_mel: 18.650753021240234 (18.650753021240234) + | > loss_duration: 1.9127126932144165 (1.9127126932144165) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8837685585021973 (3.2361443042755127) + | > loss_feat: nan (nan) + | > loss_mel: 17.59251594543457 (18.121634483337402) + | > loss_duration: 1.786305546760559 (1.8495091199874878) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003427743911743164 (+0.00032329559326171875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2361443042755127 (+0.2222062349319458) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.121634483337402 (+0.6478567123413086) + | > avg_loss_duration: 1.8495091199874878 (+0.012711584568023682) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 682/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:57:45)  + + --> TIME: 2025-04-19 09:57:53 -- STEP: 24/142 -- GLOBAL_STEP: 476150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.828144073486328 (17.764679431915283) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018365555838235795 + | > current_lr_1: 0.00018365555838235795 + | > step_time: 0.3177 (0.30741460124651593) + | > loader_time: 0.005 (0.004777848720550537) + + + --> TIME: 2025-04-19 09:58:02 -- STEP: 49/142 -- GLOBAL_STEP: 476175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.408733367919922 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.53725242614746 (17.970625235109914) + | > loss_duration: 1.5940923690795898 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018365555838235795 + | > current_lr_1: 0.00018365555838235795 + | > step_time: 0.3436 (0.3207281560313945) + | > loader_time: 0.0058 (0.005129079429470763) + + + --> TIME: 2025-04-19 09:58:11 -- STEP: 74/142 -- GLOBAL_STEP: 476200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5407187938690186 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.426380157470703 (18.01137368743484) + | > loss_duration: 1.6504895687103271 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018365555838235795 + | > current_lr_1: 0.00018365555838235795 + | > step_time: 0.3695 (0.33326304281080094) + | > loader_time: 0.0062 (0.005390038361420503) + + + --> TIME: 2025-04-19 09:58:21 -- STEP: 99/142 -- GLOBAL_STEP: 476225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.112071990966797 (18.03589037692908) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018365555838235795 + | > current_lr_1: 0.00018365555838235795 + | > step_time: 0.3954 (0.3455571068657768) + | > loader_time: 0.0079 (0.005706921972409643) + + + --> TIME: 2025-04-19 09:58:31 -- STEP: 124/142 -- GLOBAL_STEP: 476250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.338638305664062 (17.965419684686953) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018365555838235795 + | > current_lr_1: 0.00018365555838235795 + | > step_time: 0.415 (0.3573421470580561) + | > loader_time: 0.0071 (0.0059371782887366505) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.917116165161133 (2.917116165161133) + | > loss_feat: nan (nan) + | > loss_mel: 19.625625610351562 (19.625625610351562) + | > loss_duration: 1.824689507484436 (1.824689507484436) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3039841651916504 (3.3039841651916504) + | > loss_feat: nan (nan) + | > loss_mel: 17.62605857849121 (17.62605857849121) + | > loss_duration: 1.9007636308670044 (1.9007636308670044) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9886863231658936 (3.146335244178772) + | > loss_feat: nan (nan) + | > loss_mel: 16.670093536376953 (17.148076057434082) + | > loss_duration: 1.8040767908096313 (1.8524202108383179) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0034797191619873047 (+5.1975250244140625e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.146335244178772 (-0.08980906009674072) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.148076057434082 (-0.9735584259033203) + | > avg_loss_duration: 1.8524202108383179 (+0.002911090850830078) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 683/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:58:40)  + + --> TIME: 2025-04-19 09:58:43 -- STEP: 7/142 -- GLOBAL_STEP: 476275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.367921829223633 (18.348601749965123) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.2988 (0.2984823840005057) + | > loader_time: 0.004 (0.004251241683959961) + + + --> TIME: 2025-04-19 09:58:51 -- STEP: 32/142 -- GLOBAL_STEP: 476300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.58307456970215 (18.19106090068817) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.331 (0.31413640081882477) + | > loader_time: 0.0053 (0.005085371434688567) + + + --> TIME: 2025-04-19 09:59:00 -- STEP: 57/142 -- GLOBAL_STEP: 476325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.444005966186523 (17.984550091258267) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.3488 (0.32606882798044307) + | > loader_time: 0.0065 (0.005421274586727745) + + + --> TIME: 2025-04-19 09:59:09 -- STEP: 82/142 -- GLOBAL_STEP: 476350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.199874877929688 (17.963253184062683) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.3878 (0.33860858475289696) + | > loader_time: 0.0064 (0.00569559015878817) + + + --> TIME: 2025-04-19 09:59:19 -- STEP: 107/142 -- GLOBAL_STEP: 476375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.22779083251953 (18.03279436414487) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.3942 (0.3496024875997383) + | > loader_time: 0.007 (0.0059519126036456805) + + + --> TIME: 2025-04-19 09:59:29 -- STEP: 132/142 -- GLOBAL_STEP: 476400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.11174774169922 (17.986377427072235) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018363260143756013 + | > current_lr_1: 0.00018363260143756013 + | > step_time: 0.4209 (0.361208307020592) + | > loader_time: 0.0079 (0.0062195694807803984) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.856348991394043 (2.856348991394043) + | > loss_feat: nan (nan) + | > loss_mel: 16.405807495117188 (16.405807495117188) + | > loss_duration: 1.8262543678283691 (1.8262543678283691) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2301456928253174 (3.2301456928253174) + | > loss_feat: nan (nan) + | > loss_mel: 16.03009796142578 (16.03009796142578) + | > loss_duration: 1.9002898931503296 (1.9002898931503296) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9060189723968506 (3.068082332611084) + | > loss_feat: nan (nan) + | > loss_mel: 19.623336791992188 (17.826717376708984) + | > loss_duration: 1.7889161109924316 (1.8446030020713806) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003242015838623047 (-0.0002377033233642578) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.068082332611084 (-0.07825291156768799) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.826717376708984 (+0.6786413192749023) + | > avg_loss_duration: 1.8446030020713806 (-0.007817208766937256) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 684/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 09:59:35)  + + --> TIME: 2025-04-19 09:59:41 -- STEP: 15/142 -- GLOBAL_STEP: 476425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.595321655273438 (17.848984781901038) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.3112 (0.3054287592569987) + | > loader_time: 0.0053 (0.004518747329711914) + + + --> TIME: 2025-04-19 09:59:49 -- STEP: 40/142 -- GLOBAL_STEP: 476450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.330455780029297 (18.000487565994256) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.3359 (0.31819295287132265) + | > loader_time: 0.0057 (0.00497969388961792) + + + --> TIME: 2025-04-19 09:59:58 -- STEP: 65/142 -- GLOBAL_STEP: 476475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.734140396118164 (17.97225946279672) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.3675 (0.32975862209613505) + | > loader_time: 0.0061 (0.005285974649282601) + + + --> TIME: 2025-04-19 10:00:07 -- STEP: 90/142 -- GLOBAL_STEP: 476500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.21917724609375 (17.961777008904345) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.3777 (0.3407872623867458) + | > loader_time: 0.006 (0.005536715189615886) + + + --> TIME: 2025-04-19 10:00:17 -- STEP: 115/142 -- GLOBAL_STEP: 476525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.396421432495117 (17.90018658845321) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.4005 (0.35170464723006534) + | > loader_time: 0.0069 (0.005786095494809358) + + + --> TIME: 2025-04-19 10:00:28 -- STEP: 140/142 -- GLOBAL_STEP: 476550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.802309036254883 (17.882597460065572) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018360964736238044 + | > current_lr_1: 0.00018360964736238044 + | > step_time: 0.4491 (0.3640859603881836) + | > loader_time: 0.0074 (0.006040324483598982) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9076173305511475 (2.9076173305511475) + | > loss_feat: nan (nan) + | > loss_mel: 18.088857650756836 (18.088857650756836) + | > loss_duration: 1.823125958442688 (1.823125958442688) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.232001304626465 (3.232001304626465) + | > loss_feat: nan (nan) + | > loss_mel: 18.05215072631836 (18.05215072631836) + | > loss_duration: 1.9051049947738647 (1.9051049947738647) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.6950442790985107 (2.963522791862488) + | > loss_feat: nan (nan) + | > loss_mel: 17.854026794433594 (17.953088760375977) + | > loss_duration: 1.8006384372711182 (1.8528717160224915) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032078027725219727 (-3.421306610107422e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.963522791862488 (-0.10455954074859619) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.953088760375977 (+0.1263713836669922) + | > avg_loss_duration: 1.8528717160224915 (+0.00826871395111084) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 685/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:00:30)  + + --> TIME: 2025-04-19 10:00:38 -- STEP: 23/142 -- GLOBAL_STEP: 476575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.700197219848633 (18.093499805616297) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018358669615646013 + | > current_lr_1: 0.00018358669615646013 + | > step_time: 0.3214 (0.30955474273018213) + | > loader_time: 0.0054 (0.0047820443692414664) + + + --> TIME: 2025-04-19 10:00:47 -- STEP: 48/142 -- GLOBAL_STEP: 476600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.894819259643555 (18.116718411445618) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018358669615646013 + | > current_lr_1: 0.00018358669615646013 + | > step_time: 0.3372 (0.32080445686976117) + | > loader_time: 0.0058 (0.005206704139709472) + + + --> TIME: 2025-04-19 10:00:56 -- STEP: 73/142 -- GLOBAL_STEP: 476625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.96435546875 (17.93136361527117) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018358669615646013 + | > current_lr_1: 0.00018358669615646013 + | > step_time: 0.3684 (0.3336878280117087) + | > loader_time: 0.0061 (0.005615387877372845) + + + --> TIME: 2025-04-19 10:01:05 -- STEP: 98/142 -- GLOBAL_STEP: 476650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.814870834350586 (17.88280299245096) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018358669615646013 + | > current_lr_1: 0.00018358669615646013 + | > step_time: 0.3884 (0.3441175781950659) + | > loader_time: 0.0068 (0.005880655074606137) + + + --> TIME: 2025-04-19 10:01:16 -- STEP: 123/142 -- GLOBAL_STEP: 476675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.28827476501465 (17.862779175362952) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018358669615646013 + | > current_lr_1: 0.00018358669615646013 + | > step_time: 0.4142 (0.35541334190988927) + | > loader_time: 0.0074 (0.006163606798745752) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.0264487266540527 (3.0264487266540527) + | > loss_feat: nan (nan) + | > loss_mel: 17.87967300415039 (17.87967300415039) + | > loss_duration: 1.8220394849777222 (1.8220394849777222) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2868058681488037 (3.2868058681488037) + | > loss_feat: nan (nan) + | > loss_mel: 19.015710830688477 (19.015710830688477) + | > loss_duration: 1.909474492073059 (1.909474492073059) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9479434490203857 (3.1173746585845947) + | > loss_feat: nan (nan) + | > loss_mel: 18.841455459594727 (18.9285831451416) + | > loss_duration: 1.788493275642395 (1.848983883857727) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032966136932373047 (+8.881092071533203e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1173746585845947 (+0.15385186672210693) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.9285831451416 (+0.975494384765625) + | > avg_loss_duration: 1.848983883857727 (-0.0038878321647644043) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 686/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:01:25)  + + --> TIME: 2025-04-19 10:01:28 -- STEP: 6/142 -- GLOBAL_STEP: 476700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.867919921875 (17.823937098185223) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.3104 (0.3121240933736165) + | > loader_time: 0.0041 (0.00413207213083903) + + + --> TIME: 2025-04-19 10:01:36 -- STEP: 31/142 -- GLOBAL_STEP: 476725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.501436233520508 (17.796425357941658) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.3271 (0.31352017771813173) + | > loader_time: 0.0053 (0.004802103965513169) + + + --> TIME: 2025-04-19 10:01:45 -- STEP: 56/142 -- GLOBAL_STEP: 476750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.79247283935547 (17.922892076628553) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.3483 (0.32532983592578335) + | > loader_time: 0.0057 (0.005232717309679306) + + + --> TIME: 2025-04-19 10:01:54 -- STEP: 81/142 -- GLOBAL_STEP: 476775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.641294479370117 (17.97949015064004) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.37 (0.3373788492179211) + | > loader_time: 0.0067 (0.005487106464527273) + + + --> TIME: 2025-04-19 10:02:04 -- STEP: 106/142 -- GLOBAL_STEP: 476800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.06801986694336 (17.988214195899232) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.3886 (0.3484835782141056) + | > loader_time: 0.007 (0.005779095415799125) + + + --> TIME: 2025-04-19 10:02:14 -- STEP: 131/142 -- GLOBAL_STEP: 476825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.46796989440918 (17.911441002183274) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018356374781944057 + | > current_lr_1: 0.00018356374781944057 + | > step_time: 0.4357 (0.36164170185118233) + | > loader_time: 0.0069 (0.006048786731166695) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.125269651412964 (3.125269651412964) + | > loss_feat: nan (nan) + | > loss_mel: 19.339143753051758 (19.339143753051758) + | > loss_duration: 1.8246632814407349 (1.8246632814407349) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1830456256866455 (3.1830456256866455) + | > loss_feat: nan (nan) + | > loss_mel: 18.111528396606445 (18.111528396606445) + | > loss_duration: 1.8947384357452393 (1.8947384357452393) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9074666500091553 (3.0452561378479004) + | > loss_feat: nan (nan) + | > loss_mel: 16.875900268554688 (17.493714332580566) + | > loss_duration: 1.7969889640808105 (1.845863699913025) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003788471221923828 (+0.0004918575286865234) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0452561378479004 (-0.07211852073669434) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.493714332580566 (-1.4348688125610352) + | > avg_loss_duration: 1.845863699913025 (-0.0031201839447021484) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 687/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:02:21)  + + --> TIME: 2025-04-19 10:02:26 -- STEP: 14/142 -- GLOBAL_STEP: 476850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.581763505935669 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.42958641052246 (17.394291264670233) + | > loss_duration: 1.640953779220581 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.306 (0.30136704444885254) + | > loader_time: 0.0052 (0.004632983888898577) + + + --> TIME: 2025-04-19 10:02:34 -- STEP: 39/142 -- GLOBAL_STEP: 476875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.465456485748291 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.41678810119629 (17.764045739785217) + | > loss_duration: 1.6088335514068604 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.3315 (0.3146664729485146) + | > loader_time: 0.0058 (0.0050535446558243186) + + + --> TIME: 2025-04-19 10:02:43 -- STEP: 64/142 -- GLOBAL_STEP: 476900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.39412498474121 (17.938248708844185) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.3598 (0.3268609009683133) + | > loader_time: 0.0062 (0.005402587354183197) + + + --> TIME: 2025-04-19 10:02:52 -- STEP: 89/142 -- GLOBAL_STEP: 476925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.596189498901367 (17.888043596503433) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.3745 (0.3386596722549269) + | > loader_time: 0.0065 (0.005700987376523819) + + + --> TIME: 2025-04-19 10:03:02 -- STEP: 114/142 -- GLOBAL_STEP: 476950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.971832275390625 (17.866578026821735) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.3983 (0.3503258270129824) + | > loader_time: 0.0071 (0.005980307595771654) + + + --> TIME: 2025-04-19 10:03:13 -- STEP: 139/142 -- GLOBAL_STEP: 476975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.108806610107422 (17.813923808310523) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018354080235096314 + | > current_lr_1: 0.00018354080235096314 + | > step_time: 0.4383 (0.36503077239441367) + | > loader_time: 0.0078 (0.006269110192497856) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.969818115234375 (2.969818115234375) + | > loss_feat: nan (nan) + | > loss_mel: 17.162694931030273 (17.162694931030273) + | > loss_duration: 1.8412933349609375 (1.8412933349609375) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3763790130615234 (3.3763790130615234) + | > loss_feat: nan (nan) + | > loss_mel: 17.592985153198242 (17.592985153198242) + | > loss_duration: 1.9181149005889893 (1.9181149005889893) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9606711864471436 (3.1685250997543335) + | > loss_feat: nan (nan) + | > loss_mel: 18.262399673461914 (17.927692413330078) + | > loss_duration: 1.774672269821167 (1.8463935852050781) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038013458251953125 (+1.2874603271484375e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1685250997543335 (+0.1232689619064331) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.927692413330078 (+0.4339780807495117) + | > avg_loss_duration: 1.8463935852050781 (+0.0005298852920532227) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 688/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:03:16)  + + --> TIME: 2025-04-19 10:03:24 -- STEP: 22/142 -- GLOBAL_STEP: 477000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.64297866821289 (17.72027622569691) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018351785975066927 + | > current_lr_1: 0.00018351785975066927 + | > step_time: 0.3174 (0.3076531887054443) + | > loader_time: 0.0053 (0.004958033561706543) + + + --> TIME: 2025-04-19 10:03:32 -- STEP: 47/142 -- GLOBAL_STEP: 477025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.686842679977417 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.647613525390625 (17.953178121688516) + | > loss_duration: 1.5858068466186523 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018351785975066927 + | > current_lr_1: 0.00018351785975066927 + | > step_time: 0.3408 (0.32013502019516954) + | > loader_time: 0.0059 (0.005308967955569004) + + + --> TIME: 2025-04-19 10:03:41 -- STEP: 72/142 -- GLOBAL_STEP: 477050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.061697006225586 (17.949833234151196) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018351785975066927 + | > current_lr_1: 0.00018351785975066927 + | > step_time: 0.3669 (0.33226475450727666) + | > loader_time: 0.0065 (0.005594859520594279) + + + --> TIME: 2025-04-19 10:03:51 -- STEP: 97/142 -- GLOBAL_STEP: 477075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.044893264770508 (17.9715745866913) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018351785975066927 + | > current_lr_1: 0.00018351785975066927 + | > step_time: 0.3842 (0.34328520175107974) + | > loader_time: 0.0066 (0.005835466778155454) + + + --> TIME: 2025-04-19 10:04:01 -- STEP: 122/142 -- GLOBAL_STEP: 477100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.561134338378906 (17.944061404368902) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018351785975066927 + | > current_lr_1: 0.00018351785975066927 + | > step_time: 0.4264 (0.3544878373380567) + | > loader_time: 0.0069 (0.006037286070526623) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9330060482025146 (2.9330060482025146) + | > loss_feat: nan (nan) + | > loss_mel: 18.515390396118164 (18.515390396118164) + | > loss_duration: 1.8208787441253662 (1.8208787441253662) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2852139472961426 (3.2852139472961426) + | > loss_feat: nan (nan) + | > loss_mel: 14.656393051147461 (14.656393051147461) + | > loss_duration: 1.9009016752243042 (1.9009016752243042) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9112682342529297 (3.098241090774536) + | > loss_feat: nan (nan) + | > loss_mel: 18.028459548950195 (16.342426300048828) + | > loss_duration: 1.7960443496704102 (1.8484730124473572) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003763556480407715 (-3.7789344787597656e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.098241090774536 (-0.07028400897979736) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.342426300048828 (-1.58526611328125) + | > avg_loss_duration: 1.8484730124473572 (+0.0020794272422790527) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 689/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:04:11)  + + --> TIME: 2025-04-19 10:04:13 -- STEP: 5/142 -- GLOBAL_STEP: 477125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4034643173217773 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.3775634765625 (17.391089630126952) + | > loss_duration: 1.61307692527771 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.2978 (0.29701056480407717) + | > loader_time: 0.0042 (0.004316854476928711) + + + --> TIME: 2025-04-19 10:04:21 -- STEP: 30/142 -- GLOBAL_STEP: 477150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.0725040435791 (17.743964131673177) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.3324 (0.3140952825546265) + | > loader_time: 0.0055 (0.005076924959818522) + + + --> TIME: 2025-04-19 10:04:30 -- STEP: 55/142 -- GLOBAL_STEP: 477175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.05060386657715 (17.64258863275701) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.3557 (0.3284181248057972) + | > loader_time: 0.0062 (0.00546608404679732) + + + --> TIME: 2025-04-19 10:04:40 -- STEP: 80/142 -- GLOBAL_STEP: 477200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6675596237182617 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.41724395751953 (17.744402718544006) + | > loss_duration: 1.6634719371795654 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.3951 (0.34229772984981527) + | > loader_time: 0.0064 (0.005838003754615784) + + + --> TIME: 2025-04-19 10:04:50 -- STEP: 105/142 -- GLOBAL_STEP: 477225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.272987365722656 (17.803238024030417) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.4063 (0.35415954589843746) + | > loader_time: 0.0066 (0.006083779107956659) + + + --> TIME: 2025-04-19 10:05:00 -- STEP: 130/142 -- GLOBAL_STEP: 477250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.59161376953125 (17.79238542410044) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018349492001820043 + | > current_lr_1: 0.00018349492001820043 + | > step_time: 0.436 (0.36687569251427277) + | > loader_time: 0.0078 (0.006376284819382887) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.753662586212158 (2.753662586212158) + | > loss_feat: nan (nan) + | > loss_mel: 19.66469955444336 (19.66469955444336) + | > loss_duration: 1.8437490463256836 (1.8437490463256836) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.1734063625335693 (3.1734063625335693) + | > loss_feat: nan (nan) + | > loss_mel: 17.872722625732422 (17.872722625732422) + | > loss_duration: 1.9077459573745728 (1.9077459573745728) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.89656925201416 (3.0349878072738647) + | > loss_feat: nan (nan) + | > loss_mel: 16.97972297668457 (17.426222801208496) + | > loss_duration: 1.7897104024887085 (1.8487281799316406) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003390669822692871 (-0.00037288665771484375) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0349878072738647 (-0.06325328350067139) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.426222801208496 (+1.083796501159668) + | > avg_loss_duration: 1.8487281799316406 (+0.00025516748428344727) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 690/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:05:07)  + + --> TIME: 2025-04-19 10:05:12 -- STEP: 13/142 -- GLOBAL_STEP: 477275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.458888292312622 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.734474182128906 (18.198600915762093) + | > loss_duration: 1.6413285732269287 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.3065 (0.309984775689932) + | > loader_time: 0.0059 (0.004755258560180664) + + + --> TIME: 2025-04-19 10:05:20 -- STEP: 38/142 -- GLOBAL_STEP: 477300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.85442543029785 (18.014166179456208) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.3307 (0.31758957787563924) + | > loader_time: 0.0057 (0.005198227731805099) + + + --> TIME: 2025-04-19 10:05:29 -- STEP: 63/142 -- GLOBAL_STEP: 477325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.9102671146392822 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.106576919555664 (18.128930742778483) + | > loss_duration: 1.6707607507705688 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.3638 (0.3283034619830904) + | > loader_time: 0.0061 (0.0055191971006847565) + + + --> TIME: 2025-04-19 10:05:39 -- STEP: 88/142 -- GLOBAL_STEP: 477350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.91404151916504 (18.060475501147195) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.3721 (0.3394587446342816) + | > loader_time: 0.0068 (0.005775795741514726) + + + --> TIME: 2025-04-19 10:05:48 -- STEP: 113/142 -- GLOBAL_STEP: 477375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.816505432128906 (18.115748430775337) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.3992 (0.3497760570154782) + | > loader_time: 0.007 (0.006009876200583129) + + + --> TIME: 2025-04-19 10:05:59 -- STEP: 138/142 -- GLOBAL_STEP: 477400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7775957584381104 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.86304473876953 (18.061348279317226) + | > loss_duration: 1.735074520111084 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018347198315319814 + | > current_lr_1: 0.00018347198315319814 + | > step_time: 0.429 (0.3617253113484038) + | > loader_time: 0.0075 (0.006266789159912994) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8667755126953125 (2.8667755126953125) + | > loss_feat: nan (nan) + | > loss_mel: 19.419347763061523 (19.419347763061523) + | > loss_duration: 1.823652982711792 (1.823652982711792) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3493175506591797 (3.3493175506591797) + | > loss_feat: nan (nan) + | > loss_mel: 19.91815757751465 (19.91815757751465) + | > loss_duration: 1.9063681364059448 (1.9063681364059448) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.5074143409729004 (2.92836594581604) + | > loss_feat: nan (nan) + | > loss_mel: 18.194990158081055 (19.05657386779785) + | > loss_duration: 1.794896125793457 (1.850632131099701) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00352323055267334 (+0.00013256072998046875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.92836594581604 (-0.10662186145782471) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.05657386779785 (+1.6303510665893555) + | > avg_loss_duration: 1.850632131099701 (+0.0019039511680603027) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 691/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:06:02)  + + --> TIME: 2025-04-19 10:06:09 -- STEP: 21/142 -- GLOBAL_STEP: 477425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.890899658203125 (17.623381887163436) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018344904915530398 + | > current_lr_1: 0.00018344904915530398 + | > step_time: 0.3179 (0.30554546628679546) + | > loader_time: 0.0049 (0.0045424415951683395) + + + --> TIME: 2025-04-19 10:06:18 -- STEP: 46/142 -- GLOBAL_STEP: 477450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.34769630432129 (17.905680034471597) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018344904915530398 + | > current_lr_1: 0.00018344904915530398 + | > step_time: 0.3394 (0.3177235852117124) + | > loader_time: 0.0053 (0.004962351011193316) + + + --> TIME: 2025-04-19 10:06:27 -- STEP: 71/142 -- GLOBAL_STEP: 477475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.84621810913086 (18.03064725096797) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018344904915530398 + | > current_lr_1: 0.00018344904915530398 + | > step_time: 0.3627 (0.3298555998735025) + | > loader_time: 0.0064 (0.005272932455573284) + + + --> TIME: 2025-04-19 10:06:36 -- STEP: 96/142 -- GLOBAL_STEP: 477500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5804901123046875 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.410133361816406 (18.113225380579618) + | > loss_duration: 1.706363320350647 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018344904915530398 + | > current_lr_1: 0.00018344904915530398 + | > step_time: 0.3952 (0.3421432798107465) + | > loader_time: 0.0065 (0.00558067113161087) + + + --> TIME: 2025-04-19 10:06:47 -- STEP: 121/142 -- GLOBAL_STEP: 477525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.7900550365448 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.488298416137695 (18.092431170881284) + | > loss_duration: 1.7247729301452637 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018344904915530398 + | > current_lr_1: 0.00018344904915530398 + | > step_time: 0.4289 (0.35698052871325786) + | > loader_time: 0.0073 (0.005863924657017731) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.069617986679077 (3.069617986679077) + | > loss_feat: nan (nan) + | > loss_mel: 18.669797897338867 (18.669797897338867) + | > loss_duration: 1.8473122119903564 (1.8473122119903564) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.377362012863159 (3.377362012863159) + | > loss_feat: nan (nan) + | > loss_mel: 17.934404373168945 (17.934404373168945) + | > loss_duration: 1.9036061763763428 (1.9036061763763428) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8118555545806885 (3.094608783721924) + | > loss_feat: nan (nan) + | > loss_mel: 20.100194931030273 (19.01729965209961) + | > loss_duration: 1.7819538116455078 (1.8427799940109253) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036237239837646484 (+0.0001004934310913086) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.094608783721924 (+0.1662428379058838) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.01729965209961 (-0.03927421569824219) + | > avg_loss_duration: 1.8427799940109253 (-0.007852137088775635) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 692/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:06:58)  + + --> TIME: 2025-04-19 10:07:00 -- STEP: 4/142 -- GLOBAL_STEP: 477550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.237293243408203 (17.671767234802246) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.2946 (0.2982569932937622) + | > loader_time: 0.0051 (0.004738807678222656) + + + --> TIME: 2025-04-19 10:07:08 -- STEP: 29/142 -- GLOBAL_STEP: 477575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.568641424179077 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.89027976989746 (17.711326862203663) + | > loss_duration: 1.6023765802383423 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.324 (0.30994261544326257) + | > loader_time: 0.0052 (0.004818118851760338) + + + --> TIME: 2025-04-19 10:07:16 -- STEP: 54/142 -- GLOBAL_STEP: 477600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.65625762939453 (17.8539959942853) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.3461 (0.32250957577316863) + | > loader_time: 0.0059 (0.0051911142137315534) + + + --> TIME: 2025-04-19 10:07:26 -- STEP: 79/142 -- GLOBAL_STEP: 477625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.556478500366211 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.995193481445312 (17.96763828132726) + | > loss_duration: 1.6743108034133911 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.3617 (0.33647021764441387) + | > loader_time: 0.0068 (0.005595020101040225) + + + --> TIME: 2025-04-19 10:07:35 -- STEP: 104/142 -- GLOBAL_STEP: 477650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.511035919189453 (17.952560333105232) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.3897 (0.3475531958616696) + | > loader_time: 0.0071 (0.005843568306702834) + + + --> TIME: 2025-04-19 10:07:46 -- STEP: 129/142 -- GLOBAL_STEP: 477675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.789216995239258 (17.892953089041306) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018342611802415955 + | > current_lr_1: 0.00018342611802415955 + | > step_time: 0.4317 (0.3603470639664997) + | > loader_time: 0.0079 (0.006107256394024043) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.802755832672119 (2.802755832672119) + | > loss_feat: nan (nan) + | > loss_mel: 19.812259674072266 (19.812259674072266) + | > loss_duration: 1.8236817121505737 (1.8236817121505737) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3415887355804443 (3.3415887355804443) + | > loss_feat: nan (nan) + | > loss_mel: 19.702787399291992 (19.702787399291992) + | > loss_duration: 1.9050636291503906 (1.9050636291503906) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.961345672607422 (3.151467204093933) + | > loss_feat: nan (nan) + | > loss_mel: 17.043922424316406 (18.3733549118042) + | > loss_duration: 1.8035192489624023 (1.8542914390563965) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003522515296936035 (-0.00010120868682861328) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.151467204093933 (+0.05685842037200928) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.3733549118042 (-0.6439447402954102) + | > avg_loss_duration: 1.8542914390563965 (+0.011511445045471191) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 693/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:07:53)  + + --> TIME: 2025-04-19 10:07:57 -- STEP: 12/142 -- GLOBAL_STEP: 477700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3307037353515625 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.670398712158203 (17.988356749216717) + | > loss_duration: 1.628652572631836 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.3052 (0.3014365832010905) + | > loader_time: 0.0059 (0.004717111587524413) + + + --> TIME: 2025-04-19 10:08:06 -- STEP: 37/142 -- GLOBAL_STEP: 477725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4501945972442627 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.631534576416016 (18.136786125801713) + | > loss_duration: 1.5628883838653564 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.3344 (0.3191018362303038) + | > loader_time: 0.0057 (0.005153501355970228) + + + --> TIME: 2025-04-19 10:08:15 -- STEP: 62/142 -- GLOBAL_STEP: 477750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.472778558731079 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.3512020111084 (18.055226941262536) + | > loss_duration: 1.6798498630523682 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.3674 (0.3325503872286889) + | > loader_time: 0.0062 (0.005500455056467364) + + + --> TIME: 2025-04-19 10:08:25 -- STEP: 87/142 -- GLOBAL_STEP: 477775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.687421798706055 (18.079042336036434) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.3766 (0.3480886508678568) + | > loader_time: 0.0066 (0.00579336867935356) + + + --> TIME: 2025-04-19 10:08:35 -- STEP: 112/142 -- GLOBAL_STEP: 477800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.32402801513672 (18.04835689919336) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.4023 (0.3567477954285487) + | > loader_time: 0.007 (0.00601623739515032) + + + --> TIME: 2025-04-19 10:08:45 -- STEP: 137/142 -- GLOBAL_STEP: 477825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.589427947998047 (18.008922806621474) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018340318975940652 + | > current_lr_1: 0.00018340318975940652 + | > step_time: 0.4284 (0.36743371678094805) + | > loader_time: 0.0079 (0.006249631408357271) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.755779504776001 (2.755779504776001) + | > loss_feat: nan (nan) + | > loss_mel: 18.636037826538086 (18.636037826538086) + | > loss_duration: 1.8328745365142822 (1.8328745365142822) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3465499877929688 (3.3465499877929688) + | > loss_feat: nan (nan) + | > loss_mel: 15.996005058288574 (15.996005058288574) + | > loss_duration: 1.9375383853912354 (1.9375383853912354) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8482887744903564 (3.0974193811416626) + | > loss_feat: nan (nan) + | > loss_mel: 18.993322372436523 (17.49466371536255) + | > loss_duration: 1.7796664237976074 (1.8586024045944214) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003174424171447754 (-0.00034809112548828125) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0974193811416626 (-0.05404782295227051) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.49466371536255 (-0.8786911964416504) + | > avg_loss_duration: 1.8586024045944214 (+0.004310965538024902) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 694/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:08:49)  + + --> TIME: 2025-04-19 10:08:56 -- STEP: 20/142 -- GLOBAL_STEP: 477850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.182941436767578 (17.9474515914917) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018338026436068658 + | > current_lr_1: 0.00018338026436068658 + | > step_time: 0.3201 (0.3090822219848633) + | > loader_time: 0.0055 (0.004833292961120605) + + + --> TIME: 2025-04-19 10:09:04 -- STEP: 45/142 -- GLOBAL_STEP: 477875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.530559539794922 (17.889946916368267) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018338026436068658 + | > current_lr_1: 0.00018338026436068658 + | > step_time: 0.3428 (0.3208209832509359) + | > loader_time: 0.0056 (0.005250120162963867) + + + --> TIME: 2025-04-19 10:09:13 -- STEP: 70/142 -- GLOBAL_STEP: 477900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.608787536621094 (17.903453527178083) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018338026436068658 + | > current_lr_1: 0.00018338026436068658 + | > step_time: 0.3641 (0.3329360280718122) + | > loader_time: 0.006 (0.005524236815316336) + + + --> TIME: 2025-04-19 10:09:23 -- STEP: 95/142 -- GLOBAL_STEP: 477925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.708023071289062 (17.9823934454667) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018338026436068658 + | > current_lr_1: 0.00018338026436068658 + | > step_time: 0.3858 (0.3432782173156738) + | > loader_time: 0.0071 (0.005784742455733447) + + + --> TIME: 2025-04-19 10:09:33 -- STEP: 120/142 -- GLOBAL_STEP: 477950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.674633026123047 (17.92838017940521) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018338026436068658 + | > current_lr_1: 0.00018338026436068658 + | > step_time: 0.4108 (0.35461214979489647) + | > loader_time: 0.0069 (0.006033901373545327) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.907412528991699 (2.907412528991699) + | > loss_feat: nan (nan) + | > loss_mel: 19.53169822692871 (19.53169822692871) + | > loss_duration: 1.8175193071365356 (1.8175193071365356) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2329704761505127 (3.2329704761505127) + | > loss_feat: nan (nan) + | > loss_mel: 19.613191604614258 (19.613191604614258) + | > loss_duration: 1.881769061088562 (1.881769061088562) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8488364219665527 (3.0409034490585327) + | > loss_feat: nan (nan) + | > loss_mel: 19.611042022705078 (19.612116813659668) + | > loss_duration: 1.8001413345336914 (1.8409551978111267) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037589073181152344 (+0.0005844831466674805) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0409034490585327 (-0.05651593208312988) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 19.612116813659668 (+2.117453098297119) + | > avg_loss_duration: 1.8409551978111267 (-0.017647206783294678) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 695/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:09:44)  + + --> TIME: 2025-04-19 10:09:45 -- STEP: 3/142 -- GLOBAL_STEP: 477975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.93559455871582 (18.034699122111004) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.3133 (0.305803378423055) + | > loader_time: 0.0046 (0.00419163703918457) + + + --> TIME: 2025-04-19 10:09:53 -- STEP: 28/142 -- GLOBAL_STEP: 478000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.42046356201172 (17.90207290649414) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.3298 (0.3129392777170454) + | > loader_time: 0.0056 (0.005032471248081752) + + + --> TIME: 2025-04-19 10:10:02 -- STEP: 53/142 -- GLOBAL_STEP: 478025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.12909698486328 (18.030187120977434) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.3643 (0.3295035002366552) + | > loader_time: 0.0076 (0.005537410952010245) + + + --> TIME: 2025-04-19 10:10:12 -- STEP: 78/142 -- GLOBAL_STEP: 478050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.640716552734375 (17.86423580463116) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.3723 (0.3473566678854135) + | > loader_time: 0.007 (0.00590950403458033) + + + --> TIME: 2025-04-19 10:10:23 -- STEP: 103/142 -- GLOBAL_STEP: 478075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.626664400100708 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.74660301208496 (17.79705426299456) + | > loss_duration: 1.694183349609375 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.3905 (0.3624162095264323) + | > loader_time: 0.0074 (0.006350144599247906) + + + --> TIME: 2025-04-19 10:10:33 -- STEP: 128/142 -- GLOBAL_STEP: 478100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.006044387817383 (17.788209252059453) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018335734182764148 + | > current_lr_1: 0.00018335734182764148 + | > step_time: 0.4129 (0.3707573693245649) + | > loader_time: 0.0076 (0.006540324538946153) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9710216522216797 (2.9710216522216797) + | > loss_feat: nan (nan) + | > loss_mel: 18.009227752685547 (18.009227752685547) + | > loss_duration: 1.831252098083496 (1.831252098083496) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2935469150543213 (3.2935469150543213) + | > loss_feat: nan (nan) + | > loss_mel: 16.660171508789062 (16.660171508789062) + | > loss_duration: 1.9017534255981445 (1.9017534255981445) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8396379947662354 (3.0665924549102783) + | > loss_feat: nan (nan) + | > loss_mel: 18.221094131469727 (17.440632820129395) + | > loss_duration: 1.8159900903701782 (1.8588717579841614) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003480076789855957 (-0.00027883052825927734) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0665924549102783 (+0.025689005851745605) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.440632820129395 (-2.1714839935302734) + | > avg_loss_duration: 1.8588717579841614 (+0.017916560173034668) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 696/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:10:40)  + + --> TIME: 2025-04-19 10:10:45 -- STEP: 11/142 -- GLOBAL_STEP: 478125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.517848014831543 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.021774291992188 (17.464631860906426) + | > loss_duration: 1.602769136428833 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.3039 (0.3036170656030828) + | > loader_time: 0.0056 (0.004885305057872425) + + + --> TIME: 2025-04-19 10:10:53 -- STEP: 36/142 -- GLOBAL_STEP: 478150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.565732955932617 (17.796998818715416) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.3296 (0.31589476929770577) + | > loader_time: 0.0053 (0.005418214533064101) + + + --> TIME: 2025-04-19 10:11:02 -- STEP: 61/142 -- GLOBAL_STEP: 478175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.03529930114746 (17.795452258625964) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.3595 (0.3275423597116939) + | > loader_time: 0.0061 (0.005804171327684748) + + + --> TIME: 2025-04-19 10:11:11 -- STEP: 86/142 -- GLOBAL_STEP: 478200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.581029176712036 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.20254898071289 (17.823500112045647) + | > loss_duration: 1.6802340745925903 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.3778 (0.3395106459772865) + | > loader_time: 0.0066 (0.006078248800233354) + + + --> TIME: 2025-04-19 10:11:21 -- STEP: 111/142 -- GLOBAL_STEP: 478225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.43035888671875 (17.783927908888824) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.3994 (0.3499503371951817) + | > loader_time: 0.0074 (0.006334541080234288) + + + --> TIME: 2025-04-19 10:11:32 -- STEP: 136/142 -- GLOBAL_STEP: 478250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.557716369628906 (17.809744161718044) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018333442215991301 + | > current_lr_1: 0.00018333442215991301 + | > step_time: 0.4388 (0.3627330362796784) + | > loader_time: 0.0092 (0.006607409785775579) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.998971939086914 (2.998971939086914) + | > loss_feat: nan (nan) + | > loss_mel: 20.04143714904785 (20.04143714904785) + | > loss_duration: 1.8355469703674316 (1.8355469703674316) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.5954532623291016 (3.5954532623291016) + | > loss_feat: nan (nan) + | > loss_mel: 15.977259635925293 (15.977259635925293) + | > loss_duration: 1.8980789184570312 (1.8980789184570312) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9256577491760254 (3.2605555057525635) + | > loss_feat: nan (nan) + | > loss_mel: 17.21256446838379 (16.59491205215454) + | > loss_duration: 1.777880311012268 (1.8379796147346497) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037555694580078125 (+0.00027549266815185547) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2605555057525635 (+0.19396305084228516) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.59491205215454 (-0.8457207679748535) + | > avg_loss_duration: 1.8379796147346497 (-0.02089214324951172) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 697/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:11:36)  + + --> TIME: 2025-04-19 10:11:42 -- STEP: 19/142 -- GLOBAL_STEP: 478275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.70709991455078 (17.93001245197497) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000183311505357143 + | > current_lr_1: 0.000183311505357143 + | > step_time: 0.3132 (0.3054304248408267) + | > loader_time: 0.0051 (0.005112635461907638) + + + --> TIME: 2025-04-19 10:11:51 -- STEP: 44/142 -- GLOBAL_STEP: 478300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.795406341552734 (18.010295867919925) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000183311505357143 + | > current_lr_1: 0.000183311505357143 + | > step_time: 0.3433 (0.32315722920677875) + | > loader_time: 0.007 (0.005572519519112327) + + + --> TIME: 2025-04-19 10:12:00 -- STEP: 69/142 -- GLOBAL_STEP: 478325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6230454444885254 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.60221290588379 (17.884313583374034) + | > loss_duration: 1.712984561920166 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000183311505357143 + | > current_lr_1: 0.000183311505357143 + | > step_time: 0.3827 (0.3377566233925197) + | > loader_time: 0.0068 (0.005953833676766658) + + + --> TIME: 2025-04-19 10:12:10 -- STEP: 94/142 -- GLOBAL_STEP: 478350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.472089767456055 (17.985078284081006) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000183311505357143 + | > current_lr_1: 0.000183311505357143 + | > step_time: 0.3847 (0.3505075307602578) + | > loader_time: 0.0065 (0.006273974763586166) + + + --> TIME: 2025-04-19 10:12:21 -- STEP: 119/142 -- GLOBAL_STEP: 478375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.323471069335938 (17.990970194840642) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.000183311505357143 + | > current_lr_1: 0.000183311505357143 + | > step_time: 0.4176 (0.362771817616054) + | > loader_time: 0.0077 (0.00657049347372616) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9966208934783936 (2.9966208934783936) + | > loss_feat: nan (nan) + | > loss_mel: 19.206369400024414 (19.206369400024414) + | > loss_duration: 1.8350436687469482 (1.8350436687469482) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4608352184295654 (3.4608352184295654) + | > loss_feat: nan (nan) + | > loss_mel: 18.914508819580078 (18.914508819580078) + | > loss_duration: 1.9256176948547363 (1.9256176948547363) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.00754451751709 (3.2341898679733276) + | > loss_feat: nan (nan) + | > loss_mel: 18.91521644592285 (18.914862632751465) + | > loss_duration: 1.7854321002960205 (1.8555248975753784) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004035592079162598 (+0.00028002262115478516) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.2341898679733276 (-0.02636563777923584) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.914862632751465 (+2.319950580596924) + | > avg_loss_duration: 1.8555248975753784 (+0.01754528284072876) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 698/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:12:32)  + + --> TIME: 2025-04-19 10:12:34 -- STEP: 2/142 -- GLOBAL_STEP: 478400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5125110149383545 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.986223220825195 (18.782034873962402) + | > loss_duration: 1.6423808336257935 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.2984 (0.3007814884185791) + | > loader_time: 0.004 (0.003980398178100586) + + + --> TIME: 2025-04-19 10:12:42 -- STEP: 27/142 -- GLOBAL_STEP: 478425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.626859664916992 (17.840657975938576) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.3291 (0.3148089956354213) + | > loader_time: 0.0055 (0.005254754313716182) + + + --> TIME: 2025-04-19 10:12:51 -- STEP: 52/142 -- GLOBAL_STEP: 478450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.98894500732422 (17.730532022622914) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.3427 (0.3431328672629137) + | > loader_time: 0.0059 (0.006146522668691782) + + + --> TIME: 2025-04-19 10:13:00 -- STEP: 77/142 -- GLOBAL_STEP: 478475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.624719619750977 (17.799380711146767) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.3653 (0.3489711996796844) + | > loader_time: 0.0067 (0.006270582025701349) + + + --> TIME: 2025-04-19 10:13:10 -- STEP: 102/142 -- GLOBAL_STEP: 478500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.89324378967285 (17.81892444573197) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.3938 (0.35646907254761345) + | > loader_time: 0.0071 (0.006449458645839317) + + + --> TIME: 2025-04-19 10:13:21 -- STEP: 127/142 -- GLOBAL_STEP: 478525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.041513442993164 (17.781692144438974) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018328859141897335 + | > current_lr_1: 0.00018328859141897335 + | > step_time: 0.4167 (0.3663549704814521) + | > loader_time: 0.0076 (0.006677708287877361) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.925807237625122 (2.925807237625122) + | > loss_feat: nan (nan) + | > loss_mel: 17.513662338256836 (17.513662338256836) + | > loss_duration: 1.8472967147827148 (1.8472967147827148) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.336449384689331 (3.336449384689331) + | > loss_feat: nan (nan) + | > loss_mel: 16.710241317749023 (16.710241317749023) + | > loss_duration: 1.9093929529190063 (1.9093929529190063) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8462131023406982 (3.0913312435150146) + | > loss_feat: nan (nan) + | > loss_mel: 15.528566360473633 (16.119403839111328) + | > loss_duration: 1.8004677295684814 (1.854930341243744) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003430485725402832 (-0.0006051063537597656) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.0913312435150146 (-0.142858624458313) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.119403839111328 (-2.7954587936401367) + | > avg_loss_duration: 1.854930341243744 (-0.0005945563316345215) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 699/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:13:28)  + + --> TIME: 2025-04-19 10:13:32 -- STEP: 10/142 -- GLOBAL_STEP: 478550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.28486442565918 (18.330350875854492) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.3307 (0.3161593198776245) + | > loader_time: 0.0077 (0.005488729476928711) + + + --> TIME: 2025-04-19 10:13:41 -- STEP: 35/142 -- GLOBAL_STEP: 478575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.73604965209961 (18.109283992222377) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.3398 (0.327033485685076) + | > loader_time: 0.0062 (0.005895941598074776) + + + --> TIME: 2025-04-19 10:13:50 -- STEP: 60/142 -- GLOBAL_STEP: 478600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.619236707687378 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.647680282592773 (17.933649237950647) + | > loss_duration: 1.670398235321045 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.3676 (0.33809032440185555) + | > loader_time: 0.0066 (0.006070991357167561) + + + --> TIME: 2025-04-19 10:14:00 -- STEP: 85/142 -- GLOBAL_STEP: 478625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6994619369506836 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.58270835876465 (17.887811324175676) + | > loss_duration: 1.6471970081329346 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.3836 (0.3506964711581959) + | > loader_time: 0.0083 (0.006347479539759019) + + + --> TIME: 2025-04-19 10:14:10 -- STEP: 110/142 -- GLOBAL_STEP: 478650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.764406204223633 (17.838990402221686) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.4097 (0.36238107247786083) + | > loader_time: 0.0075 (0.006576824188232422) + + + --> TIME: 2025-04-19 10:14:21 -- STEP: 135/142 -- GLOBAL_STEP: 478675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.207658767700195 (17.82184723748102) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018326568034504598 + | > current_lr_1: 0.00018326568034504598 + | > step_time: 0.4317 (0.37213406032986107) + | > loader_time: 0.0083 (0.006818707784016927) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.873091459274292 (2.873091459274292) + | > loss_feat: nan (nan) + | > loss_mel: 16.91850471496582 (16.91850471496582) + | > loss_duration: 1.8528838157653809 (1.8528838157653809) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.159491539001465 (3.159491539001465) + | > loss_feat: nan (nan) + | > loss_mel: 17.589452743530273 (17.589452743530273) + | > loss_duration: 1.8952646255493164 (1.8952646255493164) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.667405366897583 (2.913448452949524) + | > loss_feat: nan (nan) + | > loss_mel: 16.27703094482422 (16.933241844177246) + | > loss_duration: 1.7946791648864746 (1.8449718952178955) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032806396484375 (-0.00014984607696533203) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.913448452949524 (-0.17788279056549072) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 16.933241844177246 (+0.813838005065918) + | > avg_loss_duration: 1.8449718952178955 (-0.009958446025848389) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 700/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:14:25)  + + --> TIME: 2025-04-19 10:14:31 -- STEP: 18/142 -- GLOBAL_STEP: 478700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.126266479492188 (17.647038247850205) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018324277213500285 + | > current_lr_1: 0.00018324277213500285 + | > step_time: 0.3111 (0.30471649434831405) + | > loader_time: 0.0054 (0.005092713567945693) + + + --> TIME: 2025-04-19 10:14:40 -- STEP: 43/142 -- GLOBAL_STEP: 478725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.864792823791504 (17.820227001988606) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018324277213500285 + | > current_lr_1: 0.00018324277213500285 + | > step_time: 0.3419 (0.31874253583508866) + | > loader_time: 0.0056 (0.005460533984871799) + + + --> TIME: 2025-04-19 10:14:49 -- STEP: 68/142 -- GLOBAL_STEP: 478750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.935060501098633 (17.8585544053246) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018324277213500285 + | > current_lr_1: 0.00018324277213500285 + | > step_time: 0.3632 (0.32979168611414295) + | > loader_time: 0.0066 (0.005760725806741154) + + + --> TIME: 2025-04-19 10:14:58 -- STEP: 93/142 -- GLOBAL_STEP: 478775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.49112319946289 (17.87213322424119) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018324277213500285 + | > current_lr_1: 0.00018324277213500285 + | > step_time: 0.3746 (0.3408510838785479) + | > loader_time: 0.007 (0.006032856561804333) + + + --> TIME: 2025-04-19 10:15:08 -- STEP: 118/142 -- GLOBAL_STEP: 478800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.737037658691406 (17.799005168979456) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018324277213500285 + | > current_lr_1: 0.00018324277213500285 + | > step_time: 0.418 (0.35167656106463935) + | > loader_time: 0.0074 (0.006316138526140635) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.101754665374756 (3.101754665374756) + | > loss_feat: nan (nan) + | > loss_mel: 18.809551239013672 (18.809551239013672) + | > loss_duration: 1.834570050239563 (1.834570050239563) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.371723175048828 (3.371723175048828) + | > loss_feat: nan (nan) + | > loss_mel: 17.825223922729492 (17.825223922729492) + | > loss_duration: 1.900503396987915 (1.900503396987915) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8906471729278564 (3.1311851739883423) + | > loss_feat: nan (nan) + | > loss_mel: 19.036785125732422 (18.431004524230957) + | > loss_duration: 1.7983014583587646 (1.8494024276733398) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033559799194335938 (+7.534027099609375e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1311851739883423 (+0.21773672103881836) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.431004524230957 (+1.497762680053711) + | > avg_loss_duration: 1.8494024276733398 (+0.004430532455444336) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 701/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:15:20)  + + --> TIME: 2025-04-19 10:15:21 -- STEP: 1/142 -- GLOBAL_STEP: 478825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.455617904663086 (18.455617904663086) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.3015 (0.3014535903930664) + | > loader_time: 0.004 (0.004034757614135742) + + + --> TIME: 2025-04-19 10:15:29 -- STEP: 26/142 -- GLOBAL_STEP: 478850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.0031795501709 (18.05424620554997) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.3386 (0.3103576715175922) + | > loader_time: 0.0059 (0.005164797489459698) + + + --> TIME: 2025-04-19 10:15:38 -- STEP: 51/142 -- GLOBAL_STEP: 478875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.708833694458008 (18.042556145611925) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.3527 (0.3287597170063095) + | > loader_time: 0.0062 (0.005624850591023763) + + + --> TIME: 2025-04-19 10:15:47 -- STEP: 76/142 -- GLOBAL_STEP: 478900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.04052734375 (18.118131562283164) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.38 (0.34465055716665177) + | > loader_time: 0.0069 (0.0060515905681409335) + + + --> TIME: 2025-04-19 10:15:57 -- STEP: 101/142 -- GLOBAL_STEP: 478925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.022029876708984 (18.076345811976076) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.3919 (0.3561875631313514) + | > loader_time: 0.0073 (0.006277457322224532) + + + --> TIME: 2025-04-19 10:16:08 -- STEP: 126/142 -- GLOBAL_STEP: 478950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.00947380065918 (17.99089459010534) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018321986678848597 + | > current_lr_1: 0.00018321986678848597 + | > step_time: 0.4318 (0.3677689310104129) + | > loader_time: 0.0077 (0.0065868619888547865) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.689845085144043 (2.689845085144043) + | > loss_feat: nan (nan) + | > loss_mel: 18.547754287719727 (18.547754287719727) + | > loss_duration: 1.8308700323104858 (1.8308700323104858) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4252099990844727 (3.4252099990844727) + | > loss_feat: nan (nan) + | > loss_mel: 18.12859344482422 (18.12859344482422) + | > loss_duration: 1.8856496810913086 (1.8856496810913086) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7797811031341553 (3.102495551109314) + | > loss_feat: nan (nan) + | > loss_mel: 17.48103904724121 (17.804816246032715) + | > loss_duration: 1.8234022855758667 (1.8545259833335876) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003641366958618164 (+0.0002853870391845703) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.102495551109314 (-0.02868962287902832) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.804816246032715 (-0.6261882781982422) + | > avg_loss_duration: 1.8545259833335876 (+0.005123555660247803) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 702/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:16:17)  + + --> TIME: 2025-04-19 10:16:20 -- STEP: 9/142 -- GLOBAL_STEP: 478975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.520309448242188 (18.13020812140571) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.302 (0.30748213662041557) + | > loader_time: 0.0046 (0.004773139953613281) + + + --> TIME: 2025-04-19 10:16:29 -- STEP: 34/142 -- GLOBAL_STEP: 479000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6061370372772217 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.248640060424805 (17.854237949146942) + | > loss_duration: 1.6118838787078857 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.331 (0.31673697864308076) + | > loader_time: 0.0056 (0.005378386553596048) + + + --> TIME: 2025-04-19 10:16:37 -- STEP: 59/142 -- GLOBAL_STEP: 479025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.582054138183594 (17.79296272083864) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.3507 (0.3272522101968022) + | > loader_time: 0.0064 (0.005654840146080922) + + + --> TIME: 2025-04-19 10:16:47 -- STEP: 84/142 -- GLOBAL_STEP: 479050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.572065591812134 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.442975997924805 (17.8739028544653) + | > loss_duration: 1.6950714588165283 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.3709 (0.339484277225676) + | > loader_time: 0.0063 (0.006038058371770948) + + + --> TIME: 2025-04-19 10:16:57 -- STEP: 109/142 -- GLOBAL_STEP: 479075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.15427589416504 (17.961496711870947) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.4001 (0.35479908470713767) + | > loader_time: 0.0078 (0.006485256580037807) + + + --> TIME: 2025-04-19 10:17:08 -- STEP: 134/142 -- GLOBAL_STEP: 479100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.925859451293945 (17.91925366956796) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001831969643051374 + | > current_lr_1: 0.0001831969643051374 + | > step_time: 0.4157 (0.3663350895269593) + | > loader_time: 0.0078 (0.006715073514340528) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8793468475341797 (2.8793468475341797) + | > loss_feat: nan (nan) + | > loss_mel: 16.186681747436523 (16.186681747436523) + | > loss_duration: 1.8339582681655884 (1.8339582681655884) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.328134059906006 (3.328134059906006) + | > loss_feat: nan (nan) + | > loss_mel: 19.188438415527344 (19.188438415527344) + | > loss_duration: 1.9258848428726196 (1.9258848428726196) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9744346141815186 (3.151284337043762) + | > loss_feat: nan (nan) + | > loss_mel: 17.17247772216797 (18.180458068847656) + | > loss_duration: 1.7659022808074951 (1.8458935618400574) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003908872604370117 (+0.0002675056457519531) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.151284337043762 (+0.04878878593444824) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.180458068847656 (+0.3756418228149414) + | > avg_loss_duration: 1.8458935618400574 (-0.008632421493530273) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 703/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:17:13)  + + --> TIME: 2025-04-19 10:17:19 -- STEP: 17/142 -- GLOBAL_STEP: 479125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.5470027923584 (17.991997326121606) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018317406468459923 + | > current_lr_1: 0.00018317406468459923 + | > step_time: 0.3179 (0.31473781080806956) + | > loader_time: 0.0054 (0.005741680369657629) + + + --> TIME: 2025-04-19 10:17:28 -- STEP: 42/142 -- GLOBAL_STEP: 479150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.638381958007812 (18.1087965284075) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018317406468459923 + | > current_lr_1: 0.00018317406468459923 + | > step_time: 0.3487 (0.3307559149605887) + | > loader_time: 0.0064 (0.005991316977001372) + + + --> TIME: 2025-04-19 10:17:37 -- STEP: 67/142 -- GLOBAL_STEP: 479175 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.694293975830078 (18.076854335727976) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018317406468459923 + | > current_lr_1: 0.00018317406468459923 + | > step_time: 0.369 (0.34466544194007986) + | > loader_time: 0.0083 (0.0062648395993816315) + + + --> TIME: 2025-04-19 10:17:47 -- STEP: 92/142 -- GLOBAL_STEP: 479200 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.48810386657715 (18.082721150439717) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018317406468459923 + | > current_lr_1: 0.00018317406468459923 + | > step_time: 0.4035 (0.3556431920632072) + | > loader_time: 0.0073 (0.006503872249437415) + + + --> TIME: 2025-04-19 10:17:58 -- STEP: 117/142 -- GLOBAL_STEP: 479225 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.69635772705078 (18.014626861637467) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018317406468459923 + | > current_lr_1: 0.00018317406468459923 + | > step_time: 0.4253 (0.36743873612493533) + | > loader_time: 0.0075 (0.006791762816600311) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.798736810684204 (2.798736810684204) + | > loss_feat: nan (nan) + | > loss_mel: 19.616329193115234 (19.616329193115234) + | > loss_duration: 1.811436414718628 (1.811436414718628) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.2356014251708984 (3.2356014251708984) + | > loss_feat: nan (nan) + | > loss_mel: 18.194046020507812 (18.194046020507812) + | > loss_duration: 1.8976620435714722 (1.8976620435714722) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9233622550964355 (3.079481840133667) + | > loss_feat: nan (nan) + | > loss_mel: 19.4259090423584 (18.809977531433105) + | > loss_duration: 1.8181259632110596 (1.8578940033912659) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0032314062118530273 (-0.0006774663925170898) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.079481840133667 (-0.07180249691009521) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.809977531433105 (+0.6295194625854492) + | > avg_loss_duration: 1.8578940033912659 (+0.012000441551208496) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 704/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:18:10)  + + --> TIME: 2025-04-19 10:18:11 -- STEP: 0/142 -- GLOBAL_STEP: 479250 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.725393295288086 (18.725393295288086) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.3382 (0.3381843566894531) + | > loader_time: 0.3276 (0.3276400566101074) + + + --> TIME: 2025-04-19 10:18:19 -- STEP: 25/142 -- GLOBAL_STEP: 479275 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5418105125427246 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.736087799072266 (18.14123046875) + | > loss_duration: 1.6677231788635254 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.3323 (0.31291915893554695) + | > loader_time: 0.0063 (0.0052350425720214845) + + + --> TIME: 2025-04-19 10:18:28 -- STEP: 50/142 -- GLOBAL_STEP: 479300 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.602386474609375 (17.953791007995612) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.3488 (0.335774440765381) + | > loader_time: 0.006 (0.0058112478256225585) + + + --> TIME: 2025-04-19 10:18:37 -- STEP: 75/142 -- GLOBAL_STEP: 479325 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.800148010253906 (17.905808703104658) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.3747 (0.3435408401489259) + | > loader_time: 0.0069 (0.006026541392008463) + + + --> TIME: 2025-04-19 10:18:47 -- STEP: 100/142 -- GLOBAL_STEP: 479350 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.769899368286133 (17.928617982864374) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.3895 (0.3520891356468202) + | > loader_time: 0.0074 (0.006287930011749265) + + + --> TIME: 2025-04-19 10:18:57 -- STEP: 125/142 -- GLOBAL_STEP: 479375 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.217985153198242 (17.901172531127923) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018315116792651365 + | > current_lr_1: 0.00018315116792651365 + | > step_time: 0.4126 (0.36225200653076184) + | > loader_time: 0.0076 (0.006536340713500975) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.7487878799438477 (2.7487878799438477) + | > loss_feat: nan (nan) + | > loss_mel: 17.757410049438477 (17.757410049438477) + | > loss_duration: 1.8221884965896606 (1.8221884965896606) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.3892059326171875 (3.3892059326171875) + | > loss_feat: nan (nan) + | > loss_mel: 18.905746459960938 (18.905746459960938) + | > loss_duration: 1.8936232328414917 (1.8936232328414917) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.011401653289795 (3.200303792953491) + | > loss_feat: nan (nan) + | > loss_mel: 19.037206649780273 (18.971476554870605) + | > loss_duration: 1.791637659072876 (1.8426304459571838) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003399968147277832 (+0.0001685619354248047) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.200303792953491 (+0.12082195281982422) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.971476554870605 (+0.1614990234375) + | > avg_loss_duration: 1.8426304459571838 (-0.015263557434082031) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 705/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:19:06)  + + --> TIME: 2025-04-19 10:19:09 -- STEP: 8/142 -- GLOBAL_STEP: 479400 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.53157615661621 (18.06847071647644) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.3003 (0.29984548687934875) + | > loader_time: 0.0054 (0.004652947187423706) + + + --> TIME: 2025-04-19 10:19:18 -- STEP: 33/142 -- GLOBAL_STEP: 479425 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.303939819335938 (17.691718101501465) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.3828 (0.33079101822592993) + | > loader_time: 0.0061 (0.005678436972878196) + + + --> TIME: 2025-04-19 10:19:27 -- STEP: 58/142 -- GLOBAL_STEP: 479450 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.165264129638672 (17.903707701584388) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.3631 (0.3410012598695426) + | > loader_time: 0.0071 (0.005898389323004361) + + + --> TIME: 2025-04-19 10:19:36 -- STEP: 83/142 -- GLOBAL_STEP: 479475 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.6421422958374023 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.893203735351562 (17.889382580676717) + | > loss_duration: 1.7254669666290283 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.3868 (0.35213162238339346) + | > loader_time: 0.0067 (0.00616307143705437) + + + --> TIME: 2025-04-19 10:19:47 -- STEP: 108/142 -- GLOBAL_STEP: 479500 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.926353454589844 (17.812127272288006) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.402 (0.36369980485351) + | > loader_time: 0.0073 (0.006466850086494729) + + + --> TIME: 2025-04-19 10:19:58 -- STEP: 133/142 -- GLOBAL_STEP: 479525 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.875455856323242 (17.852527360270795) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018312827403052281 + | > current_lr_1: 0.00018312827403052281 + | > step_time: 0.4347 (0.37674616333237265) + | > loader_time: 0.0082 (0.006725044178783446) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9239425659179688 (2.9239425659179688) + | > loss_feat: nan (nan) + | > loss_mel: 19.876585006713867 (19.876585006713867) + | > loss_duration: 1.827856183052063 (1.827856183052063) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4357051849365234 (3.4357051849365234) + | > loss_feat: nan (nan) + | > loss_mel: 17.719219207763672 (17.719219207763672) + | > loss_duration: 1.8797918558120728 (1.8797918558120728) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8700172901153564 (3.15286123752594) + | > loss_feat: nan (nan) + | > loss_mel: 18.662063598632812 (18.190641403198242) + | > loss_duration: 1.7924365997314453 (1.836114227771759) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038298368453979492 (+0.0004298686981201172) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.15286123752594 (-0.04744255542755127) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 18.190641403198242 (-0.7808351516723633) + | > avg_loss_duration: 1.836114227771759 (-0.006516218185424805) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 706/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:20:03)  + + --> TIME: 2025-04-19 10:20:09 -- STEP: 16/142 -- GLOBAL_STEP: 479550 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.3862380981445312 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.99120330810547 (17.850828170776367) + | > loss_duration: 1.590522289276123 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.3213 (0.30790190398693085) + | > loader_time: 0.0056 (0.005095422267913818) + + + --> TIME: 2025-04-19 10:20:17 -- STEP: 41/142 -- GLOBAL_STEP: 479575 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.56427001953125 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.947479248046875 (18.017328076246308) + | > loss_duration: 1.6024653911590576 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.3365 (0.3208451561811494) + | > loader_time: 0.0063 (0.005503695185591534) + + + --> TIME: 2025-04-19 10:20:26 -- STEP: 66/142 -- GLOBAL_STEP: 479600 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.712614059448242 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.20901870727539 (17.95280642942948) + | > loss_duration: 1.6088567972183228 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.3611 (0.3323853304891875) + | > loader_time: 0.0067 (0.005847797249302719) + + + --> TIME: 2025-04-19 10:20:36 -- STEP: 91/142 -- GLOBAL_STEP: 479625 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.214021682739258 (17.833715407402956) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.3775 (0.3434003578437553) + | > loader_time: 0.0071 (0.006148747035435268) + + + --> TIME: 2025-04-19 10:20:46 -- STEP: 116/142 -- GLOBAL_STEP: 479650 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.211151123046875 (17.83065966079975) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.4042 (0.3536628155872739) + | > loader_time: 0.0076 (0.006416109101525669) + + + --> TIME: 2025-04-19 10:20:56 -- STEP: 141/142 -- GLOBAL_STEP: 479675 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.332168579101562 (17.774512608846027) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018310538299626898 + | > current_lr_1: 0.00018310538299626898 + | > step_time: 0.1808 (0.36380115975724886) + | > loader_time: 0.0027 (0.0066074733193039045) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9881277084350586 (2.9881277084350586) + | > loss_feat: nan (nan) + | > loss_mel: 18.579689025878906 (18.579689025878906) + | > loss_duration: 1.8425555229187012 (1.8425555229187012) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4646990299224854 (3.4646990299224854) + | > loss_feat: nan (nan) + | > loss_mel: 18.118192672729492 (18.118192672729492) + | > loss_duration: 1.911635160446167 (1.911635160446167) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.808607578277588 (3.1366533041000366) + | > loss_feat: nan (nan) + | > loss_mel: 17.780656814575195 (17.949424743652344) + | > loss_duration: 1.803598165512085 (1.857616662979126) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003343820571899414 (-0.00048601627349853516) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.1366533041000366 (-0.01620793342590332) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.949424743652344 (-0.24121665954589844) + | > avg_loss_duration: 1.857616662979126 (+0.021502435207366943) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 707/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:20:59)  + + --> TIME: 2025-04-19 10:21:07 -- STEP: 24/142 -- GLOBAL_STEP: 479700 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.37220573425293 (18.16014544169108) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018308249482339444 + | > current_lr_1: 0.00018308249482339444 + | > step_time: 0.3332 (0.3116915722688039) + | > loader_time: 0.0059 (0.005339701970418294) + + + --> TIME: 2025-04-19 10:21:16 -- STEP: 49/142 -- GLOBAL_STEP: 479725 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.4693849086761475 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.35881233215332 (17.98571111718003) + | > loss_duration: 1.5894567966461182 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018308249482339444 + | > current_lr_1: 0.00018308249482339444 + | > step_time: 0.3552 (0.3287006203009158) + | > loader_time: 0.0063 (0.0057040428628726885) + + + --> TIME: 2025-04-19 10:21:25 -- STEP: 74/142 -- GLOBAL_STEP: 479750 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.638087272644043 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.68497657775879 (17.921540118552542) + | > loss_duration: 1.6548702716827393 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018308249482339444 + | > current_lr_1: 0.00018308249482339444 + | > step_time: 0.3769 (0.3425697152679031) + | > loader_time: 0.0068 (0.006029551093642775) + + + --> TIME: 2025-04-19 10:21:35 -- STEP: 99/142 -- GLOBAL_STEP: 479775 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5739710330963135 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.92483901977539 (17.986538848491634) + | > loss_duration: 1.6803333759307861 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018308249482339444 + | > current_lr_1: 0.00018308249482339444 + | > step_time: 0.3906 (0.3552876096783262) + | > loader_time: 0.007 (0.00625534491105513) + + + --> TIME: 2025-04-19 10:21:46 -- STEP: 124/142 -- GLOBAL_STEP: 479800 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.165245056152344 (17.969398021697994) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018308249482339444 + | > current_lr_1: 0.00018308249482339444 + | > step_time: 0.425 (0.36851843326322486) + | > loader_time: 0.0137 (0.006633150962091261) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.947507381439209 (2.947507381439209) + | > loss_feat: nan (nan) + | > loss_mel: 15.766499519348145 (15.766499519348145) + | > loss_duration: 1.821759819984436 (1.821759819984436) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.524505853652954 (3.524505853652954) + | > loss_feat: nan (nan) + | > loss_mel: 19.083127975463867 (19.083127975463867) + | > loss_duration: 1.8934478759765625 (1.8934478759765625) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.786115884780884 (3.155310869216919) + | > loss_feat: nan (nan) + | > loss_mel: 16.061752319335938 (17.572440147399902) + | > loss_duration: 1.7807159423828125 (1.8370819091796875) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003575563430786133 (+0.00023174285888671875) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.155310869216919 (+0.018657565116882324) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.572440147399902 (-0.3769845962524414) + | > avg_loss_duration: 1.8370819091796875 (-0.020534753799438477) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 708/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:21:55)  + + --> TIME: 2025-04-19 10:21:58 -- STEP: 7/142 -- GLOBAL_STEP: 479825 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 18.41487693786621 (18.11138698032924) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.322 (0.3075862612043108) + | > loader_time: 0.0069 (0.005366768155779157) + + + --> TIME: 2025-04-19 10:22:07 -- STEP: 32/142 -- GLOBAL_STEP: 479850 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.81564712524414 (18.086699068546302) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.3314 (0.31871238350868225) + | > loader_time: 0.006 (0.00569087266921997) + + + --> TIME: 2025-04-19 10:22:15 -- STEP: 57/142 -- GLOBAL_STEP: 479875 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.862390518188477 (17.99993203815661) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.3515 (0.3282944946958309) + | > loader_time: 0.006 (0.005922125096906696) + + + --> TIME: 2025-04-19 10:22:25 -- STEP: 82/142 -- GLOBAL_STEP: 479900 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.634560585021973 (17.96386867616235) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.3745 (0.3400577161370254) + | > loader_time: 0.0074 (0.0061306633600374544) + + + --> TIME: 2025-04-19 10:22:35 -- STEP: 107/142 -- GLOBAL_STEP: 479925 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.131813049316406 (17.965764892435516) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.395 (0.3542658859324232) + | > loader_time: 0.007 (0.006535761824278074) + + + --> TIME: 2025-04-19 10:22:46 -- STEP: 132/142 -- GLOBAL_STEP: 479950 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 15.934038162231445 (17.941994175766446) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.0001830596095115415 + | > current_lr_1: 0.0001830596095115415 + | > step_time: 0.4237 (0.36544577100060205) + | > loader_time: 0.0072 (0.00672120036500873) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9780163764953613 (2.9780163764953613) + | > loss_feat: nan (nan) + | > loss_mel: 18.628461837768555 (18.628461837768555) + | > loss_duration: 1.8518867492675781 (1.8518867492675781) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.4060046672821045 (3.4060046672821045) + | > loss_feat: nan (nan) + | > loss_mel: 17.574594497680664 (17.574594497680664) + | > loss_duration: 1.917394757270813 (1.917394757270813) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.8493101596832275 (3.127657413482666) + | > loss_feat: nan (nan) + | > loss_mel: 17.638837814331055 (17.60671615600586) + | > loss_duration: 1.803694486618042 (1.8605446219444275) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003618597984313965 (+4.303455352783203e-05) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 3.127657413482666 (-0.02765345573425293) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.60671615600586 (+0.03427600860595703) + | > avg_loss_duration: 1.8605446219444275 (+0.02346271276473999) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 709/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:22:51)  + + --> TIME: 2025-04-19 10:22:57 -- STEP: 15/142 -- GLOBAL_STEP: 479975 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.309436798095703 (18.322523880004884) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.3079 (0.3032933870951335) + | > loader_time: 0.0054 (0.004752333958943685) + + + --> TIME: 2025-04-19 10:23:05 -- STEP: 40/142 -- GLOBAL_STEP: 480000 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 16.821393966674805 (18.054298400878906) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.3483 (0.32482091188430784) + | > loader_time: 0.0057 (0.0054958522319793705) + + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_480000.pth + + --> TIME: 2025-04-19 10:23:16 -- STEP: 65/142 -- GLOBAL_STEP: 480025 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.66341209411621 (18.110572961660537) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.3717 (0.3409972961132342) + | > loader_time: 0.0066 (0.006095783527080829) + + + --> TIME: 2025-04-19 10:23:26 -- STEP: 90/142 -- GLOBAL_STEP: 480050 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.587690353393555 (18.039061164855966) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.3781 (0.3498505221472847) + | > loader_time: 0.0072 (0.006348779466417101) + + + --> TIME: 2025-04-19 10:23:36 -- STEP: 115/142 -- GLOBAL_STEP: 480075 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 19.05148696899414 (18.013180401014246) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.4135 (0.35953654206317404) + | > loader_time: 0.0085 (0.006706851461659307) + + + --> TIME: 2025-04-19 10:23:47 -- STEP: 140/142 -- GLOBAL_STEP: 480100 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.277769088745117 (17.978858055387224) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018303672706035257 + | > current_lr_1: 0.00018303672706035257 + | > step_time: 0.4374 (0.37424882480076394) + | > loader_time: 0.0084 (0.00704345532826015) + + + > EVALUATION  + + --> STEP: 0 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.052715301513672 (3.052715301513672) + | > loss_feat: nan (nan) + | > loss_mel: 18.533790588378906 (18.533790588378906) + | > loss_duration: 1.8525347709655762 (1.8525347709655762) + | > loss_1: nan (nan) + + --> STEP: 1 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 3.085998058319092 (3.085998058319092) + | > loss_feat: nan (nan) + | > loss_mel: 17.168542861938477 (17.168542861938477) + | > loss_duration: 1.8713973760604858 (1.8713973760604858) + | > loss_1: nan (nan) + + --> STEP: 2 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > loss_gen: nan (nan) + | > loss_kl: 2.9127025604248047 (2.9993503093719482) + | > loss_feat: nan (nan) + | > loss_mel: 17.08501625061035 (17.126779556274414) + | > loss_duration: 1.781105399131775 (1.8262513875961304) + | > loss_1: nan (nan) + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0033545494079589844 (-0.00026404857635498047) + | > avg_loss_disc: nan (+nan) + | > avg_loss_disc_real_0: nan (+nan) + | > avg_loss_disc_real_1: nan (+nan) + | > avg_loss_disc_real_2: nan (+nan) + | > avg_loss_disc_real_3: nan (+nan) + | > avg_loss_disc_real_4: nan (+nan) + | > avg_loss_disc_real_5: nan (+nan) + | > avg_loss_0: nan (+nan) + | > avg_loss_gen: nan (+nan) + | > avg_loss_kl: 2.9993503093719482 (-0.12830710411071777) + | > avg_loss_feat: nan (+nan) + | > avg_loss_mel: 17.126779556274414 (-0.4799365997314453) + | > avg_loss_duration: 1.8262513875961304 (-0.03429323434829712) + | > avg_loss_1: nan (+nan) + + + > EPOCH: 710/1000 + --> /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000 + + > TRAINING (2025-04-19 10:23:49)  + + --> TIME: 2025-04-19 10:23:58 -- STEP: 23/142 -- GLOBAL_STEP: 480125 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: 2.5969033241271973 (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.331146240234375 (17.735680828923766) + | > loss_duration: 1.613903522491455 (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018301384746947002 + | > current_lr_1: 0.00018301384746947002 + | > step_time: 0.3195 (0.31220491036124853) + | > loader_time: 0.0049 (0.005230406056279721) + + + --> TIME: 2025-04-19 10:24:06 -- STEP: 48/142 -- GLOBAL_STEP: 480150 + | > loss_disc: nan (nan) + | > loss_disc_real_0: nan (nan) + | > loss_disc_real_1: nan (nan) + | > loss_disc_real_2: nan (nan) + | > loss_disc_real_3: nan (nan) + | > loss_disc_real_4: nan (nan) + | > loss_disc_real_5: nan (nan) + | > loss_0: nan (nan) + | > grad_norm_0: 0 (0.0) + | > loss_gen: nan (nan) + | > loss_kl: nan (nan) + | > loss_feat: nan (nan) + | > loss_mel: 17.730634689331055 (17.79601675271988) + | > loss_duration: nan (nan) + | > amp_scaler: 0.0 (0.0) + | > loss_1: nan (nan) + | > grad_norm_1: 0 (0.0) + | > current_lr_0: 0.00018301384746947002 + | > current_lr_1: 0.00018301384746947002 + | > step_time: 0.3482 (0.3226629942655563) + | > loader_time: 0.0059 (0.00556454062461853) + + > Keyboard interrupt detected. + > Saving model before exiting... + + > CHECKPOINT : /home/fatima-w/workpj/coqui/TTS/recipes/ljspeech/vits_tts/vits_farsi-April-18-2025_11+00PM-0000000/checkpoint_480160.pth