Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+peav_sortformer_train/checkpoints/peav_sortformer_train.nemo filter=lfs diff=lfs merge=lfs -text

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1314-epoch=0-last.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168fcc556f339689efb16b22d8d6222a7eb76e971f0a19d81d9599f0b5c5ef96
+size 1251559602

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1314-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ced2d1f4a2f261033d596a69d55b7097eb95c48ebee8517dc46d7fc89df98d3
+size 1251559602

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1348-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52aecebedd98ce1bb126732390254bd12caaaed91698800be432849bcc370e0e
+size 1251559602

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1382-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2345c2778a51324a64f7bafd924f804da4bf2918abb17186531f0d1f65da2333
+size 1251559602

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1458-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1023cbc9cad4e83bd782fc2e6bd9429facf9a4d18b7a2a88274b0bd7039ff9
+size 1251559027

peav_sortformer_train/checkpoints/peav_sortformer_train--val_der=0.1609-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:873ebc0a3928e07d4e42efc2c42c772e9a3a76de3c12619de3c57030316c2add
+size 1251559474

peav_sortformer_train/checkpoints/peav_sortformer_train.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbfcc298079572797332202d997c1a9db7a969b0b71a414903cc0965e8c31fff
+size 500469760

peav_sortformer_train/cmd-args.log

@@ -0,0 +1 @@
+scripts/peav_sortformer_train.py --config-path=../configs --config-name=peav_sortformer_4spk.yaml exp_manager.name=peav_sortformer_train exp_manager.exp_dir=./peav_sortformer_train

peav_sortformer_train/git-info.log

@@ -0,0 +1 @@
+commit hash: b1e2e995a240deef69a6b3a9e40693059b976a6c

peav_sortformer_train/lightning_logs.txt

@@ -0,0 +1,23 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+   | Name                | Type                | Params | Mode 
+---------------------------------------------------------------------
+0  | dac_vae             | DacEncoderVAE       | 27.7 M | train
+1  | data_proj           | Linear              | 99.1 K | train
+2  | encoder             | PEAVEncoderWrapper  | 89.1 M | train
+3  | sortformer_modules  | SortformerModules   | 187 K  | train
+4  | transformer_encoder | TransformerEncoder  | 8.0 M  | train
+5  | loss                | BCELoss             | 0      | train
+6  | _accuracy_test      | MultiBinaryAccuracy | 0      | train
+7  | _accuracy_train     | MultiBinaryAccuracy | 0      | train
+8  | _accuracy_valid     | MultiBinaryAccuracy | 0      | train
+9  | _accuracy_test_ats  | MultiBinaryAccuracy | 0      | train
+10 | _accuracy_train_ats | MultiBinaryAccuracy | 0      | train
+11 | _accuracy_valid_ats | MultiBinaryAccuracy | 0      | train
+---------------------------------------------------------------------
+93.8 M    Trainable params
+31.2 M    Non-trainable params
+125 M     Total params
+500.237   Total estimated model params size (MB)
+545       Modules in train mode
+0         Modules in eval mode

peav_sortformer_train/nemo_error_log.txt

@@ -0,0 +1,8 @@
+[NeMo W 2026-04-13 11:11:25 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:25 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo W 2026-04-13 11:11:25 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-13 11:11:25 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo W 2026-04-13 11:12:11 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=15` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-13 11:12:54 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=15` in the `DataLoader` to improve performance.
+    

peav_sortformer_train/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,254 @@
+[NeMo I 2026-04-13 11:11:25 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo I 2026-04-13 11:11:25 exp_manager:594] ExpManager schema
+[NeMo I 2026-04-13 11:11:25 exp_manager:595] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'run_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1}, 'create_early_stopping_callback': False, 'create_ipl_epoch_stopper_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'ipl_epoch_stopper_callback_params': {'enable_stop': True, 'stop_every_n_epochs': 1}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo W 2026-04-13 11:11:25 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:25 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:25 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:25 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:25 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:25 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:25 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:25 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:25 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:25 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:11:30 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7828e016d130>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo W 2026-04-13 11:12:11 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=15` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-13 11:12:54 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=15` in the `DataLoader` to improve performance.
+    
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118162.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.535194.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395654.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.733094.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493985.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072822.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771407.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.213656.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.942046.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.637489.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543324.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.729116.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.408545.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822026.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9141016.

peav_sortformer_train/nemo_log_globalrank-1_localrank-1.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:39 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:39 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:39 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:39 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:39 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:39 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:44 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:10 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x79ec3de84830>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118205.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351932.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395537.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.7330906.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493804.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072822.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771307.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.213653.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420345.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374886.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543212.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291036.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.4085367.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822011.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.914093.

peav_sortformer_train/nemo_log_globalrank-2_localrank-2.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:40 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:40 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:40 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:40 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:40 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:40 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:45 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:45 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:45 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:45 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:07 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x73feafc4b740>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118193.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351932.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395525.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.7330956.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493802.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.307285.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771292.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136545.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420338.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374915.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543217.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291129.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.4085355.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822057.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9140923.

peav_sortformer_train/nemo_log_globalrank-3_localrank-3.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:38 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:38 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:38 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:38 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:38 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:38 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:38 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:43 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:43 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:43 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:43 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:11:53 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x713d95e20740>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118226.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351937.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395597.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.733098.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493806.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072832.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.977131.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136538.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420352.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374905.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543234.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291148.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.408538.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.182203.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9140947.

peav_sortformer_train/nemo_log_globalrank-4_localrank-4.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:39 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:39 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:39 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:39 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:39 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:39 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:45 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:45 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:06 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7c7c4a659460>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118166.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351925.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395554.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.733096.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493816.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072834.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771283.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136545.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420424.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374896.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543217.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291138.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.4085386.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822033.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9140968.

peav_sortformer_train/nemo_log_globalrank-5_localrank-5.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:39 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:39 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:39 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:39 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:39 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:39 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:44 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:11 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x703b76086360>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118183.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351942.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.239556.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.7330983.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.449381.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072872.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771423.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136571.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420366.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.637491.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543243.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291174.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.4085383.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822045.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9140959.

peav_sortformer_train/nemo_log_globalrank-6_localrank-6.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:39 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:39 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:39 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:39 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:39 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:39 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:44 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:10 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x72cf44392b40>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.111822.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351958.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395577.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.7330983.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.4493845.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.307287.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771316.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136576.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.9420383.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374958.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543286.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291193.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.408541.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822076.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9140973.

peav_sortformer_train/nemo_log_globalrank-7_localrank-7.txt

@@ -0,0 +1,248 @@
+[NeMo I 2026-04-13 11:11:39 peav_sortformer_train:31] Hydra config: name: PEAVSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      peav_checkpoint: pe-av-small-16-frame
+      peav_pretrained: true
+      peav_freeze_layers: 0
+      rttm_unit_10ms_frame_count: 4
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 768
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.04
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.04
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./peav_sortformer_train
+      name: peav_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 11:11:39 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 11:11:39 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :peav_sortformer_train/peav_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 11:11:39 exp_manager:655] Experiments will be logged at peav_sortformer_train/peav_sortformer_train
+[NeMo I 2026-04-13 11:11:39 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 11:11:39 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 11:11:39 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:264] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 11:11:44 peav_sortformer_model:176] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.04, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 11:11:44 peav_sortformer_model:65] Loading PEAV model: pe-av-small-16-frame, pretrained=True
+[NeMo I 2026-04-13 11:12:08 peav_sortformer_model:82] PEAV encoder: 12 layers total, first 0 frozen, last 12 trainable
+[NeMo I 2026-04-13 11:12:11 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 11:12:11 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x78cb16b8e900>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 13:14:33 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776086073.1118307.
+[NeMo I 2026-04-13 15:17:20 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093440.5351996.
+[NeMo I 2026-04-13 15:17:21 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776093441.2395585.
+[NeMo I 2026-04-13 17:20:10 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100810.7331011.
+[NeMo I 2026-04-13 17:20:11 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776100811.449385.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.3072908.
+[NeMo I 2026-04-13 19:24:08 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776108248.9771328.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.2136598.
+[NeMo I 2026-04-13 21:26:58 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776115618.942039.
+[NeMo I 2026-04-13 23:29:47 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122987.6374946.
+[NeMo I 2026-04-13 23:29:48 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776122988.3543284.
+[NeMo I 2026-04-14 01:35:29 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130529.7291188.
+[NeMo I 2026-04-14 01:35:30 nemo_model_checkpoint:573] Checkpoint save for step 14000 started at 1776130530.4085422.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.1822095.
+[NeMo I 2026-04-14 03:39:20 nemo_model_checkpoint:573] Checkpoint save for step 16000 started at 1776137960.9141161.

peav_sortformer_train/version_0/events.out.tfevents.1776078731.260d0e88e8f5.93828.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00b94e3a28aa5883807f2d77d3b01f5e75bf50b51dac7fb16aebfa15e046686b
+size 9138653

peav_sortformer_train/version_0/hparams.yaml

@@ -0,0 +1,161 @@
+cfg:
+  sample_rate: 48000
+  pil_weight: 0.5
+  ats_weight: 0.5
+  max_num_of_spks: 4
+  peav_checkpoint: pe-av-small-16-frame
+  peav_pretrained: true
+  peav_freeze_layers: 0
+  rttm_unit_10ms_frame_count: 4
+  der_collar: 0.25
+  der_ignore_overlap: true
+  model_defaults:
+    fc_d_model: 768
+    tf_d_model: 192
+  train_ds:
+    manifest_filepath: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 45
+    shift_sec: 4
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: true
+    num_workers: 1
+    validation_mode: false
+    use_hf_streaming: true
+    hf_dataset_path: humanify/real_dia_dataset
+    hf_configs:
+    - CHiME6
+    - Dipco
+    - ICSI
+    - M3SD
+    - NOTSOFAR
+    - aishell4
+    - aishell5
+    - alimeeting
+    - ami_ihm
+    - ami_sdm
+    - callhome
+    - msdwild
+    - voxconverse
+    hf_split: train
+    shuffle_seed: 42
+    shuffle_buffer_size: 4
+    prefetch_factor: 4
+    persistent_workers: true
+    prefetch_rows: 8
+    use_lhotse: false
+    use_bucketing: true
+    num_buckets: 10
+    bucket_duration_bins:
+    - 10
+    - 20
+    - 30
+    - 40
+    - 50
+    - 60
+    - 70
+    - 80
+    - 90
+    pin_memory: true
+    min_duration: 10
+    max_duration: 90
+    batch_duration: 400
+    quadratic_duration: 1200
+    bucket_buffer_size: 20000
+    window_stride: 0.04
+    subsampling_factor: 1
+  validation_ds:
+    manifest_filepath: null
+    is_tarred: false
+    tarred_audio_filepaths: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 45
+    shift_sec: 4
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: false
+    num_workers: 1
+    validation_mode: true
+    use_hf_streaming: true
+    hf_dataset_path: humanify/real_dia_dataset
+    hf_configs:
+    - alm_benchmark
+    hf_split: train
+    shuffle_seed: 42
+    shuffle_buffer_size: 100
+    prefetch_factor: 4
+    persistent_workers: true
+    prefetch_rows: 4
+    use_lhotse: false
+    use_bucketing: false
+    drop_last: false
+    pin_memory: true
+    window_stride: 0.04
+    subsampling_factor: 1
+  test_ds:
+    manifest_filepath: null
+    is_tarred: false
+    tarred_audio_filepaths: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 90
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: false
+    seq_eval_mode: true
+    num_workers: 1
+    validation_mode: true
+    use_lhotse: false
+    use_bucketing: false
+    drop_last: false
+    pin_memory: true
+    window_stride: 0.04
+    subsampling_factor: 1
+  sortformer_modules:
+    _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+    num_spks: 4
+    dropout_rate: 0.1
+    fc_d_model: 768
+    tf_d_model: 192
+    subsampling_factor: 1
+  encoder:
+    d_model: 768
+    subsampling_factor: 1
+  transformer_encoder:
+    _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+    num_layers: 18
+    hidden_size: 192
+    inner_size: 768
+    num_attention_heads: 8
+    attn_score_dropout: 0.5
+    attn_layer_dropout: 0.5
+    ffn_dropout: 0.5
+    hidden_act: relu
+    pre_ln: false
+    pre_ln_final_layer_norm: true
+  loss:
+    _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+    weight: null
+    reduction: mean
+  lr: 2.0e-05
+  optim:
+    name: adamw
+    lr: 2.0e-05
+    betas:
+    - 0.9
+    - 0.98
+    weight_decay: 0.001
+    sched:
+      name: InverseSquareRootAnnealing
+      warmup_steps: 2500
+      warmup_ratio: null
+      min_lr: 1.0e-06