Upload folder using huggingface_hub

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.5475-epoch=0-last.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:983df478cabe165ab622a9ee0d94cd07981c1a4bedf8abbafe3903554d9a8d9b
+size 920132478

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.5475-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2ac83b80a349a0950950419f2084f3529d1bb086c4a32ddd524f36d360eede
+size 920132478

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.5632-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c048f1c92fe4593462fcd82c7dd958147b92f1274db9306de6dbcc43cd18c46
+size 920132286

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.5934-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b63d1d860ac8dd26bea4f25705e02425c83cbf1808801a76819b30c354f8beed
+size 920131839

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.9860-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19d6b2d2b344445596915b774cfed53da4ae89f78df0c3c77b1f74e42e34c62
+size 920131456

openflam_sortformer_train/checkpoints/openflam_sortformer_train--val_der=0.9965-epoch=0.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb14be6ffe43bc10ba930914e66471d825201cb53e72fbbd4f3d8719d1083e05
+size 920131009

openflam_sortformer_train/cmd-args.log

@@ -0,0 +1 @@
+scripts/openflam_sortformer_train.py --config-path=../configs --config-name=openflam_sortformer_4spk.yaml exp_manager.name=openflam_sortformer_train exp_manager.exp_dir=./openflam_sortformer_train

openflam_sortformer_train/git-info.log

@@ -0,0 +1,13 @@
+commit hash: 5ecc0a3c86af359b2f459bbb39f69a6fefc4bc40
+diff --git a/.gitignore b/.gitignore
+index 982a755..0707177 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -10,6 +10,7 @@ __pycache__/
+ peav_sortformer_train/
+ dasheng_sortformer_train/
+ peav_sortformer_v2_train
++openflam_sortformer_train/
+ 
+ debug_hf_training_window/
+ 

openflam_sortformer_train/lightning_logs.txt

@@ -0,0 +1,26 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+   | Name                        | Type                   | Params | Mode 
+--------------------------------------------------------------------------------
+0  | htsat_spectrogram_extractor | Spectrogram            | 1.1 M  | train
+1  | htsat_logmel_extractor      | LogmelFilterBank       | 32.8 K | train
+2  | htsat_bn0                   | BatchNorm2d            | 128    | eval 
+3  | htsat_patch_embed           | PatchEmbed             | 2.4 K  | train
+4  | htsat_pos_drop              | Dropout                | 0      | train
+5  | encoder                     | OpenFLAMEncoderWrapper | 67.8 M | train
+6  | sortformer_modules          | SortformerModules      | 236 K  | train
+7  | transformer_encoder         | TransformerEncoder     | 8.0 M  | train
+8  | loss                        | BCELoss                | 0      | train
+9  | _accuracy_test              | MultiBinaryAccuracy    | 0      | train
+10 | _accuracy_train             | MultiBinaryAccuracy    | 0      | train
+11 | _accuracy_valid             | MultiBinaryAccuracy    | 0      | train
+12 | _accuracy_test_ats          | MultiBinaryAccuracy    | 0      | train
+13 | _accuracy_train_ats         | MultiBinaryAccuracy    | 0      | train
+14 | _accuracy_valid_ats         | MultiBinaryAccuracy    | 0      | train
+--------------------------------------------------------------------------------
+76.1 M    Trainable params
+1.1 M     Non-trainable params
+77.2 M    Total params
+308.606   Total estimated model params size (MB)
+567       Modules in train mode
+1         Modules in eval mode

openflam_sortformer_train/nemo_error_log.txt

@@ -0,0 +1,8 @@
+[NeMo W 2026-04-13 18:48:50 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:48:50 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo W 2026-04-13 18:48:50 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-13 18:48:50 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo W 2026-04-13 18:49:43 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-13 18:50:30 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance.
+    

openflam_sortformer_train/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,251 @@
+[NeMo I 2026-04-13 18:48:50 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo I 2026-04-13 18:48:50 exp_manager:594] ExpManager schema
+[NeMo I 2026-04-13 18:48:50 exp_manager:595] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'run_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1}, 'create_early_stopping_callback': False, 'create_ipl_epoch_stopper_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'ipl_epoch_stopper_callback_params': {'enable_stop': True, 'stop_every_n_epochs': 1}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo W 2026-04-13 18:48:50 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:48:50 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:48:50 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:48:50 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:48:50 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:48:50 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:48:50 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:48:50 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:48:50 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:48:50 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:11 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7fd2ee2d5940>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo W 2026-04-13 18:49:43 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-13 18:50:30 nemo_logging:364] /venv/main/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=7` in the `DataLoader` to improve performance.
+    
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453387.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900334.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641889.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533197.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908803.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.0131054.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344173.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.0732732.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.742586.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947825.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.5716007.

openflam_sortformer_train/nemo_log_globalrank-1_localrank-1.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:23 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:23 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:23 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:23 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:23 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:23 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:23 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:28 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:42 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7f9f5130a780>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453328.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900246.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641915.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533032.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908696.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.0131009.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344163.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.0732656.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425637.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947732.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.5715966.

openflam_sortformer_train/nemo_log_globalrank-2_localrank-2.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:22 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:22 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:22 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:22 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:22 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:22 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:22 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:27 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:42 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7fadd270ede0>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.445336.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900196.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641746.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533016.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908696.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.013098.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344163.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.073263.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425504.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947706.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.57159.

openflam_sortformer_train/nemo_log_globalrank-3_localrank-3.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:22 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:22 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:22 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:22 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:22 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:22 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:22 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:27 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:40 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7f2ab4b2f1d0>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.445329.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900208.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641736.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533004.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908863.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.0130968.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344135.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.0732584.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425501.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.89477.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.5715883.

openflam_sortformer_train/nemo_log_globalrank-4_localrank-4.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:23 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:23 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:23 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:23 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:23 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:23 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:23 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:28 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:28 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:42 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7fa4985c0560>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453347.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900336.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641777.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533013.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.190873.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.013099.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344116.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.073265.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425628.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947723.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.571588.

openflam_sortformer_train/nemo_log_globalrank-5_localrank-5.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:22 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:22 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:22 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:22 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:22 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:22 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:22 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:27 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:38 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7f1f5bc1b6e0>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453282.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900138.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.164175.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533056.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908715.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.0131006.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344132.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.073263.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425544.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947694.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.5715895.

openflam_sortformer_train/nemo_log_globalrank-6_localrank-6.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:22 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:22 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:22 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:22 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:22 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:22 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:22 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:27 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:39 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7f3350a98bf0>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453306.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.490022.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.16418.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533128.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.190873.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.0130994.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.634412.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.0732641.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425518.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947701.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.5715954.

openflam_sortformer_train/nemo_log_globalrank-7_localrank-7.txt

@@ -0,0 +1,245 @@
+[NeMo I 2026-04-13 18:49:22 openflam_sortformer_train:31] Hydra config: name: OpenFLAMSortformerDiarizer
+    num_workers: 1
+    batch_size: 24
+    model:
+      sample_rate: 48000
+      pil_weight: 0.5
+      ats_weight: 0.5
+      max_num_of_spks: 4
+      openflam_model_name: v1-base
+      openflam_pretrained: true
+      openflam_freeze_layers: 0
+      openflam_ckpt_cache_dir: /tmp/openflam
+      rttm_unit_10ms_frame_count: 31
+      der_collar: 0.25
+      der_ignore_overlap: true
+      model_defaults:
+        fc_d_model: 1024
+        tf_d_model: 192
+      train_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: true
+        num_workers: ${num_workers}
+        validation_mode: false
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - CHiME6
+        - Dipco
+        - ICSI
+        - M3SD
+        - NOTSOFAR
+        - aishell4
+        - aishell5
+        - alimeeting
+        - ami_ihm
+        - ami_sdm
+        - callhome
+        - msdwild
+        - voxconverse
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 4
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 8
+        use_lhotse: false
+        use_bucketing: true
+        num_buckets: 10
+        bucket_duration_bins:
+        - 10
+        - 20
+        - 30
+        - 40
+        - 50
+        - 60
+        - 70
+        - 80
+        - 90
+        pin_memory: true
+        min_duration: 10
+        max_duration: 90
+        batch_duration: 400
+        quadratic_duration: 1200
+        bucket_buffer_size: 20000
+        window_stride: 0.3125
+        subsampling_factor: 1
+      validation_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 45
+        shift_sec: 4
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_hf_streaming: true
+        hf_dataset_path: humanify/real_dia_dataset
+        hf_configs:
+        - alm_benchmark
+        hf_split: train
+        shuffle_seed: 42
+        shuffle_buffer_size: 100
+        prefetch_factor: 4
+        persistent_workers: true
+        prefetch_rows: 4
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      test_ds:
+        manifest_filepath: null
+        is_tarred: false
+        tarred_audio_filepaths: null
+        sample_rate: ${model.sample_rate}
+        num_spks: ${model.max_num_of_spks}
+        session_len_sec: 90
+        soft_label_thres: 0.5
+        soft_targets: false
+        labels: null
+        batch_size: ${batch_size}
+        shuffle: false
+        seq_eval_mode: true
+        num_workers: ${num_workers}
+        validation_mode: true
+        use_lhotse: false
+        use_bucketing: false
+        drop_last: false
+        pin_memory: true
+        window_stride: 0.3125
+        subsampling_factor: 1
+      sortformer_modules:
+        _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+        num_spks: ${model.max_num_of_spks}
+        dropout_rate: 0.1
+        fc_d_model: ${model.model_defaults.fc_d_model}
+        tf_d_model: ${model.model_defaults.tf_d_model}
+        subsampling_factor: 1
+      encoder:
+        d_model: ${model.model_defaults.fc_d_model}
+        subsampling_factor: 1
+      transformer_encoder:
+        _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+        num_layers: 18
+        hidden_size: ${model.model_defaults.tf_d_model}
+        inner_size: 768
+        num_attention_heads: 8
+        attn_score_dropout: 0.5
+        attn_layer_dropout: 0.5
+        ffn_dropout: 0.5
+        hidden_act: relu
+        pre_ln: false
+        pre_ln_final_layer_norm: true
+      loss:
+        _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+        weight: null
+        reduction: mean
+      lr: 2.0e-05
+      optim:
+        name: adamw
+        lr: ${model.lr}
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: InverseSquareRootAnnealing
+          warmup_steps: 2500
+          warmup_ratio: null
+          min_lr: 1.0e-06
+    trainer:
+      devices: 8
+      accelerator: gpu
+      precision: bf16-mixed
+      max_epochs: -1
+      max_steps: 16000
+      num_nodes: 1
+      strategy: ddp_find_unused_parameters_true
+      accumulate_grad_batches: 1
+      deterministic: false
+      enable_checkpointing: false
+      logger: false
+      log_every_n_steps: 1
+      val_check_interval: 2000
+      num_sanity_val_steps: 0
+    exp_manager:
+      use_datetime_version: false
+      exp_dir: ./openflam_sortformer_train
+      name: openflam_sortformer_train
+      resume_if_exists: true
+      resume_from_checkpoint: null
+      resume_ignore_no_checkpoint: true
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      create_wandb_logger: false
+      checkpoint_callback_params:
+        monitor: val_der
+        mode: min
+        save_top_k: 5
+        every_n_train_steps: 2000
+        every_n_epochs: 0
+      wandb_logger_kwargs:
+        resume: true
+        name: null
+        project: null
+    
+[NeMo W 2026-04-13 18:49:22 exp_manager:1177] No version folders would be created under the log folder as 'resume_if_exists' is enabled.
+[NeMo W 2026-04-13 18:49:22 exp_manager:1022] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :openflam_sortformer_train/openflam_sortformer_train/checkpoints. Training from scratch.
+[NeMo I 2026-04-13 18:49:22 exp_manager:655] Experiments will be logged at openflam_sortformer_train/openflam_sortformer_train
+[NeMo I 2026-04-13 18:49:22 exp_manager:1262] TensorboardLogger has been set up
+[NeMo W 2026-04-13 18:49:22 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 16000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-13 18:49:22 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['CHiME6', 'Dipco', 'ICSI', 'M3SD', 'NOTSOFAR', 'aishell4', 'aishell5', 'alimeeting', 'ami_ihm', 'ami_sdm', 'callhome', 'msdwild', 'voxconverse'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:371] HF streaming dataloader: humanify/real_dia_dataset configs=['alm_benchmark'] split=train (shuffle_seed=42, rank_batch_size=24, world_size=8)
+[NeMo W 2026-04-13 18:49:27 openflam_sortformer_model:284] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'is_tarred': False, 'tarred_audio_filepaths': None, 'sample_rate': 48000, 'num_spks': 4, 'session_len_sec': 90, 'soft_label_thres': 0.5, 'soft_targets': False, 'labels': None, 'batch_size': 24, 'shuffle': False, 'seq_eval_mode': True, 'num_workers': 1, 'validation_mode': True, 'use_lhotse': False, 'use_bucketing': False, 'drop_last': False, 'pin_memory': True, 'window_stride': 0.3125, 'subsampling_factor': 1, 'num_speakers': 4}
+[NeMo I 2026-04-13 18:49:27 openflam_sortformer_model:132] Loading OpenFLAM model: v1-base, pretrained=True
+[NeMo I 2026-04-13 18:49:38 openflam_sortformer_model:182] OpenFLAM HTSAT encoder: 4 stages total, first 0 frozen, last 4 trainable
+[NeMo I 2026-04-13 18:49:43 modelPT:830] Optimizer config = AdamW (
+    Parameter Group 0
+        amsgrad: False
+        betas: (0.9, 0.98)
+        capturable: False
+        decoupled_weight_decay: True
+        differentiable: False
+        eps: 1e-08
+        foreach: None
+        fused: None
+        lr: 2e-05
+        maximize: False
+        weight_decay: 0.001
+    )
+[NeMo I 2026-04-13 18:49:43 lr_scheduler:995] Scheduler "<nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing object at 0x7f82856aaba0>" 
+    will be used during training (effective maximum steps = 16000) - 
+    Parameters : 
+    (warmup_steps: 2500
+    warmup_ratio: null
+    min_lr: 1.0e-06
+    max_steps: 16000
+    )
+[NeMo I 2026-04-13 20:13:39 nemo_model_checkpoint:573] Checkpoint save for step 2000 started at 1776111219.4453285.
+[NeMo I 2026-04-13 21:38:50 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116330.4900186.
+[NeMo I 2026-04-13 21:38:52 nemo_model_checkpoint:573] Checkpoint save for step 4000 started at 1776116332.1641943.
+[NeMo I 2026-04-13 23:06:23 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121583.5533047.
+[NeMo I 2026-04-13 23:06:25 nemo_model_checkpoint:573] Checkpoint save for step 6000 started at 1776121585.1908693.
+[NeMo I 2026-04-14 00:34:32 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126872.013106.
+[NeMo I 2026-04-14 00:34:33 nemo_model_checkpoint:573] Checkpoint save for step 8000 started at 1776126873.6344168.
+[NeMo I 2026-04-14 02:01:04 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132064.0732667.
+[NeMo I 2026-04-14 02:01:05 nemo_model_checkpoint:573] Checkpoint save for step 10000 started at 1776132065.7425685.
+[NeMo I 2026-04-14 03:26:54 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137214.8947742.
+[NeMo I 2026-04-14 03:26:56 nemo_model_checkpoint:573] Checkpoint save for step 12000 started at 1776137216.571593.

openflam_sortformer_train/version_0/events.out.tfevents.1776106183.aa77be2546cc.6641.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b012681cc9b2db7e02c55703a59a4e2907b43281c8fc25798c0fc1cf62e14bb
+size 7667179

openflam_sortformer_train/version_0/hparams.yaml

@@ -0,0 +1,162 @@
+cfg:
+  sample_rate: 48000
+  pil_weight: 0.5
+  ats_weight: 0.5
+  max_num_of_spks: 4
+  openflam_model_name: v1-base
+  openflam_pretrained: true
+  openflam_freeze_layers: 0
+  openflam_ckpt_cache_dir: /tmp/openflam
+  rttm_unit_10ms_frame_count: 31
+  der_collar: 0.25
+  der_ignore_overlap: true
+  model_defaults:
+    fc_d_model: 1024
+    tf_d_model: 192
+  train_ds:
+    manifest_filepath: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 45
+    shift_sec: 4
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: true
+    num_workers: 1
+    validation_mode: false
+    use_hf_streaming: true
+    hf_dataset_path: humanify/real_dia_dataset
+    hf_configs:
+    - CHiME6
+    - Dipco
+    - ICSI
+    - M3SD
+    - NOTSOFAR
+    - aishell4
+    - aishell5
+    - alimeeting
+    - ami_ihm
+    - ami_sdm
+    - callhome
+    - msdwild
+    - voxconverse
+    hf_split: train
+    shuffle_seed: 42
+    shuffle_buffer_size: 4
+    prefetch_factor: 4
+    persistent_workers: true
+    prefetch_rows: 8
+    use_lhotse: false
+    use_bucketing: true
+    num_buckets: 10
+    bucket_duration_bins:
+    - 10
+    - 20
+    - 30
+    - 40
+    - 50
+    - 60
+    - 70
+    - 80
+    - 90
+    pin_memory: true
+    min_duration: 10
+    max_duration: 90
+    batch_duration: 400
+    quadratic_duration: 1200
+    bucket_buffer_size: 20000
+    window_stride: 0.3125
+    subsampling_factor: 1
+  validation_ds:
+    manifest_filepath: null
+    is_tarred: false
+    tarred_audio_filepaths: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 45
+    shift_sec: 4
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: false
+    num_workers: 1
+    validation_mode: true
+    use_hf_streaming: true
+    hf_dataset_path: humanify/real_dia_dataset
+    hf_configs:
+    - alm_benchmark
+    hf_split: train
+    shuffle_seed: 42
+    shuffle_buffer_size: 100
+    prefetch_factor: 4
+    persistent_workers: true
+    prefetch_rows: 4
+    use_lhotse: false
+    use_bucketing: false
+    drop_last: false
+    pin_memory: true
+    window_stride: 0.3125
+    subsampling_factor: 1
+  test_ds:
+    manifest_filepath: null
+    is_tarred: false
+    tarred_audio_filepaths: null
+    sample_rate: 48000
+    num_spks: 4
+    session_len_sec: 90
+    soft_label_thres: 0.5
+    soft_targets: false
+    labels: null
+    batch_size: 24
+    shuffle: false
+    seq_eval_mode: true
+    num_workers: 1
+    validation_mode: true
+    use_lhotse: false
+    use_bucketing: false
+    drop_last: false
+    pin_memory: true
+    window_stride: 0.3125
+    subsampling_factor: 1
+  sortformer_modules:
+    _target_: nemo.collections.asr.modules.sortformer_modules.SortformerModules
+    num_spks: 4
+    dropout_rate: 0.1
+    fc_d_model: 1024
+    tf_d_model: 192
+    subsampling_factor: 1
+  encoder:
+    d_model: 1024
+    subsampling_factor: 1
+  transformer_encoder:
+    _target_: nemo.collections.asr.modules.transformer.transformer_encoders.TransformerEncoder
+    num_layers: 18
+    hidden_size: 192
+    inner_size: 768
+    num_attention_heads: 8
+    attn_score_dropout: 0.5
+    attn_layer_dropout: 0.5
+    ffn_dropout: 0.5
+    hidden_act: relu
+    pre_ln: false
+    pre_ln_final_layer_norm: true
+  loss:
+    _target_: nemo.collections.asr.losses.bce_loss.BCELoss
+    weight: null
+    reduction: mean
+  lr: 2.0e-05
+  optim:
+    name: adamw
+    lr: 2.0e-05
+    betas:
+    - 0.9
+    - 0.98
+    weight_decay: 0.001
+    sched:
+      name: InverseSquareRootAnnealing
+      warmup_steps: 2500
+      warmup_ratio: null
+      min_lr: 1.0e-06