2512

src/dataset_sample.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0464770415073c7af8d9a44792c39d89e49d48c667d3c37d52e255e92f80fb57
-size 8209446
+oid sha256:0cdeb27a316f7c3949d5c721062ad7837b5527d3c7fda70129b51b993a087cf4
+size 26286441

src/sdxs_create.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7e7a0f7dec0f14e09690f5c4cf83505b42361436f3eec7a0edfb3c6ccf3bf91
-size 37958
+oid sha256:64795c45e866b9672710f79a3633a814e7a701c52906c59df458f3bf37478af1
+size 43355

src/sdxs_create_07.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fad9774fdede172d40365fe08c322c5fdc637039a27759e7fb93690f18a47e78
-size 6624
+oid sha256:7b24d9182db53377431320496f8f5fba1a74c3c09dd011a2cf1d3ff32f8aba58
+size 6648

test.ipynb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ba7c8adff3e90b5b271d5dc6aadb62d4651634146bcbc08aca865ae003eabd5
-size 8915079
+oid sha256:7ee6048c4a59ddefaf190e5398f0dd7586746b0959db16971a118ca0a8e6e269
+size 8882423

train.py

@@ -29,7 +29,7 @@ from transformers import AutoTokenizer, AutoModel
 # --------------------------- Параметры ---------------------------
 ds_path = "/workspace/sdxs/datasets/768"
 project = "unet"
-batch_size = 16
+batch_size = 36
 base_learning_rate = 2e-5 #2.7e-5
 min_learning_rate = 2e-6 #2.7e-5
 num_epochs = 10
@@ -341,6 +341,14 @@ if limit > 0:
 else:
     dataset = load_from_disk(ds_path)
 
+dataset = dataset.filter(
+    lambda x: [not (path.startswith("/workspace/ds/animesfw") or path.startswith("/workspace/ds/d4/animesfw"))  for path in x["image_path"]],
+    batched=True,
+    batch_size=10000, # обрабатываем по 10к строк за раз
+    num_proc=8
+)
+print(f"Осталось примеров после фильтрации: {len(dataset)}")
+
 # --- [UPDATED] Collate Function ---
 def collate_fn_simple(batch):
     # 1. Латенты (VAE)
@@ -374,7 +382,8 @@ batch_sampler = DistributedResolutionBatchSampler(
     )
 
 dataloader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn_simple)
-print("Total samples", len(dataloader))
+if accelerator.is_main_process:
+    print("Total samples", len(dataloader))
 dataloader = accelerator.prepare(dataloader)
 
 start_epoch = 0
@@ -733,11 +742,13 @@ for epoch in range(start_epoch, start_epoch + num_epochs):
                             wandb.log(log_data, step=global_step)
                         if use_comet_ml:
                             comet_experiment.log_metrics(log_data, step=global_step)
-    
+
                     if global_step % sample_interval == 0:
                         # Передаем tuple (emb, mask) для негатива
-                        #if epoch % 10 == 0:
-                        generate_and_save_samples(fixed_samples, (uncond_emb, uncond_mask), global_step)
+                        if save_model:
+                            generate_and_save_samples(fixed_samples, (uncond_emb, uncond_mask), global_step)
+                        elif epoch % 10 == 0:
+                            generate_and_save_samples(fixed_samples, (uncond_emb, uncond_mask), global_step)
                         last_n = sample_interval
                         
                         if save_model:

unet/config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb10c12478625b355068dbfce29e8db3c8a52d03e10508452e4feb5404545584
-size 1876
+oid sha256:80410b994cba2961422e6bade67a2a6a41abaf90e6dd89c0226402f45ef113cb
+size 1892

unet/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1538c42c6b7bad115da2b554b11a8295271c4758deb119711b6a35c1b8744955
-size 7444321360
+oid sha256:7e5b6db0af234e3191d65afd39653d779a2e49bd2377b3fc4e12c3b68df3fded
+size 3722280432