Upload folder using huggingface_hub

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "BioMistral/BioMistral-7B",
+  "_name_or_path": "cognitivecomputations/WestLake-7B-v2-laser",
   "architectures": [
     "MixtralForCausalLM"
   ],
@@ -17,8 +17,9 @@
   "num_experts_per_tok": 2,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "num_local_experts": 2,
+  "num_local_experts": 6,
   "output_router_logits": false,
+  "pad_token_id": 2,
   "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,
   "router_aux_loss_coef": 0.001,
@@ -27,6 +28,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.45.0.dev0",
-  "use_cache": false,
+  "unsloth_version": "2024.1",
+  "use_cache": true,
   "vocab_size": 32000
 }

mergekit_moe_config.yml

@@ -0,0 +1,50 @@
+base_model: cognitivecomputations/WestLake-7B-v2-laser
+gate_mode: hidden
+dtype: bfloat16
+experts:
+  - source_model: mlabonne/Daredevil-7B
+    positive_prompts:
+      - "Answer this question from the ARC (Argument Reasoning Comprehension)."
+      - "Use common sense and logical reasoning skills."
+    negative_prompts:
+      - "nonsense"
+      - "irrational"
+      - "math"
+      - "code"
+  - source_model: cognitivecomputations/WestLake-7B-v2-laser
+    positive_prompts:
+      - "Answer this question from the Winogrande test."
+      - "Use advanced knowledge of culture and humanity"
+    negative_prompts:
+      - "ignorance"
+      - "uninformed"
+      - "creativity"
+  - source_model: mlabonne/Daredevil-7B
+    positive_prompts:
+      - "Calculate the answer to this math problem"
+      - "My mathematical capabilities are strong, allowing me to handle complex mathematical queries"
+      - "solve for"
+    negative_prompts:
+      - "incorrect"
+      - "inaccurate"
+      - "creativity"
+  - source_model: BioMistral/BioMistral-7B
+    positive_prompts:
+      - "Provide a detailed explanation based on medical literature."
+      - "Apply clinical reasoning to answer this healthcare question."
+      - "Use biomedical knowledge to solve this problem."
+      - "Interpret these clinical findings in the context of patient care."
+      - "Leverage your understanding of healthcare systems to advise on policy."
+    negative_prompts:
+      - "vague"
+      - "non-scientific"
+      - "unrelated to healthcare"
+      - "simplistic understanding"
+      - "ignore clinical guidelines"
+  - source_model: HuggingFaceH4/zephyr-7b-beta
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Screening"
+  - source_model: mistralai/Mistral-7B-Instruct-v0.2
+    positive_prompts:
+      - "This is a prompt that is demonstrative of what expert model excels at Diagnosis"
+

model-00001-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6034a413902dfaeb921d6531a61acf06c435393c770ab5b569c8c39a0dc68ba4
+size 933250496

model-00002-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481b6b66e3dbb202a2d98a18efca0fedf64fe9672a625a02f2d1b766840132f3
+size 939525160

model-00003-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21702f4ace9530fdcc3ac6c919db0693ccbc9d5f2366baa7429280c5e8d4abc0
+size 905987672

model-00004-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76f7ce04f41037ee045c9d4a08713521f8e7740e7d08b1d9f3595045431045a4
+size 939525160

model-00005-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d053faafc8482a97872958b3c8ddb7d878add6e8fd7d2dc4c271f2f2f636b6c3
+size 989865456

model-00006-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:095e05412f23b7b15154bb097cc9caf7da6b2cbf5843570ac086214938890b1d
+size 973088024

model-00007-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a55f57eee49d62beb038fe5e2b3ae79ce7383bcc03e626a624b65d88c458c45
+size 939525160

model-00008-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46a78e8daf3ff2e6f8f8f02470b74de2364ab9ef1e434864dcd82353215740e2
+size 905987672

model-00009-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24758679b261b41d8fb1e4eacf52b31d6e6812eedc12802e134433fcb3360339
+size 939525160

model-00010-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1287c98b13850128d9ae59d6b371c94797ca8716aba9d1c659b5fd7933add6e2
+size 905987672

model-00011-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae7c6b82cd419a1870681f2142f717c9e294d918a8c77400ce2842d5d26b8deb
+size 939525160

model-00012-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:177fce161ae81c9665313bdb59d1ea31d936d3f11417f7307a751d4731afbb9d
+size 989865456

model-00013-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9675408d57fa4e55fd6cd0eae543644e2b832a2c3b1cba3ce4c5ceb089c9b40
+size 973088024

model-00014-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3887f92feaa81cfef0b301b55a7b0900665efdb8026953e88722dea2d20846c7
+size 939525160

model-00015-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c421cbeffc51002079737f45ed881c143dd22037a11799c1c51706513a83a81b
+size 905987672

model-00016-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474fa101c29f5fc9807af0ad2011399572e6a941166604c2141f3f84e87a845f
+size 939525160

model-00017-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d607f37024a8f31f6984f9a0f245b70c4006080d5fe7c2c8bc3039d5ff494e5f
+size 905987672

model-00018-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5fe63fac3bfa00b102c2ea0b229acbf0ddc1d7b2e71fa11f32f267af6f6ec1a
+size 939525160

model-00019-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0be28b743ff83ddf721ece853681d20fdec3162890e3583d374a0247e2642e39
+size 989865456

model-00020-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:920951043aec184da4273f2997b1b64c748ccd5456b0cea9d5433f2c2d0bcaa1
+size 973088024

model-00021-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:413fb9b89166f279f4a3993acc61143c9fbe2c67871257ffc86d5a9288f91bc7
+size 939525160

model-00022-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67701bdf7a68de6968410225f12cc4e1d9bedd25d8ed4018e45842f5756e01cc
+size 905987672

model-00023-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00a01cfb43af601d488a8b566b9dbf116ccd6ba9d115734b22d23dec817f3330
+size 939525160

model-00024-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7031dcaa8ee2bf94389254664bb42966ed87db31e75a09d009098b93aa774d3
+size 905987680

model-00025-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45b5f40bf445be914564ebf35c4c479b6ee2398753dd7ed8b608f6b5b2e174c1
+size 939525168

model-00026-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7b6e44799e92b6afe66835f2ab17614b9722f262fa34332c2fac1d424bebc7d
+size 989865472

model-00027-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28beb94314cbadf7cd47d7ef7fef929e5ae9ab9f83a156e36fd291eb8cf9591
+size 973088032

model-00028-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba286b739256b80779268ad15657c25e8c80b5eebc4b72c7bd629b686289f51c
+size 939525168

model-00029-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0252941feb60e97d9ccc3fde43f2dfb538d944444f475464cfe41fbdb1d1990a
+size 905987688

model-00030-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512621ddab928bfc0b809865238ab2f544a86b461af7c67b88c244d36a386feb
+size 939525168

model-00031-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7607b4eda659f423c2746b280ddaf25d13a0f02406852b61dc0fe7d9e47480c
+size 905987688

model-00032-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818c97697fc86444c108122ce8c56981c910558d4aa9c3ca3421a22b70241a14
+size 939525168

model-00033-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b8adbcf266765e7a28423ca8d6133bd24d2be798b8f008ff31937e584801e2
+size 989865472

model-00034-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1957dfe3e1b52f837adf6eccbfc3a91789c1193afc25d604ac0e31dc62320641
+size 973088032

model-00035-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ebefc2b41a8f432126c5c1dd61ccb291806e0346fc8f0798a0ddb14a0c2fa80
+size 939525168

model-00036-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7fe895f8fbf84113d86c167e09fb042f0b6af31fca5deee72a124f0a040fab7
+size 905987688

model-00037-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54d9ffd1dd204fbc9ddb8bea6f367e77dcff3cb68f5918d0ec679dc996d74c2f
+size 939525168

model-00038-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bffb88ceb546bb3c61ed260b795e9c3c03a08c934bfae434394b9f42aadbdff
+size 905987688

model-00039-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f3986dfaaf54e3489bb1667e8a51cfb68d31e46101fe4cf4c3d61f58a71148
+size 939525168

model-00040-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe142a9c15a5dfbcbcdf6d4ddc5885b20dd986e24571548bd911e1c51a95328d
+size 989865472

model-00041-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd97d3da8595694faeb16cdbd565bfbf0044de32b3858d831b7eab3efc881766
+size 973088032

model-00042-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21bf2264e2fd272cf349c0e88c64e2049449292fea65d56b434d9b68e281e7d5
+size 939525168

model-00043-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:303f725329bb1c1fa8e615b442d68a7f17fc54e75db8de04dfdf7b8bb92adb52
+size 905987688

model-00044-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91c8537567ec135ed10b54264c798f3bbd92aadd7bd136d7c88d8f2cc1ad5a11
+size 939525168

model-00045-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31947a97ca4950f63434813ba34b238267624a55bb6830375917ebfde105de3b
+size 905987688

model-00046-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b474c7e1311917c58012022d7335255b5d33fca04c890a2463b60719a6759460
+size 939525168

model-00047-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09c273a2987980f6a60c92d9b896d505daa6e4de777b6a11bf4502dcf13954ba
+size 989865472

model-00048-of-00076.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56bfe7a3d8e47363df9c317381d39ab000d285325a34c7b653b227c65571b9f8
+size 973088032