removed old files

.gitattributes

@@ -1,2 +0,0 @@
-*.jpg filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,8 +0,0 @@
-env
-env_old
-*.bak
-*.pyc
-__pycache__
-results
-# excluding user settings for the GUI frontend
-configs/settings.yaml

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Rupesh Sreeraman
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

THIRD-PARTY-LICENSES

@@ -1,143 +0,0 @@
-stablediffusion.cpp - MIT
-
-OpenVINO stablediffusion engine - Apache 2
-
-SD Turbo -  STABILITY AI NON-COMMERCIAL RESEARCH COMMUNITY LICENSE AGREEMENT
-
-MIT License
-
-Copyright (c) 2023 leejet
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-ERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-Definitions.
-
-"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
-
-"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
-
-"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
-
-"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
-
-"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
-
-Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
-
-Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
-
-Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
-
-(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
-
-(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
-
-(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
-
-(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
-
-You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
-
-Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
-
-Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
-
-Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
-
-Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
-
-Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work.
-
-  To apply the Apache License to your work, attach the following
-  boilerplate notice, with the fields enclosed by brackets "[]"
-  replaced with your own identifying information. (Don't include
-  the brackets!)  The text should be enclosed in the appropriate
-  comment syntax for the file format. We also recommend that a
-  file or class name and description of purpose be included on the
-  same "printed page" as the copyright notice for easier
-  identification within third-party archives.
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
-
-   <http://www.apache.org/licenses/LICENSE-2.0>
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
-
-STABILITY AI NON-COMMERCIAL RESEARCH COMMUNITY LICENSE AGREEMENT
-Dated: November 28, 2023
-
-By using or distributing any portion or element of the Models, Software, Software Products or Derivative Works, you agree to be bound by this Agreement.
-
-"Agreement" means this Stable Non-Commercial Research Community License Agreement.
-
-“AUP” means the Stability AI Acceptable Use Policy available at <https://stability.ai/use-policy>, as may be updated from time to time.
-
-"Derivative Work(s)” means (a) any derivative work of the Software Products as recognized by U.S. copyright laws and (b) any modifications to a Model, and any other model created which is based on or derived from the Model or the Model’s output. For clarity, Derivative Works do not include the output of any Model.
-
-“Documentation” means any specifications, manuals, documentation, and other written information provided by Stability AI related to the Software.
-
-"Licensee" or "you" means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity's behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
-
-“Model(s)" means, collectively, Stability AI’s proprietary models and algorithms, including machine-learning models, trained model weights and other elements of the foregoing, made available under this Agreement.
-
-“Non-Commercial Uses” means exercising any of the rights granted herein for the purpose of research or non-commercial purposes. Non-Commercial Uses does not include any production use of the Software Products or any Derivative Works.
-
-"Stability AI" or "we" means Stability AI Ltd. and its affiliates.
-
-"Software" means Stability AI’s proprietary software made available under this Agreement.
-
-“Software Products” means the Models, Software and Documentation, individually or in any combination.
-
-1.     License Rights and Redistribution. 
-
-a.  Subject to your compliance with this Agreement, the AUP (which is hereby incorporated herein by reference), and the Documentation, Stability AI grants you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty free and limited license under Stability AI’s intellectual property or other rights owned or controlled by Stability AI embodied in the Software Products to use, reproduce, distribute, and create Derivative Works of, the Software Products, in each case for Non-Commercial Uses only.
-
-b.  You may not use the Software Products or Derivative Works to enable third parties to use the Software Products or Derivative Works as part of your hosted service or via your APIs, whether you are adding substantial additional functionality thereto or not. Merely distributing the Software Products or Derivative Works for download online without offering any related service (ex. by distributing the Models on HuggingFace) is not a violation of this subsection. If you wish to use the Software Products or any Derivative Works for commercial or production use or you wish to make the Software Products or any Derivative Works available to third parties via your hosted service or your APIs, contact Stability AI at <https://stability.ai/contact>.
-
-c.  If you distribute or make the Software Products, or any Derivative Works thereof, available to a third party, the Software Products, Derivative Works, or any portion thereof, respectively, will remain subject to this Agreement and you must (i) provide a copy of this Agreement to such third party, and (ii) retain the following attribution notice within a "Notice" text file distributed as a part of such copies: "This Stability AI Model is licensed under the Stability AI Non-Commercial Research Community License, Copyright (c) Stability AI Ltd. All Rights Reserved.” If you create a Derivative Work of a Software Product, you may add your own attribution notices to the Notice file included with the Software Product, provided that you clearly indicate which attributions apply to the Software Product and you must state in the NOTICE file that you changed the Software Product and how it was modified.
-
-2.     Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SOFTWARE PRODUCTS  AND ANY OUTPUT AND RESULTS THERE FROM ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SOFTWARE PRODUCTS, DERIVATIVE WORKS OR ANY OUTPUT OR RESULTS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SOFTWARE PRODUCTS, DERIVATIVE WORKS AND ANY OUTPUT AND RESULTS. 
-
-3.     Limitation of Liability. IN NO EVENT WILL STABILITY AI OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF STABILITY AI OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. 
-
-4.     Intellectual Property.
-
-a.  No trademark licenses are granted under this Agreement, and in connection with the Software Products or Derivative Works, neither Stability AI nor Licensee may use any name or mark owned by or associated with the other or any of its affiliates, except as required for reasonable and customary use in describing and redistributing the Software Products or Derivative Works.
-
-b.  Subject to Stability AI’s ownership of the Software Products and Derivative Works made by or for Stability AI, with respect to any Derivative Works that are made by you, as between you and Stability AI, you are and will be the owner of such Derivative Works
-
-c.  If you institute litigation or other proceedings against Stability AI (including a cross-claim or counterclaim in a lawsuit) alleging that the Software Products, Derivative Works or associated outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Stability AI from and against any claim by any third party arising out of or related to your use or distribution of the Software Products or Derivative Works in violation of this Agreement.
-
-5.      Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Software Products and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Stability AI may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of any Software Products or Derivative Works. Sections 2-4 shall survive the termination of this Agreement.

app.py

@@ -1,4 +0,0 @@
-from frontend.webui.hf_demo import start_demo_text_to_image
-
-print("Starting HF demo text to image")
-start_demo_text_to_image(False)

app_settings.py

@@ -1,81 +0,0 @@
-import yaml
-from os import path, makedirs
-from models.settings import Settings
-from paths import FastStableDiffusionPaths
-from utils import get_models_from_text_file
-from constants import (
-    OPENVINO_LCM_MODELS_FILE,
-    LCM_LORA_MODELS_FILE,
-    SD_MODELS_FILE,
-    LCM_MODELS_FILE,
-)
-from copy import deepcopy
-
-
-class AppSettings:
-    def __init__(self):
-        self.config_path = FastStableDiffusionPaths().get_app_settings_path()
-        self._stable_diffsuion_models = ["Lykon/dreamshaper-8"]
-        self._lcm_lora_models = ["latent-consistency/lcm-lora-sdv1-5"]
-        self._openvino_lcm_models = ["rupeshs/sd-turbo-openvino"]
-        self._lcm_models = ["stabilityai/sd-turbo"]
-
-    @property
-    def settings(self):
-        return self._config
-
-    @property
-    def stable_diffsuion_models(self):
-        return self._stable_diffsuion_models
-
-    @property
-    def openvino_lcm_models(self):
-        return self._openvino_lcm_models
-
-    @property
-    def lcm_models(self):
-        return self._lcm_models
-
-    @property
-    def lcm_lora_models(self):
-        return self._lcm_lora_models
-
-    def load(self, skip_file=False):
-        if skip_file:
-            print("Skipping config file")
-            settings_dict = self._load_default()
-            self._config = Settings.parse_obj(settings_dict)
-        else:
-            if not path.exists(self.config_path):
-                base_dir = path.dirname(self.config_path)
-                if not path.exists(base_dir):
-                    makedirs(base_dir)
-                try:
-                    print("Settings not found creating default settings")
-                    with open(self.config_path, "w") as file:
-                        yaml.dump(
-                            self._load_default(),
-                            file,
-                        )
-                except Exception as ex:
-                    print(f"Error in creating settings : {ex}")
-                    exit()
-            try:
-                with open(self.config_path) as file:
-                    settings_dict = yaml.safe_load(file)
-                    self._config = Settings.parse_obj(settings_dict)
-            except Exception as ex:
-                print(f"Error in loading settings : {ex}")
-
-    def save(self):
-        try:
-            with open(self.config_path, "w") as file:
-                tmp_cfg = deepcopy(self._config)
-                tmp_cfg.lcm_diffusion_setting.init_image = None
-                yaml.dump(tmp_cfg.dict(), file)
-        except Exception as ex:
-            print(f"Error in saving settings : {ex}")
-
-    def _load_default(self) -> dict:
-        defult_config = Settings()
-        return defult_config.dict()

backend/device.py

@@ -1,23 +0,0 @@
-import platform
-from constants import DEVICE
-import torch
-import openvino as ov
-
-core = ov.Core()
-
-
-def is_openvino_device() -> bool:
-    if DEVICE.lower() == "cpu" or DEVICE.lower()[0] == "g":
-        return True
-    else:
-        return False
-
-
-def get_device_name() -> str:
-    if DEVICE == "cuda" or DEVICE == "mps":
-        default_gpu_index = torch.cuda.current_device()
-        return torch.cuda.get_device_name(default_gpu_index)
-    elif platform.system().lower() == "darwin":
-        return platform.processor()
-    elif is_openvino_device():
-        return core.get_property(DEVICE.upper(), "FULL_DEVICE_NAME")

backend/image_saver.py

@@ -1,40 +0,0 @@
-from os import path, mkdir
-from typing import Any
-from uuid import uuid4
-from backend.models.lcmdiffusion_setting import LCMDiffusionSetting
-import json
-
-
-class ImageSaver:
-    @staticmethod
-    def save_images(
-        output_path: str,
-        images: Any,
-        folder_name: str = "",
-        format: str = ".png",
-        lcm_diffusion_setting: LCMDiffusionSetting = None,
-    ) -> None:
-        gen_id = uuid4()
-
-        for index, image in enumerate(images):
-            if not path.exists(output_path):
-                mkdir(output_path)
-
-            if folder_name:
-                out_path = path.join(
-                    output_path,
-                    folder_name,
-                )
-            else:
-                out_path = output_path
-
-            if not path.exists(out_path):
-                mkdir(out_path)
-            image.save(path.join(out_path, f"{gen_id}-{index+1}{format}"))
-        if lcm_diffusion_setting:
-            with open(path.join(out_path, f"{gen_id}.json"), "w") as json_file:
-                json.dump(
-                    lcm_diffusion_setting.model_dump(exclude="init_image"),
-                    json_file,
-                    indent=4,
-                )

backend/lcm_models.py

@@ -1,11 +0,0 @@
-from typing import List
-from constants import LCM_DEFAULT_MODEL
-
-
-def get_available_models() -> List:
-    models = [
-        LCM_DEFAULT_MODEL,
-        "latent-consistency/lcm-sdxl",
-        "latent-consistency/lcm-ssd-1b",
-    ]
-    return models

backend/lcm_text_to_image.py

@@ -1,355 +0,0 @@
-from typing import Any
-from diffusers import LCMScheduler
-import torch
-from backend.models.lcmdiffusion_setting import LCMDiffusionSetting
-import numpy as np
-from constants import DEVICE
-from backend.models.lcmdiffusion_setting import LCMLora
-from backend.device import is_openvino_device
-from backend.openvino.pipelines import (
-    get_ov_text_to_image_pipeline,
-    ov_load_taesd,
-    get_ov_image_to_image_pipeline,
-)
-from backend.pipelines.lcm import (
-    get_lcm_model_pipeline,
-    load_taesd,
-    get_image_to_image_pipeline,
-)
-from backend.pipelines.lcm_lora import get_lcm_lora_pipeline
-from backend.models.lcmdiffusion_setting import DiffusionTask
-from image_ops import resize_pil_image
-from math import ceil
-
-
-class LCMTextToImage:
-    def __init__(
-        self,
-        device: str = "cpu",
-    ) -> None:
-        self.pipeline = None
-        self.use_openvino = False
-        self.device = ""
-        self.previous_model_id = None
-        self.previous_use_tae_sd = False
-        self.previous_use_lcm_lora = False
-        self.previous_ov_model_id = ""
-        self.previous_safety_checker = False
-        self.previous_use_openvino = False
-        self.img_to_img_pipeline = None
-        self.is_openvino_init = False
-        self.task_type = DiffusionTask.text_to_image
-        self.torch_data_type = (
-            torch.float32 if is_openvino_device() or DEVICE == "mps" else torch.float16
-        )
-        print(f"Torch datatype : {self.torch_data_type}")
-
-    def _pipeline_to_device(self):
-        print(f"Pipeline device : {DEVICE}")
-        print(f"Pipeline dtype : {self.torch_data_type}")
-        self.pipeline.to(
-            torch_device=DEVICE,
-            torch_dtype=self.torch_data_type,
-        )
-
-    def _add_freeu(self):
-        pipeline_class = self.pipeline.__class__.__name__
-        if isinstance(self.pipeline.scheduler, LCMScheduler):
-            if pipeline_class == "StableDiffusionPipeline":
-                print("Add FreeU - SD")
-                self.pipeline.enable_freeu(
-                    s1=0.9,
-                    s2=0.2,
-                    b1=1.2,
-                    b2=1.4,
-                )
-            elif pipeline_class == "StableDiffusionXLPipeline":
-                print("Add FreeU - SDXL")
-                self.pipeline.enable_freeu(
-                    s1=0.6,
-                    s2=0.4,
-                    b1=1.1,
-                    b2=1.2,
-                )
-
-    def _update_lcm_scheduler_params(self):
-        if isinstance(self.pipeline.scheduler, LCMScheduler):
-            self.pipeline.scheduler = LCMScheduler.from_config(
-                self.pipeline.scheduler.config,
-                beta_start=0.001,
-                beta_end=0.01,
-            )
-
-    def init(
-        self,
-        device: str = "cpu",
-        lcm_diffusion_setting: LCMDiffusionSetting = LCMDiffusionSetting(),
-    ) -> None:
-        self.device = device
-        self.use_openvino = lcm_diffusion_setting.use_openvino
-        model_id = lcm_diffusion_setting.lcm_model_id
-        use_local_model = lcm_diffusion_setting.use_offline_model
-        use_tiny_auto_encoder = lcm_diffusion_setting.use_tiny_auto_encoder
-        use_lora = lcm_diffusion_setting.use_lcm_lora
-        lcm_lora: LCMLora = lcm_diffusion_setting.lcm_lora
-        ov_model_id = lcm_diffusion_setting.openvino_lcm_model_id
-
-        if lcm_diffusion_setting.diffusion_task == DiffusionTask.image_to_image.value:
-            lcm_diffusion_setting.init_image = resize_pil_image(
-                lcm_diffusion_setting.init_image,
-                lcm_diffusion_setting.image_width,
-                lcm_diffusion_setting.image_height,
-            )
-
-        if (
-            self.pipeline is None
-            or self.previous_model_id != model_id
-            or self.previous_use_tae_sd != use_tiny_auto_encoder
-            or self.previous_lcm_lora_base_id != lcm_lora.base_model_id
-            or self.previous_lcm_lora_id != lcm_lora.lcm_lora_id
-            or self.previous_use_lcm_lora != use_lora
-            or self.previous_ov_model_id != ov_model_id
-            or self.previous_safety_checker != lcm_diffusion_setting.use_safety_checker
-            or self.previous_use_openvino != lcm_diffusion_setting.use_openvino
-            or self.previous_task_type != lcm_diffusion_setting.diffusion_task
-        ):
-            if self.use_openvino and is_openvino_device():
-                if self.pipeline:
-                    del self.pipeline
-                    self.pipeline = None
-                self.is_openvino_init = True
-                if (
-                    lcm_diffusion_setting.diffusion_task
-                    == DiffusionTask.text_to_image.value
-                ):
-                    print(f"***** Init Text to image (OpenVINO) - {ov_model_id} *****")
-                    self.pipeline = get_ov_text_to_image_pipeline(
-                        ov_model_id,
-                        use_local_model,
-                    )
-                elif (
-                    lcm_diffusion_setting.diffusion_task
-                    == DiffusionTask.image_to_image.value
-                ):
-                    print(f"***** Image to image (OpenVINO) - {ov_model_id} *****")
-                    self.pipeline = get_ov_image_to_image_pipeline(
-                        ov_model_id,
-                        use_local_model,
-                    )
-            else:
-                if self.pipeline:
-                    del self.pipeline
-                    self.pipeline = None
-                if self.img_to_img_pipeline:
-                    del self.img_to_img_pipeline
-                    self.img_to_img_pipeline = None
-
-                if use_lora:
-                    print(
-                        f"***** Init LCM-LoRA pipeline - {lcm_lora.base_model_id} *****"
-                    )
-                    self.pipeline = get_lcm_lora_pipeline(
-                        lcm_lora.base_model_id,
-                        lcm_lora.lcm_lora_id,
-                        use_local_model,
-                        torch_data_type=self.torch_data_type,
-                    )
-                else:
-                    print(f"***** Init LCM Model pipeline - {model_id} *****")
-                    self.pipeline = get_lcm_model_pipeline(
-                        model_id,
-                        use_local_model,
-                    )
-
-                if (
-                    lcm_diffusion_setting.diffusion_task
-                    == DiffusionTask.image_to_image.value
-                ):
-                    self.img_to_img_pipeline = get_image_to_image_pipeline(
-                        self.pipeline
-                    )
-                self._pipeline_to_device()
-
-            if use_tiny_auto_encoder:
-                if self.use_openvino and is_openvino_device():
-                    print("Using Tiny Auto Encoder (OpenVINO)")
-                    ov_load_taesd(
-                        self.pipeline,
-                        use_local_model,
-                    )
-                else:
-                    print("Using Tiny Auto Encoder")
-                    if (
-                        lcm_diffusion_setting.diffusion_task
-                        == DiffusionTask.text_to_image.value
-                    ):
-                        load_taesd(
-                            self.pipeline,
-                            use_local_model,
-                            self.torch_data_type,
-                        )
-                    elif (
-                        lcm_diffusion_setting.diffusion_task
-                        == DiffusionTask.image_to_image.value
-                    ):
-                        load_taesd(
-                            self.img_to_img_pipeline,
-                            use_local_model,
-                            self.torch_data_type,
-                        )
-
-            if (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.image_to_image.value
-                and lcm_diffusion_setting.use_openvino
-            ):
-                self.pipeline.scheduler = LCMScheduler.from_config(
-                    self.pipeline.scheduler.config,
-                )
-            else:
-                self._update_lcm_scheduler_params()
-
-            if use_lora:
-                self._add_freeu()
-
-            self.previous_model_id = model_id
-            self.previous_ov_model_id = ov_model_id
-            self.previous_use_tae_sd = use_tiny_auto_encoder
-            self.previous_lcm_lora_base_id = lcm_lora.base_model_id
-            self.previous_lcm_lora_id = lcm_lora.lcm_lora_id
-            self.previous_use_lcm_lora = use_lora
-            self.previous_safety_checker = lcm_diffusion_setting.use_safety_checker
-            self.previous_use_openvino = lcm_diffusion_setting.use_openvino
-            self.previous_task_type = lcm_diffusion_setting.diffusion_task
-            if (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.text_to_image.value
-            ):
-                print(f"Pipeline : {self.pipeline}")
-            elif (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.image_to_image.value
-            ):
-                if self.use_openvino and is_openvino_device():
-                    print(f"Pipeline : {self.pipeline}")
-                else:
-                    print(f"Pipeline : {self.img_to_img_pipeline}")
-
-    def generate(
-        self,
-        lcm_diffusion_setting: LCMDiffusionSetting,
-        reshape: bool = False,
-    ) -> Any:
-        guidance_scale = lcm_diffusion_setting.guidance_scale
-        img_to_img_inference_steps = lcm_diffusion_setting.inference_steps
-        check_step_value = int(
-            lcm_diffusion_setting.inference_steps * lcm_diffusion_setting.strength
-        )
-        if (
-            lcm_diffusion_setting.diffusion_task == DiffusionTask.image_to_image.value
-            and check_step_value < 1
-        ):
-            img_to_img_inference_steps = ceil(1 / lcm_diffusion_setting.strength)
-            print(
-                f"Strength: {lcm_diffusion_setting.strength},{img_to_img_inference_steps}"
-            )
-
-        if lcm_diffusion_setting.use_seed:
-            cur_seed = lcm_diffusion_setting.seed
-            if self.use_openvino:
-                np.random.seed(cur_seed)
-            else:
-                torch.manual_seed(cur_seed)
-
-        is_openvino_pipe = lcm_diffusion_setting.use_openvino and is_openvino_device()
-        if is_openvino_pipe:
-            print("Using OpenVINO")
-            if reshape and not self.is_openvino_init:
-                print("Reshape and compile")
-                self.pipeline.reshape(
-                    batch_size=-1,
-                    height=lcm_diffusion_setting.image_height,
-                    width=lcm_diffusion_setting.image_width,
-                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
-                )
-                self.pipeline.compile()
-
-            if self.is_openvino_init:
-                self.is_openvino_init = False
-
-        if not lcm_diffusion_setting.use_safety_checker:
-            self.pipeline.safety_checker = None
-            if (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.image_to_image.value
-                and not is_openvino_pipe
-            ):
-                self.img_to_img_pipeline.safety_checker = None
-
-        if (
-            not lcm_diffusion_setting.use_lcm_lora
-            and not lcm_diffusion_setting.use_openvino
-            and lcm_diffusion_setting.guidance_scale != 1.0
-        ):
-            print("Not using LCM-LoRA so setting guidance_scale 1.0")
-            guidance_scale = 1.0
-
-        if lcm_diffusion_setting.use_openvino:
-            if (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.text_to_image.value
-            ):
-                result_images = self.pipeline(
-                    prompt=lcm_diffusion_setting.prompt,
-                    negative_prompt=lcm_diffusion_setting.negative_prompt,
-                    num_inference_steps=lcm_diffusion_setting.inference_steps,
-                    guidance_scale=guidance_scale,
-                    width=lcm_diffusion_setting.image_width,
-                    height=lcm_diffusion_setting.image_height,
-                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
-                ).images
-            elif (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.image_to_image.value
-            ):
-                result_images = self.pipeline(
-                    image=lcm_diffusion_setting.init_image,
-                    strength=lcm_diffusion_setting.strength,
-                    prompt=lcm_diffusion_setting.prompt,
-                    negative_prompt=lcm_diffusion_setting.negative_prompt,
-                    num_inference_steps=img_to_img_inference_steps * 3,
-                    guidance_scale=guidance_scale,
-                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
-                ).images
-
-        else:
-            if (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.text_to_image.value
-            ):
-                result_images = self.pipeline(
-                    prompt=lcm_diffusion_setting.prompt,
-                    negative_prompt=lcm_diffusion_setting.negative_prompt,
-                    num_inference_steps=lcm_diffusion_setting.inference_steps,
-                    guidance_scale=guidance_scale,
-                    width=lcm_diffusion_setting.image_width,
-                    height=lcm_diffusion_setting.image_height,
-                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
-                ).images
-            elif (
-                lcm_diffusion_setting.diffusion_task
-                == DiffusionTask.image_to_image.value
-            ):
-                result_images = self.img_to_img_pipeline(
-                    image=lcm_diffusion_setting.init_image,
-                    strength=lcm_diffusion_setting.strength,
-                    prompt=lcm_diffusion_setting.prompt,
-                    negative_prompt=lcm_diffusion_setting.negative_prompt,
-                    num_inference_steps=img_to_img_inference_steps,
-                    guidance_scale=guidance_scale,
-                    width=lcm_diffusion_setting.image_width,
-                    height=lcm_diffusion_setting.image_height,
-                    num_images_per_prompt=lcm_diffusion_setting.number_of_images,
-                ).images
-
-        return result_images

backend/lcmdiffusion/pipelines/openvino/lcm_ov_pipeline.py

@@ -1,447 +0,0 @@
-# https://huggingface.co/deinferno/LCM_Dreamshaper_v7-openvino
-
-import inspect
-
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple, Union, Dict, Any, Callable, OrderedDict
-
-import numpy as np
-import openvino
-import torch
-
-from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
-from optimum.intel.openvino.modeling_diffusion import (
-    OVStableDiffusionPipeline,
-    OVModelUnet,
-    OVModelVaeDecoder,
-    OVModelTextEncoder,
-    OVModelVaeEncoder,
-    VaeImageProcessor,
-)
-from optimum.utils import (
-    DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER,
-    DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER,
-    DIFFUSION_MODEL_UNET_SUBFOLDER,
-    DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER,
-    DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER,
-)
-
-
-from diffusers import logging
-
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
-
-
-class LCMOVModelUnet(OVModelUnet):
-    def __call__(
-        self,
-        sample: np.ndarray,
-        timestep: np.ndarray,
-        encoder_hidden_states: np.ndarray,
-        timestep_cond: Optional[np.ndarray] = None,
-        text_embeds: Optional[np.ndarray] = None,
-        time_ids: Optional[np.ndarray] = None,
-    ):
-        self._compile()
-
-        inputs = {
-            "sample": sample,
-            "timestep": timestep,
-            "encoder_hidden_states": encoder_hidden_states,
-        }
-
-        if timestep_cond is not None:
-            inputs["timestep_cond"] = timestep_cond
-        if text_embeds is not None:
-            inputs["text_embeds"] = text_embeds
-        if time_ids is not None:
-            inputs["time_ids"] = time_ids
-
-        outputs = self.request(inputs, shared_memory=True)
-        return list(outputs.values())
-
-
-class OVLatentConsistencyModelPipeline(OVStableDiffusionPipeline):
-    def __init__(
-        self,
-        vae_decoder: openvino.runtime.Model,
-        text_encoder: openvino.runtime.Model,
-        unet: openvino.runtime.Model,
-        config: Dict[str, Any],
-        tokenizer: "CLIPTokenizer",
-        scheduler: Union["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"],
-        feature_extractor: Optional["CLIPFeatureExtractor"] = None,
-        vae_encoder: Optional[openvino.runtime.Model] = None,
-        text_encoder_2: Optional[openvino.runtime.Model] = None,
-        tokenizer_2: Optional["CLIPTokenizer"] = None,
-        device: str = "CPU",
-        dynamic_shapes: bool = True,
-        compile: bool = True,
-        ov_config: Optional[Dict[str, str]] = None,
-        model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None,
-        **kwargs,
-    ):
-        self._internal_dict = config
-        self._device = device.upper()
-        self.is_dynamic = dynamic_shapes
-        self.ov_config = ov_config if ov_config is not None else {}
-        self._model_save_dir = (
-            Path(model_save_dir.name)
-            if isinstance(model_save_dir, TemporaryDirectory)
-            else model_save_dir
-        )
-        self.vae_decoder = OVModelVaeDecoder(vae_decoder, self)
-        self.unet = LCMOVModelUnet(unet, self)
-        self.text_encoder = (
-            OVModelTextEncoder(text_encoder, self) if text_encoder is not None else None
-        )
-        self.text_encoder_2 = (
-            OVModelTextEncoder(
-                text_encoder_2,
-                self,
-                model_name=DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER,
-            )
-            if text_encoder_2 is not None
-            else None
-        )
-        self.vae_encoder = (
-            OVModelVaeEncoder(vae_encoder, self) if vae_encoder is not None else None
-        )
-
-        if "block_out_channels" in self.vae_decoder.config:
-            self.vae_scale_factor = 2 ** (
-                len(self.vae_decoder.config["block_out_channels"]) - 1
-            )
-        else:
-            self.vae_scale_factor = 8
-
-        self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
-
-        self.tokenizer = tokenizer
-        self.tokenizer_2 = tokenizer_2
-        self.scheduler = scheduler
-        self.feature_extractor = feature_extractor
-        self.safety_checker = None
-        self.preprocessors = []
-
-        if self.is_dynamic:
-            self.reshape(batch_size=-1, height=-1, width=-1, num_images_per_prompt=-1)
-
-        if compile:
-            self.compile()
-
-        sub_models = {
-            DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER: self.text_encoder,
-            DIFFUSION_MODEL_UNET_SUBFOLDER: self.unet,
-            DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER: self.vae_decoder,
-            DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER: self.vae_encoder,
-            DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER: self.text_encoder_2,
-        }
-        for name in sub_models.keys():
-            self._internal_dict[name] = (
-                ("optimum", sub_models[name].__class__.__name__)
-                if sub_models[name] is not None
-                else (None, None)
-            )
-
-        self._internal_dict.pop("vae", None)
-
-    def _reshape_unet(
-        self,
-        model: openvino.runtime.Model,
-        batch_size: int = -1,
-        height: int = -1,
-        width: int = -1,
-        num_images_per_prompt: int = -1,
-        tokenizer_max_length: int = -1,
-    ):
-        if batch_size == -1 or num_images_per_prompt == -1:
-            batch_size = -1
-        else:
-            batch_size = batch_size * num_images_per_prompt
-
-        height = height // self.vae_scale_factor if height > 0 else height
-        width = width // self.vae_scale_factor if width > 0 else width
-        shapes = {}
-        for inputs in model.inputs:
-            shapes[inputs] = inputs.get_partial_shape()
-            if inputs.get_any_name() == "timestep":
-                shapes[inputs][0] = 1
-            elif inputs.get_any_name() == "sample":
-                in_channels = self.unet.config.get("in_channels", None)
-                if in_channels is None:
-                    in_channels = shapes[inputs][1]
-                    if in_channels.is_dynamic:
-                        logger.warning(
-                            "Could not identify `in_channels` from the unet configuration, to statically reshape the unet please provide a configuration."
-                        )
-                        self.is_dynamic = True
-
-                shapes[inputs] = [batch_size, in_channels, height, width]
-            elif inputs.get_any_name() == "timestep_cond":
-                shapes[inputs] = [batch_size, inputs.get_partial_shape()[1]]
-            elif inputs.get_any_name() == "text_embeds":
-                shapes[inputs] = [
-                    batch_size,
-                    self.text_encoder_2.config["projection_dim"],
-                ]
-            elif inputs.get_any_name() == "time_ids":
-                shapes[inputs] = [batch_size, inputs.get_partial_shape()[1]]
-            else:
-                shapes[inputs][0] = batch_size
-                shapes[inputs][1] = tokenizer_max_length
-        model.reshape(shapes)
-        return model
-
-    def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=np.float32):
-        """
-        see https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298
-        Args:
-        timesteps: np.array: generate embedding vectors at these timesteps
-        embedding_dim: int: dimension of the embeddings to generate
-        dtype: data type of the generated embeddings
-
-        Returns:
-        embedding vectors with shape `(len(timesteps), embedding_dim)`
-        """
-        assert len(w.shape) == 1
-        w = w * 1000.0
-
-        half_dim = embedding_dim // 2
-        emb = np.log(np.array(10000.0)) / (half_dim - 1)
-        emb = np.exp(np.arange(half_dim, dtype=dtype) * -emb)
-        emb = w.astype(dtype)[:, None] * emb[None, :]
-        emb = np.concatenate([np.sin(emb), np.cos(emb)], axis=1)
-        if embedding_dim % 2 == 1:  # zero pad
-            emb = np.pad(emb, (0, 1))
-        assert emb.shape == (w.shape[0], embedding_dim)
-        return emb
-
-    # Adapted from https://github.com/huggingface/optimum/blob/15b8d1eed4d83c5004d3b60f6b6f13744b358f01/optimum/pipelines/diffusers/pipeline_stable_diffusion.py#L201
-    def __call__(
-        self,
-        prompt: Optional[Union[str, List[str]]] = None,
-        height: Optional[int] = None,
-        width: Optional[int] = None,
-        num_inference_steps: int = 4,
-        original_inference_steps: int = None,
-        guidance_scale: float = 7.5,
-        num_images_per_prompt: int = 1,
-        eta: float = 0.0,
-        generator: Optional[np.random.RandomState] = None,
-        latents: Optional[np.ndarray] = None,
-        prompt_embeds: Optional[np.ndarray] = None,
-        output_type: str = "pil",
-        return_dict: bool = True,
-        callback: Optional[Callable[[int, int, np.ndarray], None]] = None,
-        callback_steps: int = 1,
-        guidance_rescale: float = 0.0,
-    ):
-        r"""
-        Function invoked when calling the pipeline for generation.
-
-        Args:
-            prompt (`Optional[Union[str, List[str]]]`, defaults to None):
-                The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
-                instead.
-            height (`Optional[int]`, defaults to None):
-                The height in pixels of the generated image.
-            width (`Optional[int]`, defaults to None):
-                The width in pixels of the generated image.
-            num_inference_steps (`int`, defaults to 4):
-                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
-                expense of slower inference.
-            original_inference_steps (`int`, *optional*):
-                The original number of inference steps use to generate a linearly-spaced timestep schedule, from which
-                we will draw `num_inference_steps` evenly spaced timesteps from as our final timestep schedule,
-                following the Skipping-Step method in the paper (see Section 4.3). If not set this will default to the
-                scheduler's `original_inference_steps` attribute.
-            guidance_scale (`float`, defaults to 7.5):
-                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
-                `guidance_scale` is defined as `w` of equation 2. of [Imagen
-                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
-                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
-                usually at the expense of lower image quality.
-            num_images_per_prompt (`int`, defaults to 1):
-                The number of images to generate per prompt.
-            eta (`float`, defaults to 0.0):
-                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
-                [`schedulers.DDIMScheduler`], will be ignored for others.
-            generator (`Optional[np.random.RandomState]`, defaults to `None`)::
-                A np.random.RandomState to make generation deterministic.
-            latents (`Optional[np.ndarray]`, defaults to `None`):
-                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
-                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
-                tensor will ge generated by sampling using the supplied random `generator`.
-            prompt_embeds (`Optional[np.ndarray]`, defaults to `None`):
-                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
-                provided, text embeddings will be generated from `prompt` input argument.
-            output_type (`str`, defaults to `"pil"`):
-                The output format of the generate image. Choose between
-                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
-            return_dict (`bool`, defaults to `True`):
-                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
-                plain tuple.
-            callback (Optional[Callable], defaults to `None`):
-                A function that will be called every `callback_steps` steps during inference. The function will be
-                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
-            callback_steps (`int`, defaults to 1):
-                The frequency at which the `callback` function will be called. If not specified, the callback will be
-                called at every step.
-            guidance_rescale (`float`, defaults to 0.0):
-                Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
-                Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
-                [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
-                Guidance rescale factor should fix overexposure when using zero terminal SNR.
-
-        Returns:
-            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
-            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
-            When returning a tuple, the first element is a list with the generated images, and the second element is a
-            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
-            (nsfw) content, according to the `safety_checker`.
-        """
-        height = (
-            height or self.unet.config.get("sample_size", 64) * self.vae_scale_factor
-        )
-        width = width or self.unet.config.get("sample_size", 64) * self.vae_scale_factor
-
-        # check inputs. Raise error if not correct
-        self.check_inputs(
-            prompt, height, width, callback_steps, None, prompt_embeds, None
-        )
-
-        # define call parameters
-        if isinstance(prompt, str):
-            batch_size = 1
-        elif isinstance(prompt, list):
-            batch_size = len(prompt)
-        else:
-            batch_size = prompt_embeds.shape[0]
-
-        if generator is None:
-            generator = np.random
-
-        # Create torch.Generator instance with same state as np.random.RandomState
-        torch_generator = torch.Generator().manual_seed(
-            int(generator.get_state()[1][0])
-        )
-
-        # do_classifier_free_guidance = guidance_scale > 1.0
-
-        # NOTE: when a LCM is distilled from an LDM via latent consistency distillation (Algorithm 1) with guided
-        # distillation, the forward pass of the LCM learns to approximate sampling from the LDM using CFG with the
-        # unconditional prompt "" (the empty string). Due to this, LCMs currently do not support negative prompts.
-        prompt_embeds = self._encode_prompt(
-            prompt,
-            num_images_per_prompt,
-            False,
-            negative_prompt=None,
-            prompt_embeds=prompt_embeds,
-            negative_prompt_embeds=None,
-        )
-
-        # set timesteps
-        self.scheduler.set_timesteps(
-            num_inference_steps,
-            "cpu",
-            original_inference_steps=original_inference_steps,
-        )
-        timesteps = self.scheduler.timesteps
-
-        latents = self.prepare_latents(
-            batch_size * num_images_per_prompt,
-            self.unet.config.get("in_channels", 4),
-            height,
-            width,
-            prompt_embeds.dtype,
-            generator,
-            latents,
-        )
-
-        # Get Guidance Scale Embedding
-        w = np.tile(guidance_scale - 1, batch_size * num_images_per_prompt)
-        w_embedding = self.get_guidance_scale_embedding(
-            w, embedding_dim=self.unet.config.get("time_cond_proj_dim", 256)
-        )
-
-        # Adapted from diffusers to extend it for other runtimes than ORT
-        timestep_dtype = self.unet.input_dtype.get("timestep", np.float32)
-
-        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
-        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
-        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
-        # and should be between [0, 1]
-        accepts_eta = "eta" in set(
-            inspect.signature(self.scheduler.step).parameters.keys()
-        )
-        extra_step_kwargs = {}
-        if accepts_eta:
-            extra_step_kwargs["eta"] = eta
-
-        accepts_generator = "generator" in set(
-            inspect.signature(self.scheduler.step).parameters.keys()
-        )
-        if accepts_generator:
-            extra_step_kwargs["generator"] = torch_generator
-
-        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
-        for i, t in enumerate(self.progress_bar(timesteps)):
-            # predict the noise residual
-            timestep = np.array([t], dtype=timestep_dtype)
-
-            noise_pred = self.unet(
-                sample=latents,
-                timestep=timestep,
-                timestep_cond=w_embedding,
-                encoder_hidden_states=prompt_embeds,
-            )[0]
-
-            # compute the previous noisy sample x_t -> x_t-1
-            latents, denoised = self.scheduler.step(
-                torch.from_numpy(noise_pred),
-                t,
-                torch.from_numpy(latents),
-                **extra_step_kwargs,
-                return_dict=False,
-            )
-
-            latents, denoised = latents.numpy(), denoised.numpy()
-
-            # call the callback, if provided
-            if i == len(timesteps) - 1 or (
-                (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0
-            ):
-                if callback is not None and i % callback_steps == 0:
-                    callback(i, t, latents)
-
-        if output_type == "latent":
-            image = latents
-            has_nsfw_concept = None
-        else:
-            denoised /= self.vae_decoder.config.get("scaling_factor", 0.18215)
-            # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1
-            image = np.concatenate(
-                [
-                    self.vae_decoder(latent_sample=denoised[i : i + 1])[0]
-                    for i in range(latents.shape[0])
-                ]
-            )
-            image, has_nsfw_concept = self.run_safety_checker(image)
-
-        if has_nsfw_concept is None:
-            do_denormalize = [True] * image.shape[0]
-        else:
-            do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]
-
-        image = self.image_processor.postprocess(
-            image, output_type=output_type, do_denormalize=do_denormalize
-        )
-
-        if not return_dict:
-            return (image, has_nsfw_concept)
-
-        return StableDiffusionPipelineOutput(
-            images=image, nsfw_content_detected=has_nsfw_concept
-        )

backend/lcmdiffusion/pipelines/openvino/lcm_scheduler.py

@@ -1,576 +0,0 @@
-# Copyright 2023 Stanford University Team and The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
-# and https://github.com/hojonathanho/diffusion
-
-import math
-from dataclasses import dataclass
-from typing import List, Optional, Tuple, Union
-
-import numpy as np
-import torch
-
-from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.utils import BaseOutput, logging
-from diffusers.utils.torch_utils import randn_tensor
-from diffusers.schedulers.scheduling_utils import SchedulerMixin
-
-
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
-
-
-@dataclass
-class LCMSchedulerOutput(BaseOutput):
-    """
-    Output class for the scheduler's `step` function output.
-
-    Args:
-        prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
-            Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
-            denoising loop.
-        pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
-            The predicted denoised sample `(x_{0})` based on the model output from the current timestep.
-            `pred_original_sample` can be used to preview progress or for guidance.
-    """
-
-    prev_sample: torch.FloatTensor
-    denoised: Optional[torch.FloatTensor] = None
-
-
-# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
-def betas_for_alpha_bar(
-    num_diffusion_timesteps,
-    max_beta=0.999,
-    alpha_transform_type="cosine",
-):
-    """
-    Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of
-    (1-beta) over time from t = [0,1].
-
-    Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up
-    to that part of the diffusion process.
-
-
-    Args:
-        num_diffusion_timesteps (`int`): the number of betas to produce.
-        max_beta (`float`): the maximum beta to use; use values lower than 1 to
-                     prevent singularities.
-        alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar.
-                     Choose from `cosine` or `exp`
-
-    Returns:
-        betas (`np.ndarray`): the betas used by the scheduler to step the model outputs
-    """
-    if alpha_transform_type == "cosine":
-
-        def alpha_bar_fn(t):
-            return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2
-
-    elif alpha_transform_type == "exp":
-
-        def alpha_bar_fn(t):
-            return math.exp(t * -12.0)
-
-    else:
-        raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}")
-
-    betas = []
-    for i in range(num_diffusion_timesteps):
-        t1 = i / num_diffusion_timesteps
-        t2 = (i + 1) / num_diffusion_timesteps
-        betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta))
-    return torch.tensor(betas, dtype=torch.float32)
-
-
-# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr
-def rescale_zero_terminal_snr(betas: torch.FloatTensor) -> torch.FloatTensor:
-    """
-    Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1)
-
-
-    Args:
-        betas (`torch.FloatTensor`):
-            the betas that the scheduler is being initialized with.
-
-    Returns:
-        `torch.FloatTensor`: rescaled betas with zero terminal SNR
-    """
-    # Convert betas to alphas_bar_sqrt
-    alphas = 1.0 - betas
-    alphas_cumprod = torch.cumprod(alphas, dim=0)
-    alphas_bar_sqrt = alphas_cumprod.sqrt()
-
-    # Store old values.
-    alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
-    alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()
-
-    # Shift so the last timestep is zero.
-    alphas_bar_sqrt -= alphas_bar_sqrt_T
-
-    # Scale so the first timestep is back to the old value.
-    alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T)
-
-    # Convert alphas_bar_sqrt to betas
-    alphas_bar = alphas_bar_sqrt**2  # Revert sqrt
-    alphas = alphas_bar[1:] / alphas_bar[:-1]  # Revert cumprod
-    alphas = torch.cat([alphas_bar[0:1], alphas])
-    betas = 1 - alphas
-
-    return betas
-
-
-class LCMScheduler(SchedulerMixin, ConfigMixin):
-    """
-    `LCMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with
-    non-Markovian guidance.
-
-    This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. [`~ConfigMixin`] takes care of storing all config
-    attributes that are passed in the scheduler's `__init__` function, such as `num_train_timesteps`. They can be
-    accessed via `scheduler.config.num_train_timesteps`. [`SchedulerMixin`] provides general loading and saving
-    functionality via the [`SchedulerMixin.save_pretrained`] and [`~SchedulerMixin.from_pretrained`] functions.
-
-    Args:
-        num_train_timesteps (`int`, defaults to 1000):
-            The number of diffusion steps to train the model.
-        beta_start (`float`, defaults to 0.0001):
-            The starting `beta` value of inference.
-        beta_end (`float`, defaults to 0.02):
-            The final `beta` value.
-        beta_schedule (`str`, defaults to `"linear"`):
-            The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from
-            `linear`, `scaled_linear`, or `squaredcos_cap_v2`.
-        trained_betas (`np.ndarray`, *optional*):
-            Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`.
-        original_inference_steps (`int`, *optional*, defaults to 50):
-            The default number of inference steps used to generate a linearly-spaced timestep schedule, from which we
-            will ultimately take `num_inference_steps` evenly spaced timesteps to form the final timestep schedule.
-        clip_sample (`bool`, defaults to `True`):
-            Clip the predicted sample for numerical stability.
-        clip_sample_range (`float`, defaults to 1.0):
-            The maximum magnitude for sample clipping. Valid only when `clip_sample=True`.
-        set_alpha_to_one (`bool`, defaults to `True`):
-            Each diffusion step uses the alphas product value at that step and at the previous one. For the final step
-            there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
-            otherwise it uses the alpha value at step 0.
-        steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps. You can use a combination of `offset=1` and
-            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
-            Diffusion.
-        prediction_type (`str`, defaults to `epsilon`, *optional*):
-            Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
-            `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen
-            Video](https://imagen.research.google/video/paper.pdf) paper).
-        thresholding (`bool`, defaults to `False`):
-            Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such
-            as Stable Diffusion.
-        dynamic_thresholding_ratio (`float`, defaults to 0.995):
-            The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.
-        sample_max_value (`float`, defaults to 1.0):
-            The threshold value for dynamic thresholding. Valid only when `thresholding=True`.
-        timestep_spacing (`str`, defaults to `"leading"`):
-            The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
-            Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
-        rescale_betas_zero_snr (`bool`, defaults to `False`):
-            Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
-            dark samples instead of limiting it to samples with medium brightness. Loosely related to
-            [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506).
-    """
-
-    order = 1
-
-    @register_to_config
-    def __init__(
-        self,
-        num_train_timesteps: int = 1000,
-        beta_start: float = 0.00085,
-        beta_end: float = 0.012,
-        beta_schedule: str = "scaled_linear",
-        trained_betas: Optional[Union[np.ndarray, List[float]]] = None,
-        original_inference_steps: int = 50,
-        clip_sample: bool = False,
-        clip_sample_range: float = 1.0,
-        set_alpha_to_one: bool = True,
-        steps_offset: int = 0,
-        prediction_type: str = "epsilon",
-        thresholding: bool = False,
-        dynamic_thresholding_ratio: float = 0.995,
-        sample_max_value: float = 1.0,
-        timestep_spacing: str = "leading",
-        rescale_betas_zero_snr: bool = False,
-    ):
-        if trained_betas is not None:
-            self.betas = torch.tensor(trained_betas, dtype=torch.float32)
-        elif beta_schedule == "linear":
-            self.betas = torch.linspace(
-                beta_start, beta_end, num_train_timesteps, dtype=torch.float32
-            )
-        elif beta_schedule == "scaled_linear":
-            # this schedule is very specific to the latent diffusion model.
-            self.betas = (
-                torch.linspace(
-                    beta_start**0.5,
-                    beta_end**0.5,
-                    num_train_timesteps,
-                    dtype=torch.float32,
-                )
-                ** 2
-            )
-        elif beta_schedule == "squaredcos_cap_v2":
-            # Glide cosine schedule
-            self.betas = betas_for_alpha_bar(num_train_timesteps)
-        else:
-            raise NotImplementedError(
-                f"{beta_schedule} does is not implemented for {self.__class__}"
-            )
-
-        # Rescale for zero SNR
-        if rescale_betas_zero_snr:
-            self.betas = rescale_zero_terminal_snr(self.betas)
-
-        self.alphas = 1.0 - self.betas
-        self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)
-
-        # At every step in ddim, we are looking into the previous alphas_cumprod
-        # For the final step, there is no previous alphas_cumprod because we are already at 0
-        # `set_alpha_to_one` decides whether we set this parameter simply to one or
-        # whether we use the final alpha of the "non-previous" one.
-        self.final_alpha_cumprod = (
-            torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0]
-        )
-
-        # standard deviation of the initial noise distribution
-        self.init_noise_sigma = 1.0
-
-        # setable values
-        self.num_inference_steps = None
-        self.timesteps = torch.from_numpy(
-            np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)
-        )
-
-        self._step_index = None
-
-    # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index
-    def _init_step_index(self, timestep):
-        if isinstance(timestep, torch.Tensor):
-            timestep = timestep.to(self.timesteps.device)
-
-        index_candidates = (self.timesteps == timestep).nonzero()
-
-        # The sigma index that is taken for the **very** first `step`
-        # is always the second index (or the last index if there is only 1)
-        # This way we can ensure we don't accidentally skip a sigma in
-        # case we start in the middle of the denoising schedule (e.g. for image-to-image)
-        if len(index_candidates) > 1:
-            step_index = index_candidates[1]
-        else:
-            step_index = index_candidates[0]
-
-        self._step_index = step_index.item()
-
-    @property
-    def step_index(self):
-        return self._step_index
-
-    def scale_model_input(
-        self, sample: torch.FloatTensor, timestep: Optional[int] = None
-    ) -> torch.FloatTensor:
-        """
-        Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
-        current timestep.
-
-        Args:
-            sample (`torch.FloatTensor`):
-                The input sample.
-            timestep (`int`, *optional*):
-                The current timestep in the diffusion chain.
-        Returns:
-            `torch.FloatTensor`:
-                A scaled input sample.
-        """
-        return sample
-
-    # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
-    def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor:
-        """
-        "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
-        prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
-        s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
-        pixels from saturation at each step. We find that dynamic thresholding results in significantly better
-        photorealism as well as better image-text alignment, especially when using very large guidance weights."
-
-        https://arxiv.org/abs/2205.11487
-        """
-        dtype = sample.dtype
-        batch_size, channels, *remaining_dims = sample.shape
-
-        if dtype not in (torch.float32, torch.float64):
-            sample = (
-                sample.float()
-            )  # upcast for quantile calculation, and clamp not implemented for cpu half
-
-        # Flatten sample for doing quantile calculation along each image
-        sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
-
-        abs_sample = sample.abs()  # "a certain percentile absolute pixel value"
-
-        s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
-        s = torch.clamp(
-            s, min=1, max=self.config.sample_max_value
-        )  # When clamped to min=1, equivalent to standard clipping to [-1, 1]
-        s = s.unsqueeze(1)  # (batch_size, 1) because clamp will broadcast along dim=0
-        sample = (
-            torch.clamp(sample, -s, s) / s
-        )  # "we threshold xt0 to the range [-s, s] and then divide by s"
-
-        sample = sample.reshape(batch_size, channels, *remaining_dims)
-        sample = sample.to(dtype)
-
-        return sample
-
-    def set_timesteps(
-        self,
-        num_inference_steps: int,
-        device: Union[str, torch.device] = None,
-        original_inference_steps: Optional[int] = None,
-    ):
-        """
-        Sets the discrete timesteps used for the diffusion chain (to be run before inference).
-
-        Args:
-            num_inference_steps (`int`):
-                The number of diffusion steps used when generating samples with a pre-trained model.
-            device (`str` or `torch.device`, *optional*):
-                The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
-            original_inference_steps (`int`, *optional*):
-                The original number of inference steps, which will be used to generate a linearly-spaced timestep
-                schedule (which is different from the standard `diffusers` implementation). We will then take
-                `num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as
-                our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute.
-        """
-
-        if num_inference_steps > self.config.num_train_timesteps:
-            raise ValueError(
-                f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"
-                f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
-                f" maximal {self.config.num_train_timesteps} timesteps."
-            )
-
-        self.num_inference_steps = num_inference_steps
-        original_steps = (
-            original_inference_steps
-            if original_inference_steps is not None
-            else self.original_inference_steps
-        )
-
-        if original_steps > self.config.num_train_timesteps:
-            raise ValueError(
-                f"`original_steps`: {original_steps} cannot be larger than `self.config.train_timesteps`:"
-                f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
-                f" maximal {self.config.num_train_timesteps} timesteps."
-            )
-
-        if num_inference_steps > original_steps:
-            raise ValueError(
-                f"`num_inference_steps`: {num_inference_steps} cannot be larger than `original_inference_steps`:"
-                f" {original_steps} because the final timestep schedule will be a subset of the"
-                f" `original_inference_steps`-sized initial timestep schedule."
-            )
-
-        # LCM Timesteps Setting
-        # Currently, only linear spacing is supported.
-        c = self.config.num_train_timesteps // original_steps
-        # LCM Training Steps Schedule
-        lcm_origin_timesteps = np.asarray(list(range(1, original_steps + 1))) * c - 1
-        skipping_step = len(lcm_origin_timesteps) // num_inference_steps
-        # LCM Inference Steps Schedule
-        timesteps = lcm_origin_timesteps[::-skipping_step][:num_inference_steps]
-
-        self.timesteps = torch.from_numpy(timesteps.copy()).to(
-            device=device, dtype=torch.long
-        )
-
-        self._step_index = None
-
-    def get_scalings_for_boundary_condition_discrete(self, t):
-        self.sigma_data = 0.5  # Default: 0.5
-
-        # By dividing 0.1: This is almost a delta function at t=0.
-        c_skip = self.sigma_data**2 / ((t / 0.1) ** 2 + self.sigma_data**2)
-        c_out = (t / 0.1) / ((t / 0.1) ** 2 + self.sigma_data**2) ** 0.5
-        return c_skip, c_out
-
-    def step(
-        self,
-        model_output: torch.FloatTensor,
-        timestep: int,
-        sample: torch.FloatTensor,
-        generator: Optional[torch.Generator] = None,
-        return_dict: bool = True,
-    ) -> Union[LCMSchedulerOutput, Tuple]:
-        """
-        Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
-        process from the learned model outputs (most often the predicted noise).
-
-        Args:
-            model_output (`torch.FloatTensor`):
-                The direct output from learned diffusion model.
-            timestep (`float`):
-                The current discrete timestep in the diffusion chain.
-            sample (`torch.FloatTensor`):
-                A current instance of a sample created by the diffusion process.
-            generator (`torch.Generator`, *optional*):
-                A random number generator.
-            return_dict (`bool`, *optional*, defaults to `True`):
-                Whether or not to return a [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] or `tuple`.
-        Returns:
-            [`~schedulers.scheduling_utils.LCMSchedulerOutput`] or `tuple`:
-                If return_dict is `True`, [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] is returned, otherwise a
-                tuple is returned where the first element is the sample tensor.
-        """
-        if self.num_inference_steps is None:
-            raise ValueError(
-                "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
-            )
-
-        if self.step_index is None:
-            self._init_step_index(timestep)
-
-        # 1. get previous step value
-        prev_step_index = self.step_index + 1
-        if prev_step_index < len(self.timesteps):
-            prev_timestep = self.timesteps[prev_step_index]
-        else:
-            prev_timestep = timestep
-
-        # 2. compute alphas, betas
-        alpha_prod_t = self.alphas_cumprod[timestep]
-        alpha_prod_t_prev = (
-            self.alphas_cumprod[prev_timestep]
-            if prev_timestep >= 0
-            else self.final_alpha_cumprod
-        )
-
-        beta_prod_t = 1 - alpha_prod_t
-        beta_prod_t_prev = 1 - alpha_prod_t_prev
-
-        # 3. Get scalings for boundary conditions
-        c_skip, c_out = self.get_scalings_for_boundary_condition_discrete(timestep)
-
-        # 4. Compute the predicted original sample x_0 based on the model parameterization
-        if self.config.prediction_type == "epsilon":  # noise-prediction
-            predicted_original_sample = (
-                sample - beta_prod_t.sqrt() * model_output
-            ) / alpha_prod_t.sqrt()
-        elif self.config.prediction_type == "sample":  # x-prediction
-            predicted_original_sample = model_output
-        elif self.config.prediction_type == "v_prediction":  # v-prediction
-            predicted_original_sample = (
-                alpha_prod_t.sqrt() * sample - beta_prod_t.sqrt() * model_output
-            )
-        else:
-            raise ValueError(
-                f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or"
-                " `v_prediction` for `LCMScheduler`."
-            )
-
-        # 5. Clip or threshold "predicted x_0"
-        if self.config.thresholding:
-            predicted_original_sample = self._threshold_sample(
-                predicted_original_sample
-            )
-        elif self.config.clip_sample:
-            predicted_original_sample = predicted_original_sample.clamp(
-                -self.config.clip_sample_range, self.config.clip_sample_range
-            )
-
-        # 6. Denoise model output using boundary conditions
-        denoised = c_out * predicted_original_sample + c_skip * sample
-
-        # 7. Sample and inject noise z ~ N(0, I) for MultiStep Inference
-        # Noise is not used for one-step sampling.
-        if len(self.timesteps) > 1:
-            noise = randn_tensor(
-                model_output.shape, generator=generator, device=model_output.device
-            )
-            prev_sample = (
-                alpha_prod_t_prev.sqrt() * denoised + beta_prod_t_prev.sqrt() * noise
-            )
-        else:
-            prev_sample = denoised
-
-        # upon completion increase step index by one
-        self._step_index += 1
-
-        if not return_dict:
-            return (prev_sample, denoised)
-
-        return LCMSchedulerOutput(prev_sample=prev_sample, denoised=denoised)
-
-    # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise
-    def add_noise(
-        self,
-        original_samples: torch.FloatTensor,
-        noise: torch.FloatTensor,
-        timesteps: torch.IntTensor,
-    ) -> torch.FloatTensor:
-        # Make sure alphas_cumprod and timestep have same device and dtype as original_samples
-        alphas_cumprod = self.alphas_cumprod.to(
-            device=original_samples.device, dtype=original_samples.dtype
-        )
-        timesteps = timesteps.to(original_samples.device)
-
-        sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
-        sqrt_alpha_prod = sqrt_alpha_prod.flatten()
-        while len(sqrt_alpha_prod.shape) < len(original_samples.shape):
-            sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
-
-        sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
-        sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
-        while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape):
-            sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
-
-        noisy_samples = (
-            sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
-        )
-        return noisy_samples
-
-    # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity
-    def get_velocity(
-        self,
-        sample: torch.FloatTensor,
-        noise: torch.FloatTensor,
-        timesteps: torch.IntTensor,
-    ) -> torch.FloatTensor:
-        # Make sure alphas_cumprod and timestep have same device and dtype as sample
-        alphas_cumprod = self.alphas_cumprod.to(
-            device=sample.device, dtype=sample.dtype
-        )
-        timesteps = timesteps.to(sample.device)
-
-        sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
-        sqrt_alpha_prod = sqrt_alpha_prod.flatten()
-        while len(sqrt_alpha_prod.shape) < len(sample.shape):
-            sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
-
-        sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
-        sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
-        while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape):
-            sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
-
-        velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
-        return velocity
-
-    def __len__(self):
-        return self.config.num_train_timesteps

backend/models/lcmdiffusion_setting.py

@@ -1,39 +0,0 @@
-from typing import Optional, Any
-from enum import Enum
-from pydantic import BaseModel
-from constants import LCM_DEFAULT_MODEL, LCM_DEFAULT_MODEL_OPENVINO
-
-
-class LCMLora(BaseModel):
-    base_model_id: str = "Lykon/dreamshaper-8"
-    lcm_lora_id: str = "latent-consistency/lcm-lora-sdv1-5"
-
-
-class DiffusionTask(str, Enum):
-    """Diffusion task types"""
-
-    text_to_image = "text_to_image"
-    image_to_image = "image_to_image"
-
-
-class LCMDiffusionSetting(BaseModel):
-    lcm_model_id: str = LCM_DEFAULT_MODEL
-    openvino_lcm_model_id: str = LCM_DEFAULT_MODEL_OPENVINO
-    use_offline_model: bool = False
-    use_lcm_lora: bool = False
-    lcm_lora: Optional[LCMLora] = LCMLora()
-    use_tiny_auto_encoder: bool = False
-    use_openvino: bool = False
-    prompt: str = ""
-    negative_prompt: str = ""
-    init_image: Any = None
-    strength: Optional[float] = 0.6
-    image_height: Optional[int] = 512
-    image_width: Optional[int] = 512
-    inference_steps: Optional[int] = 1
-    guidance_scale: Optional[float] = 1
-    number_of_images: Optional[int] = 1
-    seed: Optional[int] = 123123
-    use_seed: bool = False
-    use_safety_checker: bool = False
-    diffusion_task: str = DiffusionTask.text_to_image.value

backend/openvino/custom_ov_model_vae_decoder.py

@@ -1,21 +0,0 @@
-from backend.device import is_openvino_device
-
-if is_openvino_device():
-    from optimum.intel.openvino.modeling_diffusion import OVModelVaeDecoder
-
-
-class CustomOVModelVaeDecoder(OVModelVaeDecoder):
-    def __init__(
-        self,
-        model,
-        parent_model,
-        ov_config=None,
-        model_dir=None,
-    ):
-        super(OVModelVaeDecoder, self).__init__(
-            model,
-            parent_model,
-            ov_config,
-            "vae_decoder",
-            model_dir,
-        )

backend/openvino/pipelines.py

@@ -1,75 +0,0 @@
-from constants import DEVICE, LCM_DEFAULT_MODEL_OPENVINO
-from backend.tiny_decoder import get_tiny_decoder_vae_model
-from typing import Any
-from backend.device import is_openvino_device
-from paths import get_base_folder_name
-
-if is_openvino_device():
-    from huggingface_hub import snapshot_download
-    from optimum.intel.openvino.modeling_diffusion import OVBaseModel
-
-    from optimum.intel.openvino.modeling_diffusion import (
-        OVStableDiffusionPipeline,
-        OVStableDiffusionImg2ImgPipeline,
-        OVStableDiffusionXLPipeline,
-        OVStableDiffusionXLImg2ImgPipeline,
-    )
-    from backend.openvino.custom_ov_model_vae_decoder import CustomOVModelVaeDecoder
-
-
-def ov_load_taesd(
-    pipeline: Any,
-    use_local_model: bool = False,
-):
-    taesd_dir = snapshot_download(
-        repo_id=get_tiny_decoder_vae_model(pipeline.__class__.__name__),
-        local_files_only=use_local_model,
-    )
-    pipeline.vae_decoder = CustomOVModelVaeDecoder(
-        model=OVBaseModel.load_model(f"{taesd_dir}/vae_decoder/openvino_model.xml"),
-        parent_model=pipeline,
-        model_dir=taesd_dir,
-    )
-
-
-def get_ov_text_to_image_pipeline(
-    model_id: str = LCM_DEFAULT_MODEL_OPENVINO,
-    use_local_model: bool = False,
-) -> Any:
-    if "xl" in get_base_folder_name(model_id).lower():
-        pipeline = OVStableDiffusionXLPipeline.from_pretrained(
-            model_id,
-            local_files_only=use_local_model,
-            ov_config={"CACHE_DIR": ""},
-            device=DEVICE.upper(),
-        )
-    else:
-        pipeline = OVStableDiffusionPipeline.from_pretrained(
-            model_id,
-            local_files_only=use_local_model,
-            ov_config={"CACHE_DIR": ""},
-            device=DEVICE.upper(),
-        )
-
-    return pipeline
-
-
-def get_ov_image_to_image_pipeline(
-    model_id: str = LCM_DEFAULT_MODEL_OPENVINO,
-    use_local_model: bool = False,
-) -> Any:
-    if "xl" in get_base_folder_name(model_id).lower():
-        pipeline = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(
-            model_id,
-            local_files_only=use_local_model,
-            ov_config={"CACHE_DIR": ""},
-            device=DEVICE.upper(),
-        )
-    else:
-        pipeline = OVStableDiffusionImg2ImgPipeline.from_pretrained(
-            model_id,
-            local_files_only=use_local_model,
-            ov_config={"CACHE_DIR": ""},
-            device=DEVICE.upper(),
-        )
-    return pipeline

backend/pipelines/lcm.py

@@ -1,90 +0,0 @@
-from constants import LCM_DEFAULT_MODEL
-from diffusers import (
-    DiffusionPipeline,
-    AutoencoderTiny,
-    UNet2DConditionModel,
-    LCMScheduler,
-)
-import torch
-from backend.tiny_decoder import get_tiny_decoder_vae_model
-from typing import Any
-from diffusers import (
-    LCMScheduler,
-    StableDiffusionImg2ImgPipeline,
-    StableDiffusionXLImg2ImgPipeline,
-)
-
-
-def _get_lcm_pipeline_from_base_model(
-    lcm_model_id: str,
-    base_model_id: str,
-    use_local_model: bool,
-):
-    pipeline = None
-    unet = UNet2DConditionModel.from_pretrained(
-        lcm_model_id,
-        torch_dtype=torch.float32,
-        local_files_only=use_local_model,
-    )
-    pipeline = DiffusionPipeline.from_pretrained(
-        base_model_id,
-        unet=unet,
-        torch_dtype=torch.float32,
-        local_files_only=use_local_model,
-    )
-    pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config)
-    return pipeline
-
-
-def load_taesd(
-    pipeline: Any,
-    use_local_model: bool = False,
-    torch_data_type: torch.dtype = torch.float32,
-):
-    vae_model = get_tiny_decoder_vae_model(pipeline.__class__.__name__)
-    pipeline.vae = AutoencoderTiny.from_pretrained(
-        vae_model,
-        torch_dtype=torch_data_type,
-        local_files_only=use_local_model,
-    )
-
-
-def get_lcm_model_pipeline(
-    model_id: str = LCM_DEFAULT_MODEL,
-    use_local_model: bool = False,
-):
-    pipeline = None
-    if model_id == "latent-consistency/lcm-sdxl":
-        pipeline = _get_lcm_pipeline_from_base_model(
-            model_id,
-            "stabilityai/stable-diffusion-xl-base-1.0",
-            use_local_model,
-        )
-
-    elif model_id == "latent-consistency/lcm-ssd-1b":
-        pipeline = _get_lcm_pipeline_from_base_model(
-            model_id,
-            "segmind/SSD-1B",
-            use_local_model,
-        )
-    else:
-        pipeline = DiffusionPipeline.from_pretrained(
-            model_id,
-            local_files_only=use_local_model,
-        )
-
-    return pipeline
-
-
-def get_image_to_image_pipeline(pipeline: Any) -> Any:
-    components = pipeline.components
-    pipeline_class = pipeline.__class__.__name__
-    if (
-        pipeline_class == "LatentConsistencyModelPipeline"
-        or pipeline_class == "StableDiffusionPipeline"
-    ):
-        return StableDiffusionImg2ImgPipeline(**components)
-    elif pipeline_class == "StableDiffusionXLPipeline":
-        return StableDiffusionXLImg2ImgPipeline(**components)
-    else:
-        raise Exception(f"Unknown pipeline {pipeline_class}")

backend/pipelines/lcm_lora.py

@@ -1,25 +0,0 @@
-from diffusers import DiffusionPipeline, LCMScheduler
-import torch
-
-
-def get_lcm_lora_pipeline(
-    base_model_id: str,
-    lcm_lora_id: str,
-    use_local_model: bool,
-    torch_data_type: torch.dtype,
-):
-    pipeline = DiffusionPipeline.from_pretrained(
-        base_model_id,
-        torch_dtype=torch_data_type,
-        local_files_only=use_local_model,
-    )
-    pipeline.load_lora_weights(
-        lcm_lora_id,
-        local_files_only=use_local_model,
-    )
-    if "lcm" in lcm_lora_id.lower():
-        print("LCM LoRA model detected so using recommended LCMScheduler")
-        pipeline.scheduler = LCMScheduler.from_config(pipeline.scheduler.config)
-    pipeline.fuse_lora()
-    pipeline.unet.to(memory_format=torch.channels_last)
-    return pipeline

backend/safety_check.py

@@ -1,17 +0,0 @@
-from transformers import pipeline
-
-
-def is_safe_image(
-    classifier,
-    image,
-):
-    pred = classifier(image)
-    nsfw_score = 0
-    normal_score = 0
-    for label in pred:
-        if label["label"] == "nsfw":
-            nsfw_score = label["score"]
-        elif label["label"] == "normal":
-            normal_score = label["score"]
-    print(f"nsfw_score: {nsfw_score}, normal_score: {normal_score}")
-    return normal_score > nsfw_score

backend/tiny_decoder.py

@@ -1,30 +0,0 @@
-from constants import (
-    TAESD_MODEL,
-    TAESDXL_MODEL,
-    TAESD_MODEL_OPENVINO,
-    TAESDXL_MODEL_OPENVINO,
-)
-
-
-def get_tiny_decoder_vae_model(pipeline_class) -> str:
-    print(f"Pipeline class : {pipeline_class}")
-    if (
-        pipeline_class == "LatentConsistencyModelPipeline"
-        or pipeline_class == "StableDiffusionPipeline"
-        or pipeline_class == "StableDiffusionImg2ImgPipeline"
-    ):
-        return TAESD_MODEL
-    elif (
-        pipeline_class == "StableDiffusionXLPipeline"
-        or pipeline_class == "StableDiffusionXLImg2ImgPipeline"
-    ):
-        return TAESDXL_MODEL
-    elif (
-        pipeline_class == "OVStableDiffusionPipeline"
-        or pipeline_class == "OVStableDiffusionImg2ImgPipeline"
-    ):
-        return TAESD_MODEL_OPENVINO
-    elif pipeline_class == "OVStableDiffusionXLPipeline":
-        return TAESDXL_MODEL_OPENVINO
-    else:
-        raise Exception("No valid pipeline class found!")

benchmark-openvino.bat

@@ -1,23 +0,0 @@
-@echo off
-setlocal
-
-set "PYTHON_COMMAND=python"
-
-call python --version > nul 2>&1
-if %errorlevel% equ 0 (
-    echo Python command check :OK
-) else (
-    echo "Error: Python command not found, please install Python (Recommended : Python 3.10 or Python 3.11) and try again"
-    pause
-    exit /b 1
-    
-)
-
-:check_python_version
-for /f "tokens=2" %%I in ('%PYTHON_COMMAND% --version 2^>^&1') do (
-    set "python_version=%%I"
-)
-
-echo Python version: %python_version%
-
-call "%~dp0env\Scripts\activate.bat"  && %PYTHON_COMMAND%  src/app.py -b --use_openvino  --openvino_lcm_model_id "rupeshs/sd-turbo-openvino"

benchmark.bat

@@ -1,23 +0,0 @@
-@echo off
-setlocal
-
-set "PYTHON_COMMAND=python"
-
-call python --version > nul 2>&1
-if %errorlevel% equ 0 (
-    echo Python command check :OK
-) else (
-    echo "Error: Python command not found, please install Python (Recommended : Python 3.10 or Python 3.11) and try again"
-    pause
-    exit /b 1
-    
-)
-
-:check_python_version
-for /f "tokens=2" %%I in ('%PYTHON_COMMAND% --version 2^>^&1') do (
-    set "python_version=%%I"
-)
-
-echo Python version: %python_version%
-
-call "%~dp0env\Scripts\activate.bat"  && %PYTHON_COMMAND%  src/app.py -b

configs/lcm-lora-models.txt

@@ -1,4 +0,0 @@
-latent-consistency/lcm-lora-sdv1-5
-latent-consistency/lcm-lora-sdxl
-latent-consistency/lcm-lora-ssd-1b
-rupeshs/hypersd-sd1-5-1-step-lora

configs/lcm-models.txt

@@ -1,8 +0,0 @@
-stabilityai/sd-turbo
-rupeshs/sdxs-512-0.9-orig-vae
-rupeshs/hyper-sd-sdxl-1-step
-rupeshs/SDXL-Lightning-2steps
-stabilityai/sdxl-turbo
-SimianLuo/LCM_Dreamshaper_v7
-latent-consistency/lcm-sdxl
-latent-consistency/lcm-ssd-1b

configs/openvino-lcm-models.txt

@@ -1,9 +0,0 @@
-rupeshs/sd-turbo-openvino
-rupeshs/sdxs-512-0.9-openvino
-rupeshs/hyper-sd-sdxl-1-step-openvino-int8
-rupeshs/SDXL-Lightning-2steps-openvino-int8
-rupeshs/sdxl-turbo-openvino-int8
-rupeshs/LCM-dreamshaper-v7-openvino
-Disty0/LCM_SoteMix
-rupeshs/FLUX.1-schnell-openvino-int4
-rupeshs/sd15-lcm-square-openvino-int8

configs/stable-diffusion-models.txt

@@ -1,7 +0,0 @@
-Lykon/dreamshaper-8
-Fictiverse/Stable_Diffusion_PaperCut_Model
-stabilityai/stable-diffusion-xl-base-1.0
-runwayml/stable-diffusion-v1-5
-segmind/SSD-1B
-stablediffusionapi/anything-v5
-prompthero/openjourney-v4

constants.py

@@ -1,25 +0,0 @@
-from os import environ, cpu_count
-
-cpu_cores = cpu_count()
-cpus = cpu_cores // 2 if cpu_cores else 0
-APP_VERSION = "v1.0.0 beta 200"
-LCM_DEFAULT_MODEL = "stabilityai/sd-turbo"
-LCM_DEFAULT_MODEL_OPENVINO = "rupeshs/sd-turbo-openvino"
-APP_NAME = "FastSD CPU"
-APP_SETTINGS_FILE = "settings.yaml"
-RESULTS_DIRECTORY = "results"
-CONFIG_DIRECTORY = "configs"
-DEVICE = environ.get("DEVICE", "cpu")
-SD_MODELS_FILE = "stable-diffusion-models.txt"
-LCM_LORA_MODELS_FILE = "lcm-lora-models.txt"
-OPENVINO_LCM_MODELS_FILE = "openvino-lcm-models.txt"
-TAESD_MODEL = "madebyollin/taesd"
-TAESDXL_MODEL = "madebyollin/taesdxl"
-TAESD_MODEL_OPENVINO = "rupeshs/taesd-ov"
-LCM_MODELS_FILE = "lcm-models.txt"
-TAESDXL_MODEL_OPENVINO = "rupeshs/taesdxl-openvino"
-LORA_DIRECTORY = "lora_models"
-CONTROLNET_DIRECTORY = "controlnet_models"
-MODELS_DIRECTORY = "models"
-GGUF_THREADS = environ.get("GGUF_THREADS", cpus)
-TAEF1_MODEL_OPENVINO = "rupeshs/taef1-openvino"

context.py

@@ -1,47 +0,0 @@
-from typing import Any
-from app_settings import Settings
-from models.interface_types import InterfaceType
-from backend.lcm_text_to_image import LCMTextToImage
-from time import perf_counter
-from backend.image_saver import ImageSaver
-from pprint import pprint
-
-
-class Context:
-    def __init__(
-        self,
-        interface_type: InterfaceType,
-        device="cpu",
-    ):
-        self.interface_type = interface_type
-        self.lcm_text_to_image = LCMTextToImage(device)
-
-    def generate_text_to_image(
-        self,
-        settings: Settings,
-        reshape: bool = False,
-        device: str = "cpu",
-    ) -> Any:
-        tick = perf_counter()
-        from state import get_settings
-
-        get_settings().save()
-        pprint(settings.lcm_diffusion_setting.model_dump())
-        if not settings.lcm_diffusion_setting.lcm_lora:
-            return None
-        self.lcm_text_to_image.init(
-            device,
-            settings.lcm_diffusion_setting,
-        )
-        images = self.lcm_text_to_image.generate(
-            settings.lcm_diffusion_setting,
-            reshape,
-        )
-        elapsed = perf_counter() - tick
-        # ImageSaver.save_images(
-        #     settings.results_path,
-        #     images=images,
-        #     lcm_diffusion_setting=settings.lcm_diffusion_setting,
-        # )
-        print(f"Latency : {elapsed:.2f} seconds")
-        return images

controlnet_models/Readme.txt

@@ -1,3 +0,0 @@
-Place your ControlNet models in this folder.
-You can download controlnet model (.safetensors) from  https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/tree/main
-E.g: https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/blob/main/control_v11p_sd15_canny_fp16.safetensors