| "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"/home/rnmohan/anaconda3/lib/python3.11/site-packages/transformers/utils/generic.py:260: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\\n\",\n \" torch.utils._pytree._register_pytree_node(\\n\",\n \"/home/rnmohan/anaconda3/lib/python3.11/site-packages/transformers/utils/generic.py:260: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\\n\",\n \" torch.utils._pytree._register_pytree_node(\\n\",\n \"/home/rnmohan/anaconda3/lib/python3.11/site-packages/diffusers/models/transformers/transformer_2d.py:34: FutureWarning: `Transformer2DModelOutput` is deprecated and will be removed in version 1.0.0. Importing `Transformer2DModelOutput` from `diffusers.models.transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.modeling_outputs import Transformer2DModelOutput`, instead.\\n\",\n \" deprecate(\\\"Transformer2DModelOutput\\\", \\\"1.0.0\\\", deprecation_message)\\n\"\n ]\n }\n ],\n \"source\": [\n \"import torch\\n\",\n \"from torchvision import transforms\\n\",\n \"\\n\",\n \"from diffusers.models import MotionAdapter\\n\",\n \"from diffusers import AnimateDiffSDXLPipeline, DDIMScheduler\\n\",\n \"from diffusers.utils import export_to_gif\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"device = (\\n\",\n \" \\\"mps\\\"\\n\",\n \" if torch.backends.mps.is_available()\\n\",\n \" else \\\"cuda\\\"\\n\",\n \" if torch.cuda.is_available()\\n\",\n \" else \\\"cpu\\\"\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## first model\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"adapter = MotionAdapter.from_pretrained(\\\"guoyww/animatediff-motion-adapter-sdxl-beta\\\", torch_dtype=torch.float16)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# load StableDiffusion1.5-based finetuned model\\n\",\n \"\\n\",\n \"#model_id = \\\"SG161222/Realistic_Vision_V5.1_noVAE\\\"\\n\",\n \"model_id = \\\"stabilityai/stable-diffusion-xl-base-1.0\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"scheduler = DDIMScheduler.from_pretrained(\\n\",\n \" model_id,\\n\",\n \" subfolder=\\\"scheduler\\\",\\n\",\n \" clip_sample=False,\\n\",\n \" timestep_spacing=\\\"linspace\\\",\\n\",\n \" beta_schedule=\\\"linear\\\",\\n\",\n \" steps_offset=1,\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"preprocess = transforms.Compose([\\n\",\n \" transforms.Resize((256, 256)),\\n\",\n \" transforms.ToTensor(),\\n\",\n \" transforms.Normalize([0.5], [0.5])\\n\",\n \"])\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"pipe = AnimateDiffSDXLPipeline.from_pretrained(\\n\",\n \" model_id,\\n\",\n \" motion_adapter=adapter,\\n\",\n \" scheduler=scheduler,\\n\",\n \" torch_dtype=torch.float16,\\n\",\n \" variant=\\\"fp16\\\",\\n\",\n \").to(device)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"pipe.enable_vae_slicing()\\n\",\n \"pipe.enable_vae_tiling()\\n\",\n \"\\n\",\n \"output = pipe(\\n\",\n \" prompt=\\\"a purple sunset in the ocean reflection, realistic, high quality\\\",\\n\",\n \" negative_prompt=\\\"low quality, worst quality\\\",\\n\",\n \" num_inference_steps=50,\\n\",\n \" guidance_scale=8,\\n\",\n \" width=1024,\\n\",\n \" height=1024,\\n\",\n \" num_frames=16,\\n\",\n \")\\n\",\n \"\\n\",\n \"frames = output.frames[0]\\n\",\n \"export_to_gif(frames, \\\"animation.gif\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## static image to GIF experimentation (i2v)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import torch\\n\",\n \"from diffusers import I2VGenXLPipeline\\n\",\n \"from diffusers.utils import export_to_gif, load_image\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"device = (\\n\",\n \" \\\"mps\\\"\\n\",\n \" if torch.backends.mps.is_available()\\n\",\n \" else \\\"cuda\\\"\\n\",\n \" if torch.cuda.is_available()\\n\",\n \" else \\\"cpu\\\"\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"6e6b7fccec24437eb8f4aeaee3cfdfdd\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"model_index.json: 0%| | 0.00/555 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"d53edf92a4f24ebc887bfd6c3635ada2\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Fetching 15 files: 0%| | 0/15 [00:00<?, ?it/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"5daa0d33798f40ba8e7e81efdd9fba1b\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"image_encoder/config.json: 0%| | 0.00/563 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"62e13bd4999742aca80309277aab4f6e\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"scheduler/scheduler_config.json: 0%| | 0.00/507 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"fb8d6659202f404492a5f4e770c47b33\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)ature_extractor/preprocessor_config.json: 0%| | 0.00/466 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"bd3b24cacee8495c8e2a05e2ab4e073b\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"tokenizer/special_tokens_map.json: 0%| | 0.00/588 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"3e3ab362cb2441e99a88ec5fa6e072ac\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"text_encoder/config.json: 0%| | 0.00/601 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"f9fbbae8d8704442bb30bb45c107e86b\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"tokenizer/merges.txt: 0%| | 0.00/525k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"6b98e2e6341d4008a52e2d91b12a03db\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"image_encoder/model.fp16.safetensors: 0%| | 0.00/1.26G [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"d88c9b1fc0c2473abe8b212eb78f2fb7\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"text_encoder/model.fp16.safetensors: 0%| | 0.00/706M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"051888bdb83c4397b35bf94e0929a7f0\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"unet/config.json: 0%| | 0.00/627 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"f055e4dc8610482b9e4dd654dca5ec97\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"vae/config.json: 0%| | 0.00/637 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"7f1fb6b3b0bc4f8b90831944a5a666cc\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"tokenizer/vocab.json: 0%| | 0.00/1.06M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"00dc9b1a24224b36a055c64b07bc5eb7\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)diffusion_pytorch_model.fp16.safetensors: 0%| | 0.00/2.84G [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"288cd93032e643d29a88c6cc27405d90\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)diffusion_pytorch_model.fp16.safetensors: 0%| | 0.00/167M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"7fc0e0d9795f4c579560cabd0116b516\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"tokenizer/tokenizer_config.json: 0%| | 0.00/705 [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"e745da0fe65f4abca0b91b2d7aa05575\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Loading pipeline components...: 0%| | 0/7 [00:00<?, ?it/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"pipeline = I2VGenXLPipeline.from_pretrained(\\n\",\n \" \\\"ali-vilab/i2vgen-xl\\\", torch_dtype=torch.float16, variant=\\\"fp16\\\"\\n\",\n \").to(device)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"image_loc = \\\"snow.jpg\\\"\\n\",\n \"image = load_image(image_loc).convert(\\\"RGB\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"prompt = \\\"Timelapse of a snowy cabin\\\"\\n\",\n \"negative_prompt = \\\"Distorted, discontinuous, Ugly, blurry, low resolution, motionless, static, disfigured, disconnected limbs, Ugly faces, incomplete arms\\\"\\n\",\n \"generator = torch.manual_seed(8888)\\n\",\n \"\\n\",\n \"frames = pipeline(\\n\",\n \" prompt=prompt,\\n\",\n \" image=image,\\n\",\n \" num_inference_steps=50,\\n\",\n \" negative_prompt=negative_prompt,\\n\",\n \" guidance_scale=9.0,\\n\",\n \" generator=generator,\\n\",\n \").frames[0]\\n\",\n \"video_path = export_to_gif(frames, \\\"i2v.gif\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"base\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.5\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n", |
