LoRA Fine Tuning

Fine-tuning models from inputs.

Overview of LoRA Fine Tuning

LoRA stands for Low Rank Adaptation and is a mathematical technique to reduce the number of parameters that are trained in a model - so instead of fine-tuning all the weights that constitute the weight matrix of the pre-trained large language model, two smaller matrices that approximate this larger matrix are fine-tuned.

What this means in practice is that instead of needing to build a custom model off all the original input data material (and needed many many GPUs to do so), LoRA means you can fine-tune an existing model (such as SDXL 0.9 Stable Diffusion) to be biased towards a certain result on just one GPU.

For example, an open source Stable Diffusion model can be fine-tuned to produce images in the style of Claude Monet paintings using LoRA. Fun fact: This is how Waterlily.ai trains artist models - look how good the results are even without an up to date Stable Diffusion model like SDXL0.9!

Original Claude Monet Pictures used as additional data and added to a Stable Diffusion Model by LoRA

The results of running Stable Diffusion with the LoRA trained Model.

A Claude Monet data set is available in zip form on IPFS here: ipfs://bafybeiglwmvudxxethpi46el3o5m44lrki2sjzgs7whvb6xaz6e65wm7am One of the easiest ways to upload a dataset to IPFS is to use web3.storage.

[CLI] Running LoRA Fine Tuning [coming soon]

Ensure you have installed all requirements [CLI] Install Run Requirements

To run a LoRA fine-tuning job, just provide the training data for the job to the command:

lilypad run lora_training:v0.1.7-lilypad1 '{images_cid: "bafybeiah7ib5mhzlckolwlkwquzf772wl6jdbhtbuvnbuo5arq7pcs4ubm", seed: 3}'

NB: the params above should be yaml eg. {seed: 42, 'images_cid': 'Qm...'} where images_cid contains an images.zip with training images in it.

This will output a result model CID, which can then be used to generate new images in this particular style:

lilypad run lora_inference:v0.1.7-lilypad1 '{lora_cid: <CID result from above>, prompt: "an astronaut riding a unicorn in the style of <s1><s2>", seed: 3}'

[Smart Contract] Running LoRA Fine Tuning [coming soon]

Ensure you have set up your Metamask for Lalechuza Network and have funded your wallet. Setting up Metamask & Funding your Wallet from Faucet

LoRA Module Code

See the code repo here

https://github.com/bacalhau-project/lilypad-modicum/blob/main/src/python/modules/lora.py

import yaml
import textwrap

IMAGE = "quay.io/lukemarsden/lora:v0.0.4"
MODEL_NAME = "runwayml/stable-diffusion-v1-5"

# Don't want to make this variable since the job is fixed price
NUM_IMAGES = 10

def _lora_training(params: str):
    if params.startswith("{"):
        params = yaml.safe_load(params)
    else:
        raise Exception("Please set params to yaml like {seed: 42, 'images_cid': 'Qm...'} where images_cid contains an images.zip with training images in")

    seed = params.get("seed", 42)
    images_cid = params["images_cid"]

    return {
        "APIVersion": "V1beta1",
        "Metadata": {
            "CreatedAt": "0001-01-01T00:00:00Z",
            "Requester": {}
        },
        "Spec": {
            "Deal": {
                "Concurrency": 1
            },
            "Docker": {
                "EnvironmentVariables": [
                    f"RANDOM_SEED={seed}",
                ],
                "Entrypoint": [
                    "bash", "-c", f'(cd /input && unzip images.zip && rm images.zip) && lora_pti --pretrained_model_name_or_path={MODEL_NAME} --instance_data_dir=/input --output_dir=/output --train_text_encoder --resolution=512 --train_batch_size=1 --gradient_accumulation_steps=4 --scale_lr --learning_rate_unet=1e-4 --learning_rate_text=1e-5 --learning_rate_ti=5e-4 --color_jitter --lr_scheduler="linear" --lr_warmup_steps=0 --placeholder_tokens="<s1>|<s2>" --use_template="style" --save_steps=100 --max_train_steps_ti=1000 --max_train_steps_tuning=1000 --perform_inversion=True --clip_ti_decay --weight_decay_ti=0.000 --weight_decay_lora=0.001 --continue_inversion --continue_inversion_lr=1e-4 --device="cuda:0" --lora_rank=1'
                ],
                "Image": IMAGE,
            },
            "Engine": "Docker",
            "Language": {
                "JobContext": {}
            },
            "Network": {
                "Type": "None"
            },
            "PublisherSpec": {
                "Type": "Estuary"
            },
            "Resources": {
                "GPU": "1"
            },
            "Timeout": 1800,
            "Verifier": "Noop",
            "Wasm": {
                "EntryModule": {}
            },
            "inputs": [
                {
                    "CID": images_cid,
                    "Name": "lora_input",
                    "StorageSource": "IPFS",
                    "path": "/input",

                },
            ],
            "outputs": [
                {
                    "Name": "output",
                    "StorageSource": "IPFS",
                    "path": "/output"
                }
            ]
        }
    }

def _lora_inference(params: str):
    if params.startswith("{"):
        params = yaml.safe_load(params)
    else:
        raise Exception("Please set params to yaml like {seed: 42, 'lora_cid': 'Qm...', "+
                        "prompt: 'an astronaut in the style of <s1><s2>'} "+
                        "where lora_cid is the output cid of the above step")

    # TODO add a default we pin
    lora_cid = params["lora_cid"]
    seed = params.get("seed", 42)
    prompt = params.get("prompt", "question mark floating in space")
    finetune_weighting = params.get("finetune_weighting", 0.5)

    return {
        "APIVersion": "V1beta1",
        "Metadata": {
            "CreatedAt": "0001-01-01T00:00:00Z",
            "Requester": {}
        },
        "Spec": {
            "Deal": {
                "Concurrency": 1
            },
            "Docker": {
                "EnvironmentVariables": [
                    f"PROMPT={prompt}",
                    f"RANDOM_SEED={seed}",
                    f"FINETUNE_WEIGHTING={finetune_weighting}",
                    "HF_HUB_OFFLINE=1",
                ],
                "Entrypoint": [
                    'python3',
                    '-c',
                    # dedent
                    textwrap.dedent(f"""
                        from diffusers import StableDiffusionPipeline, EulerAncestralDiscreteScheduler
                        import os
                        import torch
                        from lora_diffusion import tune_lora_scale, patch_pipe

                        model_id = "{MODEL_NAME}"

                        pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to(
                            "cuda"
                        )
                        pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)

                        prompt = os.getenv("PROMPT")
                        seed = int(os.getenv("RANDOM_SEED"))
                        torch.manual_seed(seed)

                        os.system("find /input")
                        patch_pipe(
                            pipe,
                            "/input/output/final_lora.safetensors",
                            patch_text=True,
                            patch_ti=True,
                            patch_unet=True,
                        )

                        coeff = float(os.getenv("FINETUNE_WEIGHTING", 0.5))
                        tune_lora_scale(pipe.unet, coeff)
                        tune_lora_scale(pipe.text_encoder, coeff)

                        image = pipe(prompt, num_inference_steps=50, guidance_scale=7).images[0]
                        image.save(f"/output/image-{{seed}}.jpg")
                        image
                        """)
                ],
                "Image": IMAGE,
            },
            "Engine": "Docker",
            "Language": {
                "JobContext": {}
            },
            "Network": {
                "Type": "None"
            },
            "PublisherSpec": {
                "Type": "Estuary"
            },
            "Resources": {
                "GPU": "1"
            },
            "Timeout": 1800,
            "Verifier": "Noop",
            "Wasm": {
                "EntryModule": {}
            },
            "inputs": [
                {
                    "CID": lora_cid,
                    "Name": "lora_input",
                    "StorageSource": "IPFS",
                    "path": "/input",

                },
            ],
            "outputs": [
                {
                    "Name": "output",
                    "StorageSource": "IPFS",
                    "path": "/output"
                },
            ]
        }
    }