LoRA Merge模型合并：多领域专家融合实战指南

# 导入依赖库
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# 加载基础模型和分词器
base_model_name = "LLaMA-2-7b-chat"
base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float16, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(base_model_name)

# 定义要融合的多个LoRA适配器路径
lora_paths = [
    "./lora-adapter-customer-service",
    "./lora-adapter-code",
    "./lora-adapter-copywriting"
]
# 融合权重，对应每个LoRA的占比
merge_weights = [0.4, 0.3, 0.3]

# 逐个加载LoRA并融合权重
for idx, lora_path in enumerate(lora_paths):
    peft_model = PeftModel.from_pretrained(base_model, lora_path)
    # 将LoRA权重合并到基础模型
    peft_model.merge_and_unload()
    # 按比例缩放融合后的权重
    for name, param in peft_model.named_parameters():
        if "lora" in name:
            param.data *= merge_weights[idx]
    base_model = peft_model

# 保存融合后的模型
base_model.save_pretrained("./merged-multi-domain-model")
tokenizer.save_pretrained("./merged-multi-domain-model")

# 输出说明：运行后得到的merged-multi-domain-model可直接用于推理，无需额外加载LoRA结构，同时具备三个领域的能力

我们在进行多领域LoRA合并之前，必须先完成权重的预处理工作，其中统一所有LoRA的权重维度是最基础也最关键的一步。不同领域、不同训练者产出的LoRA权重，往往存在rank设置不一致、部分层被剪枝、张量形状不匹配的问题，如果直接尝试合并，几乎一定会报维度不匹配的错误。我们首先要做的是遍历所有待合并LoRA的权重文件，将所有线性层、卷积层的LoRA权重张量形状对齐，要么统一缩放到相同的rank，要么裁剪掉多余的冗余维度，从根源上避免合并时的报错问题。

统一维度之后，我们还需要过滤掉低贡献度的冗余LoRA层，这一步直接决定了合并后模型的效果上限。我们可以用领域对应的验证集，计算每个LoRA层对最终生成效果的贡献度：将某一层的LoRA权重临时置零，观察验证集上的指标下降幅度，如果下降幅度低于1%的阈值，就说明这层是冗余的，不仅不会提升效果，还可能和其他LoRA的参数产生冲突。我们直接把这些低贡献层从权重文件中删除，既能减少合并后模型的体积，还能避免不同LoRA在冗余层上的参数互相抵消，提升融合后的效果稳定性。

最后一步是按领域能力标签对预处理后的权重文件进行分类管理，避免后续合并时出现领域冲突的问题。我们给每个LoRA权重打上的标签需要包含基座模型版本、训练领域、数据集规模、rank参数、验证集基准指标这些核心信息，比如“SDXL_base1.0_国风插画_12万张_rank64_FID27.3”这样的标签，能让我们在筛选合并对象时快速判断哪些LoRA是互补的，哪些是冲突的。比如我们把二次元绘图LoRA和写实摄影LoRA混合合并，出来的效果必然是风格混乱，提前按标签分类就能避免这种低级错误。

# 输入：待预处理的LoRA文件夹路径、贡献度阈值、目标rank
# 输出：预处理后的LoRA文件夹、权重标签清单
from diffusers import load_lora_weights
import torch

def preprocess_lora(lora_dir, threshold=0.01, target_rank=64):
    processed_loras = []
    for lora_file in os.listdir(lora_dir):
        if lora_file.endswith(".safetensors"):
            lora_path = os.path.join(lora_dir, lora_file)
            # 加载LoRA权重
            lora_state_dict = load_lora_weights(lora_path)
            # 统一维度：截断或补零到目标rank
            aligned_state_dict = {}
            for key, value in lora_state_dict.items():
                if "lora_down" in key:
                    current_rank = value.shape[0]
                    if current_rank < target_rank:
                        # 补零对齐
                        padded = torch.zeros(target_rank, value.shape[1])
                        padded[:current_rank, :] = value
                        aligned_state_dict[key] = padded
                    elif current_rank > target_rank:
                        # 截断对齐
                        aligned_state_dict[key] = value[:target_rank, :]
                    else:
                        aligned_state_dict[key] = value
                elif "lora_up" in key:
                    current_rank = value.shape[1]
                    if current_rank < target_rank:
                        padded = torch.zeros(value.shape[0], target_rank)
                        padded[:, :current_rank] = value
                        aligned_state_dict[key] = padded
                    elif current_rank > target_rank:
                        aligned_state_dict[key] = value[:, :target_rank]
                    else:
                        aligned_state_dict[key] = value
                else:
                    aligned_state_dict[key] = value
            # 过滤低贡献层（此处省略贡献度计算逻辑，可替换为实际验证逻辑）
            filtered_state_dict = filter_low_contribution_layers(aligned_state_dict, threshold)
            # 保存预处理后的LoRA
            save_path = os.path.join("processed_lora", f"processed_{lora_file}")
            torch.save(filtered_state_dict, save_path)
            # 生成标签（示例）
            tag = f"SDXL_base1.0_{lora_file.split('_')[0]}_rank{target_rank}"
            processed_loras.append({"path": save_path, "tag": tag})
    return processed_loras

# 输入示例：调用preprocess_lora("./raw_loras", threshold=0.01, target_rank=64)
# 输出说明：返回预处理后的LoRA路径列表和对应标签，可直接用于后续合并流程

# 线性合并输入示例
from peft import PeftModel
import torch

base_model = load_base_model("stable-diffusion-v1-5") # 加载基座模型
lora_paths = ["lora_anime_character.safetensors", "lora_watercolor.safetensors"]
merge_weights = [0.6, 0.4] # 对应两个LoRA的合并权重

# 按权重加和LoRA参数
merged_lora_state = {}
for lora_path, weight in zip(lora_paths, merge_weights):
    lora_state = torch.load(lora_path, map_location="cpu")
    for key in lora_state:
        merged_lora_state[key] = merged_lora_state.get(key, 0) + lora_state[key] * weight

# 加载合并后的LoRA权重
base_model.load_state_dict(merged_lora_state, strict=False)
# 输出说明：得到合并后的模型，同时具备二次元角色和水彩画风特征，若特征冲突可能出现风格混杂

# 导入依赖库
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel, merge_and_unload

# 1. 加载基础大模型底座
base_model_path = "Qwen/Qwen2.5-14B-Instruct"
model = AutoModelForCausalLM.from_pretrained(base_model_path, torch_dtype="auto", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(base_model_path)

# 2. 加载多领域LoRA权重
lora_paths = [
    "your-org/code-lora-qwen2.5-14b",
    "your-org/math-lora-qwen2.5-14b",
    "your-org/writing-lora-qwen2.5-14b"
]
model = PeftModel.from_pretrained(model, lora_paths[0])
for lora_path in lora_paths[1:]:
    model.load_adapter(lora_path, adapter_name=lora_path.split("/")[-1])

# 3. 配置合并权重并执行合并
merge_config = {
    "code-lora-qwen2.5-14b": 0.4,
    "math-lora-qwen2.5-14b": 0.3,
    "writing-lora-qwen2.5-14b": 0.3
}
model.merge_and_unload(merge_config=merge_config)

# 4. 保存合并后的全量模型
merged_model_path = "./qwen2.5-14b-multi-expert-merged"
model.save_pretrained(merged_model_path)
tokenizer.save_pretrained(merged_model_path)
print(f"合并完成，模型已保存至：{merged_model_path}")

# 输入：两个待合并的LoRA路径、基座模型路径、输出路径
# 输出：合并后的全量模型保存到指定路径
from diffusers import StableDiffusionXLPipeline
from peft import PeftModel, PeftConfig
import torch

# 加载基座模型
base_model_path = "stabilityai/stable-diffusion-xl-base-1.0"
pipe = StableDiffusionXLPipeline.from_pretrained(base_model_path, torch_dtype=torch.float16)
pipe = pipe.to("cuda")

# 加载两个待合并的LoRA
lora_path_1 = "./lora_expert_writing"
lora_path_2 = "./lora_expert_anime"
pipe.unet.load_lora_weights(lora_path_1)
pipe.unet.load_lora_weights(lora_path_2)

# 合并LoRA权重到基座模型
pipe.unet.merge_and_unload()

# 保存合并后的全量模型
pipe.save_pretrained("./merged_sdxl_expert_model")
print("合并完成，模型已保存到 ./merged_sdxl_expert_model")