@dataclass
class PEFTHelper:
"""A helper class for PEFT configurations, specifically designed for LoRA.
This class handles configuration validation, compatibility checks for
various LoRA implementations.
"""
# Required fields
r: int
lora_alpha: int
target_modules: list[str] | str
bias: Literal["none"] = field(default="none")
modules_to_save: list[str] | None = field(default=None)
# True to use Rank-Stabilized LoRA (rsLoRA, see: https://arxiv.org/abs/2312.03732)
use_rslora: bool = field(default=False)
# True to use Weight-Decomposed Low-Rank Adaptation (DoRA, see: https://arxiv.org/abs/2402.09353)
use_dora: bool = field(default=False)
# PEFT features that change the adapter's semantics and that vLLM does not
# implement. They are read only to reject such adapters.
lora_bias: bool = field(default=False)
init_lora_weights: bool | str = field(default=True)
alora_invocation_tokens: list[int] | None = field(default=None)
layer_replication: list[list[int]] | None = field(default=None)
use_bdlora: dict | None = field(default=None)
use_qalora: bool = field(default=False)
# Per-module overrides of `r` and `lora_alpha`, keyed by module name pattern
rank_pattern: dict[str, int] = field(default_factory=dict)
alpha_pattern: dict[str, float] = field(default_factory=dict)
# Extra vllm field, start with 'vllm_' to avoid conflict
vllm_lora_scaling_factor: float = field(default=1.0)
vllm_max_position_embeddings: int | None = field(default=False)
def _validate_features(self) -> list[str]:
"""Check if there are any unsupported LoRA features."""
error_msg = []
if self.modules_to_save:
unsupported_modules = [
m for m in self.modules_to_save if m not in ["classifier", "score"]
]
if unsupported_modules:
error_msg.append(
"vLLM only supports modules_to_save being either None "
'or ["classifier", "score"] for classification models. '
f"Unsupported modules_to_save: {unsupported_modules}"
)
if self.use_dora:
error_msg.append("vLLM does not yet support DoRA.")
if self.lora_bias:
error_msg.append("vLLM does not support LoRA bias (lora_bias).")
if not self._init_is_supported():
error_msg.append(
f"vLLM does not support init_lora_weights={self.init_lora_weights!r}."
" Inits such as PiSSA, OLoRA, CorDA and LoftQ modify the base model"
" weights when PEFT loads the adapter. Convert it to a regular LoRA"
" adapter with PEFT's path_initial_model_for_weight_conversion, or"
" set init_lora_weights to true if the served base model already"
" contains the modified weights."
)
if self.alora_invocation_tokens:
error_msg.append("vLLM does not support Activated LoRA (aLoRA).")
if self.layer_replication:
error_msg.append("vLLM does not support layer_replication.")
if self.use_bdlora:
error_msg.append("vLLM does not support block-diagonal LoRA (BD-LoRA).")
if self.use_qalora:
error_msg.append("vLLM does not support QALoRA.")
return error_msg
def _init_is_supported(self) -> bool:
# These inits only set the initial adapter weights, so a saved adapter
# loads in PEFT as a plain LoRA. lora_ga modifies the base weights only
# during training, when its gradients are attached.
init = self.init_lora_weights
return not isinstance(init, str) or init.lower() in (
"gaussian",
"eva",
"orthogonal",
"mica",
"lora_ga",
)
def __post_init__(self):
if self.r <= 0:
raise ValueError(f"LoRA rank `r` must be a positive integer, got {self.r}.")
self.rank_pattern = self.rank_pattern or {}
self.alpha_pattern = self.alpha_pattern or {}
for key, rank in self.rank_pattern.items():
if not isinstance(rank, int) or rank <= 0:
raise ValueError(
f"LoRA rank for `{key}` in `rank_pattern` must be a positive "
f"integer, got {rank}."
)
if self.use_rslora:
logger.info_once("Loading LoRA weights trained with rsLoRA.")
self.vllm_lora_scaling_factor = self._scaling(self.r, self.lora_alpha)
def _scaling(self, rank: int, alpha: float) -> float:
if self.use_rslora:
return alpha / math.sqrt(rank)
return alpha / rank
@staticmethod
def _match_pattern(pattern: dict, name: str):
# Same rule as PEFT's `get_pattern_key`: a key matches if it is a suffix
# of the module name starting at a "." boundary; the first match wins.
for key, value in pattern.items():
if re.match(rf"(.*\.)?({key})$", name):
return value
return None
def get_rank_and_scaling(self, module_name: str) -> tuple[int, float]:
"""Return the rank and scaling PEFT uses for `module_name`, taking
`rank_pattern` and `alpha_pattern` into account.
`module_name` is the name in the adapter checkpoint, without the
`base_model.model.` prefix and before any weights mapping.
"""
if not self.rank_pattern and not self.alpha_pattern:
return self.r, self.vllm_lora_scaling_factor
# PEFT stores fused MoE expert LoRAs (`target_parameters`) as
# `experts.base_layer` (gate_up_proj) and `experts` (down_proj), but
# matches the patterns against the parameter names.
if module_name.endswith(".experts.base_layer"):
module_name = module_name.removesuffix(".base_layer") + ".gate_up_proj"
elif module_name.endswith(".experts"):
module_name += ".down_proj"
rank = self._match_pattern(self.rank_pattern, module_name)
if rank is None:
rank = self.r
alpha = self._match_pattern(self.alpha_pattern, module_name)
if alpha is None:
alpha = self.lora_alpha
return rank, self._scaling(rank, alpha)
@classmethod
def from_dict(cls, config_dict: dict) -> "PEFTHelper":
# Get all field information from the class
class_fields = {f.name: f for f in fields(cls)}
# Check for required fields
required_fields = {
name
for name, f in class_fields.items()
if f.default is MISSING and f.default_factory is MISSING
}
# Identify any missing required fields
missing_fields = required_fields - set(config_dict.keys())
if missing_fields:
raise ValueError(f"Missing required configuration fields: {missing_fields}")
# Filter out fields that aren't defined in the class
filtered_dict = {k: v for k, v in config_dict.items() if k in class_fields}
return cls(**filtered_dict)
@classmethod
def from_local_dir(
cls,
lora_path: str,
max_position_embeddings: int | None,
) -> "PEFTHelper":
lora_config_path = os.path.join(lora_path, "adapter_config.json")
with open(lora_config_path) as f:
config = json.load(f)
config["vllm_max_position_embeddings"] = max_position_embeddings
return cls.from_dict(config)
def validate_legal(self, lora_config: LoRAConfig) -> None:
"""Validates the LoRA configuration settings against application
constraints and requirements.
"""
error_msg = self._validate_features()
max_rank = max([self.r, *self.rank_pattern.values()])
if max_rank > lora_config.max_lora_rank:
error_msg.append(
f"LoRA rank {max_rank} is greater than max_lora_rank"
f" {lora_config.max_lora_rank}."
)
if self.bias != "none":
error_msg.append("Adapter bias is not supported.")
if error_msg:
raise ValueError(f"{' '.join(error_msg)}")