Huggingface peft error message AttributeError: 'Linear8bitLt' object has no attribute 'state'

Question

I'm loading a huggingface dataset and Mistral-7B-Instruct-v0.1 to finetune for detection of sentiment. I'm running the following notebook, but updated it to use the above model and dataset instead. I cannot figure out what the cause of the error is and there doesn't seem to be an existing post on this issue in this context. Is the issue that I need to convert the target variable from int64 to floating point? Is the problem something else entirely?

FYI, I have upgraded the peft and torch libraries, since I came across that suggestion elsewhere.

Here is the code:

model = prepare_model_for_int8_training(model)
config = LoraConfig(
    r=LORA_R,
    lora_alpha=LORA_ALPHA,
    target_modules=LORA_TARGET_MODULES,
    lora_dropout=LORA_DROPOUT,
    bias="none",
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, config)
model.print_trainable_parameters()

Here is the error message:

 ---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[77], line 10
      1 model = prepare_model_for_int8_training(model)
      2 config = LoraConfig(
      3     r=LORA_R,
      4     lora_alpha=LORA_ALPHA,
   (...)
      8     task_type="CAUSAL_LM",
      9 )
---> 10 model = get_peft_model(model, config)
     11 model.print_trainable_parameters()

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/mapping.py:133, in get_peft_model(model, peft_config, adapter_name, mixed)
    131 if peft_config.is_prompt_learning:
    132     peft_config = _prepare_prompt_learning_config(peft_config, model_config)
--> 133 return MODEL_TYPE_TO_PEFT_MODEL_MAPPING[peft_config.task_type](model, peft_config, adapter_name=adapter_name)

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/peft_model.py:994, in PeftModelForCausalLM.__init__(self, model, peft_config, adapter_name)
    993 def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
--> 994     super().__init__(model, peft_config, adapter_name)
    995     self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/peft_model.py:123, in PeftModel.__init__(self, model, peft_config, adapter_name)
    121     self._peft_config = None
    122     cls = PEFT_TYPE_TO_MODEL_MAPPING[peft_config.peft_type]
--> 123     self.base_model = cls(model, {adapter_name: peft_config}, adapter_name)
    124     self.set_additional_trainable_modules(peft_config, adapter_name)
    126 self.config = getattr(self.base_model, "config", {"model_type": "custom"})

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/tuners/lora/model.py:115, in LoraModel.__init__(self, model, config, adapter_name)
    114 def __init__(self, model, config, adapter_name) -> None:
--> 115     super().__init__(model, config, adapter_name)

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/tuners/tuners_utils.py:95, in BaseTuner.__init__(self, model, peft_config, adapter_name)
     92 if not hasattr(self, "config"):
     93     self.config = {"model_type": "custom"}
---> 95 self.inject_adapter(self.model, adapter_name)
     97 # Copy the peft_config in the injected model.
     98 self.model.peft_config = self.peft_config

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/tuners/tuners_utils.py:252, in BaseTuner.inject_adapter(self, model, adapter_name)
    245     parent, target, target_name = _get_submodules(model, key)
    247     optional_kwargs = {
    248         "loaded_in_8bit": getattr(model, "is_loaded_in_8bit", False),
    249         "loaded_in_4bit": getattr(model, "is_loaded_in_4bit", False),
    250         "current_key": key,
    251     }
--> 252     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, **optional_kwargs)
    254 if not is_target_modules_in_base_model:
    255     raise ValueError(
    256         f"Target modules {peft_config.target_modules} not found in the base model. "
    257         f"Please check the target modules and try again."
    258     )

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/tuners/lora/model.py:196, in LoraModel._create_and_replace(self, lora_config, adapter_name, target, target_name, parent, current_key, **optional_kwargs)
    188     target.update_layer(
    189         adapter_name,
    190         r,
   (...)
    193         lora_config.init_lora_weights,
    194     )
    195 else:
--> 196     new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)
    197     if adapter_name != self.active_adapter:
    198         # adding an additional adapter: it is not automatically trainable
    199         new_module.requires_grad_(False)

File ~/Projects/llmtest1/lib/python3.10/site-packages/peft/tuners/lora/model.py:271, in LoraModel._create_new_module(lora_config, adapter_name, target, **kwargs)
    267 if loaded_in_8bit and isinstance(target_base_layer, bnb.nn.Linear8bitLt):
    268     eightbit_kwargs = kwargs.copy()
    269     eightbit_kwargs.update(
    270         {
--> 271             "has_fp16_weights": target.state.has_fp16_weights,
    272             "memory_efficient_backward": target.state.memory_efficient_backward,
    273             "threshold": target.state.threshold,
    274             "index": target.index,
    275         }
    276     )
    277     new_module = Linear8bitLt(target, adapter_name, **eightbit_kwargs)
    278 elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target_base_layer, bnb.nn.Linear4bit):

File ~/Projects/llmtest1/lib/python3.10/site-packages/torch/nn/modules/module.py:1614, in Module.__getattr__(self, name)
   1612     if name in modules:
   1613         return modules[name]
-> 1614 raise AttributeError("'{}' object has no attribute '{}'".format(
   1615     type(self).__name__, name))

AttributeError: 'Linear8bitLt' object has no attribute 'state'

Answer 1

I was able to resolve this error message by changing the transformer class from LlamaForCausalLM to AutoModelForSeq2SeqLM when calling from_pretrained(). The class must be compatible with the pretrained model being loaded.