updated lora

Author: levy-tech-spark

config.yaml

@@ -17,5 +17,6 @@ training:
   save_frequency: 100
   warmup_epochs: 5
   use_lora: true
-  use_lora_layers: true
+  use_lora_layers: true # This applies lora to only bbox pred layer and few transformer decoder layers the number of trainable parameters in this case will be < 1% of total parameters
+                        # if it is false then it applies 
   visualization_frequency: 5

groundingdino/util/lora.py

@@ -64,7 +64,8 @@ def add_lora_to_model(model, rank=8):
             "query",
             "key",
             "value",
-            "dense"
+            "dense",    
+            "bbox_embed",
         ],
         lora_dropout=0.1,
         bias="none",
@@ -102,7 +103,7 @@ def add_lora_to_layers(model, rank=32):
         r=rank,
         lora_alpha=rank,
         target_modules=["0", "1", "2"],  # MLP layer indices
-        lora_dropout=0.1,
+        lora_dropout=1e-3,
         bias="none",
     )