multimodalart HF Staff commited on
Commit
7fdd68b
·
verified ·
1 Parent(s): cdac5b2

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. UniICL/modeling/uniicl/bagel.py +133 -0
UniICL/modeling/uniicl/bagel.py CHANGED
@@ -100,6 +100,14 @@ class Bagel(PreTrainedModel):
100
  self.config = config
101
  self._init_weights()
102
 
 
 
 
 
 
 
 
 
103
  def _init_weights(self):
104
  if self.config.visual_gen:
105
  nn.init.constant_(self.llm2vae.weight, 0)
@@ -559,6 +567,23 @@ class Bagel(PreTrainedModel):
559
  key_values_lens: torch.IntTensor,
560
  capm_active: bool = False,
561
  ):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
563
 
564
  extra_inputs = {}
@@ -663,6 +688,33 @@ class Bagel(PreTrainedModel):
663
  key_values_lens: torch.IntTensor,
664
  capm_active: bool = False,
665
  ):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
666
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
667
  packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
668
  packed_sequence[packed_text_indexes] = packed_text_embedding
@@ -798,6 +850,33 @@ class Bagel(PreTrainedModel):
798
  packed_key_value_indexes: torch.Tensor,
799
  capm_active: bool = False,
800
  ):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
801
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
802
  packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
803
  packed_sequence[packed_text_indexes] = packed_text_embedding
@@ -972,6 +1051,47 @@ class Bagel(PreTrainedModel):
972
  # cache_args
973
  enable_taylorseer=False,
974
  ):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
975
  if enable_taylorseer:
976
  self.language_model.model.enable_taylorseer = True
977
  model_pred_cache_dic, model_pred_current = cache_init(self, num_timesteps)
@@ -1239,6 +1359,19 @@ class Bagel(PreTrainedModel):
1239
  end_token_id: int = None,
1240
  capm_active: bool = False,
1241
  ):
 
 
 
 
 
 
 
 
 
 
 
 
 
1242
  step = 0
1243
  generated_sequence = []
1244
  curr_tokens = packed_start_tokens
 
100
  self.config = config
101
  self._init_weights()
102
 
103
+ def _move_to_device(self, kwargs_dict):
104
+ """Move all tensor values in kwargs to the model's device."""
105
+ device = next(self.parameters()).device
106
+ for k, v in kwargs_dict.items():
107
+ if isinstance(v, torch.Tensor) and v.device.type != device.type:
108
+ kwargs_dict[k] = v.to(device)
109
+ return kwargs_dict
110
+
111
  def _init_weights(self):
112
  if self.config.visual_gen:
113
  nn.init.constant_(self.llm2vae.weight, 0)
 
567
  key_values_lens: torch.IntTensor,
568
  capm_active: bool = False,
569
  ):
570
+ # Move all input tensors to model device
571
+ _local = dict(
572
+ packed_text_ids=packed_text_ids,
573
+ packed_text_position_ids=packed_text_position_ids,
574
+ text_token_lens=text_token_lens,
575
+ packed_text_indexes=packed_text_indexes,
576
+ packed_key_value_indexes=packed_key_value_indexes,
577
+ key_values_lens=key_values_lens,
578
+ )
579
+ _local = self._move_to_device(_local)
580
+ packed_text_ids = _local['packed_text_ids']
581
+ packed_text_position_ids = _local['packed_text_position_ids']
582
+ text_token_lens = _local['text_token_lens']
583
+ packed_text_indexes = _local['packed_text_indexes']
584
+ packed_key_value_indexes = _local['packed_key_value_indexes']
585
+ key_values_lens = _local['key_values_lens']
586
+
587
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
588
 
589
  extra_inputs = {}
 
688
  key_values_lens: torch.IntTensor,
689
  capm_active: bool = False,
690
  ):
691
+ # Move all input tensors to model device
692
+ _local = dict(
693
+ packed_text_ids=packed_text_ids,
694
+ packed_text_indexes=packed_text_indexes,
695
+ packed_vit_tokens=packed_vit_tokens,
696
+ packed_vit_token_indexes=packed_vit_token_indexes,
697
+ packed_vit_position_ids=packed_vit_position_ids,
698
+ vit_token_seqlens=vit_token_seqlens,
699
+ packed_position_ids=packed_position_ids,
700
+ packed_seqlens=packed_seqlens,
701
+ packed_indexes=packed_indexes,
702
+ packed_key_value_indexes=packed_key_value_indexes,
703
+ key_values_lens=key_values_lens,
704
+ )
705
+ _local = self._move_to_device(_local)
706
+ packed_text_ids = _local['packed_text_ids']
707
+ packed_text_indexes = _local['packed_text_indexes']
708
+ packed_vit_tokens = _local['packed_vit_tokens']
709
+ packed_vit_token_indexes = _local['packed_vit_token_indexes']
710
+ packed_vit_position_ids = _local['packed_vit_position_ids']
711
+ vit_token_seqlens = _local['vit_token_seqlens']
712
+ packed_position_ids = _local['packed_position_ids']
713
+ packed_seqlens = _local['packed_seqlens']
714
+ packed_indexes = _local['packed_indexes']
715
+ packed_key_value_indexes = _local['packed_key_value_indexes']
716
+ key_values_lens = _local['key_values_lens']
717
+
718
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
719
  packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
720
  packed_sequence[packed_text_indexes] = packed_text_embedding
 
850
  packed_key_value_indexes: torch.Tensor,
851
  capm_active: bool = False,
852
  ):
853
+ # Move all input tensors to model device
854
+ _local = dict(
855
+ padded_images=padded_images,
856
+ packed_vae_position_ids=packed_vae_position_ids,
857
+ packed_timesteps=packed_timesteps,
858
+ packed_vae_token_indexes=packed_vae_token_indexes,
859
+ packed_text_ids=packed_text_ids,
860
+ packed_text_indexes=packed_text_indexes,
861
+ packed_position_ids=packed_position_ids,
862
+ packed_seqlens=packed_seqlens,
863
+ packed_indexes=packed_indexes,
864
+ key_values_lens=key_values_lens,
865
+ packed_key_value_indexes=packed_key_value_indexes,
866
+ )
867
+ _local = self._move_to_device(_local)
868
+ padded_images = _local['padded_images']
869
+ packed_vae_position_ids = _local['packed_vae_position_ids']
870
+ packed_timesteps = _local['packed_timesteps']
871
+ packed_vae_token_indexes = _local['packed_vae_token_indexes']
872
+ packed_text_ids = _local['packed_text_ids']
873
+ packed_text_indexes = _local['packed_text_indexes']
874
+ packed_position_ids = _local['packed_position_ids']
875
+ packed_seqlens = _local['packed_seqlens']
876
+ packed_indexes = _local['packed_indexes']
877
+ key_values_lens = _local['key_values_lens']
878
+ packed_key_value_indexes = _local['packed_key_value_indexes']
879
+
880
  packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
881
  packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
882
  packed_sequence[packed_text_indexes] = packed_text_embedding
 
1051
  # cache_args
1052
  enable_taylorseer=False,
1053
  ):
1054
+ # Move all input tensors to model device
1055
+ _local = dict(
1056
+ packed_text_ids=packed_text_ids,
1057
+ packed_text_indexes=packed_text_indexes,
1058
+ packed_init_noises=packed_init_noises,
1059
+ packed_vae_position_ids=packed_vae_position_ids,
1060
+ packed_vae_token_indexes=packed_vae_token_indexes,
1061
+ packed_seqlens=packed_seqlens,
1062
+ packed_position_ids=packed_position_ids,
1063
+ packed_indexes=packed_indexes,
1064
+ key_values_lens=key_values_lens,
1065
+ packed_key_value_indexes=packed_key_value_indexes,
1066
+ cfg_text_packed_query_indexes=cfg_text_packed_query_indexes,
1067
+ cfg_text_packed_position_ids=cfg_text_packed_position_ids,
1068
+ cfg_text_key_values_lens=cfg_text_key_values_lens,
1069
+ cfg_text_packed_key_value_indexes=cfg_text_packed_key_value_indexes,
1070
+ cfg_img_packed_query_indexes=cfg_img_packed_query_indexes,
1071
+ cfg_img_packed_position_ids=cfg_img_packed_position_ids,
1072
+ cfg_img_key_values_lens=cfg_img_key_values_lens,
1073
+ cfg_img_packed_key_value_indexes=cfg_img_packed_key_value_indexes,
1074
+ )
1075
+ _local = self._move_to_device(_local)
1076
+ packed_text_ids = _local['packed_text_ids']
1077
+ packed_text_indexes = _local['packed_text_indexes']
1078
+ packed_init_noises = _local['packed_init_noises']
1079
+ packed_vae_position_ids = _local['packed_vae_position_ids']
1080
+ packed_vae_token_indexes = _local['packed_vae_token_indexes']
1081
+ packed_seqlens = _local['packed_seqlens']
1082
+ packed_position_ids = _local['packed_position_ids']
1083
+ packed_indexes = _local['packed_indexes']
1084
+ key_values_lens = _local['key_values_lens']
1085
+ packed_key_value_indexes = _local['packed_key_value_indexes']
1086
+ cfg_text_packed_query_indexes = _local['cfg_text_packed_query_indexes']
1087
+ cfg_text_packed_position_ids = _local['cfg_text_packed_position_ids']
1088
+ cfg_text_key_values_lens = _local['cfg_text_key_values_lens']
1089
+ cfg_text_packed_key_value_indexes = _local['cfg_text_packed_key_value_indexes']
1090
+ cfg_img_packed_query_indexes = _local['cfg_img_packed_query_indexes']
1091
+ cfg_img_packed_position_ids = _local['cfg_img_packed_position_ids']
1092
+ cfg_img_key_values_lens = _local['cfg_img_key_values_lens']
1093
+ cfg_img_packed_key_value_indexes = _local['cfg_img_packed_key_value_indexes']
1094
+
1095
  if enable_taylorseer:
1096
  self.language_model.model.enable_taylorseer = True
1097
  model_pred_cache_dic, model_pred_current = cache_init(self, num_timesteps)
 
1359
  end_token_id: int = None,
1360
  capm_active: bool = False,
1361
  ):
1362
+ # Move input tensors to model device
1363
+ _local = dict(
1364
+ packed_key_value_indexes=packed_key_value_indexes,
1365
+ key_values_lens=key_values_lens,
1366
+ packed_start_tokens=packed_start_tokens,
1367
+ packed_query_position_ids=packed_query_position_ids,
1368
+ )
1369
+ _local = self._move_to_device(_local)
1370
+ packed_key_value_indexes = _local['packed_key_value_indexes']
1371
+ key_values_lens = _local['key_values_lens']
1372
+ packed_start_tokens = _local['packed_start_tokens']
1373
+ packed_query_position_ids = _local['packed_query_position_ids']
1374
+
1375
  step = 0
1376
  generated_sequence = []
1377
  curr_tokens = packed_start_tokens