Spaces:
Runtime error
Runtime error
Upload folder using huggingface_hub
Browse files- UniICL/modeling/uniicl/bagel.py +133 -0
UniICL/modeling/uniicl/bagel.py
CHANGED
|
@@ -100,6 +100,14 @@ class Bagel(PreTrainedModel):
|
|
| 100 |
self.config = config
|
| 101 |
self._init_weights()
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
def _init_weights(self):
|
| 104 |
if self.config.visual_gen:
|
| 105 |
nn.init.constant_(self.llm2vae.weight, 0)
|
|
@@ -559,6 +567,23 @@ class Bagel(PreTrainedModel):
|
|
| 559 |
key_values_lens: torch.IntTensor,
|
| 560 |
capm_active: bool = False,
|
| 561 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 562 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 563 |
|
| 564 |
extra_inputs = {}
|
|
@@ -663,6 +688,33 @@ class Bagel(PreTrainedModel):
|
|
| 663 |
key_values_lens: torch.IntTensor,
|
| 664 |
capm_active: bool = False,
|
| 665 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 666 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 667 |
packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
|
| 668 |
packed_sequence[packed_text_indexes] = packed_text_embedding
|
|
@@ -798,6 +850,33 @@ class Bagel(PreTrainedModel):
|
|
| 798 |
packed_key_value_indexes: torch.Tensor,
|
| 799 |
capm_active: bool = False,
|
| 800 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 801 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 802 |
packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
|
| 803 |
packed_sequence[packed_text_indexes] = packed_text_embedding
|
|
@@ -972,6 +1051,47 @@ class Bagel(PreTrainedModel):
|
|
| 972 |
# cache_args
|
| 973 |
enable_taylorseer=False,
|
| 974 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 975 |
if enable_taylorseer:
|
| 976 |
self.language_model.model.enable_taylorseer = True
|
| 977 |
model_pred_cache_dic, model_pred_current = cache_init(self, num_timesteps)
|
|
@@ -1239,6 +1359,19 @@ class Bagel(PreTrainedModel):
|
|
| 1239 |
end_token_id: int = None,
|
| 1240 |
capm_active: bool = False,
|
| 1241 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1242 |
step = 0
|
| 1243 |
generated_sequence = []
|
| 1244 |
curr_tokens = packed_start_tokens
|
|
|
|
| 100 |
self.config = config
|
| 101 |
self._init_weights()
|
| 102 |
|
| 103 |
+
def _move_to_device(self, kwargs_dict):
|
| 104 |
+
"""Move all tensor values in kwargs to the model's device."""
|
| 105 |
+
device = next(self.parameters()).device
|
| 106 |
+
for k, v in kwargs_dict.items():
|
| 107 |
+
if isinstance(v, torch.Tensor) and v.device.type != device.type:
|
| 108 |
+
kwargs_dict[k] = v.to(device)
|
| 109 |
+
return kwargs_dict
|
| 110 |
+
|
| 111 |
def _init_weights(self):
|
| 112 |
if self.config.visual_gen:
|
| 113 |
nn.init.constant_(self.llm2vae.weight, 0)
|
|
|
|
| 567 |
key_values_lens: torch.IntTensor,
|
| 568 |
capm_active: bool = False,
|
| 569 |
):
|
| 570 |
+
# Move all input tensors to model device
|
| 571 |
+
_local = dict(
|
| 572 |
+
packed_text_ids=packed_text_ids,
|
| 573 |
+
packed_text_position_ids=packed_text_position_ids,
|
| 574 |
+
text_token_lens=text_token_lens,
|
| 575 |
+
packed_text_indexes=packed_text_indexes,
|
| 576 |
+
packed_key_value_indexes=packed_key_value_indexes,
|
| 577 |
+
key_values_lens=key_values_lens,
|
| 578 |
+
)
|
| 579 |
+
_local = self._move_to_device(_local)
|
| 580 |
+
packed_text_ids = _local['packed_text_ids']
|
| 581 |
+
packed_text_position_ids = _local['packed_text_position_ids']
|
| 582 |
+
text_token_lens = _local['text_token_lens']
|
| 583 |
+
packed_text_indexes = _local['packed_text_indexes']
|
| 584 |
+
packed_key_value_indexes = _local['packed_key_value_indexes']
|
| 585 |
+
key_values_lens = _local['key_values_lens']
|
| 586 |
+
|
| 587 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 588 |
|
| 589 |
extra_inputs = {}
|
|
|
|
| 688 |
key_values_lens: torch.IntTensor,
|
| 689 |
capm_active: bool = False,
|
| 690 |
):
|
| 691 |
+
# Move all input tensors to model device
|
| 692 |
+
_local = dict(
|
| 693 |
+
packed_text_ids=packed_text_ids,
|
| 694 |
+
packed_text_indexes=packed_text_indexes,
|
| 695 |
+
packed_vit_tokens=packed_vit_tokens,
|
| 696 |
+
packed_vit_token_indexes=packed_vit_token_indexes,
|
| 697 |
+
packed_vit_position_ids=packed_vit_position_ids,
|
| 698 |
+
vit_token_seqlens=vit_token_seqlens,
|
| 699 |
+
packed_position_ids=packed_position_ids,
|
| 700 |
+
packed_seqlens=packed_seqlens,
|
| 701 |
+
packed_indexes=packed_indexes,
|
| 702 |
+
packed_key_value_indexes=packed_key_value_indexes,
|
| 703 |
+
key_values_lens=key_values_lens,
|
| 704 |
+
)
|
| 705 |
+
_local = self._move_to_device(_local)
|
| 706 |
+
packed_text_ids = _local['packed_text_ids']
|
| 707 |
+
packed_text_indexes = _local['packed_text_indexes']
|
| 708 |
+
packed_vit_tokens = _local['packed_vit_tokens']
|
| 709 |
+
packed_vit_token_indexes = _local['packed_vit_token_indexes']
|
| 710 |
+
packed_vit_position_ids = _local['packed_vit_position_ids']
|
| 711 |
+
vit_token_seqlens = _local['vit_token_seqlens']
|
| 712 |
+
packed_position_ids = _local['packed_position_ids']
|
| 713 |
+
packed_seqlens = _local['packed_seqlens']
|
| 714 |
+
packed_indexes = _local['packed_indexes']
|
| 715 |
+
packed_key_value_indexes = _local['packed_key_value_indexes']
|
| 716 |
+
key_values_lens = _local['key_values_lens']
|
| 717 |
+
|
| 718 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 719 |
packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
|
| 720 |
packed_sequence[packed_text_indexes] = packed_text_embedding
|
|
|
|
| 850 |
packed_key_value_indexes: torch.Tensor,
|
| 851 |
capm_active: bool = False,
|
| 852 |
):
|
| 853 |
+
# Move all input tensors to model device
|
| 854 |
+
_local = dict(
|
| 855 |
+
padded_images=padded_images,
|
| 856 |
+
packed_vae_position_ids=packed_vae_position_ids,
|
| 857 |
+
packed_timesteps=packed_timesteps,
|
| 858 |
+
packed_vae_token_indexes=packed_vae_token_indexes,
|
| 859 |
+
packed_text_ids=packed_text_ids,
|
| 860 |
+
packed_text_indexes=packed_text_indexes,
|
| 861 |
+
packed_position_ids=packed_position_ids,
|
| 862 |
+
packed_seqlens=packed_seqlens,
|
| 863 |
+
packed_indexes=packed_indexes,
|
| 864 |
+
key_values_lens=key_values_lens,
|
| 865 |
+
packed_key_value_indexes=packed_key_value_indexes,
|
| 866 |
+
)
|
| 867 |
+
_local = self._move_to_device(_local)
|
| 868 |
+
padded_images = _local['padded_images']
|
| 869 |
+
packed_vae_position_ids = _local['packed_vae_position_ids']
|
| 870 |
+
packed_timesteps = _local['packed_timesteps']
|
| 871 |
+
packed_vae_token_indexes = _local['packed_vae_token_indexes']
|
| 872 |
+
packed_text_ids = _local['packed_text_ids']
|
| 873 |
+
packed_text_indexes = _local['packed_text_indexes']
|
| 874 |
+
packed_position_ids = _local['packed_position_ids']
|
| 875 |
+
packed_seqlens = _local['packed_seqlens']
|
| 876 |
+
packed_indexes = _local['packed_indexes']
|
| 877 |
+
key_values_lens = _local['key_values_lens']
|
| 878 |
+
packed_key_value_indexes = _local['packed_key_value_indexes']
|
| 879 |
+
|
| 880 |
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
|
| 881 |
packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
|
| 882 |
packed_sequence[packed_text_indexes] = packed_text_embedding
|
|
|
|
| 1051 |
# cache_args
|
| 1052 |
enable_taylorseer=False,
|
| 1053 |
):
|
| 1054 |
+
# Move all input tensors to model device
|
| 1055 |
+
_local = dict(
|
| 1056 |
+
packed_text_ids=packed_text_ids,
|
| 1057 |
+
packed_text_indexes=packed_text_indexes,
|
| 1058 |
+
packed_init_noises=packed_init_noises,
|
| 1059 |
+
packed_vae_position_ids=packed_vae_position_ids,
|
| 1060 |
+
packed_vae_token_indexes=packed_vae_token_indexes,
|
| 1061 |
+
packed_seqlens=packed_seqlens,
|
| 1062 |
+
packed_position_ids=packed_position_ids,
|
| 1063 |
+
packed_indexes=packed_indexes,
|
| 1064 |
+
key_values_lens=key_values_lens,
|
| 1065 |
+
packed_key_value_indexes=packed_key_value_indexes,
|
| 1066 |
+
cfg_text_packed_query_indexes=cfg_text_packed_query_indexes,
|
| 1067 |
+
cfg_text_packed_position_ids=cfg_text_packed_position_ids,
|
| 1068 |
+
cfg_text_key_values_lens=cfg_text_key_values_lens,
|
| 1069 |
+
cfg_text_packed_key_value_indexes=cfg_text_packed_key_value_indexes,
|
| 1070 |
+
cfg_img_packed_query_indexes=cfg_img_packed_query_indexes,
|
| 1071 |
+
cfg_img_packed_position_ids=cfg_img_packed_position_ids,
|
| 1072 |
+
cfg_img_key_values_lens=cfg_img_key_values_lens,
|
| 1073 |
+
cfg_img_packed_key_value_indexes=cfg_img_packed_key_value_indexes,
|
| 1074 |
+
)
|
| 1075 |
+
_local = self._move_to_device(_local)
|
| 1076 |
+
packed_text_ids = _local['packed_text_ids']
|
| 1077 |
+
packed_text_indexes = _local['packed_text_indexes']
|
| 1078 |
+
packed_init_noises = _local['packed_init_noises']
|
| 1079 |
+
packed_vae_position_ids = _local['packed_vae_position_ids']
|
| 1080 |
+
packed_vae_token_indexes = _local['packed_vae_token_indexes']
|
| 1081 |
+
packed_seqlens = _local['packed_seqlens']
|
| 1082 |
+
packed_position_ids = _local['packed_position_ids']
|
| 1083 |
+
packed_indexes = _local['packed_indexes']
|
| 1084 |
+
key_values_lens = _local['key_values_lens']
|
| 1085 |
+
packed_key_value_indexes = _local['packed_key_value_indexes']
|
| 1086 |
+
cfg_text_packed_query_indexes = _local['cfg_text_packed_query_indexes']
|
| 1087 |
+
cfg_text_packed_position_ids = _local['cfg_text_packed_position_ids']
|
| 1088 |
+
cfg_text_key_values_lens = _local['cfg_text_key_values_lens']
|
| 1089 |
+
cfg_text_packed_key_value_indexes = _local['cfg_text_packed_key_value_indexes']
|
| 1090 |
+
cfg_img_packed_query_indexes = _local['cfg_img_packed_query_indexes']
|
| 1091 |
+
cfg_img_packed_position_ids = _local['cfg_img_packed_position_ids']
|
| 1092 |
+
cfg_img_key_values_lens = _local['cfg_img_key_values_lens']
|
| 1093 |
+
cfg_img_packed_key_value_indexes = _local['cfg_img_packed_key_value_indexes']
|
| 1094 |
+
|
| 1095 |
if enable_taylorseer:
|
| 1096 |
self.language_model.model.enable_taylorseer = True
|
| 1097 |
model_pred_cache_dic, model_pred_current = cache_init(self, num_timesteps)
|
|
|
|
| 1359 |
end_token_id: int = None,
|
| 1360 |
capm_active: bool = False,
|
| 1361 |
):
|
| 1362 |
+
# Move input tensors to model device
|
| 1363 |
+
_local = dict(
|
| 1364 |
+
packed_key_value_indexes=packed_key_value_indexes,
|
| 1365 |
+
key_values_lens=key_values_lens,
|
| 1366 |
+
packed_start_tokens=packed_start_tokens,
|
| 1367 |
+
packed_query_position_ids=packed_query_position_ids,
|
| 1368 |
+
)
|
| 1369 |
+
_local = self._move_to_device(_local)
|
| 1370 |
+
packed_key_value_indexes = _local['packed_key_value_indexes']
|
| 1371 |
+
key_values_lens = _local['key_values_lens']
|
| 1372 |
+
packed_start_tokens = _local['packed_start_tokens']
|
| 1373 |
+
packed_query_position_ids = _local['packed_query_position_ids']
|
| 1374 |
+
|
| 1375 |
step = 0
|
| 1376 |
generated_sequence = []
|
| 1377 |
curr_tokens = packed_start_tokens
|