Spaces:
Sleeping
Sleeping
Silvio Galesso commited on
Commit ·
11ac3b5
1
Parent(s): 2ced563
cleanup, repo name
Browse files- app2.py +1 -1
- orbis2/evaluate/rollout_demo_v2.py +59 -84
- orbis2/models/second_stage/cd_model_v2.py +44 -29
- orbis2/networks/tokenizer/ae.py +3 -481
- orbis2/networks/tokenizer/pretrained_models.py +0 -2
- orbis2/util.py +0 -199
app2.py
CHANGED
|
@@ -165,7 +165,7 @@ MODEL_SPECS = [
|
|
| 165 |
]
|
| 166 |
|
| 167 |
# Hub repo holding the three models (root contains L1/, L2/, tok/).
|
| 168 |
-
HF_CKPT_REPO = os.environ.get("ORBIS2_HF_REPO", "sud0301/
|
| 169 |
|
| 170 |
|
| 171 |
def _missing_models(root: Path) -> list[str]:
|
|
|
|
| 165 |
]
|
| 166 |
|
| 167 |
# Hub repo holding the three models (root contains L1/, L2/, tok/).
|
| 168 |
+
HF_CKPT_REPO = os.environ.get("ORBIS2_HF_REPO", "sud0301/orbis2")
|
| 169 |
|
| 170 |
|
| 171 |
def _missing_models(root: Path) -> list[str]:
|
orbis2/evaluate/rollout_demo_v2.py
CHANGED
|
@@ -3,13 +3,13 @@ import os
|
|
| 3 |
import sys
|
| 4 |
import imageio
|
| 5 |
import logging
|
|
|
|
| 6 |
|
| 7 |
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 8 |
|
| 9 |
import numpy as np
|
| 10 |
import torch
|
| 11 |
from omegaconf import OmegaConf
|
| 12 |
-
from omegaconf.errors import ConfigTypeError
|
| 13 |
from PIL import Image
|
| 14 |
from pytorch_lightning import seed_everything
|
| 15 |
from torchvision.utils import save_image
|
|
@@ -26,8 +26,8 @@ from evaluate.utils import (
|
|
| 26 |
resolve_video_backend,
|
| 27 |
str2bool,
|
| 28 |
)
|
| 29 |
-
from util import instantiate_from_config
|
| 30 |
|
|
|
|
| 31 |
|
| 32 |
logger = logging.getLogger(__name__)
|
| 33 |
|
|
@@ -48,22 +48,13 @@ class _L1L2FrameIndexer(L2ContextMixin):
|
|
| 48 |
)
|
| 49 |
|
| 50 |
|
| 51 |
-
def resolve_context_image_size(
|
| 52 |
-
"""Return (height, width) to resize context images to, from
|
| 53 |
-
|
| 54 |
-
return int(args.height), int(args.width)
|
| 55 |
-
|
| 56 |
-
try:
|
| 57 |
-
size = OmegaConf.select(config, "data.params.validation.params.size")
|
| 58 |
-
except ConfigTypeError:
|
| 59 |
-
size = None
|
| 60 |
if size is None:
|
| 61 |
-
|
| 62 |
size = (size, size) if isinstance(size, int) else tuple(size)
|
| 63 |
-
|
| 64 |
-
height = int(args.height) if args.height is not None else int(size[0])
|
| 65 |
-
width = int(args.width) if args.width is not None else int(size[1])
|
| 66 |
-
return height, width
|
| 67 |
|
| 68 |
|
| 69 |
def require_l1l2_model(model):
|
|
@@ -78,25 +69,24 @@ def require_l1l2_model(model):
|
|
| 78 |
)
|
| 79 |
|
| 80 |
|
| 81 |
-
def resolve_l2_frame_rate(
|
| 82 |
-
"""Return the L2 sampling rate
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
return float(args.l2_frame_rate)
|
| 93 |
|
| 94 |
|
| 95 |
def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1_context_frames, l2_context_frames, height, width, backend, device):
|
| 96 |
"""Sample L1 (high-rate) and L2 (low-rate, further back) context windows from one video."""
|
| 97 |
native_fps, video_length = get_video_fps_and_length(video_path, backend)
|
| 98 |
-
frame_interval = compute_frame_interval(native_fps, l1_frame_rate, "
|
| 99 |
-
compute_frame_interval(native_fps, l2_frame_rate, "
|
| 100 |
|
| 101 |
indexer = _L1L2FrameIndexer(
|
| 102 |
frame_interval=frame_interval,
|
|
@@ -120,7 +110,7 @@ def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1
|
|
| 120 |
raise ValueError(
|
| 121 |
f"Video does not have enough lookback for L2 context: start_frame={start_frame} but "
|
| 122 |
f"at least {required_offset} frames of history are required before it. "
|
| 123 |
-
"Use a longer video
|
| 124 |
)
|
| 125 |
|
| 126 |
l1_indices, l2_indices = indexer.get_l1_and_l2_indices(start_frame, l1_context_frames)
|
|
@@ -236,23 +226,23 @@ def make_unconditional_steering(min_odo_steps, dtype, device):
|
|
| 236 |
return torch.full((1, int(min_odo_steps), 2), float("nan"), dtype=dtype, device=device)
|
| 237 |
|
| 238 |
|
| 239 |
-
def
|
| 240 |
"""Override the frozen L2 predictor's sampler steps (its `l2_pred_NFE`).
|
| 241 |
|
| 242 |
-
L1 and L2 are sampled at different step counts: --
|
| 243 |
L2's comes from the config's `l2_pred_NFE`. Setting the attribute post-init is
|
| 244 |
enough -- it is read at sample time, not at construction.
|
| 245 |
"""
|
| 246 |
-
if
|
| 247 |
return
|
| 248 |
|
| 249 |
condition_preprocessor = getattr(model, "condition_preprocessor", None)
|
| 250 |
if condition_preprocessor is None or not hasattr(condition_preprocessor, "l2_pred_NFE"):
|
| 251 |
raise TypeError(
|
| 252 |
-
"--
|
| 253 |
"`l2_pred_NFE` (so it has no separately-sampled L2 predictor)."
|
| 254 |
)
|
| 255 |
-
condition_preprocessor.l2_pred_NFE = int(
|
| 256 |
|
| 257 |
|
| 258 |
@torch.no_grad()
|
|
@@ -308,11 +298,12 @@ def generate_images(args, unknown_args):
|
|
| 308 |
)
|
| 309 |
|
| 310 |
maybe_apply_condition_preprocessor_scales(model, args.speed_scale, args.yaw_rate_scale)
|
| 311 |
-
|
| 312 |
|
| 313 |
-
height, width = resolve_context_image_size(
|
| 314 |
backend = resolve_video_backend()
|
| 315 |
-
|
|
|
|
| 316 |
|
| 317 |
l1_context_frames = int(model.vit.num_context_frames)
|
| 318 |
l2_context_frames = int(model.condition_preprocessor.num_context_frames)
|
|
@@ -320,7 +311,7 @@ def generate_images(args, unknown_args):
|
|
| 320 |
l1_tensor, l2_tensor = load_l1_l2_context(
|
| 321 |
video_path=args.video,
|
| 322 |
start_frame=args.start_frame,
|
| 323 |
-
l1_frame_rate=
|
| 324 |
l2_frame_rate=l2_frame_rate,
|
| 325 |
l1_context_frames=l1_context_frames,
|
| 326 |
l2_context_frames=l2_context_frames,
|
|
@@ -330,8 +321,8 @@ def generate_images(args, unknown_args):
|
|
| 330 |
device=args.device,
|
| 331 |
)
|
| 332 |
|
| 333 |
-
num_future_frames = get_rollout_future_frame_count(model, args.
|
| 334 |
-
frame_rate = torch.tensor(
|
| 335 |
data_batch = {"images": l1_tensor, "l2_context": l2_tensor, "frame_rate": frame_rate}
|
| 336 |
|
| 337 |
if args.steering_file is not None and args.trajectory_file is not None:
|
|
@@ -344,7 +335,7 @@ def generate_images(args, unknown_args):
|
|
| 344 |
validation_params=None,
|
| 345 |
num_condition_frames=l1_context_frames,
|
| 346 |
num_gen_frames=num_future_frames,
|
| 347 |
-
rollout_steps=args.
|
| 348 |
)
|
| 349 |
if args.steering_file is not None:
|
| 350 |
data_batch["steering"] = load_steering_trajectory(
|
|
@@ -357,7 +348,7 @@ def generate_images(args, unknown_args):
|
|
| 357 |
"required odometry length (get_required_rollout_odometry_steps returned None)."
|
| 358 |
)
|
| 359 |
odometry_steps_per_image_frame = getattr(model.condition_preprocessor, "odometry_steps_per_image_frame", 1)
|
| 360 |
-
dt = 1.0 / (
|
| 361 |
traj_xy = load_trajectory_points(args.trajectory_file)
|
| 362 |
traj_xy = resample_trajectory_by_arclength(traj_xy, min_odo_steps + 1)
|
| 363 |
speed_yawrate = trajectory_to_speed_yawrate(traj_xy, dt)
|
|
@@ -368,22 +359,6 @@ def generate_images(args, unknown_args):
|
|
| 368 |
data_batch["steering"] = make_unconditional_steering(min_odo_steps, dtype=l1_tensor.dtype, device=args.device)
|
| 369 |
data_batch["steering_format"] = STEERING_FORMAT
|
| 370 |
|
| 371 |
-
# Roll out `num_videos` futures in parallel from the same context, as a single
|
| 372 |
-
# minibatch. The context (and steering) is tiled along the batch dim; the sampler
|
| 373 |
-
# draws independent initial noise per element, so the futures diverge under one
|
| 374 |
-
# seed. Everything below is computed at the tiled batch size so condition_kwargs
|
| 375 |
-
# stays internally consistent.
|
| 376 |
-
num_videos = max(1, int(args.num_videos))
|
| 377 |
-
if num_videos > 1:
|
| 378 |
-
def _tile_batch(t):
|
| 379 |
-
return t.repeat(num_videos, *([1] * (t.dim() - 1)))
|
| 380 |
-
|
| 381 |
-
l1_tensor = _tile_batch(l1_tensor)
|
| 382 |
-
l2_tensor = _tile_batch(l2_tensor)
|
| 383 |
-
data_batch["images"] = l1_tensor
|
| 384 |
-
data_batch["l2_context"] = l2_tensor
|
| 385 |
-
data_batch["steering"] = _tile_batch(data_batch["steering"])
|
| 386 |
-
|
| 387 |
condition_kwargs = model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout")
|
| 388 |
|
| 389 |
if args.steering_file is not None:
|
|
@@ -394,17 +369,17 @@ def generate_images(args, unknown_args):
|
|
| 394 |
steering_source = "none"
|
| 395 |
logger.info(f"Steering source: {steering_source}")
|
| 396 |
logger.info(f"Steering scales: speed={args.speed_scale:g}, yaw_rate={args.yaw_rate_scale:g}")
|
| 397 |
-
logger.info(f"L1/L2 frame rates: {
|
| 398 |
logger.info(f"Saving generated images to {args.output_dir}")
|
| 399 |
|
| 400 |
autocast_enabled = args.device.startswith("cuda")
|
| 401 |
with torch.autocast(dtype=torch.float16, device_type="cuda", enabled=autocast_enabled):
|
| 402 |
_latents, gen_frames = model.roll_out(
|
| 403 |
x_0={"images": l1_tensor},
|
| 404 |
-
num_gen_frames=args.
|
| 405 |
latent_input=False,
|
| 406 |
-
NFE=args.
|
| 407 |
-
eta=
|
| 408 |
sample_with_ema=args.evaluate_ema,
|
| 409 |
num_samples=l1_tensor.size(0),
|
| 410 |
frame_rate=frame_rate.reshape(1).repeat(l1_tensor.size(0)),
|
|
@@ -417,7 +392,7 @@ def generate_images(args, unknown_args):
|
|
| 417 |
overlay_trajectory = model.condition_preprocessor.get_rollout_visualization_trajectory(
|
| 418 |
condition_kwargs=model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout"),
|
| 419 |
num_condition_frames=l1_context_frames,
|
| 420 |
-
num_gen_steps=args.
|
| 421 |
num_pred_frames=model.num_pred_frames,
|
| 422 |
)
|
| 423 |
if overlay_trajectory is not None:
|
|
@@ -445,7 +420,7 @@ def generate_images(args, unknown_args):
|
|
| 445 |
np.array(Image.open(os.path.join(seq_dir, f"frame_{f:04d}.jpg")))
|
| 446 |
for f in range(num_frames)
|
| 447 |
],
|
| 448 |
-
fps=
|
| 449 |
loop=0,
|
| 450 |
)
|
| 451 |
|
|
@@ -469,28 +444,24 @@ if __name__ == "__main__":
|
|
| 469 |
logging.basicConfig(level=logging.INFO)
|
| 470 |
|
| 471 |
parser = argparse.ArgumentParser()
|
| 472 |
-
parser.add_argument(
|
| 473 |
-
|
| 474 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 475 |
parser.add_argument("--video", type=str, required=True, help="Path to the input video file to sample L1/L2 context from.")
|
| 476 |
-
parser.add_argument("--l1_frame_rate", type=float, required=True, help="Frame rate (Hz) to sample L1 context/rollout frames at, and the generated GIF's fps.")
|
| 477 |
-
parser.add_argument("--l2_frame_rate", type=float, default=None, help="Frame rate (Hz) to sample L2 context frames at. Defaults to the frozen L2 predictor's own trained frame rate; must match it if given explicitly.")
|
| 478 |
parser.add_argument("--start_frame", type=int, default=None, help="Native-video frame index to start the L1 context window at. Defaults to the latest window that fits (the end of the video).")
|
| 479 |
-
parser.add_argument("--height", type=int, default=None, help="Height to resize context images to. Defaults to the training config's size.")
|
| 480 |
-
parser.add_argument("--width", type=int, default=None, help="Width to resize context images to. Defaults to the training config's size.")
|
| 481 |
parser.add_argument(
|
| 482 |
-
"--
|
| 483 |
type=int,
|
| 484 |
-
default=
|
| 485 |
help="Number of rollout steps to generate; each step predicts `model.num_pred_frames` future frames.",
|
| 486 |
)
|
| 487 |
parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated frames and GIF to.")
|
| 488 |
-
parser.add_argument(
|
| 489 |
-
"--num_videos",
|
| 490 |
-
type=int,
|
| 491 |
-
default=1,
|
| 492 |
-
help="Number of futures to roll out in parallel from the same context, as one minibatch.",
|
| 493 |
-
)
|
| 494 |
parser.add_argument(
|
| 495 |
"--vis_mode",
|
| 496 |
type=str,
|
|
@@ -518,15 +489,14 @@ if __name__ == "__main__":
|
|
| 518 |
default="cpu",
|
| 519 |
help="Device used for decoded rollout frames. Use 'cpu' to reduce peak GPU memory during saving.",
|
| 520 |
)
|
| 521 |
-
parser.add_argument("--
|
| 522 |
parser.add_argument(
|
| 523 |
-
"--
|
| 524 |
type=int,
|
| 525 |
default=None,
|
| 526 |
help="Number of sampler steps (NFE) for the frozen L2 abstract predictor. "
|
| 527 |
"Overrides the config's `l2_pred_NFE`; defaults to whatever the config sets.",
|
| 528 |
)
|
| 529 |
-
parser.add_argument("--eta", type=float, default=0.0, help="Stochasticity for sampling")
|
| 530 |
parser.add_argument("--evaluate_ema", "--use_ema", type=str2bool, default=True, help="If the evaluation happen with ema model")
|
| 531 |
parser.add_argument(
|
| 532 |
"--compile",
|
|
@@ -555,6 +525,11 @@ if __name__ == "__main__":
|
|
| 555 |
|
| 556 |
args, unknown = parser.parse_known_args()
|
| 557 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 558 |
args.ckpt = os.path.join(args.exp_dir, args.ckpt)
|
| 559 |
args.config = os.path.join(args.exp_dir, args.config)
|
| 560 |
if args.compile_artifacts:
|
|
|
|
| 3 |
import sys
|
| 4 |
import imageio
|
| 5 |
import logging
|
| 6 |
+
import importlib
|
| 7 |
|
| 8 |
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 9 |
|
| 10 |
import numpy as np
|
| 11 |
import torch
|
| 12 |
from omegaconf import OmegaConf
|
|
|
|
| 13 |
from PIL import Image
|
| 14 |
from pytorch_lightning import seed_everything
|
| 15 |
from torchvision.utils import save_image
|
|
|
|
| 26 |
resolve_video_backend,
|
| 27 |
str2bool,
|
| 28 |
)
|
|
|
|
| 29 |
|
| 30 |
+
from util import instantiate_from_config
|
| 31 |
|
| 32 |
logger = logging.getLogger(__name__)
|
| 33 |
|
|
|
|
| 48 |
)
|
| 49 |
|
| 50 |
|
| 51 |
+
def resolve_context_image_size(config):
|
| 52 |
+
"""Return (height, width) to resize context images to, from the inference config."""
|
| 53 |
+
size = OmegaConf.select(config, "data.params.validation.params.size")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
if size is None:
|
| 55 |
+
raise ValueError("Config is missing `data.params.validation.params.size`.")
|
| 56 |
size = (size, size) if isinstance(size, int) else tuple(size)
|
| 57 |
+
return int(size[0]), int(size[1])
|
|
|
|
|
|
|
|
|
|
| 58 |
|
| 59 |
|
| 60 |
def require_l1l2_model(model):
|
|
|
|
| 69 |
)
|
| 70 |
|
| 71 |
|
| 72 |
+
def resolve_l2_frame_rate(model):
|
| 73 |
+
"""Return the L2 sampling rate: the frozen L2 predictor's own trained rate."""
|
| 74 |
+
return float(model.condition_preprocessor.l2_predictor_frame_rate)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def resolve_l1_frame_rate(config):
|
| 78 |
+
"""Return the L1 (context/rollout) frame rate (Hz) declared in the inference config."""
|
| 79 |
+
frame_rate = OmegaConf.select(config, "data.params.validation.params.frame_rate")
|
| 80 |
+
if frame_rate is None:
|
| 81 |
+
raise ValueError("Config is missing `data.params.validation.params.frame_rate`.")
|
| 82 |
+
return float(frame_rate)
|
|
|
|
| 83 |
|
| 84 |
|
| 85 |
def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1_context_frames, l2_context_frames, height, width, backend, device):
|
| 86 |
"""Sample L1 (high-rate) and L2 (low-rate, further back) context windows from one video."""
|
| 87 |
native_fps, video_length = get_video_fps_and_length(video_path, backend)
|
| 88 |
+
frame_interval = compute_frame_interval(native_fps, l1_frame_rate, "the L1 frame rate (data.params.validation.params.frame_rate)")
|
| 89 |
+
compute_frame_interval(native_fps, l2_frame_rate, "the L2 predictor's trained frame rate")
|
| 90 |
|
| 91 |
indexer = _L1L2FrameIndexer(
|
| 92 |
frame_interval=frame_interval,
|
|
|
|
| 110 |
raise ValueError(
|
| 111 |
f"Video does not have enough lookback for L2 context: start_frame={start_frame} but "
|
| 112 |
f"at least {required_offset} frames of history are required before it. "
|
| 113 |
+
"Use a longer video or a later --start_frame."
|
| 114 |
)
|
| 115 |
|
| 116 |
l1_indices, l2_indices = indexer.get_l1_and_l2_indices(start_frame, l1_context_frames)
|
|
|
|
| 226 |
return torch.full((1, int(min_odo_steps), 2), float("nan"), dtype=dtype, device=device)
|
| 227 |
|
| 228 |
|
| 229 |
+
def maybe_apply_l2_nfe(model, l2_nfe):
|
| 230 |
"""Override the frozen L2 predictor's sampler steps (its `l2_pred_NFE`).
|
| 231 |
|
| 232 |
+
L1 and L2 are sampled at different step counts: --l1_nfe is L1's NFE, while
|
| 233 |
L2's comes from the config's `l2_pred_NFE`. Setting the attribute post-init is
|
| 234 |
enough -- it is read at sample time, not at construction.
|
| 235 |
"""
|
| 236 |
+
if l2_nfe is None:
|
| 237 |
return
|
| 238 |
|
| 239 |
condition_preprocessor = getattr(model, "condition_preprocessor", None)
|
| 240 |
if condition_preprocessor is None or not hasattr(condition_preprocessor, "l2_pred_NFE"):
|
| 241 |
raise TypeError(
|
| 242 |
+
"--l2_nfe was given but the loaded condition_preprocessor has no "
|
| 243 |
"`l2_pred_NFE` (so it has no separately-sampled L2 predictor)."
|
| 244 |
)
|
| 245 |
+
condition_preprocessor.l2_pred_NFE = int(l2_nfe)
|
| 246 |
|
| 247 |
|
| 248 |
@torch.no_grad()
|
|
|
|
| 298 |
)
|
| 299 |
|
| 300 |
maybe_apply_condition_preprocessor_scales(model, args.speed_scale, args.yaw_rate_scale)
|
| 301 |
+
maybe_apply_l2_nfe(model, args.l2_nfe)
|
| 302 |
|
| 303 |
+
height, width = resolve_context_image_size(config)
|
| 304 |
backend = resolve_video_backend()
|
| 305 |
+
l1_frame_rate = resolve_l1_frame_rate(config)
|
| 306 |
+
l2_frame_rate = resolve_l2_frame_rate(model)
|
| 307 |
|
| 308 |
l1_context_frames = int(model.vit.num_context_frames)
|
| 309 |
l2_context_frames = int(model.condition_preprocessor.num_context_frames)
|
|
|
|
| 311 |
l1_tensor, l2_tensor = load_l1_l2_context(
|
| 312 |
video_path=args.video,
|
| 313 |
start_frame=args.start_frame,
|
| 314 |
+
l1_frame_rate=l1_frame_rate,
|
| 315 |
l2_frame_rate=l2_frame_rate,
|
| 316 |
l1_context_frames=l1_context_frames,
|
| 317 |
l2_context_frames=l2_context_frames,
|
|
|
|
| 321 |
device=args.device,
|
| 322 |
)
|
| 323 |
|
| 324 |
+
num_future_frames = get_rollout_future_frame_count(model, args.num_rollout_steps)
|
| 325 |
+
frame_rate = torch.tensor(l1_frame_rate, device=args.device)
|
| 326 |
data_batch = {"images": l1_tensor, "l2_context": l2_tensor, "frame_rate": frame_rate}
|
| 327 |
|
| 328 |
if args.steering_file is not None and args.trajectory_file is not None:
|
|
|
|
| 335 |
validation_params=None,
|
| 336 |
num_condition_frames=l1_context_frames,
|
| 337 |
num_gen_frames=num_future_frames,
|
| 338 |
+
rollout_steps=args.num_rollout_steps,
|
| 339 |
)
|
| 340 |
if args.steering_file is not None:
|
| 341 |
data_batch["steering"] = load_steering_trajectory(
|
|
|
|
| 348 |
"required odometry length (get_required_rollout_odometry_steps returned None)."
|
| 349 |
)
|
| 350 |
odometry_steps_per_image_frame = getattr(model.condition_preprocessor, "odometry_steps_per_image_frame", 1)
|
| 351 |
+
dt = 1.0 / (l1_frame_rate * odometry_steps_per_image_frame)
|
| 352 |
traj_xy = load_trajectory_points(args.trajectory_file)
|
| 353 |
traj_xy = resample_trajectory_by_arclength(traj_xy, min_odo_steps + 1)
|
| 354 |
speed_yawrate = trajectory_to_speed_yawrate(traj_xy, dt)
|
|
|
|
| 359 |
data_batch["steering"] = make_unconditional_steering(min_odo_steps, dtype=l1_tensor.dtype, device=args.device)
|
| 360 |
data_batch["steering_format"] = STEERING_FORMAT
|
| 361 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 362 |
condition_kwargs = model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout")
|
| 363 |
|
| 364 |
if args.steering_file is not None:
|
|
|
|
| 369 |
steering_source = "none"
|
| 370 |
logger.info(f"Steering source: {steering_source}")
|
| 371 |
logger.info(f"Steering scales: speed={args.speed_scale:g}, yaw_rate={args.yaw_rate_scale:g}")
|
| 372 |
+
logger.info(f"L1/L2 frame rates: {l1_frame_rate:g}/{l2_frame_rate:g} Hz")
|
| 373 |
logger.info(f"Saving generated images to {args.output_dir}")
|
| 374 |
|
| 375 |
autocast_enabled = args.device.startswith("cuda")
|
| 376 |
with torch.autocast(dtype=torch.float16, device_type="cuda", enabled=autocast_enabled):
|
| 377 |
_latents, gen_frames = model.roll_out(
|
| 378 |
x_0={"images": l1_tensor},
|
| 379 |
+
num_gen_frames=args.num_rollout_steps,
|
| 380 |
latent_input=False,
|
| 381 |
+
NFE=args.l1_nfe,
|
| 382 |
+
eta=0.0,
|
| 383 |
sample_with_ema=args.evaluate_ema,
|
| 384 |
num_samples=l1_tensor.size(0),
|
| 385 |
frame_rate=frame_rate.reshape(1).repeat(l1_tensor.size(0)),
|
|
|
|
| 392 |
overlay_trajectory = model.condition_preprocessor.get_rollout_visualization_trajectory(
|
| 393 |
condition_kwargs=model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout"),
|
| 394 |
num_condition_frames=l1_context_frames,
|
| 395 |
+
num_gen_steps=args.num_rollout_steps,
|
| 396 |
num_pred_frames=model.num_pred_frames,
|
| 397 |
)
|
| 398 |
if overlay_trajectory is not None:
|
|
|
|
| 420 |
np.array(Image.open(os.path.join(seq_dir, f"frame_{f:04d}.jpg")))
|
| 421 |
for f in range(num_frames)
|
| 422 |
],
|
| 423 |
+
fps=l1_frame_rate,
|
| 424 |
loop=0,
|
| 425 |
)
|
| 426 |
|
|
|
|
| 444 |
logging.basicConfig(level=logging.INFO)
|
| 445 |
|
| 446 |
parser = argparse.ArgumentParser()
|
| 447 |
+
parser.add_argument(
|
| 448 |
+
"--exp_dir",
|
| 449 |
+
type=str,
|
| 450 |
+
default=os.environ.get("ORBIS2_MODELS_DIR"),
|
| 451 |
+
help="Path to the experiment directory, where the config and checkpoints are stored. "
|
| 452 |
+
"Defaults to the ORBIS2_MODELS_DIR environment variable.",
|
| 453 |
+
)
|
| 454 |
+
parser.add_argument("--ckpt", type=str, default="L1/checkpoints/last.ckpt", help="Path to the checkpoint file, relative to exp_dir")
|
| 455 |
+
parser.add_argument("--config", type=str, default="L1/config.yaml", help="Path to the config file, relative to exp_dir")
|
| 456 |
parser.add_argument("--video", type=str, required=True, help="Path to the input video file to sample L1/L2 context from.")
|
|
|
|
|
|
|
| 457 |
parser.add_argument("--start_frame", type=int, default=None, help="Native-video frame index to start the L1 context window at. Defaults to the latest window that fits (the end of the video).")
|
|
|
|
|
|
|
| 458 |
parser.add_argument(
|
| 459 |
+
"--num_rollout_steps",
|
| 460 |
type=int,
|
| 461 |
+
default=10,
|
| 462 |
help="Number of rollout steps to generate; each step predicts `model.num_pred_frames` future frames.",
|
| 463 |
)
|
| 464 |
parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated frames and GIF to.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 465 |
parser.add_argument(
|
| 466 |
"--vis_mode",
|
| 467 |
type=str,
|
|
|
|
| 489 |
default="cpu",
|
| 490 |
help="Device used for decoded rollout frames. Use 'cpu' to reduce peak GPU memory during saving.",
|
| 491 |
)
|
| 492 |
+
parser.add_argument("--l1_nfe", type=int, default=30, help="Number of sampler steps (NFE) for the L1 detail predictor")
|
| 493 |
parser.add_argument(
|
| 494 |
+
"--l2_nfe",
|
| 495 |
type=int,
|
| 496 |
default=None,
|
| 497 |
help="Number of sampler steps (NFE) for the frozen L2 abstract predictor. "
|
| 498 |
"Overrides the config's `l2_pred_NFE`; defaults to whatever the config sets.",
|
| 499 |
)
|
|
|
|
| 500 |
parser.add_argument("--evaluate_ema", "--use_ema", type=str2bool, default=True, help="If the evaluation happen with ema model")
|
| 501 |
parser.add_argument(
|
| 502 |
"--compile",
|
|
|
|
| 525 |
|
| 526 |
args, unknown = parser.parse_known_args()
|
| 527 |
|
| 528 |
+
if args.exp_dir is None:
|
| 529 |
+
raise ValueError(
|
| 530 |
+
"--exp_dir was not given and the ORBIS2_MODELS_DIR environment variable is not set."
|
| 531 |
+
)
|
| 532 |
+
|
| 533 |
args.ckpt = os.path.join(args.exp_dir, args.ckpt)
|
| 534 |
args.config = os.path.join(args.exp_dir, args.config)
|
| 535 |
if args.compile_artifacts:
|
orbis2/models/second_stage/cd_model_v2.py
CHANGED
|
@@ -205,21 +205,25 @@ class ConsistencyDistillationSampler(FlowMatchingSamplerEuler):
|
|
| 205 |
|
| 206 |
class ConsistencyDistillPredictorModule(PredictorModule):
|
| 207 |
"""
|
| 208 |
-
PredictorModule specialization that distills a frozen teacher
|
| 209 |
-
|
|
|
|
|
|
|
|
|
|
| 210 |
The deliverable is `ema_vit` (evaluate with sample_with_ema=True, NFE 1/2/4).
|
| 211 |
"""
|
| 212 |
|
| 213 |
# `teacher_vit.*` is intentionally absent from saved checkpoints (see
|
| 214 |
-
# on_save_checkpoint below)
|
| 215 |
-
#
|
|
|
|
| 216 |
# load_state_dict(...).missing_keys should exempt these prefixes.
|
| 217 |
checkpoint_exempt_key_prefixes = ("teacher_vit.",)
|
| 218 |
|
| 219 |
def __init__(
|
| 220 |
self,
|
| 221 |
*,
|
| 222 |
-
teacher_ckpt_path,
|
| 223 |
cd_ema_mu=0.999,
|
| 224 |
cd_weight_decay=0.0,
|
| 225 |
**kwargs,
|
|
@@ -229,34 +233,45 @@ class ConsistencyDistillPredictorModule(PredictorModule):
|
|
| 229 |
self.cd_weight_decay = cd_weight_decay
|
| 230 |
self.strict_loading = False
|
| 231 |
|
| 232 |
-
|
| 233 |
-
if not
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
|
| 256 |
def setup(self, stage=None):
|
| 257 |
super().setup(stage)
|
| 258 |
-
if
|
| 259 |
self.teacher_vit.requires_grad_(False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
|
| 261 |
def configure_optimizers(self):
|
| 262 |
params = [p for p in self.vit.parameters() if p.requires_grad]
|
|
|
|
| 205 |
|
| 206 |
class ConsistencyDistillPredictorModule(PredictorModule):
|
| 207 |
"""
|
| 208 |
+
PredictorModule specialization that distills a frozen teacher into a
|
| 209 |
+
few-step student via consistency distillation. `teacher_ckpt_path`
|
| 210 |
+
initializes the student (`vit`/`ema_vit`) from the teacher and builds the
|
| 211 |
+
frozen `teacher_vit` used by the distillation loss; it is required for
|
| 212 |
+
training/validation but optional for inference-only sample()/roll_out().
|
| 213 |
The deliverable is `ema_vit` (evaluate with sample_with_ema=True, NFE 1/2/4).
|
| 214 |
"""
|
| 215 |
|
| 216 |
# `teacher_vit.*` is intentionally absent from saved checkpoints (see
|
| 217 |
+
# on_save_checkpoint below): when teacher_ckpt_path is set, the teacher is
|
| 218 |
+
# always reloaded fresh from it at __init__ time; when it is not set,
|
| 219 |
+
# teacher_vit doesn't exist at all. Either way, eval scripts that assert on
|
| 220 |
# load_state_dict(...).missing_keys should exempt these prefixes.
|
| 221 |
checkpoint_exempt_key_prefixes = ("teacher_vit.",)
|
| 222 |
|
| 223 |
def __init__(
|
| 224 |
self,
|
| 225 |
*,
|
| 226 |
+
teacher_ckpt_path=None,
|
| 227 |
cd_ema_mu=0.999,
|
| 228 |
cd_weight_decay=0.0,
|
| 229 |
**kwargs,
|
|
|
|
| 233 |
self.cd_weight_decay = cd_weight_decay
|
| 234 |
self.strict_loading = False
|
| 235 |
|
| 236 |
+
self.teacher_vit = None
|
| 237 |
+
if teacher_ckpt_path is not None:
|
| 238 |
+
checkpoint_path = os.path.expandvars(teacher_ckpt_path)
|
| 239 |
+
if not os.path.exists(checkpoint_path):
|
| 240 |
+
raise FileNotFoundError(f"teacher_ckpt_path {checkpoint_path} does not exist.")
|
| 241 |
+
|
| 242 |
+
try:
|
| 243 |
+
state_dict = torch.load(
|
| 244 |
+
checkpoint_path, map_location="cpu", weights_only=True, mmap=True
|
| 245 |
+
)["state_dict"]
|
| 246 |
+
except (TypeError, RuntimeError):
|
| 247 |
+
state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True)["state_dict"]
|
| 248 |
+
|
| 249 |
+
ema_state_dict = {
|
| 250 |
+
key[len("ema_vit.") :]: value.clone()
|
| 251 |
+
for key, value in state_dict.items()
|
| 252 |
+
if key.startswith("ema_vit.")
|
| 253 |
+
}
|
| 254 |
+
del state_dict
|
| 255 |
+
if not ema_state_dict:
|
| 256 |
+
raise ValueError(f"No ema_vit.* keys found in teacher checkpoint {checkpoint_path}.")
|
| 257 |
+
|
| 258 |
+
self.vit.load_state_dict(ema_state_dict, strict=True)
|
| 259 |
+
self.ema_vit.load_state_dict(ema_state_dict, strict=True)
|
| 260 |
+
self.teacher_vit = deepcopy(self.vit)
|
| 261 |
+
requires_grad(self.teacher_vit, False)
|
| 262 |
+
self.teacher_vit.eval()
|
| 263 |
|
| 264 |
def setup(self, stage=None):
|
| 265 |
super().setup(stage)
|
| 266 |
+
if self.teacher_vit is not None:
|
| 267 |
self.teacher_vit.requires_grad_(False)
|
| 268 |
+
elif stage in ("fit", "validate"):
|
| 269 |
+
raise RuntimeError(
|
| 270 |
+
"ConsistencyDistillPredictorModule requires teacher_ckpt_path to be set "
|
| 271 |
+
f"when stage={stage!r} (training/validation use a frozen teacher for "
|
| 272 |
+
"consistency distillation). Pass teacher_ckpt_path in the model config, "
|
| 273 |
+
"or omit it only for inference (sample()/roll_out())."
|
| 274 |
+
)
|
| 275 |
|
| 276 |
def configure_optimizers(self):
|
| 277 |
params = [p for p in self.vit.parameters() if p.requires_grad]
|
orbis2/networks/tokenizer/ae.py
CHANGED
|
@@ -1,68 +1,15 @@
|
|
| 1 |
# pytorch_diffusion + derived encoder decoder
|
| 2 |
-
import math
|
| 3 |
-
from typing import Tuple, Union
|
| 4 |
-
|
| 5 |
-
import numpy as np
|
| 6 |
import torch
|
| 7 |
import torch.nn as nn
|
| 8 |
|
| 9 |
-
from einops import rearrange
|
| 10 |
-
from einops.layers.torch import Rearrange
|
| 11 |
from omegaconf import ListConfig
|
| 12 |
|
| 13 |
|
| 14 |
-
def get_timestep_embedding(timesteps, embedding_dim):
|
| 15 |
-
"""
|
| 16 |
-
This matches the implementation in Denoising Diffusion Probabilistic Models:
|
| 17 |
-
From Fairseq.
|
| 18 |
-
Build sinusoidal embeddings.
|
| 19 |
-
This matches the implementation in tensor2tensor, but differs slightly
|
| 20 |
-
from the description in Section 3.5 of "Attention Is All You Need".
|
| 21 |
-
"""
|
| 22 |
-
assert len(timesteps.shape) == 1
|
| 23 |
-
|
| 24 |
-
half_dim = embedding_dim // 2
|
| 25 |
-
emb = math.log(10000) / (half_dim - 1)
|
| 26 |
-
emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb)
|
| 27 |
-
emb = emb.to(device=timesteps.device)
|
| 28 |
-
emb = timesteps.float()[:, None] * emb[None, :]
|
| 29 |
-
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
|
| 30 |
-
if embedding_dim % 2 == 1: # zero pad
|
| 31 |
-
emb = torch.nn.functional.pad(emb, (0,1,0,0))
|
| 32 |
-
return emb
|
| 33 |
-
|
| 34 |
-
|
| 35 |
def nonlinearity(x):
|
| 36 |
# swish
|
| 37 |
return x*torch.sigmoid(x)
|
| 38 |
|
| 39 |
|
| 40 |
-
class SpatialNorm(nn.Module):
|
| 41 |
-
def __init__(self, f_channels, zq_channels, norm_layer=nn.GroupNorm, freeze_norm_layer=False, add_conv=False, **norm_layer_params):
|
| 42 |
-
super().__init__()
|
| 43 |
-
self.norm_layer = norm_layer(num_channels=f_channels, **norm_layer_params)
|
| 44 |
-
if freeze_norm_layer:
|
| 45 |
-
for p in self.norm_layer.parameters: # for p in self.norm_layer.parameters():
|
| 46 |
-
p.requires_grad = False
|
| 47 |
-
self.add_conv = add_conv
|
| 48 |
-
if self.add_conv:
|
| 49 |
-
self.conv = nn.Conv2d(zq_channels, zq_channels, kernel_size=3, stride=1, padding=1)
|
| 50 |
-
self.conv_y = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
|
| 51 |
-
self.conv_b = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
|
| 52 |
-
def forward(self, f, zq):
|
| 53 |
-
#print(f'f shape: {f.shape}', f'zq shape: {zq.shape}')
|
| 54 |
-
f_size = f.shape[-2:]
|
| 55 |
-
zq = torch.nn.functional.interpolate(zq, size=f_size, mode="nearest")
|
| 56 |
-
if self.add_conv:
|
| 57 |
-
zq = self.conv(zq)
|
| 58 |
-
norm_f = self.norm_layer(f)
|
| 59 |
-
new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)
|
| 60 |
-
return new_f
|
| 61 |
-
|
| 62 |
-
def SpatialNormalize(in_channels, zq_ch, add_conv):
|
| 63 |
-
return SpatialNorm(in_channels, zq_ch, norm_layer=nn.GroupNorm, freeze_norm_layer=False, add_conv=add_conv, num_groups=32, eps=1e-6, affine=True)
|
| 64 |
-
|
| 65 |
-
|
| 66 |
def Normalize(in_channels):
|
| 67 |
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
|
| 68 |
|
|
@@ -168,67 +115,6 @@ class ResnetBlock(nn.Module):
|
|
| 168 |
|
| 169 |
return x+h
|
| 170 |
|
| 171 |
-
class SpatialResnetBlock(nn.Module):
|
| 172 |
-
def __init__(self, *, in_channels, out_channels=None, conv_shortcut=False,
|
| 173 |
-
dropout, temb_channels=512, zq_ch=None, add_conv=False):
|
| 174 |
-
super().__init__()
|
| 175 |
-
self.in_channels = in_channels
|
| 176 |
-
out_channels = in_channels if out_channels is None else out_channels
|
| 177 |
-
self.out_channels = out_channels
|
| 178 |
-
self.use_conv_shortcut = conv_shortcut
|
| 179 |
-
|
| 180 |
-
self.norm1 = SpatialNormalize(in_channels, zq_ch, add_conv)
|
| 181 |
-
self.conv1 = torch.nn.Conv2d(in_channels,
|
| 182 |
-
out_channels,
|
| 183 |
-
kernel_size=3,
|
| 184 |
-
stride=1,
|
| 185 |
-
padding=1)
|
| 186 |
-
if temb_channels > 0:
|
| 187 |
-
self.temb_proj = torch.nn.Linear(temb_channels,
|
| 188 |
-
out_channels)
|
| 189 |
-
self.norm2 = SpatialNormalize(out_channels, zq_ch, add_conv)
|
| 190 |
-
self.dropout = torch.nn.Dropout(dropout)
|
| 191 |
-
self.conv2 = torch.nn.Conv2d(out_channels,
|
| 192 |
-
out_channels,
|
| 193 |
-
kernel_size=3,
|
| 194 |
-
stride=1,
|
| 195 |
-
padding=1)
|
| 196 |
-
if self.in_channels != self.out_channels:
|
| 197 |
-
if self.use_conv_shortcut:
|
| 198 |
-
self.conv_shortcut = torch.nn.Conv2d(in_channels,
|
| 199 |
-
out_channels,
|
| 200 |
-
kernel_size=3,
|
| 201 |
-
stride=1,
|
| 202 |
-
padding=1)
|
| 203 |
-
else:
|
| 204 |
-
self.nin_shortcut = torch.nn.Conv2d(in_channels,
|
| 205 |
-
out_channels,
|
| 206 |
-
kernel_size=1,
|
| 207 |
-
stride=1,
|
| 208 |
-
padding=0)
|
| 209 |
-
|
| 210 |
-
def forward(self, x, temb, zq):
|
| 211 |
-
h = x
|
| 212 |
-
h = self.norm1(h, zq)
|
| 213 |
-
h = nonlinearity(h)
|
| 214 |
-
h = self.conv1(h)
|
| 215 |
-
|
| 216 |
-
if temb is not None:
|
| 217 |
-
h = h + self.temb_proj(nonlinearity(temb))[:,:,None,None]
|
| 218 |
-
|
| 219 |
-
h = self.norm2(h, zq)
|
| 220 |
-
h = nonlinearity(h)
|
| 221 |
-
h = self.dropout(h)
|
| 222 |
-
h = self.conv2(h)
|
| 223 |
-
|
| 224 |
-
if self.in_channels != self.out_channels:
|
| 225 |
-
if self.use_conv_shortcut:
|
| 226 |
-
x = self.conv_shortcut(x)
|
| 227 |
-
else:
|
| 228 |
-
x = self.nin_shortcut(x)
|
| 229 |
-
|
| 230 |
-
return x+h
|
| 231 |
-
|
| 232 |
|
| 233 |
class AttnBlock(nn.Module):
|
| 234 |
def __init__(self, in_channels):
|
|
@@ -283,371 +169,8 @@ class AttnBlock(nn.Module):
|
|
| 283 |
h_ = self.proj_out(h_)
|
| 284 |
|
| 285 |
return x+h_
|
| 286 |
-
|
| 287 |
-
class SpatialAttnBlock(nn.Module):
|
| 288 |
-
def __init__(self, in_channels, zq_ch=None, add_conv=False):
|
| 289 |
-
super().__init__()
|
| 290 |
-
self.in_channels = in_channels
|
| 291 |
-
|
| 292 |
-
self.norm = SpatialNormalize(in_channels, zq_ch, add_conv)
|
| 293 |
-
self.q = torch.nn.Conv2d(in_channels,
|
| 294 |
-
in_channels,
|
| 295 |
-
kernel_size=1,
|
| 296 |
-
stride=1,
|
| 297 |
-
padding=0)
|
| 298 |
-
self.k = torch.nn.Conv2d(in_channels,
|
| 299 |
-
in_channels,
|
| 300 |
-
kernel_size=1,
|
| 301 |
-
stride=1,
|
| 302 |
-
padding=0)
|
| 303 |
-
self.v = torch.nn.Conv2d(in_channels,
|
| 304 |
-
in_channels,
|
| 305 |
-
kernel_size=1,
|
| 306 |
-
stride=1,
|
| 307 |
-
padding=0)
|
| 308 |
-
self.proj_out = torch.nn.Conv2d(in_channels,
|
| 309 |
-
in_channels,
|
| 310 |
-
kernel_size=1,
|
| 311 |
-
stride=1,
|
| 312 |
-
padding=0)
|
| 313 |
|
| 314 |
|
| 315 |
-
def forward(self, x, zq):
|
| 316 |
-
h_ = x
|
| 317 |
-
h_ = self.norm(h_, zq)
|
| 318 |
-
q = self.q(h_)
|
| 319 |
-
k = self.k(h_)
|
| 320 |
-
v = self.v(h_)
|
| 321 |
-
|
| 322 |
-
# compute attention
|
| 323 |
-
b,c,h,w = q.shape
|
| 324 |
-
q = q.reshape(b,c,h*w)
|
| 325 |
-
q = q.permute(0,2,1) # b,hw,c
|
| 326 |
-
k = k.reshape(b,c,h*w) # b,c,hw
|
| 327 |
-
w_ = torch.bmm(q,k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j]
|
| 328 |
-
w_ = w_ * (int(c)**(-0.5))
|
| 329 |
-
w_ = torch.nn.functional.softmax(w_, dim=2)
|
| 330 |
-
|
| 331 |
-
# attend to values
|
| 332 |
-
v = v.reshape(b,c,h*w)
|
| 333 |
-
w_ = w_.permute(0,2,1) # b,hw,hw (first hw of k, second of q)
|
| 334 |
-
h_ = torch.bmm(v,w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j]
|
| 335 |
-
h_ = h_.reshape(b,c,h,w)
|
| 336 |
-
|
| 337 |
-
h_ = self.proj_out(h_)
|
| 338 |
-
|
| 339 |
-
return x+h_
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
class Model(nn.Module):
|
| 343 |
-
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
|
| 344 |
-
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
|
| 345 |
-
resolution, use_timestep=True):
|
| 346 |
-
super().__init__()
|
| 347 |
-
self.ch = ch
|
| 348 |
-
self.temb_ch = self.ch*4
|
| 349 |
-
self.num_resolutions = len(ch_mult)
|
| 350 |
-
self.num_res_blocks = num_res_blocks
|
| 351 |
-
self.resolution = resolution
|
| 352 |
-
self.in_channels = in_channels
|
| 353 |
-
|
| 354 |
-
self.use_timestep = use_timestep
|
| 355 |
-
if self.use_timestep:
|
| 356 |
-
# timestep embedding
|
| 357 |
-
self.temb = nn.Module()
|
| 358 |
-
self.temb.dense = nn.ModuleList([
|
| 359 |
-
torch.nn.Linear(self.ch,
|
| 360 |
-
self.temb_ch),
|
| 361 |
-
torch.nn.Linear(self.temb_ch,
|
| 362 |
-
self.temb_ch),
|
| 363 |
-
])
|
| 364 |
-
|
| 365 |
-
# downsampling
|
| 366 |
-
self.conv_in = torch.nn.Conv2d(in_channels,
|
| 367 |
-
self.ch,
|
| 368 |
-
kernel_size=3,
|
| 369 |
-
stride=1,
|
| 370 |
-
padding=1)
|
| 371 |
-
|
| 372 |
-
curr_res = resolution
|
| 373 |
-
in_ch_mult = (1,)+tuple(ch_mult)
|
| 374 |
-
self.down = nn.ModuleList()
|
| 375 |
-
for i_level in range(self.num_resolutions):
|
| 376 |
-
block = nn.ModuleList()
|
| 377 |
-
attn = nn.ModuleList()
|
| 378 |
-
block_in = ch*in_ch_mult[i_level]
|
| 379 |
-
block_out = ch*ch_mult[i_level]
|
| 380 |
-
for i_block in range(self.num_res_blocks):
|
| 381 |
-
block.append(ResnetBlock(in_channels=block_in,
|
| 382 |
-
out_channels=block_out,
|
| 383 |
-
temb_channels=self.temb_ch,
|
| 384 |
-
dropout=dropout))
|
| 385 |
-
block_in = block_out
|
| 386 |
-
if curr_res in attn_resolutions:
|
| 387 |
-
attn.append(AttnBlock(block_in))
|
| 388 |
-
down = nn.Module()
|
| 389 |
-
down.block = block
|
| 390 |
-
down.attn = attn
|
| 391 |
-
if i_level != self.num_resolutions-1:
|
| 392 |
-
down.downsample = Downsample(block_in, resamp_with_conv)
|
| 393 |
-
curr_res = curr_res // 2
|
| 394 |
-
self.down.append(down)
|
| 395 |
-
|
| 396 |
-
# middle
|
| 397 |
-
self.mid = nn.Module()
|
| 398 |
-
self.mid.block_1 = ResnetBlock(in_channels=block_in,
|
| 399 |
-
out_channels=block_in,
|
| 400 |
-
temb_channels=self.temb_ch,
|
| 401 |
-
dropout=dropout)
|
| 402 |
-
self.mid.attn_1 = AttnBlock(block_in)
|
| 403 |
-
self.mid.block_2 = ResnetBlock(in_channels=block_in,
|
| 404 |
-
out_channels=block_in,
|
| 405 |
-
temb_channels=self.temb_ch,
|
| 406 |
-
dropout=dropout)
|
| 407 |
-
|
| 408 |
-
# upsampling
|
| 409 |
-
self.up = nn.ModuleList()
|
| 410 |
-
for i_level in reversed(range(self.num_resolutions)):
|
| 411 |
-
block = nn.ModuleList()
|
| 412 |
-
attn = nn.ModuleList()
|
| 413 |
-
block_out = ch*ch_mult[i_level]
|
| 414 |
-
skip_in = ch*ch_mult[i_level]
|
| 415 |
-
for i_block in range(self.num_res_blocks+1):
|
| 416 |
-
if i_block == self.num_res_blocks:
|
| 417 |
-
skip_in = ch*in_ch_mult[i_level]
|
| 418 |
-
block.append(ResnetBlock(in_channels=block_in+skip_in,
|
| 419 |
-
out_channels=block_out,
|
| 420 |
-
temb_channels=self.temb_ch,
|
| 421 |
-
dropout=dropout))
|
| 422 |
-
block_in = block_out
|
| 423 |
-
if curr_res in attn_resolutions:
|
| 424 |
-
attn.append(AttnBlock(block_in))
|
| 425 |
-
up = nn.Module()
|
| 426 |
-
up.block = block
|
| 427 |
-
up.attn = attn
|
| 428 |
-
if i_level != 0:
|
| 429 |
-
up.upsample = Upsample(block_in, resamp_with_conv)
|
| 430 |
-
curr_res = curr_res * 2
|
| 431 |
-
self.up.insert(0, up) # prepend to get consistent order
|
| 432 |
-
|
| 433 |
-
# end
|
| 434 |
-
self.norm_out = Normalize(block_in)
|
| 435 |
-
self.conv_out = torch.nn.Conv2d(block_in,
|
| 436 |
-
out_ch,
|
| 437 |
-
kernel_size=3,
|
| 438 |
-
stride=1,
|
| 439 |
-
padding=1)
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
def forward(self, x, t=None):
|
| 443 |
-
#assert x.shape[2] == x.shape[3] == self.resolution
|
| 444 |
-
|
| 445 |
-
if self.use_timestep:
|
| 446 |
-
# timestep embedding
|
| 447 |
-
assert t is not None
|
| 448 |
-
temb = get_timestep_embedding(t, self.ch)
|
| 449 |
-
temb = self.temb.dense[0](temb)
|
| 450 |
-
temb = nonlinearity(temb)
|
| 451 |
-
temb = self.temb.dense[1](temb)
|
| 452 |
-
else:
|
| 453 |
-
temb = None
|
| 454 |
-
|
| 455 |
-
# downsampling
|
| 456 |
-
hs = [self.conv_in(x)]
|
| 457 |
-
for i_level in range(self.num_resolutions):
|
| 458 |
-
for i_block in range(self.num_res_blocks):
|
| 459 |
-
h = self.down[i_level].block[i_block](hs[-1], temb)
|
| 460 |
-
if len(self.down[i_level].attn) > 0:
|
| 461 |
-
h = self.down[i_level].attn[i_block](h)
|
| 462 |
-
hs.append(h)
|
| 463 |
-
if i_level != self.num_resolutions-1:
|
| 464 |
-
hs.append(self.down[i_level].downsample(hs[-1]))
|
| 465 |
-
|
| 466 |
-
# middle
|
| 467 |
-
h = hs[-1]
|
| 468 |
-
h = self.mid.block_1(h, temb)
|
| 469 |
-
h = self.mid.attn_1(h)
|
| 470 |
-
h = self.mid.block_2(h, temb)
|
| 471 |
-
|
| 472 |
-
# upsampling
|
| 473 |
-
for i_level in reversed(range(self.num_resolutions)):
|
| 474 |
-
for i_block in range(self.num_res_blocks+1):
|
| 475 |
-
h = self.up[i_level].block[i_block](
|
| 476 |
-
torch.cat([h, hs.pop()], dim=1), temb)
|
| 477 |
-
if len(self.up[i_level].attn) > 0:
|
| 478 |
-
h = self.up[i_level].attn[i_block](h)
|
| 479 |
-
if i_level != 0:
|
| 480 |
-
h = self.up[i_level].upsample(h)
|
| 481 |
-
|
| 482 |
-
# end
|
| 483 |
-
h = self.norm_out(h)
|
| 484 |
-
h = nonlinearity(h)
|
| 485 |
-
h = self.conv_out(h)
|
| 486 |
-
return h
|
| 487 |
-
|
| 488 |
-
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
| 489 |
-
"""
|
| 490 |
-
embed_dim: output dimension for each position
|
| 491 |
-
pos: a list of positions to be encoded: size (M,)
|
| 492 |
-
out: (M, D)
|
| 493 |
-
"""
|
| 494 |
-
assert embed_dim % 2 == 0
|
| 495 |
-
omega = np.arange(embed_dim // 2, dtype=float)
|
| 496 |
-
omega /= embed_dim / 2.
|
| 497 |
-
omega = 1. / 10000**omega # (D/2,)
|
| 498 |
-
|
| 499 |
-
pos = pos.reshape(-1) # (M,)
|
| 500 |
-
out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
|
| 501 |
-
|
| 502 |
-
emb_sin = np.sin(out) # (M, D/2)
|
| 503 |
-
emb_cos = np.cos(out) # (M, D/2)
|
| 504 |
-
|
| 505 |
-
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
| 506 |
-
return emb
|
| 507 |
-
|
| 508 |
-
def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
|
| 509 |
-
assert embed_dim % 2 == 0
|
| 510 |
-
|
| 511 |
-
# use half of dimensions to encode grid_h
|
| 512 |
-
emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
|
| 513 |
-
emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
|
| 514 |
-
|
| 515 |
-
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
| 516 |
-
return emb
|
| 517 |
-
|
| 518 |
-
def get_2d_sincos_pos_embed(embed_dim, grid_size):
|
| 519 |
-
"""
|
| 520 |
-
grid_size: int or (int, int) of the grid height and width
|
| 521 |
-
return:
|
| 522 |
-
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
| 523 |
-
"""
|
| 524 |
-
grid_size = (grid_size, grid_size) if type(grid_size) != tuple else grid_size
|
| 525 |
-
grid_h = np.arange(grid_size[0], dtype=np.float32)
|
| 526 |
-
grid_w = np.arange(grid_size[1], dtype=np.float32)
|
| 527 |
-
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
| 528 |
-
grid = np.stack(grid, axis=0)
|
| 529 |
-
|
| 530 |
-
grid = grid.reshape([2, 1, grid_size[0], grid_size[1]])
|
| 531 |
-
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
|
| 532 |
-
|
| 533 |
-
return pos_embed
|
| 534 |
-
|
| 535 |
-
def init_weights(m):
|
| 536 |
-
if isinstance(m, nn.Linear):
|
| 537 |
-
# we use xavier_uniform following official JAX ViT:
|
| 538 |
-
torch.nn.init.xavier_uniform_(m.weight)
|
| 539 |
-
if m.bias is not None:
|
| 540 |
-
nn.init.constant_(m.bias, 0)
|
| 541 |
-
elif isinstance(m, nn.LayerNorm):
|
| 542 |
-
nn.init.constant_(m.bias, 0)
|
| 543 |
-
nn.init.constant_(m.weight, 1.0)
|
| 544 |
-
elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
|
| 545 |
-
with torch.no_grad():
|
| 546 |
-
w = m.weight
|
| 547 |
-
torch.nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
|
| 548 |
-
|
| 549 |
-
class PreNorm(nn.Module):
|
| 550 |
-
def __init__(self, dim: int, fn: nn.Module) -> None:
|
| 551 |
-
super().__init__()
|
| 552 |
-
self.norm = nn.LayerNorm(dim)
|
| 553 |
-
self.fn = fn
|
| 554 |
-
|
| 555 |
-
def forward(self, x: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
|
| 556 |
-
return self.fn(self.norm(x), **kwargs)
|
| 557 |
-
|
| 558 |
-
class FeedForward(nn.Module):
|
| 559 |
-
def __init__(self, dim: int, hidden_dim: int) -> None:
|
| 560 |
-
super().__init__()
|
| 561 |
-
self.net = nn.Sequential(
|
| 562 |
-
nn.Linear(dim, hidden_dim),
|
| 563 |
-
nn.Tanh(),
|
| 564 |
-
nn.Linear(hidden_dim, dim)
|
| 565 |
-
)
|
| 566 |
-
|
| 567 |
-
def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
|
| 568 |
-
return self.net(x)
|
| 569 |
-
|
| 570 |
-
class Attention(nn.Module):
|
| 571 |
-
def __init__(self, dim: int, heads: int = 8, dim_head: int = 64) -> None:
|
| 572 |
-
super().__init__()
|
| 573 |
-
inner_dim = dim_head * heads
|
| 574 |
-
project_out = not (heads == 1 and dim_head == dim)
|
| 575 |
-
|
| 576 |
-
self.heads = heads
|
| 577 |
-
self.scale = dim_head ** -0.5
|
| 578 |
-
|
| 579 |
-
self.attend = nn.Softmax(dim = -1)
|
| 580 |
-
self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
|
| 581 |
-
|
| 582 |
-
self.to_out = nn.Linear(inner_dim, dim) if project_out else nn.Identity()
|
| 583 |
-
|
| 584 |
-
def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
|
| 585 |
-
qkv = self.to_qkv(x).chunk(3, dim = -1)
|
| 586 |
-
q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)
|
| 587 |
-
|
| 588 |
-
attn = torch.matmul(q, k.transpose(-1, -2)) * self.scale
|
| 589 |
-
attn = self.attend(attn)
|
| 590 |
-
|
| 591 |
-
out = torch.matmul(attn, v)
|
| 592 |
-
out = rearrange(out, 'b h n d -> b n (h d)')
|
| 593 |
-
|
| 594 |
-
return self.to_out(out)
|
| 595 |
-
|
| 596 |
-
class Transformer(nn.Module):
|
| 597 |
-
def __init__(self, dim: int, depth: int, heads: int, dim_head: int, mlp_dim: int) -> None:
|
| 598 |
-
super().__init__()
|
| 599 |
-
self.layers = nn.ModuleList([])
|
| 600 |
-
for idx in range(depth):
|
| 601 |
-
layer = nn.ModuleList([PreNorm(dim, Attention(dim, heads=heads, dim_head=dim_head)),
|
| 602 |
-
PreNorm(dim, FeedForward(dim, mlp_dim))])
|
| 603 |
-
self.layers.append(layer)
|
| 604 |
-
self.norm = nn.LayerNorm(dim)
|
| 605 |
-
|
| 606 |
-
def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
|
| 607 |
-
for attn, ff in self.layers:
|
| 608 |
-
x = attn(x) + x
|
| 609 |
-
x = ff(x) + x
|
| 610 |
-
|
| 611 |
-
return self.norm(x)
|
| 612 |
-
|
| 613 |
-
class EncoderVIT(nn.Module):
|
| 614 |
-
def __init__(self, image_size: Union[Tuple[int, int], int], patch_size: Union[Tuple[int, int], int],
|
| 615 |
-
dim: int, depth: int, heads: int, mlp_dim: int, channels: int = 3, dim_head: int = 64) -> None:
|
| 616 |
-
super().__init__()
|
| 617 |
-
|
| 618 |
-
image_height, image_width = image_size if isinstance(image_size, tuple) \
|
| 619 |
-
else (image_size, image_size)
|
| 620 |
-
patch_height, patch_width = patch_size if isinstance(patch_size, tuple) \
|
| 621 |
-
else (patch_size, patch_size)
|
| 622 |
-
self.image_height = image_height
|
| 623 |
-
self.image_width = image_width
|
| 624 |
-
self.patch_height = patch_height
|
| 625 |
-
self.patch_width = patch_width
|
| 626 |
-
assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'
|
| 627 |
-
en_pos_embedding = get_2d_sincos_pos_embed(dim, (image_height // patch_height, image_width // patch_width))
|
| 628 |
-
|
| 629 |
-
self.num_patches = (image_height // patch_height) * (image_width // patch_width)
|
| 630 |
-
self.patch_dim = channels * patch_height * patch_width
|
| 631 |
-
|
| 632 |
-
self.to_patch_embedding = nn.Sequential(
|
| 633 |
-
nn.Conv2d(channels, dim, kernel_size=patch_size, stride=patch_size),
|
| 634 |
-
Rearrange('b c h w -> b (h w) c'),
|
| 635 |
-
)
|
| 636 |
-
self.en_pos_embedding = nn.Parameter(torch.from_numpy(en_pos_embedding).float().unsqueeze(0), requires_grad=False)
|
| 637 |
-
self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim)
|
| 638 |
-
|
| 639 |
-
self.apply(init_weights)
|
| 640 |
-
|
| 641 |
-
def forward(self, img: torch.FloatTensor) -> torch.FloatTensor:
|
| 642 |
-
x = self.to_patch_embedding(img)
|
| 643 |
-
x = x + self.en_pos_embedding
|
| 644 |
-
x = self.transformer(x) # (B, N, D)
|
| 645 |
-
#import pdb; pdb.set_trace()
|
| 646 |
-
x = x.reshape(x.shape[0], x.shape[1]//(self.image_height // self.patch_height), x.shape[1]//(self.image_width // self.patch_width), -1)
|
| 647 |
-
x = x.permute(0, 3, 1, 2).contiguous()
|
| 648 |
-
#import pdb; pdb.set_trace()
|
| 649 |
-
return x
|
| 650 |
-
|
| 651 |
class Encoder(nn.Module):
|
| 652 |
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
|
| 653 |
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
|
|
@@ -761,8 +284,8 @@ class Decoder(nn.Module):
|
|
| 761 |
# compute in_ch_mult, block_in and curr_res at lowest res
|
| 762 |
in_ch_mult = (1,)+tuple(ch_mult)
|
| 763 |
block_in = ch*ch_mult[self.num_resolutions-1]
|
| 764 |
-
|
| 765 |
-
curr_res_h, curr_res_w = (resolution[0] // 2**(self.num_resolutions-1), resolution[1] // 2**(self.num_resolutions-1)) if isinstance(resolution, (list, tuple, ListConfig)) else (resolution// 2**(self.num_resolutions-1) ,resolution// 2**(self.num_resolutions-1))
|
| 766 |
self.z_shape = (1,z_channels,curr_res_h,curr_res_w)
|
| 767 |
|
| 768 |
# z to block_in
|
|
@@ -783,7 +306,7 @@ class Decoder(nn.Module):
|
|
| 783 |
out_channels=block_in,
|
| 784 |
temb_channels=self.temb_ch,
|
| 785 |
dropout=dropout)
|
| 786 |
-
|
| 787 |
#self.decoder_dino_conv = nn.Conv2d(block_in, 768, kernel_size=1, stride=1, padding=0) # remove hard-coded emb_dim=768
|
| 788 |
|
| 789 |
# upsampling
|
|
@@ -849,4 +372,3 @@ class Decoder(nn.Module):
|
|
| 849 |
h = self.conv_out(h)
|
| 850 |
|
| 851 |
return h
|
| 852 |
-
|
|
|
|
| 1 |
# pytorch_diffusion + derived encoder decoder
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
| 4 |
|
|
|
|
|
|
|
| 5 |
from omegaconf import ListConfig
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
def nonlinearity(x):
|
| 9 |
# swish
|
| 10 |
return x*torch.sigmoid(x)
|
| 11 |
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
def Normalize(in_channels):
|
| 14 |
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
|
| 15 |
|
|
|
|
| 115 |
|
| 116 |
return x+h
|
| 117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
|
| 119 |
class AttnBlock(nn.Module):
|
| 120 |
def __init__(self, in_channels):
|
|
|
|
| 169 |
h_ = self.proj_out(h_)
|
| 170 |
|
| 171 |
return x+h_
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
class Encoder(nn.Module):
|
| 175 |
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
|
| 176 |
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
|
|
|
|
| 284 |
# compute in_ch_mult, block_in and curr_res at lowest res
|
| 285 |
in_ch_mult = (1,)+tuple(ch_mult)
|
| 286 |
block_in = ch*ch_mult[self.num_resolutions-1]
|
| 287 |
+
|
| 288 |
+
curr_res_h, curr_res_w = (resolution[0] // 2**(self.num_resolutions-1), resolution[1] // 2**(self.num_resolutions-1)) if isinstance(resolution, (list, tuple, ListConfig)) else (resolution// 2**(self.num_resolutions-1) ,resolution// 2**(self.num_resolutions-1))
|
| 289 |
self.z_shape = (1,z_channels,curr_res_h,curr_res_w)
|
| 290 |
|
| 291 |
# z to block_in
|
|
|
|
| 306 |
out_channels=block_in,
|
| 307 |
temb_channels=self.temb_ch,
|
| 308 |
dropout=dropout)
|
| 309 |
+
|
| 310 |
#self.decoder_dino_conv = nn.Conv2d(block_in, 768, kernel_size=1, stride=1, padding=0) # remove hard-coded emb_dim=768
|
| 311 |
|
| 312 |
# upsampling
|
|
|
|
| 372 |
h = self.conv_out(h)
|
| 373 |
|
| 374 |
return h
|
|
|
orbis2/networks/tokenizer/pretrained_models.py
CHANGED
|
@@ -1,8 +1,6 @@
|
|
| 1 |
import torch
|
| 2 |
import timm
|
| 3 |
from torch import nn
|
| 4 |
-
from omegaconf import ListConfig
|
| 5 |
-
from einops import rearrange
|
| 6 |
|
| 7 |
|
| 8 |
from typing import Tuple, Union
|
|
|
|
| 1 |
import torch
|
| 2 |
import timm
|
| 3 |
from torch import nn
|
|
|
|
|
|
|
| 4 |
|
| 5 |
|
| 6 |
from typing import Tuple, Union
|
orbis2/util.py
CHANGED
|
@@ -1,179 +1,4 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import hashlib
|
| 3 |
import importlib
|
| 4 |
-
import logging
|
| 5 |
-
import os
|
| 6 |
-
import requests
|
| 7 |
-
|
| 8 |
-
from tqdm import tqdm
|
| 9 |
-
from pytorch_lightning.trainer import Trainer
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
def maybe_raise_nofile_limit(target_soft=32768, logger=None):
|
| 13 |
-
"""
|
| 14 |
-
Raise RLIMIT_NOFILE soft limit for current process when possible.
|
| 15 |
-
|
| 16 |
-
Returns True if the limit was raised, False otherwise.
|
| 17 |
-
"""
|
| 18 |
-
if logger is None:
|
| 19 |
-
logger = logging.getLogger(__name__)
|
| 20 |
-
try:
|
| 21 |
-
import resource
|
| 22 |
-
except ImportError:
|
| 23 |
-
logger.warning("resource module unavailable; cannot set RLIMIT_NOFILE")
|
| 24 |
-
return False
|
| 25 |
-
try:
|
| 26 |
-
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
| 27 |
-
desired = target_soft
|
| 28 |
-
if hard not in (resource.RLIM_INFINITY, -1):
|
| 29 |
-
desired = min(desired, hard)
|
| 30 |
-
if soft >= desired:
|
| 31 |
-
return False
|
| 32 |
-
resource.setrlimit(resource.RLIMIT_NOFILE, (desired, hard))
|
| 33 |
-
logger.info("Raised RLIMIT_NOFILE from %s to %s", soft, desired)
|
| 34 |
-
return True
|
| 35 |
-
except (ValueError, PermissionError, OSError) as exc:
|
| 36 |
-
logger.warning("Unable to raise RLIMIT_NOFILE: %s", exc)
|
| 37 |
-
return False
|
| 38 |
-
|
| 39 |
-
URL_MAP = {
|
| 40 |
-
"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"
|
| 41 |
-
}
|
| 42 |
-
|
| 43 |
-
CKPT_MAP = {
|
| 44 |
-
"vgg_lpips": "vgg.pth"
|
| 45 |
-
}
|
| 46 |
-
|
| 47 |
-
MD5_MAP = {
|
| 48 |
-
"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
def download(url, local_path, chunk_size=1024):
|
| 53 |
-
os.makedirs(os.path.split(local_path)[0], exist_ok=True)
|
| 54 |
-
with requests.get(url, stream=True) as r:
|
| 55 |
-
total_size = int(r.headers.get("content-length", 0))
|
| 56 |
-
with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
|
| 57 |
-
with open(local_path, "wb") as f:
|
| 58 |
-
for data in r.iter_content(chunk_size=chunk_size):
|
| 59 |
-
if data:
|
| 60 |
-
f.write(data)
|
| 61 |
-
pbar.update(chunk_size)
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
def md5_hash(path):
|
| 65 |
-
with open(path, "rb") as f:
|
| 66 |
-
content = f.read()
|
| 67 |
-
return hashlib.md5(content).hexdigest()
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
def get_ckpt_path(name, root, check=False):
|
| 71 |
-
assert name in URL_MAP
|
| 72 |
-
path = os.path.join(root, CKPT_MAP[name])
|
| 73 |
-
if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]):
|
| 74 |
-
print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path))
|
| 75 |
-
download(URL_MAP[name], path)
|
| 76 |
-
md5 = md5_hash(path)
|
| 77 |
-
assert md5 == MD5_MAP[name], md5
|
| 78 |
-
return path
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
class KeyNotFoundError(Exception):
|
| 82 |
-
def __init__(self, cause, keys=None, visited=None):
|
| 83 |
-
self.cause = cause
|
| 84 |
-
self.keys = keys
|
| 85 |
-
self.visited = visited
|
| 86 |
-
messages = list()
|
| 87 |
-
if keys is not None:
|
| 88 |
-
messages.append("Key not found: {}".format(keys))
|
| 89 |
-
if visited is not None:
|
| 90 |
-
messages.append("Visited: {}".format(visited))
|
| 91 |
-
messages.append("Cause:\n{}".format(cause))
|
| 92 |
-
message = "\n".join(messages)
|
| 93 |
-
super().__init__(message)
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def retrieve(
|
| 97 |
-
list_or_dict, key, splitval="/", default=None, expand=True, pass_success=False
|
| 98 |
-
):
|
| 99 |
-
"""Given a nested list or dict return the desired value at key expanding
|
| 100 |
-
callable nodes if necessary and :attr:`expand` is ``True``. The expansion
|
| 101 |
-
is done in-place.
|
| 102 |
-
|
| 103 |
-
Parameters
|
| 104 |
-
----------
|
| 105 |
-
list_or_dict : list or dict
|
| 106 |
-
Possibly nested list or dictionary.
|
| 107 |
-
key : str
|
| 108 |
-
key/to/value, path like string describing all keys necessary to
|
| 109 |
-
consider to get to the desired value. List indices can also be
|
| 110 |
-
passed here.
|
| 111 |
-
splitval : str
|
| 112 |
-
String that defines the delimiter between keys of the
|
| 113 |
-
different depth levels in `key`.
|
| 114 |
-
default : obj
|
| 115 |
-
Value returned if :attr:`key` is not found.
|
| 116 |
-
expand : bool
|
| 117 |
-
Whether to expand callable nodes on the path or not.
|
| 118 |
-
|
| 119 |
-
Returns
|
| 120 |
-
-------
|
| 121 |
-
The desired value or if :attr:`default` is not ``None`` and the
|
| 122 |
-
:attr:`key` is not found returns ``default``.
|
| 123 |
-
|
| 124 |
-
Raises
|
| 125 |
-
------
|
| 126 |
-
Exception if ``key`` not in ``list_or_dict`` and :attr:`default` is
|
| 127 |
-
``None``.
|
| 128 |
-
"""
|
| 129 |
-
|
| 130 |
-
keys = key.split(splitval)
|
| 131 |
-
|
| 132 |
-
success = True
|
| 133 |
-
try:
|
| 134 |
-
visited = []
|
| 135 |
-
parent = None
|
| 136 |
-
last_key = None
|
| 137 |
-
for key in keys:
|
| 138 |
-
if callable(list_or_dict):
|
| 139 |
-
if not expand:
|
| 140 |
-
raise KeyNotFoundError(
|
| 141 |
-
ValueError(
|
| 142 |
-
"Trying to get past callable node with expand=False."
|
| 143 |
-
),
|
| 144 |
-
keys=keys,
|
| 145 |
-
visited=visited,
|
| 146 |
-
)
|
| 147 |
-
list_or_dict = list_or_dict()
|
| 148 |
-
parent[last_key] = list_or_dict
|
| 149 |
-
|
| 150 |
-
last_key = key
|
| 151 |
-
parent = list_or_dict
|
| 152 |
-
|
| 153 |
-
try:
|
| 154 |
-
if isinstance(list_or_dict, dict):
|
| 155 |
-
list_or_dict = list_or_dict[key]
|
| 156 |
-
else:
|
| 157 |
-
list_or_dict = list_or_dict[int(key)]
|
| 158 |
-
except (KeyError, IndexError, ValueError) as e:
|
| 159 |
-
raise KeyNotFoundError(e, keys=keys, visited=visited)
|
| 160 |
-
|
| 161 |
-
visited += [key]
|
| 162 |
-
# final expansion of retrieved value
|
| 163 |
-
if expand and callable(list_or_dict):
|
| 164 |
-
list_or_dict = list_or_dict()
|
| 165 |
-
parent[last_key] = list_or_dict
|
| 166 |
-
except KeyNotFoundError as e:
|
| 167 |
-
if default is None:
|
| 168 |
-
raise e
|
| 169 |
-
else:
|
| 170 |
-
list_or_dict = default
|
| 171 |
-
success = False
|
| 172 |
-
|
| 173 |
-
if not pass_success:
|
| 174 |
-
return list_or_dict
|
| 175 |
-
else:
|
| 176 |
-
return list_or_dict, success
|
| 177 |
|
| 178 |
|
| 179 |
def get_obj_from_str(string, reload=False):
|
|
@@ -184,31 +9,7 @@ def get_obj_from_str(string, reload=False):
|
|
| 184 |
return getattr(importlib.import_module(module, package=None), cls)
|
| 185 |
|
| 186 |
|
| 187 |
-
|
| 188 |
-
def get_jobid():
|
| 189 |
-
try:
|
| 190 |
-
jobid = os.environ["PBS_JOBID"].replace(".lmbtorque.informatik.uni-freiburg.de", "")
|
| 191 |
-
except KeyError:
|
| 192 |
-
try:
|
| 193 |
-
jobid = "DLC" + os.environ["SLURM_JOB_ID"]
|
| 194 |
-
except KeyError:
|
| 195 |
-
try:
|
| 196 |
-
jobid = os.uname()[1]
|
| 197 |
-
except KeyError:
|
| 198 |
-
jobid = "local"
|
| 199 |
-
return jobid
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
def nondefault_trainer_args(opt):
|
| 203 |
-
parser = argparse.ArgumentParser()
|
| 204 |
-
parser = Trainer.add_argparse_args(parser)
|
| 205 |
-
args = parser.parse_args([])
|
| 206 |
-
return sorted(k for k in vars(args) if getattr(opt, k) != getattr(args, k))
|
| 207 |
-
|
| 208 |
-
|
| 209 |
def instantiate_from_config(config):
|
| 210 |
if not "target" in config:
|
| 211 |
raise KeyError("Expected key `target` to instantiate.")
|
| 212 |
return get_obj_from_str(config["target"])(**config.get("params", dict()))
|
| 213 |
-
|
| 214 |
-
|
|
|
|
|
|
|
|
|
|
| 1 |
import importlib
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
def get_obj_from_str(string, reload=False):
|
|
|
|
| 9 |
return getattr(importlib.import_module(module, package=None), cls)
|
| 10 |
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
def instantiate_from_config(config):
|
| 13 |
if not "target" in config:
|
| 14 |
raise KeyError("Expected key `target` to instantiate.")
|
| 15 |
return get_obj_from_str(config["target"])(**config.get("params", dict()))
|
|
|
|
|
|