Silvio Galesso commited on
Commit
11ac3b5
·
1 Parent(s): 2ced563

cleanup, repo name

Browse files
app2.py CHANGED
@@ -165,7 +165,7 @@ MODEL_SPECS = [
165
  ]
166
 
167
  # Hub repo holding the three models (root contains L1/, L2/, tok/).
168
- HF_CKPT_REPO = os.environ.get("ORBIS2_HF_REPO", "sud0301/orbis2_test")
169
 
170
 
171
  def _missing_models(root: Path) -> list[str]:
 
165
  ]
166
 
167
  # Hub repo holding the three models (root contains L1/, L2/, tok/).
168
+ HF_CKPT_REPO = os.environ.get("ORBIS2_HF_REPO", "sud0301/orbis2")
169
 
170
 
171
  def _missing_models(root: Path) -> list[str]:
orbis2/evaluate/rollout_demo_v2.py CHANGED
@@ -3,13 +3,13 @@ import os
3
  import sys
4
  import imageio
5
  import logging
 
6
 
7
  sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
8
 
9
  import numpy as np
10
  import torch
11
  from omegaconf import OmegaConf
12
- from omegaconf.errors import ConfigTypeError
13
  from PIL import Image
14
  from pytorch_lightning import seed_everything
15
  from torchvision.utils import save_image
@@ -26,8 +26,8 @@ from evaluate.utils import (
26
  resolve_video_backend,
27
  str2bool,
28
  )
29
- from util import instantiate_from_config
30
 
 
31
 
32
  logger = logging.getLogger(__name__)
33
 
@@ -48,22 +48,13 @@ class _L1L2FrameIndexer(L2ContextMixin):
48
  )
49
 
50
 
51
- def resolve_context_image_size(args, config):
52
- """Return (height, width) to resize context images to, from CLI args or the training config."""
53
- if args.height is not None and args.width is not None:
54
- return int(args.height), int(args.width)
55
-
56
- try:
57
- size = OmegaConf.select(config, "data.params.validation.params.size")
58
- except ConfigTypeError:
59
- size = None
60
  if size is None:
61
- size = config.data.params.train[0].params.size
62
  size = (size, size) if isinstance(size, int) else tuple(size)
63
-
64
- height = int(args.height) if args.height is not None else int(size[0])
65
- width = int(args.width) if args.width is not None else int(size[1])
66
- return height, width
67
 
68
 
69
  def require_l1l2_model(model):
@@ -78,25 +69,24 @@ def require_l1l2_model(model):
78
  )
79
 
80
 
81
- def resolve_l2_frame_rate(args, model):
82
- """Return the L2 sampling rate, defaulting to (and validating against) the frozen L2 predictor's own rate."""
83
- model_rate = model.condition_preprocessor.l2_predictor_frame_rate
84
- if args.l2_frame_rate is None:
85
- return float(model_rate)
86
- if abs(float(args.l2_frame_rate) - float(model_rate)) > 1e-6:
87
- raise ValueError(
88
- f"--l2_frame_rate={args.l2_frame_rate:g} does not match the frozen L2 predictor's "
89
- f"trained frame rate ({model_rate:g}); the L2 image context must be sampled at the "
90
- "rate the L2 predictor was trained on."
91
- )
92
- return float(args.l2_frame_rate)
93
 
94
 
95
  def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1_context_frames, l2_context_frames, height, width, backend, device):
96
  """Sample L1 (high-rate) and L2 (low-rate, further back) context windows from one video."""
97
  native_fps, video_length = get_video_fps_and_length(video_path, backend)
98
- frame_interval = compute_frame_interval(native_fps, l1_frame_rate, "--l1_frame_rate")
99
- compute_frame_interval(native_fps, l2_frame_rate, "--l2_frame_rate")
100
 
101
  indexer = _L1L2FrameIndexer(
102
  frame_interval=frame_interval,
@@ -120,7 +110,7 @@ def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1
120
  raise ValueError(
121
  f"Video does not have enough lookback for L2 context: start_frame={start_frame} but "
122
  f"at least {required_offset} frames of history are required before it. "
123
- "Use a longer video, a later --start_frame, or a lower --l2_frame_rate."
124
  )
125
 
126
  l1_indices, l2_indices = indexer.get_l1_and_l2_indices(start_frame, l1_context_frames)
@@ -236,23 +226,23 @@ def make_unconditional_steering(min_odo_steps, dtype, device):
236
  return torch.full((1, int(min_odo_steps), 2), float("nan"), dtype=dtype, device=device)
237
 
238
 
239
- def maybe_apply_l2_num_steps(model, l2_num_steps):
240
  """Override the frozen L2 predictor's sampler steps (its `l2_pred_NFE`).
241
 
242
- L1 and L2 are sampled at different step counts: --num_steps is L1's NFE, while
243
  L2's comes from the config's `l2_pred_NFE`. Setting the attribute post-init is
244
  enough -- it is read at sample time, not at construction.
245
  """
246
- if l2_num_steps is None:
247
  return
248
 
249
  condition_preprocessor = getattr(model, "condition_preprocessor", None)
250
  if condition_preprocessor is None or not hasattr(condition_preprocessor, "l2_pred_NFE"):
251
  raise TypeError(
252
- "--l2_num_steps was given but the loaded condition_preprocessor has no "
253
  "`l2_pred_NFE` (so it has no separately-sampled L2 predictor)."
254
  )
255
- condition_preprocessor.l2_pred_NFE = int(l2_num_steps)
256
 
257
 
258
  @torch.no_grad()
@@ -308,11 +298,12 @@ def generate_images(args, unknown_args):
308
  )
309
 
310
  maybe_apply_condition_preprocessor_scales(model, args.speed_scale, args.yaw_rate_scale)
311
- maybe_apply_l2_num_steps(model, args.l2_num_steps)
312
 
313
- height, width = resolve_context_image_size(args, config)
314
  backend = resolve_video_backend()
315
- l2_frame_rate = resolve_l2_frame_rate(args, model)
 
316
 
317
  l1_context_frames = int(model.vit.num_context_frames)
318
  l2_context_frames = int(model.condition_preprocessor.num_context_frames)
@@ -320,7 +311,7 @@ def generate_images(args, unknown_args):
320
  l1_tensor, l2_tensor = load_l1_l2_context(
321
  video_path=args.video,
322
  start_frame=args.start_frame,
323
- l1_frame_rate=args.l1_frame_rate,
324
  l2_frame_rate=l2_frame_rate,
325
  l1_context_frames=l1_context_frames,
326
  l2_context_frames=l2_context_frames,
@@ -330,8 +321,8 @@ def generate_images(args, unknown_args):
330
  device=args.device,
331
  )
332
 
333
- num_future_frames = get_rollout_future_frame_count(model, args.num_gen_frames)
334
- frame_rate = torch.tensor(float(args.l1_frame_rate), device=args.device)
335
  data_batch = {"images": l1_tensor, "l2_context": l2_tensor, "frame_rate": frame_rate}
336
 
337
  if args.steering_file is not None and args.trajectory_file is not None:
@@ -344,7 +335,7 @@ def generate_images(args, unknown_args):
344
  validation_params=None,
345
  num_condition_frames=l1_context_frames,
346
  num_gen_frames=num_future_frames,
347
- rollout_steps=args.num_gen_frames,
348
  )
349
  if args.steering_file is not None:
350
  data_batch["steering"] = load_steering_trajectory(
@@ -357,7 +348,7 @@ def generate_images(args, unknown_args):
357
  "required odometry length (get_required_rollout_odometry_steps returned None)."
358
  )
359
  odometry_steps_per_image_frame = getattr(model.condition_preprocessor, "odometry_steps_per_image_frame", 1)
360
- dt = 1.0 / (args.l1_frame_rate * odometry_steps_per_image_frame)
361
  traj_xy = load_trajectory_points(args.trajectory_file)
362
  traj_xy = resample_trajectory_by_arclength(traj_xy, min_odo_steps + 1)
363
  speed_yawrate = trajectory_to_speed_yawrate(traj_xy, dt)
@@ -368,22 +359,6 @@ def generate_images(args, unknown_args):
368
  data_batch["steering"] = make_unconditional_steering(min_odo_steps, dtype=l1_tensor.dtype, device=args.device)
369
  data_batch["steering_format"] = STEERING_FORMAT
370
 
371
- # Roll out `num_videos` futures in parallel from the same context, as a single
372
- # minibatch. The context (and steering) is tiled along the batch dim; the sampler
373
- # draws independent initial noise per element, so the futures diverge under one
374
- # seed. Everything below is computed at the tiled batch size so condition_kwargs
375
- # stays internally consistent.
376
- num_videos = max(1, int(args.num_videos))
377
- if num_videos > 1:
378
- def _tile_batch(t):
379
- return t.repeat(num_videos, *([1] * (t.dim() - 1)))
380
-
381
- l1_tensor = _tile_batch(l1_tensor)
382
- l2_tensor = _tile_batch(l2_tensor)
383
- data_batch["images"] = l1_tensor
384
- data_batch["l2_context"] = l2_tensor
385
- data_batch["steering"] = _tile_batch(data_batch["steering"])
386
-
387
  condition_kwargs = model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout")
388
 
389
  if args.steering_file is not None:
@@ -394,17 +369,17 @@ def generate_images(args, unknown_args):
394
  steering_source = "none"
395
  logger.info(f"Steering source: {steering_source}")
396
  logger.info(f"Steering scales: speed={args.speed_scale:g}, yaw_rate={args.yaw_rate_scale:g}")
397
- logger.info(f"L1/L2 frame rates: {args.l1_frame_rate:g}/{l2_frame_rate:g} Hz")
398
  logger.info(f"Saving generated images to {args.output_dir}")
399
 
400
  autocast_enabled = args.device.startswith("cuda")
401
  with torch.autocast(dtype=torch.float16, device_type="cuda", enabled=autocast_enabled):
402
  _latents, gen_frames = model.roll_out(
403
  x_0={"images": l1_tensor},
404
- num_gen_frames=args.num_gen_frames,
405
  latent_input=False,
406
- NFE=args.num_steps,
407
- eta=args.eta,
408
  sample_with_ema=args.evaluate_ema,
409
  num_samples=l1_tensor.size(0),
410
  frame_rate=frame_rate.reshape(1).repeat(l1_tensor.size(0)),
@@ -417,7 +392,7 @@ def generate_images(args, unknown_args):
417
  overlay_trajectory = model.condition_preprocessor.get_rollout_visualization_trajectory(
418
  condition_kwargs=model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout"),
419
  num_condition_frames=l1_context_frames,
420
- num_gen_steps=args.num_gen_frames,
421
  num_pred_frames=model.num_pred_frames,
422
  )
423
  if overlay_trajectory is not None:
@@ -445,7 +420,7 @@ def generate_images(args, unknown_args):
445
  np.array(Image.open(os.path.join(seq_dir, f"frame_{f:04d}.jpg")))
446
  for f in range(num_frames)
447
  ],
448
- fps=args.l1_frame_rate,
449
  loop=0,
450
  )
451
 
@@ -469,28 +444,24 @@ if __name__ == "__main__":
469
  logging.basicConfig(level=logging.INFO)
470
 
471
  parser = argparse.ArgumentParser()
472
- parser.add_argument("--exp_dir", type=str, default=None, help="Path to the experiment directory, where the config and checkpoints are stored")
473
- parser.add_argument("--ckpt", type=str, default="checkpoints/last.ckpt", help="Path to the checkpoint file, relative to exp_dir")
474
- parser.add_argument("--config", type=str, default="config.yaml", help="Path to the config file, relative to exp_dir")
 
 
 
 
 
 
475
  parser.add_argument("--video", type=str, required=True, help="Path to the input video file to sample L1/L2 context from.")
476
- parser.add_argument("--l1_frame_rate", type=float, required=True, help="Frame rate (Hz) to sample L1 context/rollout frames at, and the generated GIF's fps.")
477
- parser.add_argument("--l2_frame_rate", type=float, default=None, help="Frame rate (Hz) to sample L2 context frames at. Defaults to the frozen L2 predictor's own trained frame rate; must match it if given explicitly.")
478
  parser.add_argument("--start_frame", type=int, default=None, help="Native-video frame index to start the L1 context window at. Defaults to the latest window that fits (the end of the video).")
479
- parser.add_argument("--height", type=int, default=None, help="Height to resize context images to. Defaults to the training config's size.")
480
- parser.add_argument("--width", type=int, default=None, help="Width to resize context images to. Defaults to the training config's size.")
481
  parser.add_argument(
482
- "--num_gen_frames",
483
  type=int,
484
- default=1,
485
  help="Number of rollout steps to generate; each step predicts `model.num_pred_frames` future frames.",
486
  )
487
  parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated frames and GIF to.")
488
- parser.add_argument(
489
- "--num_videos",
490
- type=int,
491
- default=1,
492
- help="Number of futures to roll out in parallel from the same context, as one minibatch.",
493
- )
494
  parser.add_argument(
495
  "--vis_mode",
496
  type=str,
@@ -518,15 +489,14 @@ if __name__ == "__main__":
518
  default="cpu",
519
  help="Device used for decoded rollout frames. Use 'cpu' to reduce peak GPU memory during saving.",
520
  )
521
- parser.add_argument("--num_steps", type=int, default=30, help="Number of sampler steps (NFE) for the L1 detail predictor")
522
  parser.add_argument(
523
- "--l2_num_steps",
524
  type=int,
525
  default=None,
526
  help="Number of sampler steps (NFE) for the frozen L2 abstract predictor. "
527
  "Overrides the config's `l2_pred_NFE`; defaults to whatever the config sets.",
528
  )
529
- parser.add_argument("--eta", type=float, default=0.0, help="Stochasticity for sampling")
530
  parser.add_argument("--evaluate_ema", "--use_ema", type=str2bool, default=True, help="If the evaluation happen with ema model")
531
  parser.add_argument(
532
  "--compile",
@@ -555,6 +525,11 @@ if __name__ == "__main__":
555
 
556
  args, unknown = parser.parse_known_args()
557
 
 
 
 
 
 
558
  args.ckpt = os.path.join(args.exp_dir, args.ckpt)
559
  args.config = os.path.join(args.exp_dir, args.config)
560
  if args.compile_artifacts:
 
3
  import sys
4
  import imageio
5
  import logging
6
+ import importlib
7
 
8
  sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
9
 
10
  import numpy as np
11
  import torch
12
  from omegaconf import OmegaConf
 
13
  from PIL import Image
14
  from pytorch_lightning import seed_everything
15
  from torchvision.utils import save_image
 
26
  resolve_video_backend,
27
  str2bool,
28
  )
 
29
 
30
+ from util import instantiate_from_config
31
 
32
  logger = logging.getLogger(__name__)
33
 
 
48
  )
49
 
50
 
51
+ def resolve_context_image_size(config):
52
+ """Return (height, width) to resize context images to, from the inference config."""
53
+ size = OmegaConf.select(config, "data.params.validation.params.size")
 
 
 
 
 
 
54
  if size is None:
55
+ raise ValueError("Config is missing `data.params.validation.params.size`.")
56
  size = (size, size) if isinstance(size, int) else tuple(size)
57
+ return int(size[0]), int(size[1])
 
 
 
58
 
59
 
60
  def require_l1l2_model(model):
 
69
  )
70
 
71
 
72
+ def resolve_l2_frame_rate(model):
73
+ """Return the L2 sampling rate: the frozen L2 predictor's own trained rate."""
74
+ return float(model.condition_preprocessor.l2_predictor_frame_rate)
75
+
76
+
77
+ def resolve_l1_frame_rate(config):
78
+ """Return the L1 (context/rollout) frame rate (Hz) declared in the inference config."""
79
+ frame_rate = OmegaConf.select(config, "data.params.validation.params.frame_rate")
80
+ if frame_rate is None:
81
+ raise ValueError("Config is missing `data.params.validation.params.frame_rate`.")
82
+ return float(frame_rate)
 
83
 
84
 
85
  def load_l1_l2_context(video_path, start_frame, l1_frame_rate, l2_frame_rate, l1_context_frames, l2_context_frames, height, width, backend, device):
86
  """Sample L1 (high-rate) and L2 (low-rate, further back) context windows from one video."""
87
  native_fps, video_length = get_video_fps_and_length(video_path, backend)
88
+ frame_interval = compute_frame_interval(native_fps, l1_frame_rate, "the L1 frame rate (data.params.validation.params.frame_rate)")
89
+ compute_frame_interval(native_fps, l2_frame_rate, "the L2 predictor's trained frame rate")
90
 
91
  indexer = _L1L2FrameIndexer(
92
  frame_interval=frame_interval,
 
110
  raise ValueError(
111
  f"Video does not have enough lookback for L2 context: start_frame={start_frame} but "
112
  f"at least {required_offset} frames of history are required before it. "
113
+ "Use a longer video or a later --start_frame."
114
  )
115
 
116
  l1_indices, l2_indices = indexer.get_l1_and_l2_indices(start_frame, l1_context_frames)
 
226
  return torch.full((1, int(min_odo_steps), 2), float("nan"), dtype=dtype, device=device)
227
 
228
 
229
+ def maybe_apply_l2_nfe(model, l2_nfe):
230
  """Override the frozen L2 predictor's sampler steps (its `l2_pred_NFE`).
231
 
232
+ L1 and L2 are sampled at different step counts: --l1_nfe is L1's NFE, while
233
  L2's comes from the config's `l2_pred_NFE`. Setting the attribute post-init is
234
  enough -- it is read at sample time, not at construction.
235
  """
236
+ if l2_nfe is None:
237
  return
238
 
239
  condition_preprocessor = getattr(model, "condition_preprocessor", None)
240
  if condition_preprocessor is None or not hasattr(condition_preprocessor, "l2_pred_NFE"):
241
  raise TypeError(
242
+ "--l2_nfe was given but the loaded condition_preprocessor has no "
243
  "`l2_pred_NFE` (so it has no separately-sampled L2 predictor)."
244
  )
245
+ condition_preprocessor.l2_pred_NFE = int(l2_nfe)
246
 
247
 
248
  @torch.no_grad()
 
298
  )
299
 
300
  maybe_apply_condition_preprocessor_scales(model, args.speed_scale, args.yaw_rate_scale)
301
+ maybe_apply_l2_nfe(model, args.l2_nfe)
302
 
303
+ height, width = resolve_context_image_size(config)
304
  backend = resolve_video_backend()
305
+ l1_frame_rate = resolve_l1_frame_rate(config)
306
+ l2_frame_rate = resolve_l2_frame_rate(model)
307
 
308
  l1_context_frames = int(model.vit.num_context_frames)
309
  l2_context_frames = int(model.condition_preprocessor.num_context_frames)
 
311
  l1_tensor, l2_tensor = load_l1_l2_context(
312
  video_path=args.video,
313
  start_frame=args.start_frame,
314
+ l1_frame_rate=l1_frame_rate,
315
  l2_frame_rate=l2_frame_rate,
316
  l1_context_frames=l1_context_frames,
317
  l2_context_frames=l2_context_frames,
 
321
  device=args.device,
322
  )
323
 
324
+ num_future_frames = get_rollout_future_frame_count(model, args.num_rollout_steps)
325
+ frame_rate = torch.tensor(l1_frame_rate, device=args.device)
326
  data_batch = {"images": l1_tensor, "l2_context": l2_tensor, "frame_rate": frame_rate}
327
 
328
  if args.steering_file is not None and args.trajectory_file is not None:
 
335
  validation_params=None,
336
  num_condition_frames=l1_context_frames,
337
  num_gen_frames=num_future_frames,
338
+ rollout_steps=args.num_rollout_steps,
339
  )
340
  if args.steering_file is not None:
341
  data_batch["steering"] = load_steering_trajectory(
 
348
  "required odometry length (get_required_rollout_odometry_steps returned None)."
349
  )
350
  odometry_steps_per_image_frame = getattr(model.condition_preprocessor, "odometry_steps_per_image_frame", 1)
351
+ dt = 1.0 / (l1_frame_rate * odometry_steps_per_image_frame)
352
  traj_xy = load_trajectory_points(args.trajectory_file)
353
  traj_xy = resample_trajectory_by_arclength(traj_xy, min_odo_steps + 1)
354
  speed_yawrate = trajectory_to_speed_yawrate(traj_xy, dt)
 
359
  data_batch["steering"] = make_unconditional_steering(min_odo_steps, dtype=l1_tensor.dtype, device=args.device)
360
  data_batch["steering_format"] = STEERING_FORMAT
361
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
362
  condition_kwargs = model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout")
363
 
364
  if args.steering_file is not None:
 
369
  steering_source = "none"
370
  logger.info(f"Steering source: {steering_source}")
371
  logger.info(f"Steering scales: speed={args.speed_scale:g}, yaw_rate={args.yaw_rate_scale:g}")
372
+ logger.info(f"L1/L2 frame rates: {l1_frame_rate:g}/{l2_frame_rate:g} Hz")
373
  logger.info(f"Saving generated images to {args.output_dir}")
374
 
375
  autocast_enabled = args.device.startswith("cuda")
376
  with torch.autocast(dtype=torch.float16, device_type="cuda", enabled=autocast_enabled):
377
  _latents, gen_frames = model.roll_out(
378
  x_0={"images": l1_tensor},
379
+ num_gen_frames=args.num_rollout_steps,
380
  latent_input=False,
381
+ NFE=args.l1_nfe,
382
+ eta=0.0,
383
  sample_with_ema=args.evaluate_ema,
384
  num_samples=l1_tensor.size(0),
385
  frame_rate=frame_rate.reshape(1).repeat(l1_tensor.size(0)),
 
392
  overlay_trajectory = model.condition_preprocessor.get_rollout_visualization_trajectory(
393
  condition_kwargs=model.condition_preprocessor.get_condition_kwargs_from_batch(data_batch, split="rollout"),
394
  num_condition_frames=l1_context_frames,
395
+ num_gen_steps=args.num_rollout_steps,
396
  num_pred_frames=model.num_pred_frames,
397
  )
398
  if overlay_trajectory is not None:
 
420
  np.array(Image.open(os.path.join(seq_dir, f"frame_{f:04d}.jpg")))
421
  for f in range(num_frames)
422
  ],
423
+ fps=l1_frame_rate,
424
  loop=0,
425
  )
426
 
 
444
  logging.basicConfig(level=logging.INFO)
445
 
446
  parser = argparse.ArgumentParser()
447
+ parser.add_argument(
448
+ "--exp_dir",
449
+ type=str,
450
+ default=os.environ.get("ORBIS2_MODELS_DIR"),
451
+ help="Path to the experiment directory, where the config and checkpoints are stored. "
452
+ "Defaults to the ORBIS2_MODELS_DIR environment variable.",
453
+ )
454
+ parser.add_argument("--ckpt", type=str, default="L1/checkpoints/last.ckpt", help="Path to the checkpoint file, relative to exp_dir")
455
+ parser.add_argument("--config", type=str, default="L1/config.yaml", help="Path to the config file, relative to exp_dir")
456
  parser.add_argument("--video", type=str, required=True, help="Path to the input video file to sample L1/L2 context from.")
 
 
457
  parser.add_argument("--start_frame", type=int, default=None, help="Native-video frame index to start the L1 context window at. Defaults to the latest window that fits (the end of the video).")
 
 
458
  parser.add_argument(
459
+ "--num_rollout_steps",
460
  type=int,
461
+ default=10,
462
  help="Number of rollout steps to generate; each step predicts `model.num_pred_frames` future frames.",
463
  )
464
  parser.add_argument("--output_dir", type=str, required=True, help="Directory to save the generated frames and GIF to.")
 
 
 
 
 
 
465
  parser.add_argument(
466
  "--vis_mode",
467
  type=str,
 
489
  default="cpu",
490
  help="Device used for decoded rollout frames. Use 'cpu' to reduce peak GPU memory during saving.",
491
  )
492
+ parser.add_argument("--l1_nfe", type=int, default=30, help="Number of sampler steps (NFE) for the L1 detail predictor")
493
  parser.add_argument(
494
+ "--l2_nfe",
495
  type=int,
496
  default=None,
497
  help="Number of sampler steps (NFE) for the frozen L2 abstract predictor. "
498
  "Overrides the config's `l2_pred_NFE`; defaults to whatever the config sets.",
499
  )
 
500
  parser.add_argument("--evaluate_ema", "--use_ema", type=str2bool, default=True, help="If the evaluation happen with ema model")
501
  parser.add_argument(
502
  "--compile",
 
525
 
526
  args, unknown = parser.parse_known_args()
527
 
528
+ if args.exp_dir is None:
529
+ raise ValueError(
530
+ "--exp_dir was not given and the ORBIS2_MODELS_DIR environment variable is not set."
531
+ )
532
+
533
  args.ckpt = os.path.join(args.exp_dir, args.ckpt)
534
  args.config = os.path.join(args.exp_dir, args.config)
535
  if args.compile_artifacts:
orbis2/models/second_stage/cd_model_v2.py CHANGED
@@ -205,21 +205,25 @@ class ConsistencyDistillationSampler(FlowMatchingSamplerEuler):
205
 
206
  class ConsistencyDistillPredictorModule(PredictorModule):
207
  """
208
- PredictorModule specialization that distills a frozen teacher (loaded from
209
- `teacher_ckpt_path`) into a few-step student via consistency distillation.
 
 
 
210
  The deliverable is `ema_vit` (evaluate with sample_with_ema=True, NFE 1/2/4).
211
  """
212
 
213
  # `teacher_vit.*` is intentionally absent from saved checkpoints (see
214
- # on_save_checkpoint below) since the teacher is always reloaded fresh from
215
- # teacher_ckpt_path at __init__ time. Eval scripts that assert on
 
216
  # load_state_dict(...).missing_keys should exempt these prefixes.
217
  checkpoint_exempt_key_prefixes = ("teacher_vit.",)
218
 
219
  def __init__(
220
  self,
221
  *,
222
- teacher_ckpt_path,
223
  cd_ema_mu=0.999,
224
  cd_weight_decay=0.0,
225
  **kwargs,
@@ -229,34 +233,45 @@ class ConsistencyDistillPredictorModule(PredictorModule):
229
  self.cd_weight_decay = cd_weight_decay
230
  self.strict_loading = False
231
 
232
- checkpoint_path = os.path.expandvars(teacher_ckpt_path)
233
- if not os.path.exists(checkpoint_path):
234
- raise FileNotFoundError(f"teacher_ckpt_path {checkpoint_path} does not exist.")
235
-
236
- try:
237
- state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True, mmap=True)["state_dict"]
238
- except (TypeError, RuntimeError):
239
- state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True)["state_dict"]
240
-
241
- ema_state_dict = {
242
- key[len("ema_vit.") :]: value.clone()
243
- for key, value in state_dict.items()
244
- if key.startswith("ema_vit.")
245
- }
246
- del state_dict
247
- if not ema_state_dict:
248
- raise ValueError(f"No ema_vit.* keys found in teacher checkpoint {checkpoint_path}.")
249
-
250
- self.vit.load_state_dict(ema_state_dict, strict=True)
251
- self.ema_vit.load_state_dict(ema_state_dict, strict=True)
252
- self.teacher_vit = deepcopy(self.vit)
253
- requires_grad(self.teacher_vit, False)
254
- self.teacher_vit.eval()
 
 
 
 
255
 
256
  def setup(self, stage=None):
257
  super().setup(stage)
258
- if hasattr(self, "teacher_vit") and self.teacher_vit is not None:
259
  self.teacher_vit.requires_grad_(False)
 
 
 
 
 
 
 
260
 
261
  def configure_optimizers(self):
262
  params = [p for p in self.vit.parameters() if p.requires_grad]
 
205
 
206
  class ConsistencyDistillPredictorModule(PredictorModule):
207
  """
208
+ PredictorModule specialization that distills a frozen teacher into a
209
+ few-step student via consistency distillation. `teacher_ckpt_path`
210
+ initializes the student (`vit`/`ema_vit`) from the teacher and builds the
211
+ frozen `teacher_vit` used by the distillation loss; it is required for
212
+ training/validation but optional for inference-only sample()/roll_out().
213
  The deliverable is `ema_vit` (evaluate with sample_with_ema=True, NFE 1/2/4).
214
  """
215
 
216
  # `teacher_vit.*` is intentionally absent from saved checkpoints (see
217
+ # on_save_checkpoint below): when teacher_ckpt_path is set, the teacher is
218
+ # always reloaded fresh from it at __init__ time; when it is not set,
219
+ # teacher_vit doesn't exist at all. Either way, eval scripts that assert on
220
  # load_state_dict(...).missing_keys should exempt these prefixes.
221
  checkpoint_exempt_key_prefixes = ("teacher_vit.",)
222
 
223
  def __init__(
224
  self,
225
  *,
226
+ teacher_ckpt_path=None,
227
  cd_ema_mu=0.999,
228
  cd_weight_decay=0.0,
229
  **kwargs,
 
233
  self.cd_weight_decay = cd_weight_decay
234
  self.strict_loading = False
235
 
236
+ self.teacher_vit = None
237
+ if teacher_ckpt_path is not None:
238
+ checkpoint_path = os.path.expandvars(teacher_ckpt_path)
239
+ if not os.path.exists(checkpoint_path):
240
+ raise FileNotFoundError(f"teacher_ckpt_path {checkpoint_path} does not exist.")
241
+
242
+ try:
243
+ state_dict = torch.load(
244
+ checkpoint_path, map_location="cpu", weights_only=True, mmap=True
245
+ )["state_dict"]
246
+ except (TypeError, RuntimeError):
247
+ state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True)["state_dict"]
248
+
249
+ ema_state_dict = {
250
+ key[len("ema_vit.") :]: value.clone()
251
+ for key, value in state_dict.items()
252
+ if key.startswith("ema_vit.")
253
+ }
254
+ del state_dict
255
+ if not ema_state_dict:
256
+ raise ValueError(f"No ema_vit.* keys found in teacher checkpoint {checkpoint_path}.")
257
+
258
+ self.vit.load_state_dict(ema_state_dict, strict=True)
259
+ self.ema_vit.load_state_dict(ema_state_dict, strict=True)
260
+ self.teacher_vit = deepcopy(self.vit)
261
+ requires_grad(self.teacher_vit, False)
262
+ self.teacher_vit.eval()
263
 
264
  def setup(self, stage=None):
265
  super().setup(stage)
266
+ if self.teacher_vit is not None:
267
  self.teacher_vit.requires_grad_(False)
268
+ elif stage in ("fit", "validate"):
269
+ raise RuntimeError(
270
+ "ConsistencyDistillPredictorModule requires teacher_ckpt_path to be set "
271
+ f"when stage={stage!r} (training/validation use a frozen teacher for "
272
+ "consistency distillation). Pass teacher_ckpt_path in the model config, "
273
+ "or omit it only for inference (sample()/roll_out())."
274
+ )
275
 
276
  def configure_optimizers(self):
277
  params = [p for p in self.vit.parameters() if p.requires_grad]
orbis2/networks/tokenizer/ae.py CHANGED
@@ -1,68 +1,15 @@
1
  # pytorch_diffusion + derived encoder decoder
2
- import math
3
- from typing import Tuple, Union
4
-
5
- import numpy as np
6
  import torch
7
  import torch.nn as nn
8
 
9
- from einops import rearrange
10
- from einops.layers.torch import Rearrange
11
  from omegaconf import ListConfig
12
 
13
 
14
- def get_timestep_embedding(timesteps, embedding_dim):
15
- """
16
- This matches the implementation in Denoising Diffusion Probabilistic Models:
17
- From Fairseq.
18
- Build sinusoidal embeddings.
19
- This matches the implementation in tensor2tensor, but differs slightly
20
- from the description in Section 3.5 of "Attention Is All You Need".
21
- """
22
- assert len(timesteps.shape) == 1
23
-
24
- half_dim = embedding_dim // 2
25
- emb = math.log(10000) / (half_dim - 1)
26
- emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb)
27
- emb = emb.to(device=timesteps.device)
28
- emb = timesteps.float()[:, None] * emb[None, :]
29
- emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
30
- if embedding_dim % 2 == 1: # zero pad
31
- emb = torch.nn.functional.pad(emb, (0,1,0,0))
32
- return emb
33
-
34
-
35
  def nonlinearity(x):
36
  # swish
37
  return x*torch.sigmoid(x)
38
 
39
 
40
- class SpatialNorm(nn.Module):
41
- def __init__(self, f_channels, zq_channels, norm_layer=nn.GroupNorm, freeze_norm_layer=False, add_conv=False, **norm_layer_params):
42
- super().__init__()
43
- self.norm_layer = norm_layer(num_channels=f_channels, **norm_layer_params)
44
- if freeze_norm_layer:
45
- for p in self.norm_layer.parameters: # for p in self.norm_layer.parameters():
46
- p.requires_grad = False
47
- self.add_conv = add_conv
48
- if self.add_conv:
49
- self.conv = nn.Conv2d(zq_channels, zq_channels, kernel_size=3, stride=1, padding=1)
50
- self.conv_y = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
51
- self.conv_b = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
52
- def forward(self, f, zq):
53
- #print(f'f shape: {f.shape}', f'zq shape: {zq.shape}')
54
- f_size = f.shape[-2:]
55
- zq = torch.nn.functional.interpolate(zq, size=f_size, mode="nearest")
56
- if self.add_conv:
57
- zq = self.conv(zq)
58
- norm_f = self.norm_layer(f)
59
- new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)
60
- return new_f
61
-
62
- def SpatialNormalize(in_channels, zq_ch, add_conv):
63
- return SpatialNorm(in_channels, zq_ch, norm_layer=nn.GroupNorm, freeze_norm_layer=False, add_conv=add_conv, num_groups=32, eps=1e-6, affine=True)
64
-
65
-
66
  def Normalize(in_channels):
67
  return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
68
 
@@ -168,67 +115,6 @@ class ResnetBlock(nn.Module):
168
 
169
  return x+h
170
 
171
- class SpatialResnetBlock(nn.Module):
172
- def __init__(self, *, in_channels, out_channels=None, conv_shortcut=False,
173
- dropout, temb_channels=512, zq_ch=None, add_conv=False):
174
- super().__init__()
175
- self.in_channels = in_channels
176
- out_channels = in_channels if out_channels is None else out_channels
177
- self.out_channels = out_channels
178
- self.use_conv_shortcut = conv_shortcut
179
-
180
- self.norm1 = SpatialNormalize(in_channels, zq_ch, add_conv)
181
- self.conv1 = torch.nn.Conv2d(in_channels,
182
- out_channels,
183
- kernel_size=3,
184
- stride=1,
185
- padding=1)
186
- if temb_channels > 0:
187
- self.temb_proj = torch.nn.Linear(temb_channels,
188
- out_channels)
189
- self.norm2 = SpatialNormalize(out_channels, zq_ch, add_conv)
190
- self.dropout = torch.nn.Dropout(dropout)
191
- self.conv2 = torch.nn.Conv2d(out_channels,
192
- out_channels,
193
- kernel_size=3,
194
- stride=1,
195
- padding=1)
196
- if self.in_channels != self.out_channels:
197
- if self.use_conv_shortcut:
198
- self.conv_shortcut = torch.nn.Conv2d(in_channels,
199
- out_channels,
200
- kernel_size=3,
201
- stride=1,
202
- padding=1)
203
- else:
204
- self.nin_shortcut = torch.nn.Conv2d(in_channels,
205
- out_channels,
206
- kernel_size=1,
207
- stride=1,
208
- padding=0)
209
-
210
- def forward(self, x, temb, zq):
211
- h = x
212
- h = self.norm1(h, zq)
213
- h = nonlinearity(h)
214
- h = self.conv1(h)
215
-
216
- if temb is not None:
217
- h = h + self.temb_proj(nonlinearity(temb))[:,:,None,None]
218
-
219
- h = self.norm2(h, zq)
220
- h = nonlinearity(h)
221
- h = self.dropout(h)
222
- h = self.conv2(h)
223
-
224
- if self.in_channels != self.out_channels:
225
- if self.use_conv_shortcut:
226
- x = self.conv_shortcut(x)
227
- else:
228
- x = self.nin_shortcut(x)
229
-
230
- return x+h
231
-
232
 
233
  class AttnBlock(nn.Module):
234
  def __init__(self, in_channels):
@@ -283,371 +169,8 @@ class AttnBlock(nn.Module):
283
  h_ = self.proj_out(h_)
284
 
285
  return x+h_
286
-
287
- class SpatialAttnBlock(nn.Module):
288
- def __init__(self, in_channels, zq_ch=None, add_conv=False):
289
- super().__init__()
290
- self.in_channels = in_channels
291
-
292
- self.norm = SpatialNormalize(in_channels, zq_ch, add_conv)
293
- self.q = torch.nn.Conv2d(in_channels,
294
- in_channels,
295
- kernel_size=1,
296
- stride=1,
297
- padding=0)
298
- self.k = torch.nn.Conv2d(in_channels,
299
- in_channels,
300
- kernel_size=1,
301
- stride=1,
302
- padding=0)
303
- self.v = torch.nn.Conv2d(in_channels,
304
- in_channels,
305
- kernel_size=1,
306
- stride=1,
307
- padding=0)
308
- self.proj_out = torch.nn.Conv2d(in_channels,
309
- in_channels,
310
- kernel_size=1,
311
- stride=1,
312
- padding=0)
313
 
314
 
315
- def forward(self, x, zq):
316
- h_ = x
317
- h_ = self.norm(h_, zq)
318
- q = self.q(h_)
319
- k = self.k(h_)
320
- v = self.v(h_)
321
-
322
- # compute attention
323
- b,c,h,w = q.shape
324
- q = q.reshape(b,c,h*w)
325
- q = q.permute(0,2,1) # b,hw,c
326
- k = k.reshape(b,c,h*w) # b,c,hw
327
- w_ = torch.bmm(q,k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j]
328
- w_ = w_ * (int(c)**(-0.5))
329
- w_ = torch.nn.functional.softmax(w_, dim=2)
330
-
331
- # attend to values
332
- v = v.reshape(b,c,h*w)
333
- w_ = w_.permute(0,2,1) # b,hw,hw (first hw of k, second of q)
334
- h_ = torch.bmm(v,w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j]
335
- h_ = h_.reshape(b,c,h,w)
336
-
337
- h_ = self.proj_out(h_)
338
-
339
- return x+h_
340
-
341
-
342
- class Model(nn.Module):
343
- def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
344
- attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
345
- resolution, use_timestep=True):
346
- super().__init__()
347
- self.ch = ch
348
- self.temb_ch = self.ch*4
349
- self.num_resolutions = len(ch_mult)
350
- self.num_res_blocks = num_res_blocks
351
- self.resolution = resolution
352
- self.in_channels = in_channels
353
-
354
- self.use_timestep = use_timestep
355
- if self.use_timestep:
356
- # timestep embedding
357
- self.temb = nn.Module()
358
- self.temb.dense = nn.ModuleList([
359
- torch.nn.Linear(self.ch,
360
- self.temb_ch),
361
- torch.nn.Linear(self.temb_ch,
362
- self.temb_ch),
363
- ])
364
-
365
- # downsampling
366
- self.conv_in = torch.nn.Conv2d(in_channels,
367
- self.ch,
368
- kernel_size=3,
369
- stride=1,
370
- padding=1)
371
-
372
- curr_res = resolution
373
- in_ch_mult = (1,)+tuple(ch_mult)
374
- self.down = nn.ModuleList()
375
- for i_level in range(self.num_resolutions):
376
- block = nn.ModuleList()
377
- attn = nn.ModuleList()
378
- block_in = ch*in_ch_mult[i_level]
379
- block_out = ch*ch_mult[i_level]
380
- for i_block in range(self.num_res_blocks):
381
- block.append(ResnetBlock(in_channels=block_in,
382
- out_channels=block_out,
383
- temb_channels=self.temb_ch,
384
- dropout=dropout))
385
- block_in = block_out
386
- if curr_res in attn_resolutions:
387
- attn.append(AttnBlock(block_in))
388
- down = nn.Module()
389
- down.block = block
390
- down.attn = attn
391
- if i_level != self.num_resolutions-1:
392
- down.downsample = Downsample(block_in, resamp_with_conv)
393
- curr_res = curr_res // 2
394
- self.down.append(down)
395
-
396
- # middle
397
- self.mid = nn.Module()
398
- self.mid.block_1 = ResnetBlock(in_channels=block_in,
399
- out_channels=block_in,
400
- temb_channels=self.temb_ch,
401
- dropout=dropout)
402
- self.mid.attn_1 = AttnBlock(block_in)
403
- self.mid.block_2 = ResnetBlock(in_channels=block_in,
404
- out_channels=block_in,
405
- temb_channels=self.temb_ch,
406
- dropout=dropout)
407
-
408
- # upsampling
409
- self.up = nn.ModuleList()
410
- for i_level in reversed(range(self.num_resolutions)):
411
- block = nn.ModuleList()
412
- attn = nn.ModuleList()
413
- block_out = ch*ch_mult[i_level]
414
- skip_in = ch*ch_mult[i_level]
415
- for i_block in range(self.num_res_blocks+1):
416
- if i_block == self.num_res_blocks:
417
- skip_in = ch*in_ch_mult[i_level]
418
- block.append(ResnetBlock(in_channels=block_in+skip_in,
419
- out_channels=block_out,
420
- temb_channels=self.temb_ch,
421
- dropout=dropout))
422
- block_in = block_out
423
- if curr_res in attn_resolutions:
424
- attn.append(AttnBlock(block_in))
425
- up = nn.Module()
426
- up.block = block
427
- up.attn = attn
428
- if i_level != 0:
429
- up.upsample = Upsample(block_in, resamp_with_conv)
430
- curr_res = curr_res * 2
431
- self.up.insert(0, up) # prepend to get consistent order
432
-
433
- # end
434
- self.norm_out = Normalize(block_in)
435
- self.conv_out = torch.nn.Conv2d(block_in,
436
- out_ch,
437
- kernel_size=3,
438
- stride=1,
439
- padding=1)
440
-
441
-
442
- def forward(self, x, t=None):
443
- #assert x.shape[2] == x.shape[3] == self.resolution
444
-
445
- if self.use_timestep:
446
- # timestep embedding
447
- assert t is not None
448
- temb = get_timestep_embedding(t, self.ch)
449
- temb = self.temb.dense[0](temb)
450
- temb = nonlinearity(temb)
451
- temb = self.temb.dense[1](temb)
452
- else:
453
- temb = None
454
-
455
- # downsampling
456
- hs = [self.conv_in(x)]
457
- for i_level in range(self.num_resolutions):
458
- for i_block in range(self.num_res_blocks):
459
- h = self.down[i_level].block[i_block](hs[-1], temb)
460
- if len(self.down[i_level].attn) > 0:
461
- h = self.down[i_level].attn[i_block](h)
462
- hs.append(h)
463
- if i_level != self.num_resolutions-1:
464
- hs.append(self.down[i_level].downsample(hs[-1]))
465
-
466
- # middle
467
- h = hs[-1]
468
- h = self.mid.block_1(h, temb)
469
- h = self.mid.attn_1(h)
470
- h = self.mid.block_2(h, temb)
471
-
472
- # upsampling
473
- for i_level in reversed(range(self.num_resolutions)):
474
- for i_block in range(self.num_res_blocks+1):
475
- h = self.up[i_level].block[i_block](
476
- torch.cat([h, hs.pop()], dim=1), temb)
477
- if len(self.up[i_level].attn) > 0:
478
- h = self.up[i_level].attn[i_block](h)
479
- if i_level != 0:
480
- h = self.up[i_level].upsample(h)
481
-
482
- # end
483
- h = self.norm_out(h)
484
- h = nonlinearity(h)
485
- h = self.conv_out(h)
486
- return h
487
-
488
- def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
489
- """
490
- embed_dim: output dimension for each position
491
- pos: a list of positions to be encoded: size (M,)
492
- out: (M, D)
493
- """
494
- assert embed_dim % 2 == 0
495
- omega = np.arange(embed_dim // 2, dtype=float)
496
- omega /= embed_dim / 2.
497
- omega = 1. / 10000**omega # (D/2,)
498
-
499
- pos = pos.reshape(-1) # (M,)
500
- out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
501
-
502
- emb_sin = np.sin(out) # (M, D/2)
503
- emb_cos = np.cos(out) # (M, D/2)
504
-
505
- emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
506
- return emb
507
-
508
- def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
509
- assert embed_dim % 2 == 0
510
-
511
- # use half of dimensions to encode grid_h
512
- emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
513
- emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
514
-
515
- emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
516
- return emb
517
-
518
- def get_2d_sincos_pos_embed(embed_dim, grid_size):
519
- """
520
- grid_size: int or (int, int) of the grid height and width
521
- return:
522
- pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
523
- """
524
- grid_size = (grid_size, grid_size) if type(grid_size) != tuple else grid_size
525
- grid_h = np.arange(grid_size[0], dtype=np.float32)
526
- grid_w = np.arange(grid_size[1], dtype=np.float32)
527
- grid = np.meshgrid(grid_w, grid_h) # here w goes first
528
- grid = np.stack(grid, axis=0)
529
-
530
- grid = grid.reshape([2, 1, grid_size[0], grid_size[1]])
531
- pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
532
-
533
- return pos_embed
534
-
535
- def init_weights(m):
536
- if isinstance(m, nn.Linear):
537
- # we use xavier_uniform following official JAX ViT:
538
- torch.nn.init.xavier_uniform_(m.weight)
539
- if m.bias is not None:
540
- nn.init.constant_(m.bias, 0)
541
- elif isinstance(m, nn.LayerNorm):
542
- nn.init.constant_(m.bias, 0)
543
- nn.init.constant_(m.weight, 1.0)
544
- elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
545
- with torch.no_grad():
546
- w = m.weight
547
- torch.nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
548
-
549
- class PreNorm(nn.Module):
550
- def __init__(self, dim: int, fn: nn.Module) -> None:
551
- super().__init__()
552
- self.norm = nn.LayerNorm(dim)
553
- self.fn = fn
554
-
555
- def forward(self, x: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
556
- return self.fn(self.norm(x), **kwargs)
557
-
558
- class FeedForward(nn.Module):
559
- def __init__(self, dim: int, hidden_dim: int) -> None:
560
- super().__init__()
561
- self.net = nn.Sequential(
562
- nn.Linear(dim, hidden_dim),
563
- nn.Tanh(),
564
- nn.Linear(hidden_dim, dim)
565
- )
566
-
567
- def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
568
- return self.net(x)
569
-
570
- class Attention(nn.Module):
571
- def __init__(self, dim: int, heads: int = 8, dim_head: int = 64) -> None:
572
- super().__init__()
573
- inner_dim = dim_head * heads
574
- project_out = not (heads == 1 and dim_head == dim)
575
-
576
- self.heads = heads
577
- self.scale = dim_head ** -0.5
578
-
579
- self.attend = nn.Softmax(dim = -1)
580
- self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
581
-
582
- self.to_out = nn.Linear(inner_dim, dim) if project_out else nn.Identity()
583
-
584
- def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
585
- qkv = self.to_qkv(x).chunk(3, dim = -1)
586
- q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)
587
-
588
- attn = torch.matmul(q, k.transpose(-1, -2)) * self.scale
589
- attn = self.attend(attn)
590
-
591
- out = torch.matmul(attn, v)
592
- out = rearrange(out, 'b h n d -> b n (h d)')
593
-
594
- return self.to_out(out)
595
-
596
- class Transformer(nn.Module):
597
- def __init__(self, dim: int, depth: int, heads: int, dim_head: int, mlp_dim: int) -> None:
598
- super().__init__()
599
- self.layers = nn.ModuleList([])
600
- for idx in range(depth):
601
- layer = nn.ModuleList([PreNorm(dim, Attention(dim, heads=heads, dim_head=dim_head)),
602
- PreNorm(dim, FeedForward(dim, mlp_dim))])
603
- self.layers.append(layer)
604
- self.norm = nn.LayerNorm(dim)
605
-
606
- def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
607
- for attn, ff in self.layers:
608
- x = attn(x) + x
609
- x = ff(x) + x
610
-
611
- return self.norm(x)
612
-
613
- class EncoderVIT(nn.Module):
614
- def __init__(self, image_size: Union[Tuple[int, int], int], patch_size: Union[Tuple[int, int], int],
615
- dim: int, depth: int, heads: int, mlp_dim: int, channels: int = 3, dim_head: int = 64) -> None:
616
- super().__init__()
617
-
618
- image_height, image_width = image_size if isinstance(image_size, tuple) \
619
- else (image_size, image_size)
620
- patch_height, patch_width = patch_size if isinstance(patch_size, tuple) \
621
- else (patch_size, patch_size)
622
- self.image_height = image_height
623
- self.image_width = image_width
624
- self.patch_height = patch_height
625
- self.patch_width = patch_width
626
- assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'
627
- en_pos_embedding = get_2d_sincos_pos_embed(dim, (image_height // patch_height, image_width // patch_width))
628
-
629
- self.num_patches = (image_height // patch_height) * (image_width // patch_width)
630
- self.patch_dim = channels * patch_height * patch_width
631
-
632
- self.to_patch_embedding = nn.Sequential(
633
- nn.Conv2d(channels, dim, kernel_size=patch_size, stride=patch_size),
634
- Rearrange('b c h w -> b (h w) c'),
635
- )
636
- self.en_pos_embedding = nn.Parameter(torch.from_numpy(en_pos_embedding).float().unsqueeze(0), requires_grad=False)
637
- self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim)
638
-
639
- self.apply(init_weights)
640
-
641
- def forward(self, img: torch.FloatTensor) -> torch.FloatTensor:
642
- x = self.to_patch_embedding(img)
643
- x = x + self.en_pos_embedding
644
- x = self.transformer(x) # (B, N, D)
645
- #import pdb; pdb.set_trace()
646
- x = x.reshape(x.shape[0], x.shape[1]//(self.image_height // self.patch_height), x.shape[1]//(self.image_width // self.patch_width), -1)
647
- x = x.permute(0, 3, 1, 2).contiguous()
648
- #import pdb; pdb.set_trace()
649
- return x
650
-
651
  class Encoder(nn.Module):
652
  def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
653
  attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
@@ -761,8 +284,8 @@ class Decoder(nn.Module):
761
  # compute in_ch_mult, block_in and curr_res at lowest res
762
  in_ch_mult = (1,)+tuple(ch_mult)
763
  block_in = ch*ch_mult[self.num_resolutions-1]
764
-
765
- curr_res_h, curr_res_w = (resolution[0] // 2**(self.num_resolutions-1), resolution[1] // 2**(self.num_resolutions-1)) if isinstance(resolution, (list, tuple, ListConfig)) else (resolution// 2**(self.num_resolutions-1) ,resolution// 2**(self.num_resolutions-1))
766
  self.z_shape = (1,z_channels,curr_res_h,curr_res_w)
767
 
768
  # z to block_in
@@ -783,7 +306,7 @@ class Decoder(nn.Module):
783
  out_channels=block_in,
784
  temb_channels=self.temb_ch,
785
  dropout=dropout)
786
-
787
  #self.decoder_dino_conv = nn.Conv2d(block_in, 768, kernel_size=1, stride=1, padding=0) # remove hard-coded emb_dim=768
788
 
789
  # upsampling
@@ -849,4 +372,3 @@ class Decoder(nn.Module):
849
  h = self.conv_out(h)
850
 
851
  return h
852
-
 
1
  # pytorch_diffusion + derived encoder decoder
 
 
 
 
2
  import torch
3
  import torch.nn as nn
4
 
 
 
5
  from omegaconf import ListConfig
6
 
7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  def nonlinearity(x):
9
  # swish
10
  return x*torch.sigmoid(x)
11
 
12
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  def Normalize(in_channels):
14
  return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
15
 
 
115
 
116
  return x+h
117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
 
119
  class AttnBlock(nn.Module):
120
  def __init__(self, in_channels):
 
169
  h_ = self.proj_out(h_)
170
 
171
  return x+h_
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
172
 
173
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  class Encoder(nn.Module):
175
  def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
176
  attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
 
284
  # compute in_ch_mult, block_in and curr_res at lowest res
285
  in_ch_mult = (1,)+tuple(ch_mult)
286
  block_in = ch*ch_mult[self.num_resolutions-1]
287
+
288
+ curr_res_h, curr_res_w = (resolution[0] // 2**(self.num_resolutions-1), resolution[1] // 2**(self.num_resolutions-1)) if isinstance(resolution, (list, tuple, ListConfig)) else (resolution// 2**(self.num_resolutions-1) ,resolution// 2**(self.num_resolutions-1))
289
  self.z_shape = (1,z_channels,curr_res_h,curr_res_w)
290
 
291
  # z to block_in
 
306
  out_channels=block_in,
307
  temb_channels=self.temb_ch,
308
  dropout=dropout)
309
+
310
  #self.decoder_dino_conv = nn.Conv2d(block_in, 768, kernel_size=1, stride=1, padding=0) # remove hard-coded emb_dim=768
311
 
312
  # upsampling
 
372
  h = self.conv_out(h)
373
 
374
  return h
 
orbis2/networks/tokenizer/pretrained_models.py CHANGED
@@ -1,8 +1,6 @@
1
  import torch
2
  import timm
3
  from torch import nn
4
- from omegaconf import ListConfig
5
- from einops import rearrange
6
 
7
 
8
  from typing import Tuple, Union
 
1
  import torch
2
  import timm
3
  from torch import nn
 
 
4
 
5
 
6
  from typing import Tuple, Union
orbis2/util.py CHANGED
@@ -1,179 +1,4 @@
1
- import argparse
2
- import hashlib
3
  import importlib
4
- import logging
5
- import os
6
- import requests
7
-
8
- from tqdm import tqdm
9
- from pytorch_lightning.trainer import Trainer
10
-
11
-
12
- def maybe_raise_nofile_limit(target_soft=32768, logger=None):
13
- """
14
- Raise RLIMIT_NOFILE soft limit for current process when possible.
15
-
16
- Returns True if the limit was raised, False otherwise.
17
- """
18
- if logger is None:
19
- logger = logging.getLogger(__name__)
20
- try:
21
- import resource
22
- except ImportError:
23
- logger.warning("resource module unavailable; cannot set RLIMIT_NOFILE")
24
- return False
25
- try:
26
- soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
27
- desired = target_soft
28
- if hard not in (resource.RLIM_INFINITY, -1):
29
- desired = min(desired, hard)
30
- if soft >= desired:
31
- return False
32
- resource.setrlimit(resource.RLIMIT_NOFILE, (desired, hard))
33
- logger.info("Raised RLIMIT_NOFILE from %s to %s", soft, desired)
34
- return True
35
- except (ValueError, PermissionError, OSError) as exc:
36
- logger.warning("Unable to raise RLIMIT_NOFILE: %s", exc)
37
- return False
38
-
39
- URL_MAP = {
40
- "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"
41
- }
42
-
43
- CKPT_MAP = {
44
- "vgg_lpips": "vgg.pth"
45
- }
46
-
47
- MD5_MAP = {
48
- "vgg_lpips": "d507d7349b931f0638a25a48a722f98a"
49
- }
50
-
51
-
52
- def download(url, local_path, chunk_size=1024):
53
- os.makedirs(os.path.split(local_path)[0], exist_ok=True)
54
- with requests.get(url, stream=True) as r:
55
- total_size = int(r.headers.get("content-length", 0))
56
- with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
57
- with open(local_path, "wb") as f:
58
- for data in r.iter_content(chunk_size=chunk_size):
59
- if data:
60
- f.write(data)
61
- pbar.update(chunk_size)
62
-
63
-
64
- def md5_hash(path):
65
- with open(path, "rb") as f:
66
- content = f.read()
67
- return hashlib.md5(content).hexdigest()
68
-
69
-
70
- def get_ckpt_path(name, root, check=False):
71
- assert name in URL_MAP
72
- path = os.path.join(root, CKPT_MAP[name])
73
- if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]):
74
- print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path))
75
- download(URL_MAP[name], path)
76
- md5 = md5_hash(path)
77
- assert md5 == MD5_MAP[name], md5
78
- return path
79
-
80
-
81
- class KeyNotFoundError(Exception):
82
- def __init__(self, cause, keys=None, visited=None):
83
- self.cause = cause
84
- self.keys = keys
85
- self.visited = visited
86
- messages = list()
87
- if keys is not None:
88
- messages.append("Key not found: {}".format(keys))
89
- if visited is not None:
90
- messages.append("Visited: {}".format(visited))
91
- messages.append("Cause:\n{}".format(cause))
92
- message = "\n".join(messages)
93
- super().__init__(message)
94
-
95
-
96
- def retrieve(
97
- list_or_dict, key, splitval="/", default=None, expand=True, pass_success=False
98
- ):
99
- """Given a nested list or dict return the desired value at key expanding
100
- callable nodes if necessary and :attr:`expand` is ``True``. The expansion
101
- is done in-place.
102
-
103
- Parameters
104
- ----------
105
- list_or_dict : list or dict
106
- Possibly nested list or dictionary.
107
- key : str
108
- key/to/value, path like string describing all keys necessary to
109
- consider to get to the desired value. List indices can also be
110
- passed here.
111
- splitval : str
112
- String that defines the delimiter between keys of the
113
- different depth levels in `key`.
114
- default : obj
115
- Value returned if :attr:`key` is not found.
116
- expand : bool
117
- Whether to expand callable nodes on the path or not.
118
-
119
- Returns
120
- -------
121
- The desired value or if :attr:`default` is not ``None`` and the
122
- :attr:`key` is not found returns ``default``.
123
-
124
- Raises
125
- ------
126
- Exception if ``key`` not in ``list_or_dict`` and :attr:`default` is
127
- ``None``.
128
- """
129
-
130
- keys = key.split(splitval)
131
-
132
- success = True
133
- try:
134
- visited = []
135
- parent = None
136
- last_key = None
137
- for key in keys:
138
- if callable(list_or_dict):
139
- if not expand:
140
- raise KeyNotFoundError(
141
- ValueError(
142
- "Trying to get past callable node with expand=False."
143
- ),
144
- keys=keys,
145
- visited=visited,
146
- )
147
- list_or_dict = list_or_dict()
148
- parent[last_key] = list_or_dict
149
-
150
- last_key = key
151
- parent = list_or_dict
152
-
153
- try:
154
- if isinstance(list_or_dict, dict):
155
- list_or_dict = list_or_dict[key]
156
- else:
157
- list_or_dict = list_or_dict[int(key)]
158
- except (KeyError, IndexError, ValueError) as e:
159
- raise KeyNotFoundError(e, keys=keys, visited=visited)
160
-
161
- visited += [key]
162
- # final expansion of retrieved value
163
- if expand and callable(list_or_dict):
164
- list_or_dict = list_or_dict()
165
- parent[last_key] = list_or_dict
166
- except KeyNotFoundError as e:
167
- if default is None:
168
- raise e
169
- else:
170
- list_or_dict = default
171
- success = False
172
-
173
- if not pass_success:
174
- return list_or_dict
175
- else:
176
- return list_or_dict, success
177
 
178
 
179
  def get_obj_from_str(string, reload=False):
@@ -184,31 +9,7 @@ def get_obj_from_str(string, reload=False):
184
  return getattr(importlib.import_module(module, package=None), cls)
185
 
186
 
187
-
188
- def get_jobid():
189
- try:
190
- jobid = os.environ["PBS_JOBID"].replace(".lmbtorque.informatik.uni-freiburg.de", "")
191
- except KeyError:
192
- try:
193
- jobid = "DLC" + os.environ["SLURM_JOB_ID"]
194
- except KeyError:
195
- try:
196
- jobid = os.uname()[1]
197
- except KeyError:
198
- jobid = "local"
199
- return jobid
200
-
201
-
202
- def nondefault_trainer_args(opt):
203
- parser = argparse.ArgumentParser()
204
- parser = Trainer.add_argparse_args(parser)
205
- args = parser.parse_args([])
206
- return sorted(k for k in vars(args) if getattr(opt, k) != getattr(args, k))
207
-
208
-
209
  def instantiate_from_config(config):
210
  if not "target" in config:
211
  raise KeyError("Expected key `target` to instantiate.")
212
  return get_obj_from_str(config["target"])(**config.get("params", dict()))
213
-
214
-
 
 
 
1
  import importlib
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
 
4
  def get_obj_from_str(string, reload=False):
 
9
  return getattr(importlib.import_module(module, package=None), cls)
10
 
11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  def instantiate_from_config(config):
13
  if not "target" in config:
14
  raise KeyError("Expected key `target` to instantiate.")
15
  return get_obj_from_str(config["target"])(**config.get("params", dict()))