RefineOnTestScene.yml

# Parameters to setup experiment.
experiment:
  # Experiment logs and checkpoints will be stored in:
  logdir: projects/VolumetricEnhance/logs/RefiningOnTestScenes
  # Seed for random number generators (for repeatability).
  randomseed: 0 # 34  # 8239
  # Number of training iterations.
  train_iters: 1500000
  # Evaluation round frequency:
  validate_every: [0.1,5000] # [Desired portion of evaluation time, compared to training time,Max training iterations between evaluation rounds]
  # Checkpoint saving frequency:
  save_every: 10.0 # Minutes (Use floating point numbers)
  # Number of training iterations after which to print progress:
  print_every: 100

# Dataset parameters.
dataset:
  # Per dataset type configurations:
  synt: # Synthetic scenes, from the NeRF dataset or using the same configuration
    root: datasets/Synthetic
    # Near clip plane (clip all depth values closer than this threshold).
    near: 2
    # Far clip plane (clip all depth values farther than this threshold).
    far:  6
    # Do not use NDC (normalized device coordinates). Usually True for
    # synthetic (Blender) datasets.
    no_ndc: True
  llff: # Real world scenes from NeRF:
    root: datasets/LLFF
    # Near clip plane (clip all depth values closer than this threshold).
    near: 0
    # Far clip plane (clip all depth values farther than this threshold).
    far:  1
    # Do not use NDC (normalized device coordinates). Usually True for
    # synthetic (Blender) datasets.
    no_ndc: False
    # If set, adding test views by interpolating camera poses, to allow rendering smoother high FPS video sequences:
    # min_eval_frames:  200
  max_scenes_eval:  2 # If set, using at most this number of evaluation scenes in evaluation rounds:
  # prob_assigned2scene_groups: False # (Default: True) When True, the probability value reflects the chance of sampling A (any) SCENE IN THE GROUP, and not the probabiliy of sampling any specific scene.
  dir:
    train:
      # Training scenes: 
      # Parameter convention:
      #   downsampling_factor,pos_feature_plane_resolution,viewdir_feature_plane_resoution,dataset_type (default:'synt'),relative probability (default:1): [scene names]
      # Adding suffix "##<integer>" to scene name allows training multiple independent feature plane sets to the same scene. E.g. ['lego','ship','lego##1','lego##2']
      # LR scenes (training and some of evaluation):
      # 8,200,32:   ['house','robot','ficus','materials','holiday','bugatti','motorbike','teddy','dragon','cola','donut','guitar','hotdog','engine','fruits','marble_fireplace','piano','plane','satellite','speakers','steamTrain','thuja','toad','triceratops','drums'] #'lego','chair',mic,ship
      # HR scenes (training only):
      2,800,32:   ['house','robot','holiday','bugatti','motorbike','teddy','dragon','cola','donut','guitar','engine','fruits','marble_fireplace','piano','plane','satellite','speakers','steamTrain','thuja','toad','triceratops','ficus','materials','hotdog','drums'] #,'lego','chair','mic','ship',
      # Evaluation scene(s) for image inconsistency loss term:
      # Should be added (with sampling probability set to 0) if not already included in LR scenes above
      8,200,32,'synt',0:   ['lego##1']
    val:
      # Evaluation scenes (Should not overlap with training scenes):
      # Use the same parameter convention as in training scenes, without sampling probability
      2,800,32:   ['lego##1']
  
  testskip: 10 # Stride for synthetic scenes: Include one evaluation image per "testskip" images in the dataset.
  llffhold: 2 # For real scenes: Number of training images to hold out for evaluation

# Model parameters.
models:
  # Path to pre-trained model checkpoint to resume (if set):
  path: projects/VolumetricEnhance/models/21TrainScenes_LRforFicusMaterialsHotdogDrums
  # Path to feature planes checkpoint for scenes missing from the pre-trained model path:
  planes_path:  projects/VolumetricEnhance/models/new_scene_planes
  use_existing_planes:  True # If True, favor feature plane sets from pre-trained model checkpoint folder over ones in planes_path folder, and keep them frozen (do not update). Default: False
  coarse:
    # Name of the torch.nn.Module class that implements the model.
    type: TwoDimPlanesModel
    # Number of planes to project to (3 and up, default is 3):
    # num_planes: 4
    # Planes interpolation method (default to bilinear):
    plane_interp: bilinear # bicubic # bilinear
    # Number of layers in the density decoder:
    dec_density_layers: 4
    # Number of layers in the rgb decoder:
    dec_rgb_layers: 4 #8
    # Nunber of channels per layer (for both density and radience branches):
    dec_channels: 128 #256 #128
    # Input to the rgb decoder, apart from view directions (if used):
    rgb_dec_input:  projections # projections, features
    # How to combine projections from different planes:
    proj_combination: avg #avg #concat #sum
    # How to combine projection from viewdir plane with the ones from position planes. Defaults to whatever is set in proj_combination:
    viewdir_proj_combination: concat_pos
    # Whether to align values of -1 and 1 with the actual grid corners when interpolating. If False, these values correspond to the center of corner array values, making the interpolation more resolution dependent.
    # align_corners: True
    # Number of hidden units in each layer of the MLP (multi-layer
    # perceptron).
    # Add a skip connection once in a while. Note: This parameter won't take affect unless num_layers > skip_connect_every:
    skip_connect_every: 3
    # Plane coniguration:
    # num_plane_channels: 24 (Default: 48)
    # num_viewdir_plane_channels: 48 (Defaults to the value in num_plane_channels)
  fine: # All missing paramers default to their corresponding setting for the coarse decoder
    # Name of the torch.nn.Module class that implements the model:
    # type: use_same
    type: TwoDimPlanesModel # Set to "use_same" to use a single model instance (same weights) for both coarse and fine models.
    # use_coarse_planes: True # Whether to use feature planes from the coarse decoder model. Default: True, False is not supported yet.
# Decoder and feature planes optimizer params:
optimizer:
  # Name of the torch.optim class used for optimization.
  type: Adam
  # Decoder model learning rate:
  lr: 5.E-4
  # Feature planes learing rate:
  # planes_lr: 1.E-3 # Defaults to value in lr

# NeRF parameters.
nerf:
  # Use viewing directions as input, in addition to the X, Y, Z coordinates.
  use_viewdirs: True
  # Normalize elevation and azimuth angle ranges to the range observed across training image frames
  # adjust_elevation_range: 1 # Default: False
  # adjust_azimuth_range: True # Default: False
  train: # Training-specific parameters:
    # Which modules to train:
    what: ['LR_planes','decoder','SR'] # Exclude modules to freeze their weights.
    # In the end-2-end learning scheme, don't propagate gradients all the way to LR planes in iterations using the SR model. Meaning, learn 
    # the LR planes corresponding to the training scenes independently of the SR model (This is anyway indepepent for the evaluation scenes,
    #  since HR images are not available for their training):
    # detach_LR_planes: False # Defaults to False
    # Number of random rays to retain from each image. These sampled rays are used for training, while the rest are discarded:
    num_random_rays: 4096 #3072
    # Size of each chunk (rays are batched into "chunks" and passed through the network):
    chunksize: 131072
    # Store and load plane parameters using hard drive to facilitate simultaneously training on many scenes with limited GPU memory:
    store_planes:
      # buffer_size:  10 # Number of different scenes to randomly choose from at each training iteration
      steps_per_buffer: 200 # Number of training iterations before re-drawing a new set of scenes
    # Whether or not to perturb the sampled depth values along each ray:
    perturb: True
    # Number of depth samples per ray for the coarse network:
    num_coarse: 64
    # Number of depth samples per ray for the fine network:
    num_fine: 64
    # Whether to render models using a white background:
    white_background: False
    # Penalty (loss) term weights:
    im_inconsistency_loss_w: 1 # Set to 0 to measure without effecting training. Default: None
    im_consistency_iters_freq:  4 # Sampling frequency for set of evaluation scenes for which image inconsistency loss is minimized.
    # Standard deviation of noise to be added to the radiance field when performing volume rendering.
    radiance_field_noise_std: 0.2
    # Sample linearly in disparity space, as opposed to in depth space.
    lindisp: False
  validation:   # Validation-specific parameters:
    chunksize: 131072
    # Whether or not to perturb the sampled depth values:
    perturb: False
    # Number of depth samples per ray for the coarse network:
    num_coarse: 64
    # Number of depth samples per ray for the fine network:
    num_fine: 64
    # Whether to render models using a white background:
    white_background: False
    # Standard deviation of noise to be added to the radiance field when performing volume rendering:
    radiance_field_noise_std: 0.
    # Sample linearly in disparity space, as opposed to in depth space.
    lindisp: False
    # Whether to add full training views to evaluation (different from the sparse ray sampling from these views during training):
    eval_train_scenes:  True # Defaults to False
super_resolution: # Feature planes Super-Resolution module parameters:
  lr: 5.E-5
  training:
    # Calculate the loss on the output of which decoder models:
    loss: fine # fine,coarse,both
  # Super-resolve feature planes before feeding into coarse decoder:
  apply_2_coarse: False #True # False
  # Super-resolve view-directions plane as well, or only positional planes:
  # SR_viewdir: False # Defaults to False, True is not supported yet.
  # Normalize feature planes (per channel) before feeding into SR module (Can only be used when training the SR model using frozen feature planes):
  # input_normalization:  True # Dafault: False
  # Add Gaussian noise to SR input, with STD relative to the plane's STD:
  # Planes interpolation method (default to fine model plane_interp):
  # plane_resize_mode:  bicubic
  model:
    # Path to pre-trained model checkpoint to resume (if set. Otherwise defaults to path set to decoder model, if set):
    path: projects/VolumetricEnhance/models/21TrainScenes_LRforFicusMaterialsHotdogDrums
    # Name of the torch.nn.Module class that implements the model.
    type: EDSR # Can be set to None to test the effect of image inconsistency loss without super-resolving planes
    # type: None
    # Number of hidden units in each layer of the MLP (multi-layer perceptron):
    hidden_size: 256
    # Number of residual blocks (originally 32):
    n_blocks: 32
    # Limit model's receptive field size. Using 1x1 convolutions once reaching the limit:
    # receptive_field_bound:  55 # Defaults to no limit
    # Effective scale factor for feature planes SR: Can be a function of the ratio between downsceling-factors in the training scenes ('sqrt','linear') or an explicit integer. Defaults to 'linear'.
    # scale_factor: 4 #sqrt #linear