checkpoint:
  monitor: monitoring_step
  mode: max
  dirpath: null
  save_top_k: 10
data:
  dataset:
    name: LRS3
    name_train: LRS3
    name_val: LRS3
    name_test: LRS3
    train_csv: null
    train_labelled_csv: null
    val_csv: null
    test_csv: null
  channel:
    obj:
      _target_: pytorchvideo.transforms.Normalize
      mean:
      - 0.421
      std:
      - 0.165
    in_video_channels: 1
    grayscale_prob: 0.0
  crop_type:
    random_crop_dim: 88
    resize_dim: 88
    random_erasing_prob: 0.0
    random_erasing_scale:
    - 0.02
    - 0.33
    name: mouth
    video_dir: cropped_mouths_uncompressed_gray_with_audio
  horizontal_flip_prob: 0.5
  use_masking: true
  mask_prob_video: 0.2
  mask_prob_audio: 0.4
  mask_length_video: 3
  mask_length_audio: ${data.mask_length_video}
  frames_per_gpu: 2400
  frames_per_gpu_labelled: 600
  frames_per_gpu_val: 500
  timemask_window: 0.4
  timemask_stride: 1
  timemask_window_audio: 0.6
  timemask_stride_audio: 2
  lrs2_video_dir: null
  lrs2_audio_dir: null
  lrs3_video_dir: null
  lrs3_audio_dir: null
  vox2_video_dir: null
  vox2_audio_dir: null
  skip_fails: true
debug:
  log_gradients: false
  deterministic: false
  profiler: null
  save_av_sample: false
decode:
  name: default
  penalty: 0.0
  ctc_weight: 0.1
  lm_weight: 0.0
  beam_size: 40
  minlenratio: 0.0
  maxlenratio: 0.0
  snr_target: 9999
logger:
  _target_: pytorch_lightning.loggers.WandbLogger
  log_model: false
  name: ${experiment_name}
  offline: false
  project: ${project_name}
  mode: online
  entity: default
logging:
  logging_interval: step
model:
  backbone:
    idim: 512
    adim: 512
    aheads: 8
    eunits: 2048
    elayers: 12
    transformer_input_layer: only_pos
    dropout_rate: 0.1
    transformer_attn_dropout_rate: 0.1
    transformer_encoder_attn_layer_type: rel_mha
    macaron_style: false
    use_cnn_module: false
    cnn_module_kernel: 31
    zero_triu: false
    a_upsample_ratio: 1
    relu_type: swish
    d_idim: 1049
    d_odim: null
    ddim: ${model.backbone.adim}
    dheads: ${model.backbone.aheads}
    dunits: ${model.backbone.eunits}
    dlayers: 6
    d_input_layer: embed
    d_avg_branches: ${model.avg_branches}
    d_soft_inputs: ${model.soft_inputs}
    lsm_weight: 0.1
    lsm_weight_ctc: 0.0
    transformer_length_normalized_loss: false
    rel_pos_type: latest
    layerscale: true
    init_values: 0.1
    ff_bn_pre: false
    post_norm: false
    gamma_zero: false
    gamma_init: 0.1
    mask_init_type: null
    ctc_type: warpctc
    drop_path: 0.0
    encoder_stride: 1
    ctc_rel_weight: ${model.ctc_rel_weight}
  predictor_2a:
    _target_: espnet.nets.pytorch_backend.transformer.encoder.Encoder
    idim: ${model.backbone.adim}
    attention_dim: 512
    attention_heads: 8
    linear_units: 2048
    num_blocks: 2
    input_layer: vanilla_linear
    dropout_rate: 0.1
    attention_dropout_rate: 0.1
    encoder_attn_layer_type: rel_mha
    macaron_style: false
    use_cnn_module: false
    zero_triu: false
    a_upsample_ratio: 1
    ff_bn_pre: true
    post_norm: false
    mask_init_type: normal
    last_linear: true
    odim: ${model.target_dim}
    multi_pred: false
    predictor_stride: 1
    conv_sub: false
    mlp_sub: false
    separate_preds: false
  predictor_a2a:
    _target_: espnet.nets.pytorch_backend.transformer.encoder.Encoder
    idim: ${model.backbone.adim}
    attention_dim: 512
    attention_heads: 8
    linear_units: 2048
    num_blocks: 2
    input_layer: token_embed
    dropout_rate: 0.1
    attention_dropout_rate: 0.1
    encoder_attn_layer_type: rel_mha
    macaron_style: false
    use_cnn_module: false
    zero_triu: false
    a_upsample_ratio: 1
    ff_bn_pre: false
    post_norm: false
    mask_init_type: normal
    last_linear: true
    odim: ${model.target_dim}
    multi_pred: false
    predictor_stride: 1
    conv_sub: false
    mlp_sub: false
    separate_preds: false
  predictor_v2a:
    _target_: espnet.nets.pytorch_backend.transformer.encoder.Encoder
    idim: ${model.backbone.adim}
    attention_dim: 512
    attention_heads: 8
    linear_units: 2048
    num_blocks: 2
    input_layer: token_embed
    dropout_rate: 0.1
    attention_dropout_rate: 0.1
    encoder_attn_layer_type: rel_mha
    macaron_style: false
    use_cnn_module: false
    zero_triu: false
    a_upsample_ratio: 1
    ff_bn_pre: false
    post_norm: false
    mask_init_type: normal
    last_linear: true
    odim: ${model.target_dim}
    multi_pred: false
    predictor_stride: 1
    conv_sub: false
    mlp_sub: false
    separate_preds: false
  momentum_scheduler:
    _target_: schedulers.warmup_cosine.WarmupCosineMomentumScheduler
    base_momentum: 0.999
    warmup_epochs: 0
    cosine_decay: true
    num_epochs: ${trainer.max_epochs}
  language_model:
    pos_enc: none
    embed_unit: 128
    att_unit: 512
    head: 8
    unit: 2048
    layer: 16
    dropout_rate: 0.0
  obj:
    _target_: models.usr.USR
  use_predictor: true
  use_global: false
  sync_batchnorm: true
  audio_steps_per_video_step: 1
  target_dropout_off: true
  masked_only_loss: false
  pretrained_model_path: /data4/jrichter/logs/usr_logs/checkpoints/base_high_resource_lrs3.pth
  transfer_only_encoder: false
  v2a_weight: 0
  unlab_rel_weight_v: 0.5
  unlab_rel_weight_a: 0.5
  ctc_rel_weight: 0.1
  v_rel_weight: 0.5
  size_average: false
  return_feats: null
  norm_targets: null
  avg_feats: true
  average_targets: false
  flip_targets: true
  average_pool_kernel: 1
  predictor_stride: 1
  keep_last_avg_pooled: false
  target_dim: ${model.backbone.adim}
  before_norm_targets: false
  after_norm_targets: true
  a2a_pred_to_a2v: false
  avg_branches: false
  pretrained_path: null
  use_audio_targets: false
  ce_loss: false
  progressive_growing: false
  conf_threshold: 0.8
  conf_threshold_a: ${model.conf_threshold}
  conf_threshold_ctc: ${model.conf_threshold}
  conf_threshold_ctc_a: ${model.conf_threshold_a}
  seqlen_max: 120
  seqlen_linear_epochs: ${trainer.max_epochs}
  soft_inputs: false
  loss_mean: false
  mask_feats_loss: false
  pretrained_lm_path: null
  avg_ckpts: 10
  pred_block_idx: 1
  detach: false
optimizer:
  optim:
    obj:
      _target_: torch.optim.AdamW
      betas:
      - 0.9
      - 0.999
    weight_decay: 0.04
    scale_sqrt: true
  base_lr: 0.001
  base_lr_other: ${optimizer.base_lr}
  base_lr_decoder: ${optimizer.base_lr}
  lr_decay_rate: 1.0
  min_lr: 1.0e-05
  cosine_decay: true
  warmup_epochs: 60
  gradient_clip_val: 3.0
  weight_decay: 0.04
  betas:
  - 0.9
  - 0.98
trainer:
  precision: 32
  accelerator: auto
  strategy: ddp_find_unused_parameters_true
  devices: auto
  profiler: ${debug.profiler}
  max_epochs: 150
  num_nodes: 1
  sync_batchnorm: ${model.sync_batchnorm}
  num_sanity_val_steps: 10
  limit_val_batches: 1.0
  accumulate_grad_batches: 1
  use_distributed_sampler: false
project_name: semi_supervised_usr
experiment_name: null
num_workers: 8
log_wandb: true
replace_sampler_ddp: false
compile_model: true
fix_seed: true
matmul_precision: highest
ckpt_path: null
gpus: null
test: false
test_avg: false
txt_file: null
slurm_job_id: null
