VideoBERT: transformer on discretized video tokens + text

Joint MLM: video/text tokens embedded into one sequence with modality-typed embeddings, one joint self-attention block (morie_geron_scaled_dot_product), softmax head over the joint vocabulary scores masked positions.

Usage

morie_geron_videobert(
  video_tokens,
  text_tokens,
  d_model = 8,
  mask_positions = NULL,
  mask_prob = 0.25,
  seed = 0
)

Arguments

video_tokens, text_tokens

Integer (0-based) token id vectors.

d_model

Embedding width.

mask_positions

Optional 0-based positions to mask.

mask_prob

Masking rate when mask_positions is NULL.

seed

LCG seed.

Value

list with loss, token_losses, attention, hidden, masked, predictions, targets, cross_modal_mass, n_video, n_text, vocab_size, estimate, n, method.

Examples

V <- c(1, 2, 3, 4, 5, 6, 7, 8)
morie_geron_videobert(V, V)
#> $loss
#> [1] 2.890217
#> 
#> $token_losses
#> [1] 2.923328 2.895511 2.854439 2.887588
#> 
#> $attention
#>             [,1]       [,2]       [,3]       [,4]       [,5]       [,6]
#>  [1,] 0.06234669 0.06260770 0.06236417 0.06250342 0.06234669 0.06211539
#>  [2,] 0.06277852 0.07070283 0.06055315 0.06024513 0.06277852 0.05750381
#>  [3,] 0.06191203 0.06344729 0.06252854 0.05957996 0.06191203 0.06397559
#>  [4,] 0.06225200 0.05777808 0.06239027 0.06299408 0.06225200 0.06507717
#>  [5,] 0.06234669 0.06260770 0.06236417 0.06250342 0.06234669 0.06211539
#>  [6,] 0.06336216 0.05738877 0.06703166 0.06363844 0.06336216 0.06865069
#>  [7,] 0.06420733 0.06135560 0.06309788 0.06265115 0.06420733 0.05622713
#>  [8,] 0.06357229 0.05642209 0.06560831 0.06399558 0.06357229 0.06669578
#>  [9,] 0.06234669 0.06260770 0.06236417 0.06250342 0.06234669 0.06211539
#> [10,] 0.06206120 0.06741296 0.06309567 0.06320145 0.06206120 0.05970870
#> [11,] 0.06323270 0.06148347 0.06301940 0.06334407 0.06323270 0.06155737
#> [12,] 0.06178512 0.06496285 0.06162171 0.06003306 0.06178512 0.06441523
#> [13,] 0.06234669 0.06260770 0.06236417 0.06250342 0.06234669 0.06211539
#> [14,] 0.06129612 0.07200129 0.06134834 0.05973281 0.06129612 0.06075968
#> [15,] 0.06088916 0.06329169 0.06097947 0.06034425 0.06088916 0.06912402
#> [16,] 0.06310664 0.05455037 0.06559004 0.06691998 0.06310664 0.06698659
#>             [,7]       [,8]       [,9]      [,10]      [,11]      [,12]
#>  [1,] 0.06261806 0.06326770 0.06234669 0.06186291 0.06238849 0.06384437
#>  [2,] 0.05941571 0.05864619 0.06277852 0.06794129 0.06461071 0.06103949
#>  [3,] 0.06725202 0.06304432 0.06191203 0.06194803 0.06134560 0.06469313
#>  [4,] 0.06590349 0.06445166 0.06225200 0.05836437 0.06081580 0.06512421
#>  [5,] 0.06261806 0.06326770 0.06234669 0.06186291 0.06238849 0.06384437
#>  [6,] 0.06212848 0.05785910 0.06336216 0.06676347 0.06273709 0.05134911
#>  [7,] 0.06419903 0.05580539 0.06420733 0.06380901 0.06054311 0.06438335
#>  [8,] 0.06368061 0.05801990 0.06357229 0.06361854 0.06211696 0.05495398
#>  [9,] 0.06261806 0.06326770 0.06234669 0.06186291 0.06238849 0.06384437
#> [10,] 0.05707315 0.06374320 0.06206120 0.06529609 0.06507455 0.06183737
#> [11,] 0.06165265 0.05972000 0.06323270 0.06379965 0.06341445 0.05983570
#> [12,] 0.06538360 0.06367125 0.06178512 0.06207593 0.06221760 0.06326694
#> [13,] 0.06261806 0.06326770 0.06234669 0.06186291 0.06238849 0.06384437
#> [14,] 0.05947580 0.06450738 0.06129612 0.06630206 0.06484587 0.06269944
#> [15,] 0.06533210 0.06814294 0.06088916 0.06011536 0.06240347 0.06267677
#> [16,] 0.06067174 0.06232906 0.06310664 0.06128292 0.06206682 0.05689002
#>            [,13]      [,14]      [,15]      [,16]
#>  [1,] 0.06234669 0.06210614 0.06324037 0.06169452
#>  [2,] 0.06277852 0.06838014 0.06154540 0.05830207
#>  [3,] 0.06191203 0.06300118 0.06097337 0.06056286
#>  [4,] 0.06225200 0.05865643 0.06424513 0.06519132
#>  [5,] 0.06234669 0.06210614 0.06324037 0.06169452
#>  [6,] 0.06336216 0.06390356 0.05484351 0.07025752
#>  [7,] 0.06420733 0.06519387 0.06493148 0.06097370
#>  [8,] 0.06357229 0.06209771 0.05840886 0.07009252
#>  [9,] 0.06234669 0.06210614 0.06324037 0.06169452
#> [10,] 0.06206120 0.06496123 0.06225734 0.05809350
#> [11,] 0.06323270 0.06319822 0.06181961 0.06422458
#> [12,] 0.06178512 0.06286713 0.06099681 0.06134743
#> [13,] 0.06234669 0.06210614 0.06324037 0.06169452
#> [14,] 0.06129612 0.06665741 0.06033019 0.05615527
#> [15,] 0.06088916 0.05993311 0.06027472 0.06382548
#> [16,] 0.06310664 0.05901063 0.06093731 0.07033796
#> 
#> $hidden
#>             [,1]       [,2]         [,3]       [,4]        [,5]        [,6]
#>  [1,] 0.07662206 0.01453336 -0.011564434 0.04364114 -0.02041569 -0.04245240
#>  [2,] 0.07277634 0.01247264 -0.011645750 0.03808767 -0.02462944 -0.03880539
#>  [3,] 0.07854634 0.01427021 -0.009037411 0.04220079 -0.01875161 -0.04237459
#>  [4,] 0.07931717 0.01562571 -0.010434029 0.04625381 -0.01699471 -0.04439898
#>  [5,] 0.07662206 0.01453336 -0.011564434 0.04364114 -0.02041569 -0.04245240
#>  [6,] 0.09326956 0.01696288 -0.009195017 0.05580598 -0.02053881 -0.04638594
#>  [7,] 0.07008954 0.01598972 -0.009279117 0.03820679 -0.02164262 -0.03824611
#>  [8,] 0.08845365 0.01688367 -0.009003701 0.05293239 -0.01905115 -0.04548572
#>  [9,] 0.07662206 0.01453336 -0.011564434 0.04364114 -0.02041569 -0.04245240
#> [10,] 0.07498822 0.01335875 -0.014173384 0.04309282 -0.02467242 -0.04143705
#> [11,] 0.07882931 0.01504123 -0.010923475 0.04574460 -0.02106253 -0.04275441
#> [12,] 0.07945841 0.01355663 -0.010038080 0.04328967 -0.01922781 -0.04281526
#> [13,] 0.07662206 0.01453336 -0.011564434 0.04364114 -0.02041569 -0.04245240
#> [14,] 0.07545417 0.01172462 -0.012943647 0.04014074 -0.02412945 -0.04076959
#> [15,] 0.08485674 0.01336893 -0.011002608 0.04795651 -0.01732796 -0.04610145
#> [16,] 0.08727396 0.01721994 -0.011923883 0.05516671 -0.01923911 -0.04686737
#>             [,7]        [,8]
#>  [1,] 0.07036297 -0.02610114
#>  [2,] 0.06400878 -0.02042306
#>  [3,] 0.06908639 -0.02426873
#>  [4,] 0.07326094 -0.02914190
#>  [5,] 0.07036297 -0.02610114
#>  [6,] 0.07463782 -0.02797226
#>  [7,] 0.06507793 -0.02357557
#>  [8,] 0.07375515 -0.02877434
#>  [9,] 0.07036297 -0.02610114
#> [10,] 0.06918738 -0.02411803
#> [11,] 0.07013524 -0.02629951
#> [12,] 0.06953923 -0.02454714
#> [13,] 0.07036297 -0.02610114
#> [14,] 0.06684391 -0.02114673
#> [15,] 0.07393949 -0.02735288
#> [16,] 0.07720313 -0.03179351
#> 
#> $masked
#> [1]  0  4  8 12
#> 
#> $predictions
#> [1] 3 3 3 3
#> 
#> $targets
#> [1]  1  5 10 14
#> 
#> $cross_modal_mass
#> [1] 0.5009885
#> 
#> $n_video
#> [1] 8
#> 
#> $n_text
#> [1] 8
#> 
#> $vocab_size
#> [1] 18
#> 
#> $estimate
#> [1] 2.890217
#> 
#> $n
#> [1] 16
#> 
#> $method
#> [1] "Joint MLM over concatenated video/text tokens with modality embeddings and shared self-attention"
#>