ViLBERT: dual-stream vision-language transformer — morie_geron

Two separate streams exchange info only through swapped queries: image queries over text keys/values -> attention_v2t; text queries over image keys/values -> attention_t2v (both via morie_geron_scaled_dot_product).

Usage

morie_geron_vilbert(image, text, d_model = 8, seed = 0)

Arguments

image: Region features (n_regions, d_v), or (H,W) map flattened to single-feature regions.
text: Token features (n_tokens, d_t), or 1-D vector of 0-based token ids.
d_model: Shared co-attention width.
seed: LCG seed.

Value

list with image_out, text_out, attention_v2t, attention_t2v, pooled, image_hidden, text_hidden, n_regions, n_tokens, estimate, n, method.

Examples

V <- c(1, 2, 3, 4, 5, 6, 7, 8)
morie_geron_vilbert(V, V)
#> $image_out
#>            [,1]        [,2]       [,3]      [,4]      [,5]       [,6]
#> [1,] -0.1737620 -0.04682726 -0.3557973 0.3357759 0.2118172 -0.4651850
#> [2,] -0.4489470 -0.08609337 -0.6591117 0.5744898 0.4316079 -0.8473989
#> [3,] -0.7238560 -0.12544159 -0.9626725 0.8134026 0.6513476 -1.2297115
#> [4,] -0.9984484 -0.16487182 -1.2665011 1.0525255 0.8710313 -1.6121292
#> [5,] -1.2726906 -0.20438313 -1.5706141 1.2918666 1.0906551 -1.9946572
#> [6,] -1.5465557 -0.24397388 -1.8750235 1.5314311 1.3102164 -2.3772988
#> [7,] -1.8200238 -0.28364180 -2.1797367 1.7712212 1.5297133 -2.7600563
#> [8,] -2.0930816 -0.32338411 -2.4847573 2.0112367 1.7491452 -3.1429303
#>            [,7]       [,8]
#> [1,] -0.1539569 -0.1264381
#> [2,] -0.4554143 -0.1540211
#> [3,] -0.7566471 -0.1816581
#> [4,] -1.0576278 -0.2093538
#> [5,] -1.3583337 -0.2371122
#> [6,] -1.6587470 -0.2649365
#> [7,] -1.9588551 -0.2928287
#> [8,] -2.2586497 -0.3207900
#> 
#> $text_out
#>           [,1]      [,2]      [,3]     [,4]      [,5]         [,6]         [,7]
#> [1,] 0.2550455 0.7470417 -1.896278 1.857987 0.8828972  0.620738901 -0.546545743
#> [2,] 1.2053002 0.5217899 -1.700161 2.244510 0.6217006 -0.070981119 -0.751442990
#> [3,] 0.4447238 0.8778800 -1.734092 2.015038 0.6370587  0.564051655  0.119963322
#> [4,] 0.6380224 1.1447919 -1.903784 2.154846 0.8283947  0.803244880  0.068518049
#> [5,] 1.3417417 1.2024944 -2.316098 2.189107 1.1343983  0.003245962 -0.344887174
#> [6,] 0.9506291 1.0783505 -2.083567 1.769197 1.3719502  0.060746782  0.004168058
#> [7,] 0.6592041 0.1848208 -1.883240 1.663175 1.1413069  0.205820174 -0.648339227
#> [8,] 1.1195228 1.1124737 -2.328864 1.693530 1.3827041  0.620926582 -0.359339862
#>            [,8]
#> [1,] -0.7245541
#> [2,] -1.5592962
#> [3,] -1.6527483
#> [4,] -1.1006000
#> [5,] -1.7689118
#> [6,] -1.8752912
#> [7,] -1.1544686
#> [8,] -1.4554904
#> 
#> $attention_v2t
#>           [,1]      [,2]      [,3]      [,4]       [,5]       [,6]      [,7]
#> [1,] 0.1334466 0.1289032 0.1292430 0.1291574 0.11537344 0.11737139 0.1291350
#> [2,] 0.1420684 0.1325593 0.1332590 0.1330825 0.10619256 0.10990233 0.1330364
#> [3,] 0.1508394 0.1359514 0.1370292 0.1367571 0.09747866 0.10263106 0.1366860
#> [4,] 0.1597344 0.1390668 0.1405388 0.1401667 0.08924652 0.09559099 0.1400697
#> [5,] 0.1687294 0.1418966 0.1437765 0.1433009 0.08150455 0.08881042 0.1431769
#> [6,] 0.1778022 0.1444357 0.1467350 0.1461527 0.07425512 0.08231233 0.1460009
#> [7,] 0.1869324 0.1466825 0.1494104 0.1487189 0.06749509 0.07611444 0.1485387
#> [8,] 0.1961019 0.1486386 0.1518020 0.1509993 0.06121637 0.07022939 0.1507902
#>            [,8]
#> [1,] 0.11736989
#> [2,] 0.10989952
#> [3,] 0.10262712
#> [4,] 0.09558610
#> [5,] 0.08880474
#> [6,] 0.08230601
#> [7,] 0.07610763
#> [8,] 0.07022220
#> 
#> $attention_t2v
#>            [,1]      [,2]      [,3]      [,4]      [,5]      [,6]       [,7]
#> [1,] 0.16586629 0.1521378 0.1395456 0.1279957 0.1174017 0.1076846 0.09877171
#> [2,] 0.11438662 0.1172691 0.1202242 0.1232537 0.1263596 0.1295438 0.13280818
#> [3,] 0.12761774 0.1268609 0.1261086 0.1253607 0.1246172 0.1238782 0.12314352
#> [4,] 0.13278091 0.1304797 0.1282184 0.1259963 0.1238127 0.1216670 0.11955843
#> [5,] 0.09788507 0.1046209 0.1118202 0.1195149 0.1277392 0.1365294 0.14592441
#> [6,] 0.10834848 0.1127327 0.1172943 0.1220406 0.1269788 0.1321169 0.13746292
#> [7,] 0.14668485 0.1398958 0.1334209 0.1272458 0.1213564 0.1157396 0.11038279
#> [8,] 0.12288913 0.1234864 0.1240866 0.1246896 0.1252957 0.1259046 0.12651655
#>            [,8]
#> [1,] 0.09059655
#> [2,] 0.13615484
#> [3,] 0.12241322
#> [4,] 0.11748641
#> [5,] 0.15596598
#> [6,] 0.14302524
#> [7,] 0.10527390
#> [8,] 0.12713144
#> 
#> $pooled
#> [1] -0.1539485  0.3369391 -1.7000187  1.5605837  0.9903715 -0.7263483 -0.7572586
#> [8] -0.8174062
#> 
#> $image_hidden
#>            [,1]        [,2]       [,3]      [,4]      [,5]       [,6]    [,7]
#> [1,] -0.2631569 -0.04002556 -0.3110497 0.2418996 0.2170099 -0.3827708 -0.2942
#> [2,] -0.5263138 -0.08005112 -0.6220994 0.4837993 0.4340198 -0.7655416 -0.5884
#> [3,] -0.7894708 -0.12007668 -0.9331491 0.7256989 0.6510297 -1.1483123 -0.8826
#> [4,] -1.0526277 -0.16010224 -1.2441988 0.9675985 0.8680395 -1.5310831 -1.1768
#> [5,] -1.3157846 -0.20012781 -1.5552485 1.2094981 1.0850494 -1.9138539 -1.4710
#> [6,] -1.5789415 -0.24015337 -1.8662982 1.4513978 1.3020593 -2.2966247 -1.7652
#> [7,] -1.8420985 -0.28017893 -2.1773479 1.6932974 1.5190692 -2.6793954 -2.0594
#> [8,] -2.1052554 -0.32020449 -2.4883976 1.9351970 1.7360791 -3.0621662 -2.3536
#>             [,8]
#> [1,] -0.02765847
#> [2,] -0.05531694
#> [3,] -0.08297540
#> [4,] -0.11063387
#> [5,] -0.13829234
#> [6,] -0.16595081
#> [7,] -0.19360927
#> [8,] -0.22126774
#> 
#> $text_hidden
#>             [,1]       [,2]       [,3]        [,4]        [,5]        [,6]
#> [1,] -0.48350676  0.1392460 -0.2495489 -0.07762223  0.09069020  0.33693366
#> [2,]  0.36088709 -0.1731245  0.1826027  0.03145911 -0.28405799 -0.39546577
#> [3,] -0.37018490  0.2072464  0.0828871 -0.12068757 -0.23705200  0.25090473
#> [4,] -0.16584897  0.4832416 -0.1114151  0.04804718 -0.03387678  0.49433931
#> [5,]  0.45773013  0.4749922 -0.3450422 -0.12772476  0.18616448 -0.33645529
#> [6,]  0.09211465  0.3718314 -0.1693611 -0.48081091  0.45106592 -0.26915661
#> [7,] -0.11614045 -0.4532533 -0.1544763 -0.36886014  0.30963468 -0.09212333
#> [8,]  0.29427807  0.4333340 -0.4888396 -0.46928447  0.49750654  0.30380782
#>             [,7]       [,8]
#> [1,] -0.26492408  0.4808460
#> [2,] -0.42945490 -0.1811193
#> [3,]  0.43070092 -0.3227259
#> [4,]  0.37504692  0.2114082
#> [5,] -0.00779956 -0.3261057
#> [6,]  0.33153319 -0.4740994
#> [7,] -0.35268808  0.1109806
#> [8,] -0.04466100 -0.1085986
#> 
#> $n_regions
#> [1] 8
#> 
#> $n_tokens
#> [1] 8
#> 
#> $estimate
#> [1] 0.1961019
#> 
#> $n
#> [1] 16
#> 
#> $method
#> [1] "Dual-stream co-attention: image queries over text keys/values and vice versa (hmsdp)"
#>