BERT encoder forward pass with MLM + NSP heads (Geron Ch 15, hmbert)
Source:R/geron_ml_native.R
morie_geron_bert.RdToken + learned positional embeddings, n_layers distinct post-layernorm
blocks, an MLM head tied to the embedding matrix, and an NSP head on the
pooled first position. Masked positions come from the same LCG draw as
Python and are returned 0-based.
Usage
morie_geron_bert(
X,
n_layers = 2,
n_heads = 2,
d_model = 8,
vocab_size = NULL,
d_ff = NULL,
mask_prob = 0.15,
seed = 0
)Value
List with hidden, mlm_loss, mlm_losses, masked_positions,
nsp_logits, attentions, embeddings, n_params, vocab_size.
Examples
V <- c(1, 2, 3, 4, 5, 6, 7, 8)
morie_geron_bert(V)
#> $hidden
#> , , 1
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.7206043 -0.756603 1.024037 1.184145 0.8523242 -0.568885 0.8013662
#> [,8]
#> [1,] -1.871561
#>
#> , , 2
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.4307573 0.8077563 0.5081968 0.691331 2.165906 0.7486641 -0.6930678
#> [,8]
#> [1,] -0.1566506
#>
#> , , 3
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] -1.519815 0.6629509 0.1961472 -1.509026 -0.6175178 1.02497 1.791691
#> [,8]
#> [1,] 0.2380827
#>
#> , , 4
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] -1.798969 -1.201311 -0.6182665 1.186278 -1.242077 -1.33348 1.019615
#> [,8]
#> [1,] 1.253153
#>
#> , , 5
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] -0.02581466 1.28938 -1.883692 -1.470411 -0.02347788 0.5355233 -0.7094825
#> [,8]
#> [1,] -0.3888582
#>
#> , , 6
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.9636581 -0.5065857 -1.033637 0.3908979 -0.7338321 1.48264 -0.6179425
#> [,8]
#> [1,] 0.7257015
#>
#> , , 7
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.4362682 -1.358069 1.00982 -0.1714439 -0.06167361 -1.037117 -1.283368
#> [,8]
#> [1,] -0.9361978
#>
#> , , 8
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.7933103 1.062482 0.7973954 -0.3017713 -0.3396518 -0.8523146 -0.3088121
#> [,8]
#> [1,] 1.13633
#>
#>
#> $mlm_loss
#> [1] 2.57099
#>
#> $mlm_losses
#> [1] 2.57099
#>
#> $masked_positions
#> $masked_positions[[1]]
#> [1] 0
#>
#>
#> $nsp_logits
#> [,1] [,2]
#> [1,] -0.2312883 0.230359
#>
#> $attentions
#> $attentions[[1]]
#> $attentions[[1]][[1]]
#> , , 1
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1250220 0.1249783 0.1250049 0.1250329 0.1250217 0.1250030 0.1249936
#> [2,] 0.1249781 0.1250015 0.1249844 0.1249926 0.1249889 0.1249828 0.1250165
#> [,8]
#> [1,] 0.1249768
#> [2,] 0.1249959
#>
#> , , 2
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1249888 0.1250217 0.1250051 0.1249835 0.1250041 0.1250186 0.1250410
#> [2,] 0.1250070 0.1249818 0.1250105 0.1250094 0.1249953 0.1250032 0.1249912
#> [,8]
#> [1,] 0.1249624
#> [2,] 0.1250148
#>
#> , , 3
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1250247 0.1249689 0.1249981 0.1250392 0.1250179 0.1249853 0.1249591
#> [2,] 0.1249798 0.1249999 0.1249843 0.1249574 0.1249942 0.1250163 0.1249901
#> [,8]
#> [1,] 0.1250215
#> [2,] 0.1250037
#>
#> , , 4
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1250160 0.1249798 0.1249947 0.1250183 0.1249981 0.1249803 0.1249534
#> [2,] 0.1249837 0.1250258 0.1249764 0.1249943 0.1249956 0.1249957 0.1250238
#> [,8]
#> [1,] 0.1250264
#> [2,] 0.1249862
#>
#> , , 5
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1250205 0.1249730 0.1250013 0.1250321 0.1250137 0.1249937 0.1249772
#> [2,] 0.1249795 0.1250048 0.1249826 0.1249887 0.1249896 0.1249916 0.1250141
#> [,8]
#> [1,] 0.1250034
#> [2,] 0.1249972
#>
#> , , 6
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1249712 0.1250285 0.1250014 0.1249616 0.1249824 0.1250143 0.1250434
#> [2,] 0.1250289 0.1249805 0.1250336 0.1250727 0.1250048 0.1249567 0.1250129
#> [,8]
#> [1,] 0.1249952
#> [2,] 0.1249992
#>
#> , , 7
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1249711 0.1250328 0.1249887 0.124956 0.1249735 0.1249865 0.1249903
#> [2,] 0.1250174 0.1250038 0.1250083 0.124990 0.1250107 0.1250322 0.1249784
#> [,8]
#> [1,] 0.1250507
#> [2,] 0.1250058
#>
#> , , 8
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1249856 0.1250170 0.1250057 0.1249765 0.1249886 0.1250183 0.1250420
#> [2,] 0.1250257 0.1250019 0.1250199 0.1249949 0.1250209 0.1250216 0.1249729
#> [,8]
#> [1,] 0.1249637
#> [2,] 0.1249973
#>
#>
#> $attentions[[1]][[2]]
#> , , 1
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1252142 0.1268387 0.1197179 0.1236128 0.1213845 0.1259620 0.1200844
#> [2,] 0.1245039 0.1220842 0.1256186 0.1274652 0.1239430 0.1232369 0.1251029
#> [,8]
#> [1,] 0.1326113
#> [2,] 0.1254987
#>
#> , , 2
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1259361 0.1228683 0.1171621 0.1276861 0.1178010 0.1228427 0.1206663
#> [2,] 0.1246810 0.1277636 0.1240373 0.1225497 0.1262632 0.1284616 0.1250002
#> [,8]
#> [1,] 0.1329623
#> [2,] 0.1253071
#>
#> , , 3
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1232218 0.1280498 0.1242587 0.1233406 0.127332 0.1270404 0.1289067
#> [2,] 0.1225943 0.1161762 0.1267401 0.1309135 0.116463 0.1167909 0.1244007
#> [,8]
#> [1,] 0.1216241
#> [2,] 0.1325402
#>
#> , , 4
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1237801 0.1275896 0.1347654 0.1214846 0.1344712 0.1281606 0.1302012
#> [2,] 0.1268205 0.1241084 0.1259017 0.1262510 0.1252450 0.1216899 0.1246638
#> [,8]
#> [1,] 0.1148118
#> [2,] 0.1224129
#>
#> , , 5
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1256326 0.1229382 0.1175375 0.1280035 0.1181909 0.1225451 0.1222191
#> [2,] 0.1228768 0.1215720 0.1255261 0.1261823 0.1202252 0.1210231 0.1258593
#> [,8]
#> [1,] 0.1315326
#> [2,] 0.1282849
#>
#> , , 6
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1273565 0.1207418 0.1206517 0.1286696 0.1176851 0.1209153 0.1190555
#> [2,] 0.1246755 0.1307740 0.1233306 0.1208721 0.1304384 0.1328421 0.1261906
#> [,8]
#> [1,] 0.1332102
#> [2,] 0.1217976
#>
#> , , 7
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.1250439 0.1214106 0.1398166 0.1265238 0.1343594 0.1234622 0.1341336
#> [2,] 0.1255129 0.1277540 0.1245701 0.1226306 0.1264725 0.1260387 0.1256019
#> [,8]
#> [1,] 0.1092106
#> [2,] 0.1227392
#>
#> , , 8
#>
#> [,1] [,2] [,3] [,4] [,5] [,6] [,7]
#> [1,] 0.123815 0.1295630 0.1260901 0.1206789 0.1287760 0.1290716 0.1247332
#> [2,] 0.128335 0.1297675 0.1242756 0.1231356 0.1309496 0.1299169 0.1231805
#> [,8]
#> [1,] 0.1240372
#> [2,] 0.1214194
#>
#>
#>
#>
#> $embeddings
#> [,1] [,2] [,3] [,4] [,5]
#> [1,] -0.04495785 -1.207056e-02 0.095013853 -0.01869603 0.03943388
#> [2,] 0.07760132 -2.318092e-02 0.024603195 0.08071895 -0.04463720
#> [3,] -0.06473834 6.938133e-02 0.008204800 -0.05809362 -0.03507191
#> [4,] 0.07391744 7.767102e-02 0.003117328 -0.08263396 -0.05034977
#> [5,] 0.03307861 2.449737e-02 -0.074803432 0.06459037 -0.07529554
#> [6,] -0.05842601 8.756461e-02 -0.077316436 -0.09416989 -0.08412606
#> [7,] -0.02424565 -4.388738e-02 0.012532736 -0.09964005 0.09900286
#> [8,] 0.08019558 5.757322e-05 0.079240013 0.02924411 0.00194957
#> [9,] -0.04909821 5.163250e-02 0.036170947 0.09260199 -0.03096029
#> [10,] 0.08100843 -4.720252e-04 -0.050611152 -0.08062622 0.08801934
#> [,6] [,7] [,8]
#> [1,] -0.071878987 0.07569324 -0.065703050
#> [2,] -0.096260815 -0.08645123 0.018073678
#> [3,] -0.022516411 -0.08123714 -0.002035520
#> [4,] -0.003575079 0.03821546 0.037901014
#> [5,] -0.061103920 0.04394522 -0.027372076
#> [6,] -0.085117201 -0.06177238 -0.026281535
#> [7,] -0.021085731 -0.07884695 -0.071909864
#> [8,] -0.044546238 -0.07929500 -0.070979707
#> [9,] 0.076789257 0.08533812 0.079661786
#> [10,] 0.033424065 0.03905640 0.009058573
#>
#> $n_params
#> [1] 1232
#>
#> $vocab_size
#> [1] 9
#>
#> $estimate
#> [1] 2.57099
#>
#> $n
#> [1] 8
#>
#> $method
#> [1] "BERT-style bidirectional encoder, 2 layers x 2 heads, MLM + NSP heads"
#>