-
Notifications
You must be signed in to change notification settings - Fork 184
/
d4bfb25f-688d-4da5-8743-33926fad4842.txt
5740 lines (5673 loc) · 383 KB
/
d4bfb25f-688d-4da5-8743-33926fad4842.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=3e-4, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5,
rank=0, world_size=1):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
self.rank = rank
self.world_size = world_size
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
# generate weight updates in distributed fashion
total_params = sum(p.numel() for p in group['params'])
updates_flat = torch.zeros(total_params, device='cuda', dtype=torch.bfloat16)
curr_idx = 0
for i, p in enumerate(group['params']):
# luckily this will perfectly distribute a transformer with multiple of 4 layers to 8 GPUs
if i % self.world_size == self.rank:
g = p.grad
if g is None:
continue
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
g = zeropower_backend(g, steps=group['backend_steps'])
g *= max(g.size(0), g.size(1))**0.5 # scale to have update.square().mean() == 1
updates_flat[curr_idx:curr_idx+p.numel()] = g.flatten()
curr_idx += p.numel()
# sync updates across devices. we are not memory-constrained so can do this simple deserialization
dist.all_reduce(updates_flat, op=dist.ReduceOp.SUM)
# deserialize and apply updates
curr_idx = 0
for p in group['params']:
g = updates_flat[curr_idx:curr_idx+p.numel()].view_as(p.data).type_as(p.data)
p.data.add_(g, alpha=-lr)
curr_idx += p.numel()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
def forward(self, x):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
cos, sin = self.rotary(q)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
def forward(self, x):
x = x + self.attn(F.rms_norm(x, (x.size(-1),)))
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
def forward(self, idx, targets=None, return_logits=True):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
for block in self.transformer.h:
x = block(x)
x = F.rms_norm(x, (x.size(-1),))
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 5100 # number of iterations to run
learning_rate : float = 0.0036
warmup_iters : int = 0
warmdown_iters : int = 1450 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# init the optimizer(s)
optimizer1 = torch.optim.AdamW(raw_model.lm_head.parameters(), lr=args.learning_rate, betas=(0.9, 0.95),
weight_decay=args.weight_decay, fused=True)
optimizer2 = Muon(raw_model.transformer.h.parameters(), lr=0.1*args.learning_rate, momentum=0.95,
rank=ddp_rank, world_size=ddp_world_size)
optimizers = [optimizer1, optimizer2]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with ctx: # of course, we'd like to use no_grad() here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss.detach()
del loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.5.0+cu124 compiled for CUDA 12.4
nvidia-smi:
Fri Oct 18 06:08:02 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 555.42.06 Driver Version: 555.42.06 CUDA Version: 12.5 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA H100 80GB HBM3 Off | 00000000:18:00.0 Off | 0 |
| N/A 33C P0 141W / 700W | 4860MiB / 81559MiB | 7% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 Off | 00000000:2A:00.0 Off | 0 |
| N/A 33C P0 121W / 700W | 4908MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 Off | 00000000:3A:00.0 Off | 0 |
| N/A 34C P0 123W / 700W | 4908MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 Off | 00000000:5D:00.0 Off | 0 |
| N/A 31C P0 124W / 700W | 4908MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 Off | 00000000:9A:00.0 Off | 0 |
| N/A 33C P0 133W / 700W | 4908MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 Off | 00000000:AB:00.0 Off | 0 |
| N/A 34C P0 128W / 700W | 4908MiB / 81559MiB | 7% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 Off | 00000000:BA:00.0 Off | 0 |
| N/A 33C P0 126W / 700W | 4908MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 Off | 00000000:DB:00.0 Off | 0 |
| N/A 33C P0 132W / 700W | 4668MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 958 C /usr/bin/python3 0MiB |
| 1 N/A N/A 959 C /usr/bin/python3 0MiB |
| 2 N/A N/A 960 C /usr/bin/python3 0MiB |
| 3 N/A N/A 961 C /usr/bin/python3 0MiB |
| 4 N/A N/A 962 C /usr/bin/python3 0MiB |
| 5 N/A N/A 963 C /usr/bin/python3 0MiB |
| 6 N/A N/A 964 C /usr/bin/python3 0MiB |
| 7 N/A N/A 965 C /usr/bin/python3 0MiB |
+-----------------------------------------------------------------------------------------+
====================================================================================================
step:0/5100 val_loss:16.0280 train_time:475ms step_avg:nanms
step:1/5100 train_loss:16.0268 train_time:35679ms step_avg:nanms
step:2/5100 train_loss:9.5039 train_time:35770ms step_avg:nanms
step:3/5100 train_loss:8.6998 train_time:35910ms step_avg:nanms
step:4/5100 train_loss:8.0755 train_time:36048ms step_avg:nanms
step:5/5100 train_loss:7.5667 train_time:36186ms step_avg:nanms
step:6/5100 train_loss:7.5466 train_time:36322ms step_avg:nanms
step:7/5100 train_loss:7.2854 train_time:36462ms step_avg:nanms
step:8/5100 train_loss:7.5010 train_time:36600ms step_avg:nanms
step:9/5100 train_loss:7.3365 train_time:36745ms step_avg:nanms
step:10/5100 train_loss:7.0160 train_time:36885ms step_avg:nanms
step:11/5100 train_loss:6.9676 train_time:83ms step_avg:nanms
step:12/5100 train_loss:6.8712 train_time:221ms step_avg:nanms
step:13/5100 train_loss:6.6573 train_time:359ms step_avg:119.69ms
step:14/5100 train_loss:6.6428 train_time:498ms step_avg:124.60ms
step:15/5100 train_loss:6.6117 train_time:638ms step_avg:127.56ms
step:16/5100 train_loss:6.5329 train_time:780ms step_avg:130.05ms
step:17/5100 train_loss:6.5378 train_time:921ms step_avg:131.57ms
step:18/5100 train_loss:6.5676 train_time:1061ms step_avg:132.63ms
step:19/5100 train_loss:6.3968 train_time:1208ms step_avg:134.17ms
step:20/5100 train_loss:6.4218 train_time:1339ms step_avg:133.88ms
step:21/5100 train_loss:6.0760 train_time:1478ms step_avg:134.34ms
step:22/5100 train_loss:6.4488 train_time:1618ms step_avg:134.79ms
step:23/5100 train_loss:6.6571 train_time:1759ms step_avg:135.33ms
step:24/5100 train_loss:6.3400 train_time:1906ms step_avg:136.14ms
step:25/5100 train_loss:6.4840 train_time:2042ms step_avg:136.11ms
step:26/5100 train_loss:6.1829 train_time:2180ms step_avg:136.23ms
step:27/5100 train_loss:6.0988 train_time:2320ms step_avg:136.48ms
step:28/5100 train_loss:6.2481 train_time:2459ms step_avg:136.59ms
step:29/5100 train_loss:5.9280 train_time:2601ms step_avg:136.87ms
step:30/5100 train_loss:6.2061 train_time:2743ms step_avg:137.13ms
step:31/5100 train_loss:6.0375 train_time:2881ms step_avg:137.17ms
step:32/5100 train_loss:6.0071 train_time:3021ms step_avg:137.31ms
step:33/5100 train_loss:5.8334 train_time:3159ms step_avg:137.37ms
step:34/5100 train_loss:6.1077 train_time:3303ms step_avg:137.62ms
step:35/5100 train_loss:6.0470 train_time:3440ms step_avg:137.58ms
step:36/5100 train_loss:6.1838 train_time:3580ms step_avg:137.67ms
step:37/5100 train_loss:6.1180 train_time:3719ms step_avg:137.76ms
step:38/5100 train_loss:6.0241 train_time:3860ms step_avg:137.86ms
step:39/5100 train_loss:5.9013 train_time:4001ms step_avg:137.96ms
step:40/5100 train_loss:5.9212 train_time:4144ms step_avg:138.13ms
step:41/5100 train_loss:5.8390 train_time:4280ms step_avg:138.08ms
step:42/5100 train_loss:5.8646 train_time:4427ms step_avg:138.34ms
step:43/5100 train_loss:5.7445 train_time:4559ms step_avg:138.17ms
step:44/5100 train_loss:5.8480 train_time:4700ms step_avg:138.24ms
step:45/5100 train_loss:5.8083 train_time:4841ms step_avg:138.30ms
step:46/5100 train_loss:5.9675 train_time:4982ms step_avg:138.38ms
step:47/5100 train_loss:5.7653 train_time:5120ms step_avg:138.39ms
step:48/5100 train_loss:5.6361 train_time:5260ms step_avg:138.43ms
step:49/5100 train_loss:5.8404 train_time:5401ms step_avg:138.50ms
step:50/5100 train_loss:5.7207 train_time:5543ms step_avg:138.57ms
step:51/5100 train_loss:5.8657 train_time:5681ms step_avg:138.56ms
step:52/5100 train_loss:5.7324 train_time:5827ms step_avg:138.74ms
step:53/5100 train_loss:5.5902 train_time:5963ms step_avg:138.68ms
step:54/5100 train_loss:5.7212 train_time:6115ms step_avg:138.97ms
step:55/5100 train_loss:5.6004 train_time:6244ms step_avg:138.76ms
step:56/5100 train_loss:5.9441 train_time:6383ms step_avg:138.76ms
step:57/5100 train_loss:5.6002 train_time:6523ms step_avg:138.78ms
step:58/5100 train_loss:5.4733 train_time:6663ms step_avg:138.81ms
step:59/5100 train_loss:5.6211 train_time:6805ms step_avg:138.88ms
step:60/5100 train_loss:5.5806 train_time:6948ms step_avg:138.95ms
step:61/5100 train_loss:5.6756 train_time:7086ms step_avg:138.94ms
step:62/5100 train_loss:5.4460 train_time:7226ms step_avg:138.96ms
step:63/5100 train_loss:5.5505 train_time:7367ms step_avg:139.00ms
step:64/5100 train_loss:5.5335 train_time:7507ms step_avg:139.02ms
step:65/5100 train_loss:5.2016 train_time:7646ms step_avg:139.02ms
step:66/5100 train_loss:5.3469 train_time:7789ms step_avg:139.09ms
step:67/5100 train_loss:5.4996 train_time:7927ms step_avg:139.06ms
step:68/5100 train_loss:5.3771 train_time:8067ms step_avg:139.09ms
step:69/5100 train_loss:5.6439 train_time:8209ms step_avg:139.14ms
step:70/5100 train_loss:5.2914 train_time:8351ms step_avg:139.18ms
step:71/5100 train_loss:5.3085 train_time:8490ms step_avg:139.17ms
step:72/5100 train_loss:5.5120 train_time:8629ms step_avg:139.18ms
step:73/5100 train_loss:5.4412 train_time:8770ms step_avg:139.21ms
step:74/5100 train_loss:5.3140 train_time:8913ms step_avg:139.26ms
step:75/5100 train_loss:5.4440 train_time:9065ms step_avg:139.46ms
step:76/5100 train_loss:5.4089 train_time:9195ms step_avg:139.31ms
step:77/5100 train_loss:5.3792 train_time:9336ms step_avg:139.35ms
step:78/5100 train_loss:5.4709 train_time:9477ms step_avg:139.37ms
step:79/5100 train_loss:5.5314 train_time:9619ms step_avg:139.40ms
step:80/5100 train_loss:5.3345 train_time:9759ms step_avg:139.42ms
step:81/5100 train_loss:5.4311 train_time:9900ms step_avg:139.43ms
step:82/5100 train_loss:5.1976 train_time:10041ms step_avg:139.45ms
step:83/5100 train_loss:5.3647 train_time:10181ms step_avg:139.47ms
step:84/5100 train_loss:5.3113 train_time:10321ms step_avg:139.48ms
step:85/5100 train_loss:5.3062 train_time:10461ms step_avg:139.49ms
step:86/5100 train_loss:5.1549 train_time:10602ms step_avg:139.50ms
step:87/5100 train_loss:5.3655 train_time:10741ms step_avg:139.50ms
step:88/5100 train_loss:5.2718 train_time:10883ms step_avg:139.53ms
step:89/5100 train_loss:5.3375 train_time:11022ms step_avg:139.51ms
step:90/5100 train_loss:5.2967 train_time:11164ms step_avg:139.55ms
step:91/5100 train_loss:5.2241 train_time:11302ms step_avg:139.53ms
step:92/5100 train_loss:5.1922 train_time:11442ms step_avg:139.53ms
step:93/5100 train_loss:5.3442 train_time:11582ms step_avg:139.54ms
step:94/5100 train_loss:5.1495 train_time:11721ms step_avg:139.54ms
step:95/5100 train_loss:5.1675 train_time:11861ms step_avg:139.54ms
step:96/5100 train_loss:5.1980 train_time:12002ms step_avg:139.55ms
step:97/5100 train_loss:5.1103 train_time:12141ms step_avg:139.56ms
step:98/5100 train_loss:5.1987 train_time:12290ms step_avg:139.66ms
step:99/5100 train_loss:5.1154 train_time:12421ms step_avg:139.57ms
step:100/5100 train_loss:5.2372 train_time:12561ms step_avg:139.57ms
step:101/5100 train_loss:5.2118 train_time:12706ms step_avg:139.63ms
step:102/5100 train_loss:5.1125 train_time:12844ms step_avg:139.61ms
step:103/5100 train_loss:5.1995 train_time:12982ms step_avg:139.59ms
step:104/5100 train_loss:5.1493 train_time:13122ms step_avg:139.60ms
step:105/5100 train_loss:5.0168 train_time:13262ms step_avg:139.60ms
step:106/5100 train_loss:5.1194 train_time:13402ms step_avg:139.60ms
step:107/5100 train_loss:5.3136 train_time:13542ms step_avg:139.60ms
step:108/5100 train_loss:5.0858 train_time:13682ms step_avg:139.61ms
step:109/5100 train_loss:4.8767 train_time:13825ms step_avg:139.64ms
step:110/5100 train_loss:5.0602 train_time:13963ms step_avg:139.63ms
step:111/5100 train_loss:5.0414 train_time:14103ms step_avg:139.64ms
step:112/5100 train_loss:5.0043 train_time:14244ms step_avg:139.64ms
step:113/5100 train_loss:5.1192 train_time:14383ms step_avg:139.64ms
step:114/5100 train_loss:5.0516 train_time:14535ms step_avg:139.76ms
step:115/5100 train_loss:4.9023 train_time:14663ms step_avg:139.65ms
step:116/5100 train_loss:5.0652 train_time:14804ms step_avg:139.66ms
step:117/5100 train_loss:4.9585 train_time:14947ms step_avg:139.69ms
step:118/5100 train_loss:4.9147 train_time:15083ms step_avg:139.66ms
step:119/5100 train_loss:5.0614 train_time:15222ms step_avg:139.65ms
step:120/5100 train_loss:5.0139 train_time:15363ms step_avg:139.66ms
step:121/5100 train_loss:4.9475 train_time:15503ms step_avg:139.67ms
step:122/5100 train_loss:4.8469 train_time:15644ms step_avg:139.68ms
step:123/5100 train_loss:4.9655 train_time:15784ms step_avg:139.68ms
step:124/5100 train_loss:4.8197 train_time:15924ms step_avg:139.69ms
step:125/5100 train_loss:5.1296 train_time:16064ms step_avg:139.69ms
step:125/5100 val_loss:4.9523 train_time:16121ms step_avg:140.18ms
step:126/5100 train_loss:4.9996 train_time:16216ms step_avg:139.79ms
step:127/5100 train_loss:4.9385 train_time:16366ms step_avg:139.88ms
step:128/5100 train_loss:5.0094 train_time:16505ms step_avg:139.87ms
step:129/5100 train_loss:4.8793 train_time:16647ms step_avg:139.89ms
step:130/5100 train_loss:5.1899 train_time:16786ms step_avg:139.88ms
step:131/5100 train_loss:4.9339 train_time:16921ms step_avg:139.84ms
step:132/5100 train_loss:4.9435 train_time:17060ms step_avg:139.83ms
step:133/5100 train_loss:4.8981 train_time:17201ms step_avg:139.84ms
step:134/5100 train_loss:4.9410 train_time:17345ms step_avg:139.88ms
step:135/5100 train_loss:4.8279 train_time:17487ms step_avg:139.90ms
step:136/5100 train_loss:4.9531 train_time:17628ms step_avg:139.90ms
step:137/5100 train_loss:4.7257 train_time:17768ms step_avg:139.91ms
step:138/5100 train_loss:4.8931 train_time:17910ms step_avg:139.92ms
step:139/5100 train_loss:4.8438 train_time:18048ms step_avg:139.91ms
step:140/5100 train_loss:4.8759 train_time:18190ms step_avg:139.92ms
step:141/5100 train_loss:4.9391 train_time:18333ms step_avg:139.95ms
step:142/5100 train_loss:4.8126 train_time:18474ms step_avg:139.96ms
step:143/5100 train_loss:4.8760 train_time:18614ms step_avg:139.96ms
step:144/5100 train_loss:4.7315 train_time:18754ms step_avg:139.95ms
step:145/5100 train_loss:4.8627 train_time:18898ms step_avg:139.99ms
step:146/5100 train_loss:4.8117 train_time:19033ms step_avg:139.95ms
step:147/5100 train_loss:4.6851 train_time:19174ms step_avg:139.96ms
step:148/5100 train_loss:4.8378 train_time:19318ms step_avg:139.99ms
step:149/5100 train_loss:4.8399 train_time:19456ms step_avg:139.97ms
step:150/5100 train_loss:4.8707 train_time:19596ms step_avg:139.97ms
step:151/5100 train_loss:4.9068 train_time:19735ms step_avg:139.97ms
step:152/5100 train_loss:4.7968 train_time:19875ms step_avg:139.96ms
step:153/5100 train_loss:4.7896 train_time:20014ms step_avg:139.96ms
step:154/5100 train_loss:4.8759 train_time:20155ms step_avg:139.96ms
step:155/5100 train_loss:4.8361 train_time:20305ms step_avg:140.03ms
step:156/5100 train_loss:4.7903 train_time:20436ms step_avg:139.97ms
step:157/5100 train_loss:4.8125 train_time:20576ms step_avg:139.97ms
step:158/5100 train_loss:4.9304 train_time:20716ms step_avg:139.97ms
step:159/5100 train_loss:4.7162 train_time:20855ms step_avg:139.97ms
step:160/5100 train_loss:4.7894 train_time:20995ms step_avg:139.96ms
step:161/5100 train_loss:4.6171 train_time:21134ms step_avg:139.96ms
step:162/5100 train_loss:4.8063 train_time:21275ms step_avg:139.97ms
step:163/5100 train_loss:4.8376 train_time:21415ms step_avg:139.97ms
step:164/5100 train_loss:4.8225 train_time:21555ms step_avg:139.97ms
step:165/5100 train_loss:4.6319 train_time:21695ms step_avg:139.97ms
step:166/5100 train_loss:4.7650 train_time:21835ms step_avg:139.97ms
step:167/5100 train_loss:4.8934 train_time:21974ms step_avg:139.96ms
step:168/5100 train_loss:4.6755 train_time:22115ms step_avg:139.97ms
step:169/5100 train_loss:4.7737 train_time:22255ms step_avg:139.97ms
step:170/5100 train_loss:4.6248 train_time:22395ms step_avg:139.97ms
step:171/5100 train_loss:4.5313 train_time:22535ms step_avg:139.97ms
step:172/5100 train_loss:4.6855 train_time:22676ms step_avg:139.97ms
step:173/5100 train_loss:4.6687 train_time:22815ms step_avg:139.97ms
step:174/5100 train_loss:4.7283 train_time:22953ms step_avg:139.96ms
step:175/5100 train_loss:4.8815 train_time:23094ms step_avg:139.97ms
step:176/5100 train_loss:4.7293 train_time:23235ms step_avg:139.97ms
step:177/5100 train_loss:4.5799 train_time:23375ms step_avg:139.97ms
step:178/5100 train_loss:4.5530 train_time:23519ms step_avg:139.99ms
step:179/5100 train_loss:4.6195 train_time:23655ms step_avg:139.97ms
step:180/5100 train_loss:4.6284 train_time:23796ms step_avg:139.98ms
step:181/5100 train_loss:4.6216 train_time:23935ms step_avg:139.97ms
step:182/5100 train_loss:4.7571 train_time:24081ms step_avg:140.01ms
step:183/5100 train_loss:4.6170 train_time:24216ms step_avg:139.98ms
step:184/5100 train_loss:4.5692 train_time:24354ms step_avg:139.97ms
step:185/5100 train_loss:4.5808 train_time:24499ms step_avg:139.99ms
step:186/5100 train_loss:4.7013 train_time:24635ms step_avg:139.97ms
step:187/5100 train_loss:4.6171 train_time:24776ms step_avg:139.98ms
step:188/5100 train_loss:4.8039 train_time:24915ms step_avg:139.97ms
step:189/5100 train_loss:4.6349 train_time:25214ms step_avg:140.86ms
step:190/5100 train_loss:4.5574 train_time:25528ms step_avg:141.82ms
step:191/5100 train_loss:4.6914 train_time:25663ms step_avg:141.78ms
step:192/5100 train_loss:4.5396 train_time:25801ms step_avg:141.76ms
step:193/5100 train_loss:4.4604 train_time:25938ms step_avg:141.74ms
step:194/5100 train_loss:4.6879 train_time:26076ms step_avg:141.72ms
step:195/5100 train_loss:4.6142 train_time:26214ms step_avg:141.70ms
step:196/5100 train_loss:4.8141 train_time:26351ms step_avg:141.67ms
step:197/5100 train_loss:4.6685 train_time:26497ms step_avg:141.69ms
step:198/5100 train_loss:4.5125 train_time:26639ms step_avg:141.69ms
step:199/5100 train_loss:4.5861 train_time:26788ms step_avg:141.73ms
step:200/5100 train_loss:4.4490 train_time:26917ms step_avg:141.67ms
step:201/5100 train_loss:4.5490 train_time:27059ms step_avg:141.67ms
step:202/5100 train_loss:4.4513 train_time:27194ms step_avg:141.64ms
step:203/5100 train_loss:4.6903 train_time:27333ms step_avg:141.62ms
step:204/5100 train_loss:4.5505 train_time:27476ms step_avg:141.63ms
step:205/5100 train_loss:4.5938 train_time:27617ms step_avg:141.63ms
step:206/5100 train_loss:4.7052 train_time:27760ms step_avg:141.63ms
step:207/5100 train_loss:4.3718 train_time:27897ms step_avg:141.61ms
step:208/5100 train_loss:4.5220 train_time:28035ms step_avg:141.59ms
step:209/5100 train_loss:4.5035 train_time:28174ms step_avg:141.58ms
step:210/5100 train_loss:4.6587 train_time:28314ms step_avg:141.57ms
step:211/5100 train_loss:4.5725 train_time:28455ms step_avg:141.57ms
step:212/5100 train_loss:4.4606 train_time:28598ms step_avg:141.57ms
step:213/5100 train_loss:4.5614 train_time:28739ms step_avg:141.57ms
step:214/5100 train_loss:4.4331 train_time:28879ms step_avg:141.56ms
step:215/5100 train_loss:4.5012 train_time:29017ms step_avg:141.55ms
step:216/5100 train_loss:4.3770 train_time:29156ms step_avg:141.53ms
step:217/5100 train_loss:4.4625 train_time:29295ms step_avg:141.52ms
step:218/5100 train_loss:4.4426 train_time:29435ms step_avg:141.51ms
step:219/5100 train_loss:4.4648 train_time:29583ms step_avg:141.54ms
step:220/5100 train_loss:4.4582 train_time:29723ms step_avg:141.54ms
step:221/5100 train_loss:4.4875 train_time:29859ms step_avg:141.51ms
step:222/5100 train_loss:4.5042 train_time:29997ms step_avg:141.50ms
step:223/5100 train_loss:4.4190 train_time:30135ms step_avg:141.48ms
step:224/5100 train_loss:4.4362 train_time:30274ms step_avg:141.47ms
step:225/5100 train_loss:4.6332 train_time:30414ms step_avg:141.46ms
step:226/5100 train_loss:4.2978 train_time:30555ms step_avg:141.46ms
step:227/5100 train_loss:4.3456 train_time:30697ms step_avg:141.46ms
step:228/5100 train_loss:4.3605 train_time:30837ms step_avg:141.45ms
step:229/5100 train_loss:4.5079 train_time:30977ms step_avg:141.45ms
step:230/5100 train_loss:4.3049 train_time:31115ms step_avg:141.43ms
step:231/5100 train_loss:4.4461 train_time:31254ms step_avg:141.42ms
step:232/5100 train_loss:4.3060 train_time:31395ms step_avg:141.42ms
step:233/5100 train_loss:4.3158 train_time:31535ms step_avg:141.41ms
step:234/5100 train_loss:4.4816 train_time:31675ms step_avg:141.41ms
step:235/5100 train_loss:4.3614 train_time:31815ms step_avg:141.40ms
step:236/5100 train_loss:4.2647 train_time:31955ms step_avg:141.40ms
step:237/5100 train_loss:4.4681 train_time:32096ms step_avg:141.39ms
step:238/5100 train_loss:4.4326 train_time:32234ms step_avg:141.38ms
step:239/5100 train_loss:4.2878 train_time:32374ms step_avg:141.37ms
step:240/5100 train_loss:4.4543 train_time:32515ms step_avg:141.37ms
step:241/5100 train_loss:4.4453 train_time:32655ms step_avg:141.36ms
step:242/5100 train_loss:4.3327 train_time:32795ms step_avg:141.36ms
step:243/5100 train_loss:4.5133 train_time:32935ms step_avg:141.35ms
step:244/5100 train_loss:4.3444 train_time:33075ms step_avg:141.35ms
step:245/5100 train_loss:4.3847 train_time:33215ms step_avg:141.34ms
step:246/5100 train_loss:4.4585 train_time:33353ms step_avg:141.33ms
step:247/5100 train_loss:4.3941 train_time:33497ms step_avg:141.34ms
step:248/5100 train_loss:4.3347 train_time:33634ms step_avg:141.32ms
step:249/5100 train_loss:4.4618 train_time:33774ms step_avg:141.32ms
step:250/5100 train_loss:4.2348 train_time:33915ms step_avg:141.31ms
step:250/5100 val_loss:4.3355 train_time:33971ms step_avg:141.54ms
step:251/5100 train_loss:4.2891 train_time:34068ms step_avg:141.36ms
step:252/5100 train_loss:4.4021 train_time:34210ms step_avg:141.36ms
step:253/5100 train_loss:4.4402 train_time:34349ms step_avg:141.35ms
step:254/5100 train_loss:4.2654 train_time:34494ms step_avg:141.37ms
step:255/5100 train_loss:4.2126 train_time:34628ms step_avg:141.34ms
step:256/5100 train_loss:4.3859 train_time:34767ms step_avg:141.33ms
step:257/5100 train_loss:4.3120 train_time:34910ms step_avg:141.34ms
step:258/5100 train_loss:4.3147 train_time:35048ms step_avg:141.32ms
step:259/5100 train_loss:4.2802 train_time:35192ms step_avg:141.33ms
step:260/5100 train_loss:4.3168 train_time:35333ms step_avg:141.33ms
step:261/5100 train_loss:4.3628 train_time:35477ms step_avg:141.34ms
step:262/5100 train_loss:4.3211 train_time:35611ms step_avg:141.31ms
step:263/5100 train_loss:4.2856 train_time:35754ms step_avg:141.32ms
step:264/5100 train_loss:4.2053 train_time:35890ms step_avg:141.30ms
step:265/5100 train_loss:4.2866 train_time:36031ms step_avg:141.30ms
step:266/5100 train_loss:4.1538 train_time:36173ms step_avg:141.30ms
step:267/5100 train_loss:4.2081 train_time:36314ms step_avg:141.30ms
step:268/5100 train_loss:4.2269 train_time:36452ms step_avg:141.29ms
step:269/5100 train_loss:4.2355 train_time:36591ms step_avg:141.28ms
step:270/5100 train_loss:4.1494 train_time:36734ms step_avg:141.29ms
step:271/5100 train_loss:4.3922 train_time:36870ms step_avg:141.27ms
step:272/5100 train_loss:4.2877 train_time:37010ms step_avg:141.26ms
step:273/5100 train_loss:4.1898 train_time:37151ms step_avg:141.26ms
step:274/5100 train_loss:4.2434 train_time:37292ms step_avg:141.26ms
step:275/5100 train_loss:4.3153 train_time:37432ms step_avg:141.25ms
step:276/5100 train_loss:4.3375 train_time:37574ms step_avg:141.26ms
step:277/5100 train_loss:4.5175 train_time:37712ms step_avg:141.24ms
step:278/5100 train_loss:4.3018 train_time:37856ms step_avg:141.25ms
step:279/5100 train_loss:4.3763 train_time:37993ms step_avg:141.24ms
step:280/5100 train_loss:4.2722 train_time:38135ms step_avg:141.24ms
step:281/5100 train_loss:4.4043 train_time:38273ms step_avg:141.23ms
step:282/5100 train_loss:4.2265 train_time:38413ms step_avg:141.22ms
step:283/5100 train_loss:4.2598 train_time:38553ms step_avg:141.22ms
step:284/5100 train_loss:4.1781 train_time:38693ms step_avg:141.21ms
step:285/5100 train_loss:4.3250 train_time:38833ms step_avg:141.21ms
step:286/5100 train_loss:4.3280 train_time:38972ms step_avg:141.20ms
step:287/5100 train_loss:4.3635 train_time:39116ms step_avg:141.21ms
step:288/5100 train_loss:4.1933 train_time:39254ms step_avg:141.20ms
step:289/5100 train_loss:4.2827 train_time:39392ms step_avg:141.19ms
step:290/5100 train_loss:4.1403 train_time:39532ms step_avg:141.18ms
step:291/5100 train_loss:4.1319 train_time:39672ms step_avg:141.18ms
step:292/5100 train_loss:4.2187 train_time:39812ms step_avg:141.18ms
step:293/5100 train_loss:4.1353 train_time:39952ms step_avg:141.17ms
step:294/5100 train_loss:4.1773 train_time:40092ms step_avg:141.17ms
step:295/5100 train_loss:4.2191 train_time:40233ms step_avg:141.17ms
step:296/5100 train_loss:4.1036 train_time:40373ms step_avg:141.16ms
step:297/5100 train_loss:4.1130 train_time:40512ms step_avg:141.16ms
step:298/5100 train_loss:4.1240 train_time:40651ms step_avg:141.15ms
step:299/5100 train_loss:4.2226 train_time:40801ms step_avg:141.18ms
step:300/5100 train_loss:4.0927 train_time:40933ms step_avg:141.15ms
step:301/5100 train_loss:4.2335 train_time:41073ms step_avg:141.14ms
step:302/5100 train_loss:4.2414 train_time:41215ms step_avg:141.15ms
step:303/5100 train_loss:4.1842 train_time:41352ms step_avg:141.13ms
step:304/5100 train_loss:4.2413 train_time:41495ms step_avg:141.14ms
step:305/5100 train_loss:4.2201 train_time:41631ms step_avg:141.12ms
step:306/5100 train_loss:4.6999 train_time:41771ms step_avg:141.12ms
step:307/5100 train_loss:4.1954 train_time:41912ms step_avg:141.12ms
step:308/5100 train_loss:4.0971 train_time:42052ms step_avg:141.11ms
step:309/5100 train_loss:4.2585 train_time:42193ms step_avg:141.11ms
step:310/5100 train_loss:4.1086 train_time:42333ms step_avg:141.11ms
step:311/5100 train_loss:4.3348 train_time:42477ms step_avg:141.12ms
step:312/5100 train_loss:4.1895 train_time:42612ms step_avg:141.10ms
step:313/5100 train_loss:4.1204 train_time:42751ms step_avg:141.09ms
step:314/5100 train_loss:4.2241 train_time:42892ms step_avg:141.09ms
step:315/5100 train_loss:4.3334 train_time:43031ms step_avg:141.09ms
step:316/5100 train_loss:4.2118 train_time:43172ms step_avg:141.09ms
step:317/5100 train_loss:4.0455 train_time:43315ms step_avg:141.09ms
step:318/5100 train_loss:4.1227 train_time:43453ms step_avg:141.08ms
step:319/5100 train_loss:4.1540 train_time:43593ms step_avg:141.08ms
step:320/5100 train_loss:4.1345 train_time:43732ms step_avg:141.07ms
step:321/5100 train_loss:4.2412 train_time:43872ms step_avg:141.07ms
step:322/5100 train_loss:4.1950 train_time:44013ms step_avg:141.07ms
step:323/5100 train_loss:4.1623 train_time:44152ms step_avg:141.06ms
step:324/5100 train_loss:4.2450 train_time:44293ms step_avg:141.06ms
step:325/5100 train_loss:4.2092 train_time:44433ms step_avg:141.06ms
step:326/5100 train_loss:4.2768 train_time:44573ms step_avg:141.05ms
step:327/5100 train_loss:4.1299 train_time:44713ms step_avg:141.05ms
step:328/5100 train_loss:4.6284 train_time:44853ms step_avg:141.05ms
step:329/5100 train_loss:4.3118 train_time:44992ms step_avg:141.04ms
step:330/5100 train_loss:4.0570 train_time:45132ms step_avg:141.04ms
step:331/5100 train_loss:3.9979 train_time:45272ms step_avg:141.03ms
step:332/5100 train_loss:4.2168 train_time:45414ms step_avg:141.04ms
step:333/5100 train_loss:4.1448 train_time:45553ms step_avg:141.03ms
step:334/5100 train_loss:4.1207 train_time:45693ms step_avg:141.03ms
step:335/5100 train_loss:4.0790 train_time:45832ms step_avg:141.02ms
step:336/5100 train_loss:4.2491 train_time:45972ms step_avg:141.02ms
step:337/5100 train_loss:4.1919 train_time:46113ms step_avg:141.02ms
step:338/5100 train_loss:4.6667 train_time:46252ms step_avg:141.01ms
step:339/5100 train_loss:4.1764 train_time:46394ms step_avg:141.01ms
step:340/5100 train_loss:4.1236 train_time:46533ms step_avg:141.01ms
step:341/5100 train_loss:4.1626 train_time:46673ms step_avg:141.01ms
step:342/5100 train_loss:4.0757 train_time:46812ms step_avg:141.00ms
step:343/5100 train_loss:4.0488 train_time:46952ms step_avg:141.00ms
step:344/5100 train_loss:4.0972 train_time:47092ms step_avg:140.99ms
step:345/5100 train_loss:4.2305 train_time:47232ms step_avg:140.99ms
step:346/5100 train_loss:4.0744 train_time:47372ms step_avg:140.99ms
step:347/5100 train_loss:4.0078 train_time:47513ms step_avg:140.99ms
step:348/5100 train_loss:4.0508 train_time:47652ms step_avg:140.98ms
step:349/5100 train_loss:4.0869 train_time:47792ms step_avg:140.98ms
step:350/5100 train_loss:4.0463 train_time:47932ms step_avg:140.98ms
step:351/5100 train_loss:3.7723 train_time:48073ms step_avg:140.98ms
step:352/5100 train_loss:4.0418 train_time:48214ms step_avg:140.98ms
step:353/5100 train_loss:4.3813 train_time:48355ms step_avg:140.98ms
step:354/5100 train_loss:3.8922 train_time:48493ms step_avg:140.97ms
step:355/5100 train_loss:4.1522 train_time:48633ms step_avg:140.96ms
step:356/5100 train_loss:4.0244 train_time:48772ms step_avg:140.96ms
step:357/5100 train_loss:4.1172 train_time:48916ms step_avg:140.97ms
step:358/5100 train_loss:4.0827 train_time:49052ms step_avg:140.95ms
step:359/5100 train_loss:4.0725 train_time:49193ms step_avg:140.96ms
step:360/5100 train_loss:4.1245 train_time:49332ms step_avg:140.95ms
step:361/5100 train_loss:3.6890 train_time:49472ms step_avg:140.95ms
step:362/5100 train_loss:4.2446 train_time:49613ms step_avg:140.95ms
step:363/5100 train_loss:4.1444 train_time:49752ms step_avg:140.94ms
step:364/5100 train_loss:4.0674 train_time:49893ms step_avg:140.94ms
step:365/5100 train_loss:3.9845 train_time:50033ms step_avg:140.94ms
step:366/5100 train_loss:4.1359 train_time:50172ms step_avg:140.93ms
step:367/5100 train_loss:4.0992 train_time:50312ms step_avg:140.93ms
step:368/5100 train_loss:4.0830 train_time:50452ms step_avg:140.93ms
step:369/5100 train_loss:4.0625 train_time:50593ms step_avg:140.93ms
step:370/5100 train_loss:3.9645 train_time:50732ms step_avg:140.92ms
step:371/5100 train_loss:4.1114 train_time:50872ms step_avg:140.92ms
step:372/5100 train_loss:3.9922 train_time:51012ms step_avg:140.92ms
step:373/5100 train_loss:3.9238 train_time:51151ms step_avg:140.91ms
step:374/5100 train_loss:4.1321 train_time:51292ms step_avg:140.91ms
step:375/5100 train_loss:4.0562 train_time:51434ms step_avg:140.91ms
step:375/5100 val_loss:4.0552 train_time:51489ms step_avg:141.07ms
step:376/5100 train_loss:4.0260 train_time:51581ms step_avg:140.93ms
step:377/5100 train_loss:4.0951 train_time:51732ms step_avg:140.96ms
step:378/5100 train_loss:4.0099 train_time:52070ms step_avg:141.49ms
step:379/5100 train_loss:4.0686 train_time:52205ms step_avg:141.48ms
step:380/5100 train_loss:4.0964 train_time:52516ms step_avg:141.94ms
step:381/5100 train_loss:4.1709 train_time:52654ms step_avg:141.92ms
step:382/5100 train_loss:4.0740 train_time:52791ms step_avg:141.91ms
step:383/5100 train_loss:4.0536 train_time:52929ms step_avg:141.90ms
step:384/5100 train_loss:4.0085 train_time:53067ms step_avg:141.89ms
step:385/5100 train_loss:4.0928 train_time:53205ms step_avg:141.88ms
step:386/5100 train_loss:4.0061 train_time:53344ms step_avg:141.87ms
step:387/5100 train_loss:4.1163 train_time:53493ms step_avg:141.89ms
step:388/5100 train_loss:4.3070 train_time:53635ms step_avg:141.89ms
step:389/5100 train_loss:4.0191 train_time:53774ms step_avg:141.88ms
step:390/5100 train_loss:4.0120 train_time:53913ms step_avg:141.88ms
step:391/5100 train_loss:4.1075 train_time:54051ms step_avg:141.87ms
step:392/5100 train_loss:4.0275 train_time:54189ms step_avg:141.86ms
step:393/5100 train_loss:4.1395 train_time:54329ms step_avg:141.85ms
step:394/5100 train_loss:3.9731 train_time:54473ms step_avg:141.86ms
step:395/5100 train_loss:4.1123 train_time:54615ms step_avg:141.86ms
step:396/5100 train_loss:3.8467 train_time:54756ms step_avg:141.86ms
step:397/5100 train_loss:4.0517 train_time:54896ms step_avg:141.85ms
step:398/5100 train_loss:4.1039 train_time:55034ms step_avg:141.84ms