e66b0dd9-9680-4e7a-85c7-ac77f1d89aa2.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# Use of FlexAttention contributed by @KoszarskyB
from torch.nn.attention.flex_attention import BlockMask, flex_attention

# -----------------------------------------------------------------------------
# Muon optimizer

def zeropower_via_svd(G, steps=None):
    U, S, V = G.svd()
    return U @ V.T

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
        backend_steps: The number of iteration steps to use in the backend, if it is iterative.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
                 backend='newtonschulz5', backend_steps=5):
        self.num_process = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ["RANK"])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
        params: "list[torch.Tensor]" = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                "params": [p for p in params if p.numel() == size],
                "update_buffer": [
                    torch.empty(size, device="cuda", dtype=torch.bfloat16)
                    for _ in range(self.num_process)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):
        for group in self.param_groups:
            lr: float = group["lr"]
            momentum: float = group["momentum"]
            nesterov: bool = group["nesterov"]
            zeropower_backend = zeropower_backends[group["backend"]]
            backend_steps: int = group["backend_steps"]
            update_buffers: "list[torch.Tensor]" = group["update_buffer"]
            # generate weight updates in distributed fashion
            params: "list[torch.Tensor]" = group["params"]
            assert len(params) % self.num_process == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.num_process]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p] 
                if "momentum_buffer" not in state:
                    state["momentum_buffer"] = torch.zeros_like(g)
                buf: torch.Tensor = state["momentum_buffer"]
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_backend(g, steps=backend_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.num_process]
            update_prev()


# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, n_head):
        super().__init__()
        assert dim % n_head == 0
        self.n_head = n_head
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        # value residual lambda
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5])) # @Grad62304977
        # rotary embeddings
        self.rotary = Rotary(dim // n_head) # dim // n_head = head_dim
        # output projection
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor, vi: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q: torch.Tensor = self.c_q(x).view(B, T, self.n_head, -1)
        k: torch.Tensor = self.c_k(x).view(B, T, self.n_head, -1)
        v: torch.Tensor = self.c_v(x).view(B, T, self.n_head, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @Grad62304977
        q, k = norm(q), norm(k) # QK norm suggested by @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim: int):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.n_embd, config.n_head)
        self.mlp = MLP(config.n_embd)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x: torch.Tensor, vi: torch.Tensor, x0: torch.Tensor, block_mask: BlockMask) -> torch.Tensor:
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    n_layer : int = 12
    n_head : int = 6 # head dim 128 suggested by @Grad62304977
    n_embd : int = 768
    lm_head_softcap : int = 30

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.n_layer = config.n_layer
        self.lm_head_softcap = config.lm_head_softcap

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.n_layer // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.n_layer - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
            # U-net structure on token value embeddings by @leloykun
            vte = nn.Embedding(config.vocab_size, config.n_embd*self.num_encoder_layers),
            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
        ))
        self.lm_head = CastedLinear(config.n_embd, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(self, idx: torch.Tensor, target: torch.Tensor, sliding_window: torch.Tensor) -> torch.Tensor:
        BLOCK_SIZE = 128
        assert idx.ndim == 1
        docs = (idx == 50256).cumsum(0)
        docs_low = docs.reshape(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.reshape(-1, BLOCK_SIZE)[:, -1].contiguous()
        def document_sliding_window_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            window_mask = q_idx - kv_idx < sliding_window
            return causal_mask & document_mask & window_mask

        S = len(idx)
        def create_sliding_window_causal_mask(S: int, sliding_window: torch.Tensor):
            kv_idx = block_idx = torch.arange(S // BLOCK_SIZE, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_mask = q_idx >= kv_idx
            document_mask = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            window_mask = q_idx - kv_idx < ((sliding_window + BLOCK_SIZE - 1) // BLOCK_SIZE)
            dense_mask = causal_mask & document_mask & window_mask
            dense_mask = dense_mask.to(torch.int32)
            num_blocks = dense_mask.sum(dim=-1).to(torch.int32)
            indices = torch.argsort(dense_mask, dim=-1, descending=True, stable=True).to(torch.int32)
            num_blocks = num_blocks[None, None, :].contiguous()
            indices = indices[None, None, :].contiguous()
            return BlockMask.from_kv_blocks(num_blocks, indices, BLOCK_SIZE=BLOCK_SIZE, mask_mod=document_sliding_window_causal)
        block_mask = create_sliding_window_causal_mask(S, sliding_window)

        # forward the GPT model itself
        x = self.transformer.wte(idx[None]) # token embeddings of shape (b, t, n_embd)
        x = norm(x) # @Grad62304977
        x0 = x
        vi = self.transformer.vte(idx[None]).chunk(self.num_encoder_layers, dim=-1)

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.transformer.h[i](x, vi[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.transformer.h[self.num_encoder_layers + i](x, vi[self.num_encoder_layers-1-i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = self.lm_head_softcap * torch.tanh(logits / self.lm_head_softcap) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(file: Path, ntok: int):
    with file.open("rb") as f:
        tokens = torch.empty(ntok, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * ntok, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, T, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.T = T

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.ntoks = [_peek_data_shard(file) for file in self.files]
        assert min(self.ntoks) >= num_processes * T + 1
        self.ntok_total = sum(self.ntoks)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.T
        self.tokens = _load_data_shard(self.files[self.current_shard], self.ntoks[self.current_shard])

    def next_batch(self):
        batch_size = self.T * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.T+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        x = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        y = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return x, y

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = str(uuid.uuid4())
    logdir = 'logs/%s/' % run_id
    # os.makedirs(logdir, exist_ok=True)
    logfile = 'logs/%s.txt' % run_id
    # create the log file
    with open(logfile, "w") as f:
        # begin the log by printing this file (the Python code)
        f.write(code)
        f.write('='*100 + '\n')
def print0(s, logonly=False):
    if master_process:
        with open(logfile, "a") as f:
            if not logonly:
                print(s)
            f.write(s+'\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# convenience variables
T = args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (T * ddp_world_size) == 0
val_steps = args.val_tokens // (T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size
assert train_accumulation_steps == 1

# load tokens
train_loader = DistributedDataLoader(args.input_bin, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, T, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
x, y = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
if hasattr(config, "coordinate_descent_tuning"):
    config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight, raw_model.transformer.vte.weight], lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_size = torch.tensor(64, dtype=torch.int32, device="cuda")
sw_size_prev = 64
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Set the sliding window size for the current step, in chunks of 64. By @fernbear.bsky.social
    sw_size =  64 * int((64 + (1792 - 64) * step / args.num_iterations) // 64)
    if sw_size != sw_size_prev:
        sliding_window_size.copy_(sw_size, non_blocking=True)
        sw_size_prev = sw_size

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                x_val, y_val = val_loader.next_batch()
                val_loss += model(x_val, y_val, sliding_window=sliding_window_size)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        # torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    loss = model(x, y, sliding_window=sliding_window_size)
    loss.backward()
    del loss
    # advance the dataset for the next batch
    x, y = train_loader.next_batch()
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

if master_process:
    print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Sun Dec  8 13:55:47 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.6     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:65:02.0 Off |                    0 |
| N/A   37C    P0              74W / 700W |      7MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:67:02.0 Off |                    0 |
| N/A   46C    P0             129W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:69:02.0 Off |                    0 |
| N/A   46C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:6B:02.0 Off |                    0 |
| N/A   39C    P0             118W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:6F:02.0 Off |                    0 |
| N/A   39C    P0             117W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:71:02.0 Off |                    0 |
| N/A   46C    P0             122W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:73:02.0 Off |                    0 |
| N/A   46C    P0             118W / 700W |    533MiB / 81559MiB |      1%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:75:02.0 Off |                    0 |
| N/A   39C    P0             124W / 700W |    533MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 3200000000 across 32 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:24209ms step_avg:nanms
step:2/1480 train_time:24296ms step_avg:nanms
step:3/1480 train_time:24434ms step_avg:nanms
step:4/1480 train_time:24575ms step_avg:nanms
step:5/1480 train_time:24717ms step_avg:nanms
step:6/1480 train_time:24859ms step_avg:nanms
step:7/1480 train_time:25000ms step_avg:nanms
step:8/1480 train_time:25143ms step_avg:nanms
step:9/1480 train_time:25287ms step_avg:nanms
step:10/1480 train_time:25431ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:427ms step_avg:142.17ms
step:14/1480 train_time:570ms step_avg:142.51ms
step:15/1480 train_time:714ms step_avg:142.87ms
step:16/1480 train_time:858ms step_avg:143.02ms
step:17/1480 train_time:1001ms step_avg:143.06ms
step:18/1480 train_time:1146ms step_avg:143.23ms
step:19/1480 train_time:1290ms step_avg:143.30ms
step:20/1480 train_time:1433ms step_avg:143.27ms
step:21/1480 train_time:1575ms step_avg:143.17ms
step:22/1480 train_time:1717ms step_avg:143.05ms
step:23/1480 train_time:1861ms step_avg:143.13ms
step:24/1480 train_time:2004ms step_avg:143.15ms
step:25/1480 train_time:2148ms step_avg:143.17ms
step:26/1480 train_time:2291ms step_avg:143.20ms
step:27/1480 train_time:2433ms step_avg:143.13ms
step:28/1480 train_time:2576ms step_avg:143.13ms
step:29/1480 train_time:2719ms step_avg:143.10ms
step:30/1480 train_time:2863ms step_avg:143.15ms
step:31/1480 train_time:3008ms step_avg:143.22ms
step:32/1480 train_time:3151ms step_avg:143.22ms
step:33/1480 train_time:3295ms step_avg:143.25ms
step:34/1480 train_time:3436ms step_avg:143.17ms
step:35/1480 train_time:3578ms step_avg:143.14ms
step:36/1480 train_time:3721ms step_avg:143.10ms
step:37/1480 train_time:3866ms step_avg:143.17ms
step:38/1480 train_time:4009ms step_avg:143.18ms
step:39/1480 train_time:4152ms step_avg:143.16ms
step:40/1480 train_time:4295ms step_avg:143.16ms
step:41/1480 train_time:4437ms step_avg:143.13ms
step:42/1480 train_time:4580ms step_avg:143.11ms
step:43/1480 train_time:4723ms step_avg:143.14ms
step:44/1480 train_time:4865ms step_avg:143.08ms
step:45/1480 train_time:5008ms step_avg:143.10ms
step:46/1480 train_time:5152ms step_avg:143.12ms
step:47/1480 train_time:5296ms step_avg:143.14ms
step:48/1480 train_time:5438ms step_avg:143.11ms
step:49/1480 train_time:5581ms step_avg:143.10ms
step:50/1480 train_time:5726ms step_avg:143.14ms
step:51/1480 train_time:5868ms step_avg:143.11ms
step:52/1480 train_time:6011ms step_avg:143.13ms
step:53/1480 train_time:6155ms step_avg:143.13ms
step:54/1480 train_time:6297ms step_avg:143.12ms
step:55/1480 train_time:6439ms step_avg:143.08ms
step:56/1480 train_time:6580ms step_avg:143.05ms
step:57/1480 train_time:6722ms step_avg:143.02ms
step:58/1480 train_time:6865ms step_avg:143.02ms
step:59/1480 train_time:7008ms step_avg:143.02ms
step:60/1480 train_time:7151ms step_avg:143.03ms
step:61/1480 train_time:7295ms step_avg:143.04ms
step:62/1480 train_time:7436ms step_avg:143.01ms
step:63/1480 train_time:7579ms step_avg:143.00ms
step:64/1480 train_time:7721ms step_avg:142.98ms
step:65/1480 train_time:7863ms step_avg:142.96ms
step:66/1480 train_time:8005ms step_avg:142.95ms
step:67/1480 train_time:8149ms step_avg:142.96ms
step:68/1480 train_time:8292ms step_avg:142.97ms
step:69/1480 train_time:8434ms step_avg:142.94ms
step:70/1480 train_time:8577ms step_avg:142.95ms
step:71/1480 train_time:8719ms step_avg:142.93ms
step:72/1480 train_time:8862ms step_avg:142.94ms
step:73/1480 train_time:9005ms step_avg:142.93ms
step:74/1480 train_time:9148ms step_avg:142.93ms
step:75/1480 train_time:9292ms step_avg:142.96ms
step:76/1480 train_time:9434ms step_avg:142.94ms
step:77/1480 train_time:9577ms step_avg:142.94ms
step:78/1480 train_time:9719ms step_avg:142.92ms
step:79/1480 train_time:9861ms step_avg:142.91ms
step:80/1480 train_time:10005ms step_avg:142.92ms
step:81/1480 train_time:10149ms step_avg:142.94ms
step:82/1480 train_time:10293ms step_avg:142.96ms
step:83/1480 train_time:10435ms step_avg:142.95ms
step:84/1480 train_time:10579ms step_avg:142.96ms
step:85/1480 train_time:10720ms step_avg:142.93ms
step:86/1480 train_time:10862ms step_avg:142.92ms
step:87/1480 train_time:11007ms step_avg:142.95ms
step:88/1480 train_time:11151ms step_avg:142.96ms
step:89/1480 train_time:11295ms step_avg:142.97ms
step:90/1480 train_time:11437ms step_avg:142.96ms
step:91/1480 train_time:11579ms step_avg:142.95ms
step:92/1480 train_time:11720ms step_avg:142.93ms
step:93/1480 train_time:11862ms step_avg:142.91ms
step:94/1480 train_time:12005ms step_avg:142.92ms
step:95/1480 train_time:12150ms step_avg:142.94ms
step:96/1480 train_time:12294ms step_avg:142.95ms
step:97/1480 train_time:12437ms step_avg:142.95ms
step:98/1480 train_time:12578ms step_avg:142.94ms
step:99/1480 train_time:12720ms step_avg:142.92ms
step:100/1480 train_time:12863ms step_avg:142.92ms
step:101/1480 train_time:13006ms step_avg:142.93ms
step:102/1480 train_time:13150ms step_avg:142.94ms
step:103/1480 train_time:13294ms step_avg:142.94ms
step:104/1480 train_time:13436ms step_avg:142.94ms
step:105/1480 train_time:13579ms step_avg:142.93ms
step:106/1480 train_time:13719ms step_avg:142.91ms
step:107/1480 train_time:13861ms step_avg:142.90ms
step:108/1480 train_time:14003ms step_avg:142.89ms
step:109/1480 train_time:14148ms step_avg:142.90ms
step:110/1480 train_time:14291ms step_avg:142.91ms
step:111/1480 train_time:14435ms step_avg:142.92ms
step:112/1480 train_time:14581ms step_avg:142.95ms
step:113/1480 train_time:14727ms step_avg:142.98ms
step:114/1480 train_time:14874ms step_avg:143.02ms
step:115/1480 train_time:15020ms step_avg:143.05ms
step:116/1480 train_time:15169ms step_avg:143.10ms
step:117/1480 train_time:15317ms step_avg:143.15ms
step:118/1480 train_time:15464ms step_avg:143.18ms
step:119/1480 train_time:15612ms step_avg:143.23ms
step:120/1480 train_time:15758ms step_avg:143.25ms
step:121/1480 train_time:15905ms step_avg:143.29ms
step:122/1480 train_time:16051ms step_avg:143.31ms
step:123/1480 train_time:16199ms step_avg:143.36ms
step:124/1480 train_time:16346ms step_avg:143.38ms
step:125/1480 train_time:16493ms step_avg:143.42ms
step:125/1480 val_loss:4.4180 train_time:16551ms step_avg:143.92ms
step:126/1480 train_time:16648ms step_avg:143.52ms
step:127/1480 train_time:16796ms step_avg:143.56ms
step:128/1480 train_time:16942ms step_avg:143.57ms
step:129/1480 train_time:17088ms step_avg:143.60ms
step:130/1480 train_time:17235ms step_avg:143.63ms
step:131/1480 train_time:17381ms step_avg:143.65ms
step:132/1480 train_time:17528ms step_avg:143.67ms
step:133/1480 train_time:17675ms step_avg:143.70ms
step:134/1480 train_time:17824ms step_avg:143.74ms
step:135/1480 train_time:17972ms step_avg:143.77ms
step:136/1480 train_time:18118ms step_avg:143.79ms
step:137/1480 train_time:18264ms step_avg:143.81ms
step:138/1480 train_time:18412ms step_avg:143.84ms
step:139/1480 train_time:18558ms step_avg:143.86ms
step:140/1480 train_time:18708ms step_avg:143.91ms
step:141/1480 train_time:18855ms step_avg:143.94ms
step:142/1480 train_time:19003ms step_avg:143.97ms
step:143/1480 train_time:19152ms step_avg:144.00ms
step:144/1480 train_time:19297ms step_avg:144.01ms
step:145/1480 train_time:19444ms step_avg:144.03ms
step:146/1480 train_time:19590ms step_avg:144.05ms
step:147/1480 train_time:19737ms step_avg:144.07ms
step:148/1480 train_time:19884ms step_avg:144.09ms
step:149/1480 train_time:20032ms step_avg:144.11ms
step:150/1480 train_time:20178ms step_avg:144.13ms
step:151/1480 train_time:20326ms step_avg:144.15ms
step:152/1480 train_time:20473ms step_avg:144.17ms
step:153/1480 train_time:20619ms step_avg:144.19ms
step:154/1480 train_time:20767ms step_avg:144.22ms
step:155/1480 train_time:20914ms step_avg:144.24ms
step:156/1480 train_time:21061ms step_avg:144.25ms
step:157/1480 train_time:21209ms step_avg:144.28ms
step:158/1480 train_time:21355ms step_avg:144.29ms
step:159/1480 train_time:21502ms step_avg:144.31ms
step:160/1480 train_time:21650ms step_avg:144.33ms
step:161/1480 train_time:21796ms step_avg:144.34ms
step:162/1480 train_time:21946ms step_avg:144.38ms
step:163/1480 train_time:22092ms step_avg:144.39ms
step:164/1480 train_time:22240ms step_avg:144.41ms
step:165/1480 train_time:22387ms step_avg:144.43ms
step:166/1480 train_time:22534ms step_avg:144.45ms
step:167/1480 train_time:22680ms step_avg:144.46ms
step:168/1480 train_time:22827ms step_avg:144.48ms
step:169/1480 train_time:22975ms step_avg:144.50ms
step:170/1480 train_time:23123ms step_avg:144.52ms
step:171/1480 train_time:23270ms step_avg:144.53ms
step:172/1480 train_time:23417ms step_avg:144.55ms
step:173/1480 train_time:23563ms step_avg:144.56ms
step:174/1480 train_time:23711ms step_avg:144.58ms
step:175/1480 train_time:23857ms step_avg:144.59ms
step:176/1480 train_time:24004ms step_avg:144.60ms
step:177/1480 train_time:24151ms step_avg:144.62ms
step:178/1480 train_time:24298ms step_avg:144.63ms
step:179/1480 train_time:24445ms step_avg:144.64ms
step:180/1480 train_time:24593ms step_avg:144.66ms
step:181/1480 train_time:24739ms step_avg:144.67ms
step:182/1480 train_time:24887ms step_avg:144.69ms
step:183/1480 train_time:25034ms step_avg:144.70ms
step:184/1480 train_time:25180ms step_avg:144.71ms
step:185/1480 train_time:25329ms step_avg:144.74ms
step:186/1480 train_time:25475ms step_avg:144.74ms
step:187/1480 train_time:25621ms step_avg:144.75ms
step:188/1480 train_time:25769ms step_avg:144.77ms
step:189/1480 train_time:25916ms step_avg:144.78ms
step:190/1480 train_time:26062ms step_avg:144.79ms
step:191/1480 train_time:26210ms step_avg:144.81ms
step:192/1480 train_time:26356ms step_avg:144.81ms
step:193/1480 train_time:26503ms step_avg:144.83ms
step:194/1480 train_time:26650ms step_avg:144.84ms
step:195/1480 train_time:26797ms step_avg:144.85ms
step:196/1480 train_time:26944ms step_avg:144.86ms
step:197/1480 train_time:27092ms step_avg:144.87ms
step:198/1480 train_time:27238ms step_avg:144.88ms
step:199/1480 train_time:27387ms step_avg:144.90ms
step:200/1480 train_time:27534ms step_avg:144.92ms
step:201/1480 train_time:27679ms step_avg:144.92ms
step:202/1480 train_time:27828ms step_avg:144.94ms
step:203/1480 train_time:27974ms step_avg:144.94ms
step:204/1480 train_time:28120ms step_avg:144.95ms
step:205/1480 train_time:28267ms step_avg:144.96ms
step:206/1480 train_time:28415ms step_avg:144.97ms
step:207/1480 train_time:28560ms step_avg:144.98ms
step:208/1480 train_time:28709ms step_avg:145.00ms
step:209/1480 train_time:28856ms step_avg:145.01ms
step:210/1480 train_time:29005ms step_avg:145.02ms
step:211/1480 train_time:29152ms step_avg:145.03ms
step:212/1480 train_time:29296ms step_avg:145.03ms
step:213/1480 train_time:29443ms step_avg:145.04ms
step:214/1480 train_time:29590ms step_avg:145.05ms
step:215/1480 train_time:29737ms step_avg:145.06ms
step:216/1480 train_time:29884ms step_avg:145.07ms
step:217/1480 train_time:30033ms step_avg:145.09ms
step:218/1480 train_time:30180ms step_avg:145.09ms
step:219/1480 train_time:30327ms step_avg:145.11ms
step:220/1480 train_time:30474ms step_avg:145.11ms
step:221/1480 train_time:30620ms step_avg:145.12ms
step:222/1480 train_time:30771ms step_avg:145.15ms
step:223/1480 train_time:30923ms step_avg:145.18ms
step:224/1480 train_time:31074ms step_avg:145.20ms
step:225/1480 train_time:31225ms step_avg:145.23ms
step:226/1480 train_time:31376ms step_avg:145.26ms
step:227/1480 train_time:31528ms step_avg:145.29ms
step:228/1480 train_time:31677ms step_avg:145.31ms
step:229/1480 train_time:31829ms step_avg:145.34ms
step:230/1480 train_time:31980ms step_avg:145.36ms
step:231/1480 train_time:32130ms step_avg:145.39ms
step:232/1480 train_time:32280ms step_avg:145.41ms
step:233/1480 train_time:32433ms step_avg:145.44ms
step:234/1480 train_time:32583ms step_avg:145.46ms
step:235/1480 train_time:32735ms step_avg:145.49ms
step:236/1480 train_time:32885ms step_avg:145.51ms
step:237/1480 train_time:33036ms step_avg:145.53ms
step:238/1480 train_time:33187ms step_avg:145.56ms
step:239/1480 train_time:33338ms step_avg:145.58ms
step:240/1480 train_time:33489ms step_avg:145.60ms
step:241/1480 train_time:33641ms step_avg:145.63ms
step:242/1480 train_time:33792ms step_avg:145.65ms
step:243/1480 train_time:33944ms step_avg:145.68ms
step:244/1480 train_time:34095ms step_avg:145.70ms
step:245/1480 train_time:34247ms step_avg:145.73ms
step:246/1480 train_time:34396ms step_avg:145.75ms
step:247/1480 train_time:34549ms step_avg:145.77ms
step:248/1480 train_time:34698ms step_avg:145.79ms
step:249/1480 train_time:34850ms step_avg:145.81ms
step:250/1480 train_time:35001ms step_avg:145.84ms
step:250/1480 val_loss:3.9930 train_time:35060ms step_avg:146.08ms
step:251/1480 train_time:35157ms step_avg:145.88ms
step:252/1480 train_time:35308ms step_avg:145.90ms
step:253/1480 train_time:35459ms step_avg:145.92ms
step:254/1480 train_time:35608ms step_avg:145.94ms
step:255/1480 train_time:35758ms step_avg:145.95ms
step:256/1480 train_time:35908ms step_avg:145.97ms
step:257/1480 train_time:36060ms step_avg:145.99ms
step:258/1480 train_time:36211ms step_avg:146.01ms
step:259/1480 train_time:36363ms step_avg:146.04ms
step:260/1480 train_time:36513ms step_avg:146.05ms
step:261/1480 train_time:36664ms step_avg:146.07ms
step:262/1480 train_time:36812ms step_avg:146.08ms
step:263/1480 train_time:36963ms step_avg:146.10ms
step:264/1480 train_time:37115ms step_avg:146.12ms
step:265/1480 train_time:37267ms step_avg:146.14ms
step:266/1480 train_time:37418ms step_avg:146.17ms
step:267/1480 train_time:37569ms step_avg:146.18ms
step:268/1480 train_time:37720ms step_avg:146.20ms
step:269/1480 train_time:37871ms step_avg:146.22ms
step:270/1480 train_time:38020ms step_avg:146.23ms
step:271/1480 train_time:38170ms step_avg:146.25ms
step:272/1480 train_time:38322ms step_avg:146.27ms
step:273/1480 train_time:38473ms step_avg:146.28ms
step:274/1480 train_time:38624ms step_avg:146.30ms
step:275/1480 train_time:38775ms step_avg:146.32ms
step:276/1480 train_time:38926ms step_avg:146.34ms
step:277/1480 train_time:39076ms step_avg:146.35ms
step:278/1480 train_time:39228ms step_avg:146.37ms
step:279/1480 train_time:39380ms step_avg:146.39ms
step:280/1480 train_time:39531ms step_avg:146.41ms
step:281/1480 train_time:39681ms step_avg:146.43ms
step:282/1480 train_time:39833ms step_avg:146.44ms
step:283/1480 train_time:39983ms step_avg:146.46ms
step:284/1480 train_time:40134ms step_avg:146.47ms
step:285/1480 train_time:40284ms step_avg:146.49ms
step:286/1480 train_time:40436ms step_avg:146.51ms
step:287/1480 train_time:40588ms step_avg:146.53ms
step:288/1480 train_time:40740ms step_avg:146.55ms
step:289/1480 train_time:40890ms step_avg:146.56ms
step:290/1480 train_time:41042ms step_avg:146.58ms
step:291/1480 train_time:41193ms step_avg:146.60ms
step:292/1480 train_time:41344ms step_avg:146.61ms
step:293/1480 train_time:41495ms step_avg:146.62ms
step:294/1480 train_time:41646ms step_avg:146.64ms
step:295/1480 train_time:41795ms step_avg:146.65ms
step:296/1480 train_time:41947ms step_avg:146.67ms
step:297/1480 train_time:42096ms step_avg:146.68ms
step:298/1480 train_time:42248ms step_avg:146.69ms
step:299/1480 train_time:42399ms step_avg:146.71ms
step:300/1480 train_time:42550ms step_avg:146.72ms
step:301/1480 train_time:42700ms step_avg:146.74ms
step:302/1480 train_time:42851ms step_avg:146.75ms
step:303/1480 train_time:43002ms step_avg:146.76ms
step:304/1480 train_time:43153ms step_avg:146.78ms
step:305/1480 train_time:43305ms step_avg:146.80ms
step:306/1480 train_time:43456ms step_avg:146.81ms
step:307/1480 train_time:43606ms step_avg:146.82ms
step:308/1480 train_time:43757ms step_avg:146.83ms
step:309/1480 train_time:43907ms step_avg:146.84ms
step:310/1480 train_time:44057ms step_avg:146.86ms
step:311/1480 train_time:44208ms step_avg:146.87ms
step:312/1480 train_time:44360ms step_avg:146.89ms
step:313/1480 train_time:44511ms step_avg:146.90ms
step:314/1480 train_time:44663ms step_avg:146.92ms
step:315/1480 train_time:44813ms step_avg:146.93ms
step:316/1480 train_time:44964ms step_avg:146.94ms
step:317/1480 train_time:45115ms step_avg:146.95ms
step:318/1480 train_time:45265ms step_avg:146.96ms
step:319/1480 train_time:45416ms step_avg:146.98ms
step:320/1480 train_time:45567ms step_avg:146.99ms
step:321/1480 train_time:45717ms step_avg:147.00ms
step:322/1480 train_time:45868ms step_avg:147.01ms
step:323/1480 train_time:46019ms step_avg:147.03ms
step:324/1480 train_time:46169ms step_avg:147.03ms
step:325/1480 train_time:46320ms step_avg:147.05ms
step:326/1480 train_time:46471ms step_avg:147.06ms
step:327/1480 train_time:46622ms step_avg:147.07ms
step:328/1480 train_time:46772ms step_avg:147.08ms
step:329/1480 train_time:46923ms step_avg:147.09ms
step:330/1480 train_time:47076ms step_avg:147.11ms
step:331/1480 train_time:47229ms step_avg:147.13ms
step:332/1480 train_time:47383ms step_avg:147.15ms
step:333/1480 train_time:47537ms step_avg:147.17ms
step:334/1480 train_time:47690ms step_avg:147.19ms
step:335/1480 train_time:47846ms step_avg:147.22ms
step:336/1480 train_time:48000ms step_avg:147.24ms
step:337/1480 train_time:48154ms step_avg:147.26ms
step:338/1480 train_time:48308ms step_avg:147.28ms
step:339/1480 train_time:48461ms step_avg:147.30ms
step:340/1480 train_time:48615ms step_avg:147.32ms
step:341/1480 train_time:48768ms step_avg:147.34ms
step:342/1480 train_time:48922ms step_avg:147.35ms
step:343/1480 train_time:49075ms step_avg:147.37ms
step:344/1480 train_time:49229ms step_avg:147.39ms
step:345/1480 train_time:49384ms step_avg:147.41ms
step:346/1480 train_time:49537ms step_avg:147.43ms
step:347/1480 train_time:49690ms step_avg:147.45ms
step:348/1480 train_time:49844ms step_avg:147.47ms
step:349/1480 train_time:49998ms step_avg:147.49ms
step:350/1480 train_time:50152ms step_avg:147.51ms
step:351/1480 train_time:50306ms step_avg:147.52ms
step:352/1480 train_time:50461ms step_avg:147.55ms
step:353/1480 train_time:50616ms step_avg:147.57ms
step:354/1480 train_time:50769ms step_avg:147.58ms
step:355/1480 train_time:50923ms step_avg:147.60ms
step:356/1480 train_time:51078ms step_avg:147.63ms
step:357/1480 train_time:51232ms step_avg:147.64ms
step:358/1480 train_time:51385ms step_avg:147.66ms
step:359/1480 train_time:51540ms step_avg:147.68ms
step:360/1480 train_time:51697ms step_avg:147.71ms
step:361/1480 train_time:51854ms step_avg:147.73ms
step:362/1480 train_time:52007ms step_avg:147.75ms
step:363/1480 train_time:52162ms step_avg:147.77ms
step:364/1480 train_time:52317ms step_avg:147.79ms
step:365/1480 train_time:52469ms step_avg:147.80ms
step:366/1480 train_time:52622ms step_avg:147.82ms
step:367/1480 train_time:52776ms step_avg:147.83ms
step:368/1480 train_time:52929ms step_avg:147.85ms
step:369/1480 train_time:53082ms step_avg:147.86ms
step:370/1480 train_time:53236ms step_avg:147.88ms
step:371/1480 train_time:53389ms step_avg:147.89ms
step:372/1480 train_time:53543ms step_avg:147.91ms
step:373/1480 train_time:53697ms step_avg:147.93ms
step:374/1480 train_time:53849ms step_avg:147.94ms
step:375/1480 train_time:54002ms step_avg:147.95ms
step:375/1480 val_loss:3.8084 train_time:54065ms step_avg:148.12ms
step:376/1480 train_time:54165ms step_avg:147.99ms
step:377/1480 train_time:54321ms step_avg:148.01ms
step:378/1480 train_time:54474ms step_avg:148.03ms
step:379/1480 train_time:54626ms step_avg:148.04ms
step:380/1480 train_time:54778ms step_avg:148.05ms
step:381/1480 train_time:54929ms step_avg:148.06ms
step:382/1480 train_time:55083ms step_avg:148.07ms
step:383/1480 train_time:55241ms step_avg:148.10ms
step:384/1480 train_time:55395ms step_avg:148.12ms
step:385/1480 train_time:55548ms step_avg:148.13ms
step:386/1480 train_time:55701ms step_avg:148.14ms
step:387/1480 train_time:55856ms step_avg:148.16ms
step:388/1480 train_time:56009ms step_avg:148.17ms
step:389/1480 train_time:56163ms step_avg:148.19ms
step:390/1480 train_time:56319ms step_avg:148.21ms
step:391/1480 train_time:56473ms step_avg:148.22ms
step:392/1480 train_time:56626ms step_avg:148.24ms
step:393/1480 train_time:56779ms step_avg:148.25ms
step:394/1480 train_time:56932ms step_avg:148.26ms
step:395/1480 train_time:57084ms step_avg:148.27ms
step:396/1480 train_time:57238ms step_avg:148.29ms
step:397/1480 train_time:57392ms step_avg:148.30ms
step:398/1480 train_time:57546ms step_avg:148.31ms
step:399/1480 train_time:57700ms step_avg:148.33ms
step:400/1480 train_time:57853ms step_avg:148.34ms
step:401/1480 train_time:58005ms step_avg:148.35ms
step:402/1480 train_time:58160ms step_avg:148.37ms
step:403/1480 train_time:58314ms step_avg:148.38ms
step:404/1480 train_time:58468ms step_avg:148.40ms
step:405/1480 train_time:58622ms step_avg:148.41ms
step:406/1480 train_time:58778ms step_avg:148.43ms
step:407/1480 train_time:58930ms step_avg:148.44ms
step:408/1480 train_time:59084ms step_avg:148.45ms
step:409/1480 train_time:59239ms step_avg:148.47ms
step:410/1480 train_time:59393ms step_avg:148.48ms
step:411/1480 train_time:59545ms step_avg:148.49ms
step:412/1480 train_time:59699ms step_avg:148.51ms
step:413/1480 train_time:59854ms step_avg:148.52ms
step:414/1480 train_time:60008ms step_avg:148.53ms
step:415/1480 train_time:60162ms step_avg:148.55ms
step:416/1480 train_time:60316ms step_avg:148.56ms
step:417/1480 train_time:60469ms step_avg:148.57ms
step:418/1480 train_time:60623ms step_avg:148.59ms
step:419/1480 train_time:60777ms step_avg:148.60ms
step:420/1480 train_time:60930ms step_avg:148.61ms
step:421/1480 train_time:61084ms step_avg:148.62ms
step:422/1480 train_time:61238ms step_avg:148.64ms
step:423/1480 train_time:61392ms step_avg:148.65ms
step:424/1480 train_time:61546ms step_avg:148.66ms
step:425/1480 train_time:61700ms step_avg:148.67ms
step:426/1480 train_time:61854ms step_avg:148.69ms
step:427/1480 train_time:62007ms step_avg:148.70ms
step:428/1480 train_time:62162ms step_avg:148.71ms
step:429/1480 train_time:62315ms step_avg:148.72ms
step:430/1480 train_time:62469ms step_avg:148.74ms
step:431/1480 train_time:62623ms step_avg:148.75ms
step:432/1480 train_time:62777ms step_avg:148.76ms
step:433/1480 train_time:62931ms step_avg:148.77ms
step:434/1480 train_time:63084ms step_avg:148.78ms
step:435/1480 train_time:63239ms step_avg:148.80ms
step:436/1480 train_time:63393ms step_avg:148.81ms
step:437/1480 train_time:63547ms step_avg:148.82ms
step:438/1480 train_time:63700ms step_avg:148.83ms
step:439/1480 train_time:63856ms step_avg:148.85ms
step:440/1480 train_time:64010ms step_avg:148.86ms
step:441/1480 train_time:64166ms step_avg:148.88ms
step:442/1480 train_time:64323ms step_avg:148.90ms
step:443/1480 train_time:64481ms step_avg:148.92ms
step:444/1480 train_time:64637ms step_avg:148.93ms
step:445/1480 train_time:64792ms step_avg:148.95ms
step:446/1480 train_time:64948ms step_avg:148.96ms
step:447/1480 train_time:65103ms step_avg:148.98ms
step:448/1480 train_time:65262ms step_avg:149.00ms
step:449/1480 train_time:65420ms step_avg:149.02ms
step:450/1480 train_time:65576ms step_avg:149.04ms
step:451/1480 train_time:65734ms step_avg:149.06ms
step:452/1480 train_time:65890ms step_avg:149.07ms
step:453/1480 train_time:66046ms step_avg:149.09ms
step:454/1480 train_time:66203ms step_avg:149.11ms
step:455/1480 train_time:66361ms step_avg:149.13ms
step:456/1480 train_time:66517ms step_avg:149.14ms
step:457/1480 train_time:66673ms step_avg:149.16ms
step:458/1480 train_time:66829ms step_avg:149.17ms
step:459/1480 train_time:66987ms step_avg:149.19ms
step:460/1480 train_time:67143ms step_avg:149.21ms
step:461/1480 train_time:67303ms step_avg:149.23ms
step:462/1480 train_time:67461ms step_avg:149.25ms
step:463/1480 train_time:67620ms step_avg:149.27ms
step:464/1480 train_time:67776ms step_avg:149.29ms
step:465/1480 train_time:67932ms step_avg:149.30ms
step:466/1480 train_time:68088ms step_avg:149.32ms
step:467/1480 train_time:68246ms step_avg:149.33ms
step:468/1480 train_time:68402ms step_avg:149.35ms
step:469/1480 train_time:68561ms step_avg:149.37ms
step:470/1480 train_time:68717ms step_avg:149.38ms
step:471/1480 train_time:68873ms step_avg:149.40ms
step:472/1480 train_time:69030ms step_avg:149.42ms
step:473/1480 train_time:69187ms step_avg:149.43ms
step:474/1480 train_time:69343ms step_avg:149.45ms
step:475/1480 train_time:69500ms step_avg:149.46ms
step:476/1480 train_time:69656ms step_avg:149.48ms
step:477/1480 train_time:69812ms step_avg:149.49ms
step:478/1480 train_time:69968ms step_avg:149.50ms
step:479/1480 train_time:70125ms step_avg:149.52ms
step:480/1480 train_time:70284ms step_avg:149.54ms
step:481/1480 train_time:70441ms step_avg:149.56ms
step:482/1480 train_time:70597ms step_avg:149.57ms
step:483/1480 train_time:70753ms step_avg:149.58ms
step:484/1480 train_time:70909ms step_avg:149.60ms
step:485/1480 train_time:71067ms step_avg:149.61ms
step:486/1480 train_time:71224ms step_avg:149.63ms
step:487/1480 train_time:71383ms step_avg:149.65ms
step:488/1480 train_time:71543ms step_avg:149.67ms
step:489/1480 train_time:71701ms step_avg:149.69ms
step:490/1480 train_time:71858ms step_avg:149.70ms
step:491/1480 train_time:72014ms step_avg:149.72ms
step:492/1480 train_time:72169ms step_avg:149.73ms
step:493/1480 train_time:72326ms step_avg:149.74ms
step:494/1480 train_time:72482ms step_avg:149.76ms
step:495/1480 train_time:72641ms step_avg:149.77ms
step:496/1480 train_time:72798ms step_avg:149.79ms
step:497/1480 train_time:72954ms step_avg:149.80ms
step:498/1480 train_time:73111ms step_avg:149.82ms
step:499/1480 train_time:73268ms step_avg:149.83ms
step:500/1480 train_time:73424ms step_avg:149.85ms
step:500/1480 val_loss:3.6904 train_time:73486ms step_avg:149.97ms
step:501/1480 train_time:73585ms step_avg:149.87ms
step:502/1480 train_time:73745ms step_avg:149.89ms
step:503/1480 train_time:73902ms step_avg:149.90ms
step:504/1480 train_time:74056ms step_avg:149.91ms
step:505/1480 train_time:74211ms step_avg:149.92ms
step:506/1480 train_time:74368ms step_avg:149.94ms
step:507/1480 train_time:74526ms step_avg:149.95ms
step:508/1480 train_time:74684ms step_avg:149.97ms
step:509/1480 train_time:74842ms step_avg:149.98ms
step:510/1480 train_time:74999ms step_avg:150.00ms
step:511/1480 train_time:75156ms step_avg:150.01ms
step:512/1480 train_time:75314ms step_avg:150.03ms
step:513/1480 train_time:75470ms step_avg:150.04ms
step:514/1480 train_time:75627ms step_avg:150.05ms
step:515/1480 train_time:75785ms step_avg:150.07ms
step:516/1480 train_time:75945ms step_avg:150.09ms
step:517/1480 train_time:76104ms step_avg:150.11ms
step:518/1480 train_time:76261ms step_avg:150.12ms
step:519/1480 train_time:76418ms step_avg:150.13ms
step:520/1480 train_time:76574ms step_avg:150.14ms
step:521/1480 train_time:76730ms step_avg:150.16ms
step:522/1480 train_time:76887ms step_avg:150.17ms
step:523/1480 train_time:77047ms step_avg:150.19ms
step:524/1480 train_time:77204ms step_avg:150.20ms
step:525/1480 train_time:77361ms step_avg:150.22ms
step:526/1480 train_time:77520ms step_avg:150.23ms
step:527/1480 train_time:77679ms step_avg:150.25ms
step:528/1480 train_time:77836ms step_avg:150.26ms
step:529/1480 train_time:77992ms step_avg:150.27ms
step:530/1480 train_time:78148ms step_avg:150.29ms
step:531/1480 train_time:78305ms step_avg:150.30ms
step:532/1480 train_time:78464ms step_avg:150.31ms
step:533/1480 train_time:78621ms step_avg:150.33ms
step:534/1480 train_time:78776ms step_avg:150.34ms
step:535/1480 train_time:78932ms step_avg:150.35ms
step:536/1480 train_time:79090ms step_avg:150.36ms
step:537/1480 train_time:79247ms step_avg:150.37ms
step:538/1480 train_time:79404ms step_avg:150.39ms
step:539/1480 train_time:79561ms step_avg:150.40ms
step:540/1480 train_time:79718ms step_avg:150.41ms
step:541/1480 train_time:79873ms step_avg:150.42ms
step:542/1480 train_time:80030ms step_avg:150.43ms
step:543/1480 train_time:80185ms step_avg:150.44ms
step:544/1480 train_time:80345ms step_avg:150.46ms
step:545/1480 train_time:80502ms step_avg:150.47ms
step:546/1480 train_time:80658ms step_avg:150.48ms
step:547/1480 train_time:80813ms step_avg:150.49ms
step:548/1480 train_time:80971ms step_avg:150.50ms
step:549/1480 train_time:81128ms step_avg:150.52ms
step:550/1480 train_time:81287ms step_avg:150.53ms
step:551/1480 train_time:81446ms step_avg:150.55ms
step:552/1480 train_time:81606ms step_avg:150.56ms
step:553/1480 train_time:81767ms step_avg:150.58ms
step:554/1480 train_time:81928ms step_avg:150.60ms
step:555/1480 train_time:82088ms step_avg:150.62ms
step:556/1480 train_time:82248ms step_avg:150.64ms
step:557/1480 train_time:82408ms step_avg:150.65ms
step:558/1480 train_time:82567ms step_avg:150.67ms
step:559/1480 train_time:82728ms step_avg:150.69ms
step:560/1480 train_time:82887ms step_avg:150.70ms
step:561/1480 train_time:83047ms step_avg:150.72ms
step:562/1480 train_time:83207ms step_avg:150.74ms
step:563/1480 train_time:83367ms step_avg:150.75ms
step:564/1480 train_time:83526ms step_avg:150.77ms
step:565/1480 train_time:83685ms step_avg:150.78ms
step:566/1480 train_time:83848ms step_avg:150.81ms
step:567/1480 train_time:84007ms step_avg:150.82ms
step:568/1480 train_time:84167ms step_avg:150.84ms
step:569/1480 train_time:84327ms step_avg:150.85ms
step:570/1480 train_time:84486ms step_avg:150.87ms
step:571/1480 train_time:84647ms step_avg:150.89ms
step:572/1480 train_time:84808ms step_avg:150.90ms
step:573/1480 train_time:84968ms step_avg:150.92ms
step:574/1480 train_time:85130ms step_avg:150.94ms
step:575/1480 train_time:85289ms step_avg:150.95ms
step:576/1480 train_time:85449ms step_avg:150.97ms
step:577/1480 train_time:85608ms step_avg:150.98ms
step:578/1480 train_time:85767ms step_avg:151.00ms
step:579/1480 train_time:85926ms step_avg:151.01ms
step:580/1480 train_time:86087ms step_avg:151.03ms
step:581/1480 train_time:86249ms step_avg:151.05ms
step:582/1480 train_time:86409ms step_avg:151.06ms
step:583/1480 train_time:86568ms step_avg:151.08ms
step:584/1480 train_time:86728ms step_avg:151.09ms
step:585/1480 train_time:86885ms step_avg:151.10ms
step:586/1480 train_time:87047ms step_avg:151.12ms
step:587/1480 train_time:87206ms step_avg:151.14ms
step:588/1480 train_time:87365ms step_avg:151.15ms
step:589/1480 train_time:87526ms step_avg:151.17ms
step:590/1480 train_time:87687ms step_avg:151.18ms
step:591/1480 train_time:87846ms step_avg:151.20ms
step:592/1480 train_time:88007ms step_avg:151.21ms
step:593/1480 train_time:88168ms step_avg:151.23ms
step:594/1480 train_time:88328ms step_avg:151.25ms
step:595/1480 train_time:88489ms step_avg:151.26ms
step:596/1480 train_time:88650ms step_avg:151.28ms
step:597/1480 train_time:88808ms step_avg:151.29ms
step:598/1480 train_time:88966ms step_avg:151.30ms
step:599/1480 train_time:89126ms step_avg:151.32ms
step:600/1480 train_time:89286ms step_avg:151.33ms
step:601/1480 train_time:89446ms step_avg:151.35ms
step:602/1480 train_time:89606ms step_avg:151.36ms
step:603/1480 train_time:89768ms step_avg:151.38ms
step:604/1480 train_time:89927ms step_avg:151.39ms
step:605/1480 train_time:90088ms step_avg:151.41ms
step:606/1480 train_time:90250ms step_avg:151.43ms
step:607/1480 train_time:90411ms step_avg:151.44ms
step:608/1480 train_time:90570ms step_avg:151.45ms
step:609/1480 train_time:90729ms step_avg:151.47ms
step:610/1480 train_time:90887ms step_avg:151.48ms
step:611/1480 train_time:91049ms step_avg:151.50ms
step:612/1480 train_time:91209ms step_avg:151.51ms
step:613/1480 train_time:91370ms step_avg:151.53ms
step:614/1480 train_time:91529ms step_avg:151.54ms
step:615/1480 train_time:91687ms step_avg:151.55ms
step:616/1480 train_time:91846ms step_avg:151.56ms
step:617/1480 train_time:92007ms step_avg:151.58ms
step:618/1480 train_time:92166ms step_avg:151.59ms
step:619/1480 train_time:92326ms step_avg:151.60ms
step:620/1480 train_time:92486ms step_avg:151.62ms
step:621/1480 train_time:92648ms step_avg:151.63ms
step:622/1480 train_time:92809ms step_avg:151.65ms
step:623/1480 train_time:92969ms step_avg:151.66ms
step:624/1480 train_time:93127ms step_avg:151.67ms
step:625/1480 train_time:93287ms step_avg:151.69ms
step:625/1480 val_loss:3.6062 train_time:93352ms step_avg:151.79ms
step:626/1480 train_time:93452ms step_avg:151.71ms
step:627/1480 train_time:93612ms step_avg:151.72ms
step:628/1480 train_time:93771ms step_avg:151.73ms
step:629/1480 train_time:93929ms step_avg:151.74ms
step:630/1480 train_time:94086ms step_avg:151.75ms
step:631/1480 train_time:94244ms step_avg:151.76ms
step:632/1480 train_time:94404ms step_avg:151.78ms
step:633/1480 train_time:94563ms step_avg:151.79ms
step:634/1480 train_time:94723ms step_avg:151.80ms
step:635/1480 train_time:94883ms step_avg:151.81ms
step:636/1480 train_time:95042ms step_avg:151.82ms
step:637/1480 train_time:95201ms step_avg:151.84ms
step:638/1480 train_time:95360ms step_avg:151.85ms
step:639/1480 train_time:95520ms step_avg:151.86ms
step:640/1480 train_time:95679ms step_avg:151.87ms
step:641/1480 train_time:95840ms step_avg:151.89ms
step:642/1480 train_time:95999ms step_avg:151.90ms
step:643/1480 train_time:96158ms step_avg:151.91ms
step:644/1480 train_time:96318ms step_avg:151.92ms
step:645/1480 train_time:96478ms step_avg:151.93ms
step:646/1480 train_time:96637ms step_avg:151.95ms
step:647/1480 train_time:96798ms step_avg:151.96ms
step:648/1480 train_time:96958ms step_avg:151.97ms
step:649/1480 train_time:97119ms step_avg:151.99ms
step:650/1480 train_time:97278ms step_avg:152.00ms
step:651/1480 train_time:97439ms step_avg:152.01ms
step:652/1480 train_time:97599ms step_avg:152.02ms
step:653/1480 train_time:97758ms step_avg:152.03ms
step:654/1480 train_time:97919ms step_avg:152.05ms
step:655/1480 train_time:98079ms step_avg:152.06ms
step:656/1480 train_time:98240ms step_avg:152.07ms
step:657/1480 train_time:98400ms step_avg:152.09ms
step:658/1480 train_time:98559ms step_avg:152.10ms
step:659/1480 train_time:98722ms step_avg:152.11ms
step:660/1480 train_time:98884ms step_avg:152.13ms
step:661/1480 train_time:99045ms step_avg:152.14ms
step:662/1480 train_time:99203ms step_avg:152.15ms
step:663/1480 train_time:99362ms step_avg:152.16ms
step:664/1480 train_time:99524ms step_avg:152.18ms
step:665/1480 train_time:99686ms step_avg:152.19ms
step:666/1480 train_time:99846ms step_avg:152.20ms
step:667/1480 train_time:100007ms step_avg:152.22ms
step:668/1480 train_time:100169ms step_avg:152.23ms
step:669/1480 train_time:100331ms step_avg:152.25ms
step:670/1480 train_time:100490ms step_avg:152.26ms
step:671/1480 train_time:100651ms step_avg:152.27ms
step:672/1480 train_time:100812ms step_avg:152.28ms
step:673/1480 train_time:100975ms step_avg:152.30ms
step:674/1480 train_time:101138ms step_avg:152.32ms
step:675/1480 train_time:101301ms step_avg:152.33ms
step:676/1480 train_time:101463ms step_avg:152.35ms
step:677/1480 train_time:101623ms step_avg:152.36ms
step:678/1480 train_time:101783ms step_avg:152.37ms
step:679/1480 train_time:101945ms step_avg:152.38ms
step:680/1480 train_time:102107ms step_avg:152.40ms
step:681/1480 train_time:102267ms step_avg:152.41ms
step:682/1480 train_time:102428ms step_avg:152.42ms
step:683/1480 train_time:102589ms step_avg:152.44ms
step:684/1480 train_time:102749ms step_avg:152.45ms
step:685/1480 train_time:102912ms step_avg:152.46ms
step:686/1480 train_time:103074ms step_avg:152.48ms
step:687/1480 train_time:103235ms step_avg:152.49ms
step:688/1480 train_time:103400ms step_avg:152.51ms
step:689/1480 train_time:103562ms step_avg:152.52ms
step:690/1480 train_time:103726ms step_avg:152.54ms
step:691/1480 train_time:103886ms step_avg:152.55ms
step:692/1480 train_time:104046ms step_avg:152.56ms
step:693/1480 train_time:104206ms step_avg:152.57ms
step:694/1480 train_time:104366ms step_avg:152.58ms
step:695/1480 train_time:104527ms step_avg:152.59ms
step:696/1480 train_time:104687ms step_avg:152.60ms
step:697/1480 train_time:104848ms step_avg:152.62ms
step:698/1480 train_time:105010ms step_avg:152.63ms
step:699/1480 train_time:105174ms step_avg:152.65ms
step:700/1480 train_time:105337ms step_avg:152.66ms
step:701/1480 train_time:105498ms step_avg:152.67ms
step:702/1480 train_time:105658ms step_avg:152.68ms
step:703/1480 train_time:105820ms step_avg:152.70ms
step:704/1480 train_time:105981ms step_avg:152.71ms
step:705/1480 train_time:106145ms step_avg:152.73ms
step:706/1480 train_time:106311ms step_avg:152.75ms
step:707/1480 train_time:106474ms step_avg:152.76ms
step:708/1480 train_time:106633ms step_avg:152.77ms
step:709/1480 train_time:106796ms step_avg:152.78ms
step:710/1480 train_time:106956ms step_avg:152.79ms
step:711/1480 train_time:107120ms step_avg:152.81ms
step:712/1480 train_time:107286ms step_avg:152.83ms
step:713/1480 train_time:107447ms step_avg:152.84ms
step:714/1480 train_time:107607ms step_avg:152.85ms
step:715/1480 train_time:107766ms step_avg:152.86ms
step:716/1480 train_time:107926ms step_avg:152.87ms
step:717/1480 train_time:108088ms step_avg:152.88ms
step:718/1480 train_time:108246ms step_avg:152.89ms
step:719/1480 train_time:108405ms step_avg:152.90ms
step:720/1480 train_time:108567ms step_avg:152.91ms
step:721/1480 train_time:108728ms step_avg:152.92ms
step:722/1480 train_time:108891ms step_avg:152.94ms
step:723/1480 train_time:109053ms step_avg:152.95ms
step:724/1480 train_time:109215ms step_avg:152.96ms
step:725/1480 train_time:109379ms step_avg:152.98ms
step:726/1480 train_time:109544ms step_avg:152.99ms
step:727/1480 train_time:109708ms step_avg:153.01ms
step:728/1480 train_time:109867ms step_avg:153.02ms
step:729/1480 train_time:110026ms step_avg:153.03ms
step:730/1480 train_time:110189ms step_avg:153.04ms
step:731/1480 train_time:110351ms step_avg:153.05ms
step:732/1480 train_time:110511ms step_avg:153.06ms
step:733/1480 train_time:110674ms step_avg:153.08ms
step:734/1480 train_time:110836ms step_avg:153.09ms
step:735/1480 train_time:110998ms step_avg:153.10ms
step:736/1480 train_time:111160ms step_avg:153.11ms
step:737/1480 train_time:111322ms step_avg:153.13ms
step:738/1480 train_time:111483ms step_avg:153.14ms
step:739/1480 train_time:111643ms step_avg:153.15ms
step:740/1480 train_time:111808ms step_avg:153.16ms
step:741/1480 train_time:111970ms step_avg:153.17ms
step:742/1480 train_time:112131ms step_avg:153.18ms
step:743/1480 train_time:112293ms step_avg:153.20ms
step:744/1480 train_time:112457ms step_avg:153.21ms
step:745/1480 train_time:112622ms step_avg:153.23ms
step:746/1480 train_time:112781ms step_avg:153.24ms
step:747/1480 train_time:112942ms step_avg:153.24ms
step:748/1480 train_time:113106ms step_avg:153.26ms
step:749/1480 train_time:113268ms step_avg:153.27ms
step:750/1480 train_time:113426ms step_avg:153.28ms
step:750/1480 val_loss:3.5502 train_time:113491ms step_avg:153.37ms
step:751/1480 train_time:113590ms step_avg:153.29ms
step:752/1480 train_time:113753ms step_avg:153.31ms
step:753/1480 train_time:113913ms step_avg:153.32ms
step:754/1480 train_time:114072ms step_avg:153.32ms
step:755/1480 train_time:114232ms step_avg:153.33ms
step:756/1480 train_time:114392ms step_avg:153.34ms
step:757/1480 train_time:114555ms step_avg:153.35ms
step:758/1480 train_time:114717ms step_avg:153.36ms
step:759/1480 train_time:114881ms step_avg:153.38ms
step:760/1480 train_time:115043ms step_avg:153.39ms
step:761/1480 train_time:115206ms step_avg:153.40ms
step:762/1480 train_time:115367ms step_avg:153.41ms
step:763/1480 train_time:115528ms step_avg:153.42ms
step:764/1480 train_time:115689ms step_avg:153.43ms
step:765/1480 train_time:115851ms step_avg:153.44ms
step:766/1480 train_time:116013ms step_avg:153.46ms
step:767/1480 train_time:116175ms step_avg:153.47ms
step:768/1480 train_time:116337ms step_avg:153.48ms
step:769/1480 train_time:116500ms step_avg:153.49ms
step:770/1480 train_time:116664ms step_avg:153.50ms
step:771/1480 train_time:116826ms step_avg:153.52ms
step:772/1480 train_time:116988ms step_avg:153.53ms
step:773/1480 train_time:117151ms step_avg:153.54ms
step:774/1480 train_time:117312ms step_avg:153.55ms
step:775/1480 train_time:117474ms step_avg:153.56ms
step:776/1480 train_time:117640ms step_avg:153.58ms
step:777/1480 train_time:117806ms step_avg:153.59ms
step:778/1480 train_time:117969ms step_avg:153.60ms
step:779/1480 train_time:118130ms step_avg:153.62ms
step:780/1480 train_time:118293ms step_avg:153.63ms
step:781/1480 train_time:118455ms step_avg:153.64ms
step:782/1480 train_time:118618ms step_avg:153.65ms
step:783/1480 train_time:118781ms step_avg:153.66ms
step:784/1480 train_time:118944ms step_avg:153.67ms
step:785/1480 train_time:119107ms step_avg:153.69ms
step:786/1480 train_time:119271ms step_avg:153.70ms
step:787/1480 train_time:119433ms step_avg:153.71ms
step:788/1480 train_time:119597ms step_avg:153.72ms
step:789/1480 train_time:119758ms step_avg:153.73ms
step:790/1480 train_time:119924ms step_avg:153.75ms
step:791/1480 train_time:120090ms step_avg:153.76ms
step:792/1480 train_time:120255ms step_avg:153.78ms
step:793/1480 train_time:120417ms step_avg:153.79ms
step:794/1480 train_time:120582ms step_avg:153.80ms
step:795/1480 train_time:120747ms step_avg:153.82ms
step:796/1480 train_time:120912ms step_avg:153.83ms
step:797/1480 train_time:121076ms step_avg:153.85ms
step:798/1480 train_time:121240ms step_avg:153.86ms
step:799/1480 train_time:121407ms step_avg:153.87ms
step:800/1480 train_time:121569ms step_avg:153.89ms
step:801/1480 train_time:121733ms step_avg:153.90ms
step:802/1480 train_time:121900ms step_avg:153.91ms
step:803/1480 train_time:122063ms step_avg:153.93ms
step:804/1480 train_time:122224ms step_avg:153.94ms
step:805/1480 train_time:122388ms step_avg:153.95ms
step:806/1480 train_time:122550ms step_avg:153.96ms
step:807/1480 train_time:122710ms step_avg:153.96ms
step:808/1480 train_time:122875ms step_avg:153.98ms
step:809/1480 train_time:123038ms step_avg:153.99ms
step:810/1480 train_time:123200ms step_avg:154.00ms
step:811/1480 train_time:123363ms step_avg:154.01ms
step:812/1480 train_time:123526ms step_avg:154.02ms
step:813/1480 train_time:123688ms step_avg:154.03ms
step:814/1480 train_time:123851ms step_avg:154.04ms
step:815/1480 train_time:124012ms step_avg:154.05ms
step:816/1480 train_time:124181ms step_avg:154.07ms
step:817/1480 train_time:124344ms step_avg:154.08ms
step:818/1480 train_time:124505ms step_avg:154.09ms
step:819/1480 train_time:124668ms step_avg:154.10ms
step:820/1480 train_time:124831ms step_avg:154.11ms
step:821/1480 train_time:124991ms step_avg:154.12ms
step:822/1480 train_time:125155ms step_avg:154.13ms
step:823/1480 train_time:125318ms step_avg:154.14ms
step:824/1480 train_time:125482ms step_avg:154.15ms
step:825/1480 train_time:125648ms step_avg:154.17ms
step:826/1480 train_time:125814ms step_avg:154.18ms
step:827/1480 train_time:125978ms step_avg:154.20ms
step:828/1480 train_time:126142ms step_avg:154.21ms
step:829/1480 train_time:126306ms step_avg:154.22ms
step:830/1480 train_time:126471ms step_avg:154.23ms
step:831/1480 train_time:126634ms step_avg:154.24ms
step:832/1480 train_time:126797ms step_avg:154.25ms
step:833/1480 train_time:126963ms step_avg:154.27ms
step:834/1480 train_time:127127ms step_avg:154.28ms
step:835/1480 train_time:127290ms step_avg:154.29ms
step:836/1480 train_time:127455ms step_avg:154.30ms
step:837/1480 train_time:127618ms step_avg:154.31ms
step:838/1480 train_time:127784ms step_avg:154.33ms
step:839/1480 train_time:127946ms step_avg:154.34ms
step:840/1480 train_time:128107ms step_avg:154.35ms
step:841/1480 train_time:128267ms step_avg:154.35ms
step:842/1480 train_time:128429ms step_avg:154.36ms
step:843/1480 train_time:128590ms step_avg:154.37ms
step:844/1480 train_time:128754ms step_avg:154.38ms
step:845/1480 train_time:128917ms step_avg:154.39ms
step:846/1480 train_time:129084ms step_avg:154.41ms
step:847/1480 train_time:129247ms step_avg:154.42ms
step:848/1480 train_time:129409ms step_avg:154.43ms
step:849/1480 train_time:129572ms step_avg:154.44ms
step:850/1480 train_time:129735ms step_avg:154.45ms
step:851/1480 train_time:129900ms step_avg:154.46ms
step:852/1480 train_time:130064ms step_avg:154.47ms
step:853/1480 train_time:130225ms step_avg:154.48ms
step:854/1480 train_time:130389ms step_avg:154.49ms
step:855/1480 train_time:130552ms step_avg:154.50ms
step:856/1480 train_time:130714ms step_avg:154.51ms
step:857/1480 train_time:130880ms step_avg:154.52ms
step:858/1480 train_time:131046ms step_avg:154.53ms
step:859/1480 train_time:131209ms step_avg:154.54ms
step:860/1480 train_time:131369ms step_avg:154.55ms
step:861/1480 train_time:131535ms step_avg:154.57ms
step:862/1480 train_time:131705ms step_avg:154.58ms
step:863/1480 train_time:131872ms step_avg:154.60ms
step:864/1480 train_time:132037ms step_avg:154.61ms
step:865/1480 train_time:132199ms step_avg:154.62ms
step:866/1480 train_time:132367ms step_avg:154.63ms
step:867/1480 train_time:132530ms step_avg:154.64ms
step:868/1480 train_time:132690ms step_avg:154.65ms
step:869/1480 train_time:132852ms step_avg:154.66ms
step:870/1480 train_time:133018ms step_avg:154.67ms
step:871/1480 train_time:133182ms step_avg:154.68ms
step:872/1480 train_time:133346ms step_avg:154.69ms
step:873/1480 train_time:133510ms step_avg:154.70ms
step:874/1480 train_time:133675ms step_avg:154.72ms
step:875/1480 train_time:133842ms step_avg:154.73ms
step:875/1480 val_loss:3.5033 train_time:133906ms step_avg:154.80ms
step:876/1480 train_time:134006ms step_avg:154.74ms
step:877/1480 train_time:134172ms step_avg:154.75ms
step:878/1480 train_time:134336ms step_avg:154.76ms
step:879/1480 train_time:134500ms step_avg:154.78ms
step:880/1480 train_time:134662ms step_avg:154.78ms
step:881/1480 train_time:134824ms step_avg:154.79ms
step:882/1480 train_time:134989ms step_avg:154.80ms
step:883/1480 train_time:135155ms step_avg:154.82ms
step:884/1480 train_time:135322ms step_avg:154.83ms
step:885/1480 train_time:135487ms step_avg:154.84ms
step:886/1480 train_time:135654ms step_avg:154.86ms
step:887/1480 train_time:135822ms step_avg:154.87ms
step:888/1480 train_time:135996ms step_avg:154.89ms
step:889/1480 train_time:136163ms step_avg:154.91ms
step:890/1480 train_time:136326ms step_avg:154.92ms
step:891/1480 train_time:136491ms step_avg:154.93ms
step:892/1480 train_time:136656ms step_avg:154.94ms
step:893/1480 train_time:136818ms step_avg:154.95ms
step:894/1480 train_time:136984ms step_avg:154.96ms
step:895/1480 train_time:137150ms step_avg:154.97ms
step:896/1480 train_time:137315ms step_avg:154.98ms
step:897/1480 train_time:137480ms step_avg:154.99ms
step:898/1480 train_time:137648ms step_avg:155.01ms
step:899/1480 train_time:137813ms step_avg:155.02ms
step:900/1480 train_time:137976ms step_avg:155.03ms
step:901/1480 train_time:138141ms step_avg:155.04ms
step:902/1480 train_time:138304ms step_avg:155.05ms
step:903/1480 train_time:138474ms step_avg:155.07ms
step:904/1480 train_time:138639ms step_avg:155.08ms
step:905/1480 train_time:138801ms step_avg:155.09ms
step:906/1480 train_time:138967ms step_avg:155.10ms
step:907/1480 train_time:139135ms step_avg:155.11ms
step:908/1480 train_time:139298ms step_avg:155.12ms
step:909/1480 train_time:139463ms step_avg:155.13ms
step:910/1480 train_time:139634ms step_avg:155.15ms
step:911/1480 train_time:139798ms step_avg:155.16ms
step:912/1480 train_time:139963ms step_avg:155.17ms
step:913/1480 train_time:140130ms step_avg:155.18ms
step:914/1480 train_time:140297ms step_avg:155.20ms
step:915/1480 train_time:140466ms step_avg:155.21ms
step:916/1480 train_time:140631ms step_avg:155.22ms
step:917/1480 train_time:140794ms step_avg:155.23ms
step:918/1480 train_time:140962ms step_avg:155.24ms
step:919/1480 train_time:141133ms step_avg:155.26ms
step:920/1480 train_time:141299ms step_avg:155.27ms
step:921/1480 train_time:141465ms step_avg:155.29ms
step:922/1480 train_time:141635ms step_avg:155.30ms
step:923/1480 train_time:141798ms step_avg:155.31ms
step:924/1480 train_time:141963ms step_avg:155.32ms
step:925/1480 train_time:142128ms step_avg:155.33ms
step:926/1480 train_time:142292ms step_avg:155.34ms
step:927/1480 train_time:142456ms step_avg:155.35ms
step:928/1480 train_time:142621ms step_avg:155.36ms
step:929/1480 train_time:142786ms step_avg:155.37ms
step:930/1480 train_time:142951ms step_avg:155.38ms
step:931/1480 train_time:143115ms step_avg:155.39ms
step:932/1480 train_time:143281ms step_avg:155.40ms
step:933/1480 train_time:143449ms step_avg:155.42ms
step:934/1480 train_time:143617ms step_avg:155.43ms
step:935/1480 train_time:143787ms step_avg:155.45ms
step:936/1480 train_time:143955ms step_avg:155.46ms
step:937/1480 train_time:144125ms step_avg:155.47ms
step:938/1480 train_time:144289ms step_avg:155.48ms
step:939/1480 train_time:144458ms step_avg:155.50ms
step:940/1480 train_time:144624ms step_avg:155.51ms
step:941/1480 train_time:144789ms step_avg:155.52ms
step:942/1480 train_time:144952ms step_avg:155.53ms
step:943/1480 train_time:145122ms step_avg:155.54ms
step:944/1480 train_time:145295ms step_avg:155.56ms
step:945/1480 train_time:145458ms step_avg:155.57ms
step:946/1480 train_time:145628ms step_avg:155.58ms
step:947/1480 train_time:145796ms step_avg:155.60ms
step:948/1480 train_time:145963ms step_avg:155.61ms
step:949/1480 train_time:146128ms step_avg:155.62ms
step:950/1480 train_time:146293ms step_avg:155.63ms
step:951/1480 train_time:146461ms step_avg:155.64ms
step:952/1480 train_time:146626ms step_avg:155.65ms
step:953/1480 train_time:146792ms step_avg:155.67ms
step:954/1480 train_time:146961ms step_avg:155.68ms
step:955/1480 train_time:147125ms step_avg:155.69ms
step:956/1480 train_time:147290ms step_avg:155.70ms
step:957/1480 train_time:147457ms step_avg:155.71ms
step:958/1480 train_time:147625ms step_avg:155.72ms
step:959/1480 train_time:147790ms step_avg:155.73ms
step:960/1480 train_time:147956ms step_avg:155.74ms
step:961/1480 train_time:148121ms step_avg:155.75ms
step:962/1480 train_time:148285ms step_avg:155.76ms
step:963/1480 train_time:148450ms step_avg:155.77ms
step:964/1480 train_time:148619ms step_avg:155.79ms
step:965/1480 train_time:148782ms step_avg:155.79ms
step:966/1480 train_time:148947ms step_avg:155.80ms
step:967/1480 train_time:149110ms step_avg:155.81ms
step:968/1480 train_time:149275ms step_avg:155.82ms
step:969/1480 train_time:149442ms step_avg:155.83ms
step:970/1480 train_time:149606ms step_avg:155.84ms
step:971/1480 train_time:149770ms step_avg:155.85ms
step:972/1480 train_time:149936ms step_avg:155.86ms
step:973/1480 train_time:150100ms step_avg:155.87ms
step:974/1480 train_time:150268ms step_avg:155.88ms
step:975/1480 train_time:150435ms step_avg:155.89ms
step:976/1480 train_time:150600ms step_avg:155.90ms
step:977/1480 train_time:150763ms step_avg:155.91ms
step:978/1480 train_time:150931ms step_avg:155.92ms
step:979/1480 train_time:151098ms step_avg:155.93ms
step:980/1480 train_time:151262ms step_avg:155.94ms
step:981/1480 train_time:151431ms step_avg:155.95ms
step:982/1480 train_time:151594ms step_avg:155.96ms
step:983/1480 train_time:151759ms step_avg:155.97ms
step:984/1480 train_time:151923ms step_avg:155.98ms
step:985/1480 train_time:152092ms step_avg:155.99ms
step:986/1480 train_time:152257ms step_avg:156.00ms
step:987/1480 train_time:152420ms step_avg:156.01ms
step:988/1480 train_time:152587ms step_avg:156.02ms
step:989/1480 train_time:152753ms step_avg:156.03ms
step:990/1480 train_time:152922ms step_avg:156.04ms
step:991/1480 train_time:153089ms step_avg:156.05ms
step:992/1480 train_time:153263ms step_avg:156.07ms
step:993/1480 train_time:153438ms step_avg:156.09ms
step:994/1480 train_time:153602ms step_avg:156.10ms
step:995/1480 train_time:153766ms step_avg:156.11ms
step:996/1480 train_time:153929ms step_avg:156.11ms
step:997/1480 train_time:154094ms step_avg:156.12ms
step:998/1480 train_time:154258ms step_avg:156.13ms
step:999/1480 train_time:154423ms step_avg:156.14ms
step:1000/1480 train_time:154592ms step_avg:156.15ms
step:1000/1480 val_loss:3.4414 train_time:154659ms step_avg:156.22ms
step:1001/1480 train_time:154759ms step_avg:156.16ms
step:1002/1480 train_time:154925ms step_avg:156.17ms
step:1003/1480 train_time:155096ms step_avg:156.19ms
step:1004/1480 train_time:155264ms step_avg:156.20ms
step:1005/1480 train_time:155432ms step_avg:156.21ms
step:1006/1480 train_time:155599ms step_avg:156.22ms
step:1007/1480 train_time:155763ms step_avg:156.23ms
step:1008/1480 train_time:155931ms step_avg:156.24ms
step:1009/1480 train_time:156105ms step_avg:156.26ms
step:1010/1480 train_time:156271ms step_avg:156.27ms
step:1011/1480 train_time:156438ms step_avg:156.28ms
step:1012/1480 train_time:156601ms step_avg:156.29ms
step:1013/1480 train_time:156771ms step_avg:156.30ms
step:1014/1480 train_time:156937ms step_avg:156.31ms
step:1015/1480 train_time:157106ms step_avg:156.32ms
step:1016/1480 train_time:157274ms step_avg:156.34ms
step:1017/1480 train_time:157443ms step_avg:156.35ms
step:1018/1480 train_time:157612ms step_avg:156.36ms
step:1019/1480 train_time:157779ms step_avg:156.37ms
step:1020/1480 train_time:157949ms step_avg:156.39ms
step:1021/1480 train_time:158115ms step_avg:156.39ms
step:1022/1480 train_time:158282ms step_avg:156.40ms
step:1023/1480 train_time:158450ms step_avg:156.42ms
step:1024/1480 train_time:158617ms step_avg:156.43ms
step:1025/1480 train_time:158788ms step_avg:156.44ms
step:1026/1480 train_time:158954ms step_avg:156.45ms
step:1027/1480 train_time:159120ms step_avg:156.46ms
step:1028/1480 train_time:159292ms step_avg:156.48ms
step:1029/1480 train_time:159469ms step_avg:156.50ms
step:1030/1480 train_time:159636ms step_avg:156.51ms
step:1031/1480 train_time:159800ms step_avg:156.51ms
step:1032/1480 train_time:159972ms step_avg:156.53ms
step:1033/1480 train_time:160139ms step_avg:156.54ms
step:1034/1480 train_time:160307ms step_avg:156.55ms
step:1035/1480 train_time:160476ms step_avg:156.56ms
step:1036/1480 train_time:160641ms step_avg:156.57ms
step:1037/1480 train_time:160808ms step_avg:156.58ms
step:1038/1480 train_time:160976ms step_avg:156.59ms
step:1039/1480 train_time:161145ms step_avg:156.60ms
step:1040/1480 train_time:161311ms step_avg:156.61ms
step:1041/1480 train_time:161479ms step_avg:156.62ms
step:1042/1480 train_time:161643ms step_avg:156.63ms
step:1043/1480 train_time:161808ms step_avg:156.64ms
step:1044/1480 train_time:161973ms step_avg:156.65ms
step:1045/1480 train_time:162142ms step_avg:156.66ms
step:1046/1480 train_time:162308ms step_avg:156.67ms
step:1047/1480 train_time:162475ms step_avg:156.68ms
step:1048/1480 train_time:162640ms step_avg:156.69ms
step:1049/1480 train_time:162806ms step_avg:156.69ms
step:1050/1480 train_time:162977ms step_avg:156.71ms
step:1051/1480 train_time:163145ms step_avg:156.72ms
step:1052/1480 train_time:163313ms step_avg:156.73ms
step:1053/1480 train_time:163478ms step_avg:156.74ms
step:1054/1480 train_time:163645ms step_avg:156.75ms
step:1055/1480 train_time:163811ms step_avg:156.76ms
step:1056/1480 train_time:163976ms step_avg:156.76ms
step:1057/1480 train_time:164141ms step_avg:156.77ms
step:1058/1480 train_time:164310ms step_avg:156.78ms
step:1059/1480 train_time:164483ms step_avg:156.80ms
step:1060/1480 train_time:164652ms step_avg:156.81ms
step:1061/1480 train_time:164815ms step_avg:156.82ms
step:1062/1480 train_time:164981ms step_avg:156.83ms
step:1063/1480 train_time:165145ms step_avg:156.83ms
step:1064/1480 train_time:165309ms step_avg:156.84ms
step:1065/1480 train_time:165477ms step_avg:156.85ms
step:1066/1480 train_time:165644ms step_avg:156.86ms
step:1067/1480 train_time:165814ms step_avg:156.87ms
step:1068/1480 train_time:165980ms step_avg:156.88ms
step:1069/1480 train_time:166151ms step_avg:156.89ms
step:1070/1480 train_time:166317ms step_avg:156.90ms
step:1071/1480 train_time:166490ms step_avg:156.92ms
step:1072/1480 train_time:166657ms step_avg:156.93ms
step:1073/1480 train_time:166819ms step_avg:156.93ms
step:1074/1480 train_time:166985ms step_avg:156.94ms
step:1075/1480 train_time:167156ms step_avg:156.95ms
step:1076/1480 train_time:167322ms step_avg:156.96ms
step:1077/1480 train_time:167488ms step_avg:156.97ms
step:1078/1480 train_time:167662ms step_avg:156.99ms
step:1079/1480 train_time:167834ms step_avg:157.00ms
step:1080/1480 train_time:168004ms step_avg:157.01ms
step:1081/1480 train_time:168170ms step_avg:157.02ms
step:1082/1480 train_time:168337ms step_avg:157.03ms
step:1083/1480 train_time:168504ms step_avg:157.04ms
step:1084/1480 train_time:168673ms step_avg:157.05ms
step:1085/1480 train_time:168842ms step_avg:157.06ms
step:1086/1480 train_time:169010ms step_avg:157.07ms
step:1087/1480 train_time:169177ms step_avg:157.08ms
step:1088/1480 train_time:169347ms step_avg:157.09ms
step:1089/1480 train_time:169518ms step_avg:157.11ms
step:1090/1480 train_time:169689ms step_avg:157.12ms
step:1091/1480 train_time:169857ms step_avg:157.13ms
step:1092/1480 train_time:170023ms step_avg:157.14ms
step:1093/1480 train_time:170192ms step_avg:157.15ms
step:1094/1480 train_time:170358ms step_avg:157.16ms
step:1095/1480 train_time:170522ms step_avg:157.16ms
step:1096/1480 train_time:170692ms step_avg:157.17ms
step:1097/1480 train_time:170861ms step_avg:157.19ms
step:1098/1480 train_time:171032ms step_avg:157.20ms
step:1099/1480 train_time:171202ms step_avg:157.21ms
step:1100/1480 train_time:171375ms step_avg:157.22ms
step:1101/1480 train_time:171544ms step_avg:157.24ms
step:1102/1480 train_time:171716ms step_avg:157.25ms
step:1103/1480 train_time:171891ms step_avg:157.26ms
step:1104/1480 train_time:172059ms step_avg:157.28ms
step:1105/1480 train_time:172229ms step_avg:157.29ms
step:1106/1480 train_time:172398ms step_avg:157.30ms
step:1107/1480 train_time:172567ms step_avg:157.31ms
step:1108/1480 train_time:172733ms step_avg:157.32ms
step:1109/1480 train_time:172899ms step_avg:157.32ms
step:1110/1480 train_time:173064ms step_avg:157.33ms
step:1111/1480 train_time:173231ms step_avg:157.34ms
step:1112/1480 train_time:173400ms step_avg:157.35ms
step:1113/1480 train_time:173580ms step_avg:157.37ms
step:1114/1480 train_time:173753ms step_avg:157.38ms
step:1115/1480 train_time:173924ms step_avg:157.40ms
step:1116/1480 train_time:174091ms step_avg:157.41ms
step:1117/1480 train_time:174264ms step_avg:157.42ms
step:1118/1480 train_time:174439ms step_avg:157.44ms
step:1119/1480 train_time:174605ms step_avg:157.44ms
step:1120/1480 train_time:174775ms step_avg:157.46ms
step:1121/1480 train_time:174944ms step_avg:157.47ms
step:1122/1480 train_time:175111ms step_avg:157.47ms
step:1123/1480 train_time:175277ms step_avg:157.48ms
step:1124/1480 train_time:175445ms step_avg:157.49ms
step:1125/1480 train_time:175612ms step_avg:157.50ms
step:1125/1480 val_loss:3.3854 train_time:175680ms step_avg:157.56ms
step:1126/1480 train_time:175786ms step_avg:157.51ms
step:1127/1480 train_time:175955ms step_avg:157.52ms
step:1128/1480 train_time:176125ms step_avg:157.54ms
step:1129/1480 train_time:176299ms step_avg:157.55ms
step:1130/1480 train_time:176470ms step_avg:157.56ms
step:1131/1480 train_time:176646ms step_avg:157.58ms
step:1132/1480 train_time:176811ms step_avg:157.59ms
step:1133/1480 train_time:176982ms step_avg:157.60ms
step:1134/1480 train_time:177153ms step_avg:157.61ms
step:1135/1480 train_time:177320ms step_avg:157.62ms
step:1136/1480 train_time:177490ms step_avg:157.63ms
step:1137/1480 train_time:177658ms step_avg:157.64ms
step:1138/1480 train_time:177830ms step_avg:157.65ms
step:1139/1480 train_time:177997ms step_avg:157.66ms
step:1140/1480 train_time:178166ms step_avg:157.67ms
step:1141/1480 train_time:178337ms step_avg:157.68ms
step:1142/1480 train_time:178507ms step_avg:157.69ms
step:1143/1480 train_time:178676ms step_avg:157.70ms
step:1144/1480 train_time:178843ms step_avg:157.71ms
step:1145/1480 train_time:179010ms step_avg:157.72ms
step:1146/1480 train_time:179180ms step_avg:157.73ms
step:1147/1480 train_time:179348ms step_avg:157.74ms
step:1148/1480 train_time:179518ms step_avg:157.75ms
step:1149/1480 train_time:179690ms step_avg:157.76ms
step:1150/1480 train_time:179858ms step_avg:157.77ms
step:1151/1480 train_time:180029ms step_avg:157.78ms
step:1152/1480 train_time:180201ms step_avg:157.79ms
step:1153/1480 train_time:180375ms step_avg:157.81ms
step:1154/1480 train_time:180541ms step_avg:157.82ms
step:1155/1480 train_time:180713ms step_avg:157.83ms
step:1156/1480 train_time:180891ms step_avg:157.85ms
step:1157/1480 train_time:181060ms step_avg:157.86ms
step:1158/1480 train_time:181227ms step_avg:157.86ms
step:1159/1480 train_time:181394ms step_avg:157.87ms
step:1160/1480 train_time:181560ms step_avg:157.88ms
step:1161/1480 train_time:181730ms step_avg:157.89ms
step:1162/1480 train_time:181901ms step_avg:157.90ms
step:1163/1480 train_time:182072ms step_avg:157.91ms
step:1164/1480 train_time:182241ms step_avg:157.92ms
step:1165/1480 train_time:182407ms step_avg:157.93ms
step:1166/1480 train_time:182575ms step_avg:157.94ms
step:1167/1480 train_time:182744ms step_avg:157.95ms
step:1168/1480 train_time:182911ms step_avg:157.95ms
step:1169/1480 train_time:183080ms step_avg:157.96ms
step:1170/1480 train_time:183248ms step_avg:157.97ms
step:1171/1480 train_time:183414ms step_avg:157.98ms
step:1172/1480 train_time:183581ms step_avg:157.99ms
step:1173/1480 train_time:183752ms step_avg:158.00ms
step:1174/1480 train_time:183934ms step_avg:158.02ms
step:1175/1480 train_time:184107ms step_avg:158.03ms
step:1176/1480 train_time:184278ms step_avg:158.04ms
step:1177/1480 train_time:184455ms step_avg:158.06ms
step:1178/1480 train_time:184624ms step_avg:158.07ms
step:1179/1480 train_time:184791ms step_avg:158.08ms
step:1180/1480 train_time:184971ms step_avg:158.10ms
step:1181/1480 train_time:185141ms step_avg:158.11ms
step:1182/1480 train_time:185309ms step_avg:158.11ms
step:1183/1480 train_time:185479ms step_avg:158.12ms
step:1184/1480 train_time:185646ms step_avg:158.13ms
step:1185/1480 train_time:185817ms step_avg:158.14ms
step:1186/1480 train_time:185989ms step_avg:158.15ms
step:1187/1480 train_time:186172ms step_avg:158.18ms
step:1188/1480 train_time:186338ms step_avg:158.18ms
step:1189/1480 train_time:186510ms step_avg:158.19ms
step:1190/1480 train_time:186676ms step_avg:158.20ms
step:1191/1480 train_time:186848ms step_avg:158.21ms
step:1192/1480 train_time:187013ms step_avg:158.22ms
step:1193/1480 train_time:187182ms step_avg:158.23ms
step:1194/1480 train_time:187351ms step_avg:158.24ms
step:1195/1480 train_time:187525ms step_avg:158.25ms
step:1196/1480 train_time:187708ms step_avg:158.27ms
step:1197/1480 train_time:187878ms step_avg:158.28ms
step:1198/1480 train_time:188059ms step_avg:158.30ms
step:1199/1480 train_time:188228ms step_avg:158.31ms
step:1200/1480 train_time:188397ms step_avg:158.32ms
step:1201/1480 train_time:188567ms step_avg:158.33ms
step:1202/1480 train_time:188748ms step_avg:158.35ms
step:1203/1480 train_time:188924ms step_avg:158.36ms
step:1204/1480 train_time:189098ms step_avg:158.37ms
step:1205/1480 train_time:189267ms step_avg:158.38ms
step:1206/1480 train_time:189433ms step_avg:158.39ms
step:1207/1480 train_time:189604ms step_avg:158.40ms
step:1208/1480 train_time:189771ms step_avg:158.41ms
step:1209/1480 train_time:189945ms step_avg:158.42ms
step:1210/1480 train_time:190119ms step_avg:158.43ms
step:1211/1480 train_time:190292ms step_avg:158.44ms
step:1212/1480 train_time:190464ms step_avg:158.46ms
step:1213/1480 train_time:190636ms step_avg:158.47ms
step:1214/1480 train_time:190813ms step_avg:158.48ms
step:1215/1480 train_time:190985ms step_avg:158.49ms
step:1216/1480 train_time:191155ms step_avg:158.50ms
step:1217/1480 train_time:191329ms step_avg:158.52ms
step:1218/1480 train_time:191499ms step_avg:158.53ms
step:1219/1480 train_time:191678ms step_avg:158.54ms
step:1220/1480 train_time:191847ms step_avg:158.55ms
step:1221/1480 train_time:192018ms step_avg:158.56ms
step:1222/1480 train_time:192187ms step_avg:158.57ms
step:1223/1480 train_time:192356ms step_avg:158.58ms
step:1224/1480 train_time:192533ms step_avg:158.59ms
step:1225/1480 train_time:192705ms step_avg:158.61ms
step:1226/1480 train_time:192878ms step_avg:158.62ms
step:1227/1480 train_time:193050ms step_avg:158.63ms
step:1228/1480 train_time:193218ms step_avg:158.64ms
step:1229/1480 train_time:193391ms step_avg:158.65ms
step:1230/1480 train_time:193572ms step_avg:158.67ms
step:1231/1480 train_time:193748ms step_avg:158.68ms
step:1232/1480 train_time:193922ms step_avg:158.69ms
step:1233/1480 train_time:194091ms step_avg:158.70ms
step:1234/1480 train_time:194261ms step_avg:158.71ms
step:1235/1480 train_time:194435ms step_avg:158.72ms
step:1236/1480 train_time:194604ms step_avg:158.73ms
step:1237/1480 train_time:194774ms step_avg:158.74ms
step:1238/1480 train_time:194958ms step_avg:158.76ms
step:1239/1480 train_time:195129ms step_avg:158.77ms
step:1240/1480 train_time:195301ms step_avg:158.78ms
step:1241/1480 train_time:195473ms step_avg:158.79ms
step:1242/1480 train_time:195643ms step_avg:158.80ms
step:1243/1480 train_time:195815ms step_avg:158.81ms
step:1244/1480 train_time:195982ms step_avg:158.82ms
step:1245/1480 train_time:196151ms step_avg:158.83ms
step:1246/1480 train_time:196319ms step_avg:158.83ms
step:1247/1480 train_time:196490ms step_avg:158.84ms
step:1248/1480 train_time:196658ms step_avg:158.85ms
step:1249/1480 train_time:196827ms step_avg:158.86ms
step:1250/1480 train_time:196996ms step_avg:158.87ms
step:1250/1480 val_loss:3.3359 train_time:197067ms step_avg:158.93ms
step:1251/1480 train_time:197176ms step_avg:158.89ms
step:1252/1480 train_time:197346ms step_avg:158.89ms
step:1253/1480 train_time:197513ms step_avg:158.90ms
step:1254/1480 train_time:197684ms step_avg:158.91ms
step:1255/1480 train_time:197872ms step_avg:158.93ms
step:1256/1480 train_time:198043ms step_avg:158.94ms
step:1257/1480 train_time:198214ms step_avg:158.95ms
step:1258/1480 train_time:198390ms step_avg:158.97ms
step:1259/1480 train_time:198561ms step_avg:158.98ms
step:1260/1480 train_time:198728ms step_avg:158.98ms
step:1261/1480 train_time:198899ms step_avg:158.99ms
step:1262/1480 train_time:199075ms step_avg:159.01ms
step:1263/1480 train_time:199249ms step_avg:159.02ms
step:1264/1480 train_time:199416ms step_avg:159.02ms
step:1265/1480 train_time:199582ms step_avg:159.03ms
step:1266/1480 train_time:199754ms step_avg:159.04ms
step:1267/1480 train_time:199924ms step_avg:159.05ms
step:1268/1480 train_time:200095ms step_avg:159.06ms
step:1269/1480 train_time:200271ms step_avg:159.07ms
step:1270/1480 train_time:200440ms step_avg:159.08ms
step:1271/1480 train_time:200612ms step_avg:159.09ms
step:1272/1480 train_time:200777ms step_avg:159.09ms
step:1273/1480 train_time:200950ms step_avg:159.11ms
step:1274/1480 train_time:201123ms step_avg:159.12ms
step:1275/1480 train_time:201291ms step_avg:159.12ms
step:1276/1480 train_time:201456ms step_avg:159.13ms
step:1277/1480 train_time:201630ms step_avg:159.14ms
step:1278/1480 train_time:201798ms step_avg:159.15ms
step:1279/1480 train_time:201971ms step_avg:159.16ms
step:1280/1480 train_time:202150ms step_avg:159.17ms
step:1281/1480 train_time:202319ms step_avg:159.18ms
step:1282/1480 train_time:202484ms step_avg:159.19ms
step:1283/1480 train_time:202655ms step_avg:159.19ms
step:1284/1480 train_time:202826ms step_avg:159.20ms
step:1285/1480 train_time:202995ms step_avg:159.21ms
step:1286/1480 train_time:203163ms step_avg:159.22ms
step:1287/1480 train_time:203336ms step_avg:159.23ms
step:1288/1480 train_time:203507ms step_avg:159.24ms
step:1289/1480 train_time:203690ms step_avg:159.26ms
step:1290/1480 train_time:203869ms step_avg:159.27ms
step:1291/1480 train_time:204043ms step_avg:159.28ms
step:1292/1480 train_time:204217ms step_avg:159.30ms
step:1293/1480 train_time:204393ms step_avg:159.31ms
step:1294/1480 train_time:204562ms step_avg:159.32ms
step:1295/1480 train_time:204734ms step_avg:159.33ms
step:1296/1480 train_time:204907ms step_avg:159.34ms
step:1297/1480 train_time:205077ms step_avg:159.34ms
step:1298/1480 train_time:205247ms step_avg:159.35ms
step:1299/1480 train_time:205417ms step_avg:159.36ms
step:1300/1480 train_time:205584ms step_avg:159.37ms
step:1301/1480 train_time:205753ms step_avg:159.37ms
step:1302/1480 train_time:205928ms step_avg:159.39ms
step:1303/1480 train_time:206103ms step_avg:159.40ms
step:1304/1480 train_time:206276ms step_avg:159.41ms
step:1305/1480 train_time:206445ms step_avg:159.42ms
step:1306/1480 train_time:206621ms step_avg:159.43ms
step:1307/1480 train_time:206789ms step_avg:159.44ms
step:1308/1480 train_time:206959ms step_avg:159.44ms
step:1309/1480 train_time:207132ms step_avg:159.45ms
step:1310/1480 train_time:207300ms step_avg:159.46ms
step:1311/1480 train_time:207469ms step_avg:159.47ms
step:1312/1480 train_time:207641ms step_avg:159.48ms
step:1313/1480 train_time:207811ms step_avg:159.49ms
step:1314/1480 train_time:207984ms step_avg:159.50ms
step:1315/1480 train_time:208154ms step_avg:159.50ms
step:1316/1480 train_time:208321ms step_avg:159.51ms
step:1317/1480 train_time:208493ms step_avg:159.52ms
step:1318/1480 train_time:208675ms step_avg:159.54ms
step:1319/1480 train_time:208850ms step_avg:159.55ms
step:1320/1480 train_time:209028ms step_avg:159.56ms
step:1321/1480 train_time:209201ms step_avg:159.57ms
step:1322/1480 train_time:209381ms step_avg:159.59ms
step:1323/1480 train_time:209555ms step_avg:159.60ms
step:1324/1480 train_time:209731ms step_avg:159.61ms
step:1325/1480 train_time:209912ms step_avg:159.63ms
step:1326/1480 train_time:210086ms step_avg:159.64ms
step:1327/1480 train_time:210256ms step_avg:159.65ms
step:1328/1480 train_time:210426ms step_avg:159.66ms
step:1329/1480 train_time:210624ms step_avg:159.68ms
step:1330/1480 train_time:210802ms step_avg:159.70ms
step:1331/1480 train_time:210972ms step_avg:159.71ms
step:1332/1480 train_time:211145ms step_avg:159.72ms
step:1333/1480 train_time:211321ms step_avg:159.73ms
step:1334/1480 train_time:211493ms step_avg:159.74ms
step:1335/1480 train_time:211661ms step_avg:159.74ms
step:1336/1480 train_time:211844ms step_avg:159.76ms
step:1337/1480 train_time:212020ms step_avg:159.77ms
step:1338/1480 train_time:212192ms step_avg:159.78ms
step:1339/1480 train_time:212365ms step_avg:159.79ms
step:1340/1480 train_time:212537ms step_avg:159.80ms
step:1341/1480 train_time:212704ms step_avg:159.81ms
step:1342/1480 train_time:212878ms step_avg:159.82ms
step:1343/1480 train_time:213049ms step_avg:159.83ms
step:1344/1480 train_time:213222ms step_avg:159.84ms
step:1345/1480 train_time:213402ms step_avg:159.85ms
step:1346/1480 train_time:213571ms step_avg:159.86ms
step:1347/1480 train_time:213740ms step_avg:159.87ms
step:1348/1480 train_time:213910ms step_avg:159.87ms
step:1349/1480 train_time:214079ms step_avg:159.88ms
step:1350/1480 train_time:214254ms step_avg:159.89ms
step:1351/1480 train_time:214427ms step_avg:159.90ms
step:1352/1480 train_time:214597ms step_avg:159.91ms
step:1353/1480 train_time:214774ms step_avg:159.92ms
step:1354/1480 train_time:214946ms step_avg:159.93ms
step:1355/1480 train_time:215115ms step_avg:159.94ms
step:1356/1480 train_time:215287ms step_avg:159.95ms
step:1357/1480 train_time:215459ms step_avg:159.95ms
step:1358/1480 train_time:215634ms step_avg:159.97ms
step:1359/1480 train_time:215805ms step_avg:159.97ms
step:1360/1480 train_time:215980ms step_avg:159.99ms
step:1361/1480 train_time:216158ms step_avg:160.00ms
step:1362/1480 train_time:216334ms step_avg:160.01ms
step:1363/1480 train_time:216514ms step_avg:160.03ms
step:1364/1480 train_time:216683ms step_avg:160.03ms
step:1365/1480 train_time:216851ms step_avg:160.04ms
step:1366/1480 train_time:217022ms step_avg:160.05ms
step:1367/1480 train_time:217195ms step_avg:160.06ms
step:1368/1480 train_time:217369ms step_avg:160.07ms
step:1369/1480 train_time:217550ms step_avg:160.08ms
step:1370/1480 train_time:217728ms step_avg:160.09ms
step:1371/1480 train_time:217900ms step_avg:160.10ms
step:1372/1480 train_time:218076ms step_avg:160.11ms
step:1373/1480 train_time:218245ms step_avg:160.12ms
step:1374/1480 train_time:218421ms step_avg:160.13ms
step:1375/1480 train_time:218592ms step_avg:160.14ms
step:1375/1480 val_loss:3.2970 train_time:218660ms step_avg:160.19ms
step:1376/1480 train_time:218764ms step_avg:160.15ms
step:1377/1480 train_time:218937ms step_avg:160.16ms
step:1378/1480 train_time:219104ms step_avg:160.16ms
step:1379/1480 train_time:219280ms step_avg:160.18ms
step:1380/1480 train_time:219454ms step_avg:160.19ms
step:1381/1480 train_time:219636ms step_avg:160.20ms
step:1382/1480 train_time:219808ms step_avg:160.21ms
step:1383/1480 train_time:219980ms step_avg:160.22ms
step:1384/1480 train_time:220159ms step_avg:160.23ms
step:1385/1480 train_time:220324ms step_avg:160.24ms
step:1386/1480 train_time:220495ms step_avg:160.24ms
step:1387/1480 train_time:220665ms step_avg:160.25ms
step:1388/1480 train_time:220834ms step_avg:160.26ms
step:1389/1480 train_time:221006ms step_avg:160.27ms
step:1390/1480 train_time:221174ms step_avg:160.27ms
step:1391/1480 train_time:221344ms step_avg:160.28ms
step:1392/1480 train_time:221519ms step_avg:160.29ms
step:1393/1480 train_time:221691ms step_avg:160.30ms
step:1394/1480 train_time:221861ms step_avg:160.30ms
step:1395/1480 train_time:222029ms step_avg:160.31ms
step:1396/1480 train_time:222198ms step_avg:160.32ms
step:1397/1480 train_time:222365ms step_avg:160.32ms
step:1398/1480 train_time:222534ms step_avg:160.33ms
step:1399/1480 train_time:222701ms step_avg:160.33ms
step:1400/1480 train_time:222875ms step_avg:160.34ms
step:1401/1480 train_time:223041ms step_avg:160.35ms
step:1402/1480 train_time:223211ms step_avg:160.35ms
step:1403/1480 train_time:223388ms step_avg:160.36ms
step:1404/1480 train_time:223559ms step_avg:160.37ms
step:1405/1480 train_time:223733ms step_avg:160.38ms
step:1406/1480 train_time:223906ms step_avg:160.39ms
step:1407/1480 train_time:224075ms step_avg:160.40ms
step:1408/1480 train_time:224244ms step_avg:160.40ms
step:1409/1480 train_time:224426ms step_avg:160.42ms
step:1410/1480 train_time:224596ms step_avg:160.43ms
step:1411/1480 train_time:224764ms step_avg:160.43ms
step:1412/1480 train_time:224935ms step_avg:160.44ms
step:1413/1480 train_time:225104ms step_avg:160.44ms
step:1414/1480 train_time:225277ms step_avg:160.45ms
step:1415/1480 train_time:225453ms step_avg:160.46ms
step:1416/1480 train_time:225641ms step_avg:160.48ms
step:1417/1480 train_time:225815ms step_avg:160.49ms
step:1418/1480 train_time:225985ms step_avg:160.50ms
step:1419/1480 train_time:226159ms step_avg:160.51ms
step:1420/1480 train_time:226333ms step_avg:160.52ms
step:1421/1480 train_time:226505ms step_avg:160.53ms
step:1422/1480 train_time:226677ms step_avg:160.54ms
step:1423/1480 train_time:226846ms step_avg:160.54ms
step:1424/1480 train_time:227023ms step_avg:160.55ms
step:1425/1480 train_time:227202ms step_avg:160.57ms
step:1426/1480 train_time:227374ms step_avg:160.57ms
step:1427/1480 train_time:227549ms step_avg:160.59ms
step:1428/1480 train_time:227719ms step_avg:160.59ms
step:1429/1480 train_time:227886ms step_avg:160.60ms
step:1430/1480 train_time:228059ms step_avg:160.61ms
step:1431/1480 train_time:228234ms step_avg:160.62ms
step:1432/1480 train_time:228412ms step_avg:160.63ms
step:1433/1480 train_time:228590ms step_avg:160.64ms
step:1434/1480 train_time:228768ms step_avg:160.65ms
step:1435/1480 train_time:228942ms step_avg:160.66ms
step:1436/1480 train_time:229116ms step_avg:160.67ms
step:1437/1480 train_time:229285ms step_avg:160.68ms
step:1438/1480 train_time:229456ms step_avg:160.68ms
step:1439/1480 train_time:229628ms step_avg:160.69ms
step:1440/1480 train_time:229797ms step_avg:160.70ms
step:1441/1480 train_time:229965ms step_avg:160.70ms
step:1442/1480 train_time:230143ms step_avg:160.71ms
step:1443/1480 train_time:230332ms step_avg:160.73ms
step:1444/1480 train_time:230502ms step_avg:160.74ms
step:1445/1480 train_time:230674ms step_avg:160.75ms
step:1446/1480 train_time:230851ms step_avg:160.76ms
step:1447/1480 train_time:231028ms step_avg:160.77ms
step:1448/1480 train_time:231199ms step_avg:160.78ms
step:1449/1480 train_time:231373ms step_avg:160.79ms
step:1450/1480 train_time:231546ms step_avg:160.80ms
step:1451/1480 train_time:231717ms step_avg:160.80ms
step:1452/1480 train_time:231889ms step_avg:160.81ms
step:1453/1480 train_time:232059ms step_avg:160.82ms
step:1454/1480 train_time:232232ms step_avg:160.83ms
step:1455/1480 train_time:232410ms step_avg:160.84ms
step:1456/1480 train_time:232582ms step_avg:160.84ms
step:1457/1480 train_time:232754ms step_avg:160.85ms
step:1458/1480 train_time:232925ms step_avg:160.86ms
step:1459/1480 train_time:233100ms step_avg:160.87ms
step:1460/1480 train_time:233272ms step_avg:160.88ms
step:1461/1480 train_time:233446ms step_avg:160.89ms
step:1462/1480 train_time:233617ms step_avg:160.89ms
step:1463/1480 train_time:233795ms step_avg:160.91ms
step:1464/1480 train_time:233969ms step_avg:160.91ms
step:1465/1480 train_time:234143ms step_avg:160.92ms
step:1466/1480 train_time:234314ms step_avg:160.93ms
step:1467/1480 train_time:234489ms step_avg:160.94ms
step:1468/1480 train_time:234660ms step_avg:160.95ms
step:1469/1480 train_time:234833ms step_avg:160.95ms
step:1470/1480 train_time:235013ms step_avg:160.97ms
step:1471/1480 train_time:235200ms step_avg:160.99ms
step:1472/1480 train_time:235380ms step_avg:161.00ms
step:1473/1480 train_time:235553ms step_avg:161.01ms
step:1474/1480 train_time:235730ms step_avg:161.02ms
step:1475/1480 train_time:235910ms step_avg:161.03ms
step:1476/1480 train_time:236082ms step_avg:161.04ms
step:1477/1480 train_time:236263ms step_avg:161.05ms
step:1478/1480 train_time:236444ms step_avg:161.07ms
step:1479/1480 train_time:236617ms step_avg:161.07ms
step:1480/1480 train_time:236790ms step_avg:161.08ms
step:1480/1480 val_loss:3.2782 train_time:236861ms step_avg:161.13ms