apply the vit-large,here is the code:
__all__ = ["ViT", "SimpleFeaturePyramid", "get_vit_lr_decay_rate"]
class SwiGLU(nn.Module):
def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.,
norm_layer=nn.LayerNorm, subln=False
):
super().__init__()
out_features = out_features or in_features
hidden_features = hidden_features or in_features
self.w1 = nn.Linear(in_features, hidden_features)
self.w2 = nn.Linear(in_features, hidden_features)
self.act = act_layer()
self.ffn_ln = norm_layer(hidden_features) if subln else nn.Identity()
self.w3 = nn.Linear(hidden_features, out_features)
self.drop = nn.Dropout(drop)
def forward(self, x):
x1 = self.w1(x)
x2 = self.w2(x)
hidden = self.act(x1) * x2
x = self.ffn_ln(hidden)
x = self.w3(x)
x = self.drop(x)
return x
class Attention(nn.Module):
def __init__(
self,
dim,
num_heads=8,
qkv_bias=True,
qk_scale=None,
attn_head_dim=None,
norm_layer=nn.LayerNorm,
rope=None,
flash_attn=True,
subln=False
):
super().__init__()
self.num_heads = num_heads
head_dim = dim // num_heads
if attn_head_dim is not None:
head_dim = attn_head_dim
all_head_dim = head_dim * self.num_heads
self.scale = qk_scale or head_dim ** -0.5
self.subln = subln
self.q_proj = nn.Linear(dim, all_head_dim, bias=False)
self.k_proj = nn.Linear(dim, all_head_dim, bias=False)
self.v_proj = nn.Linear(dim, all_head_dim, bias=False)
if qkv_bias:
self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
else:
self.q_bias = None
self.v_bias = None
self.rope = rope
self.flash_attn = flash_attn
self.proj = nn.Linear(all_head_dim, dim)
self.inner_attn_ln = norm_layer(all_head_dim) if subln else nn.Identity()
if self.flash_attn:
factory_kwargs = {'device': 'cuda', 'dtype': torch.float16}
self.inner_attn = FlashAttention(attention_dropout=0.0, **factory_kwargs)
def forward(self, x):
B, H, W, C = x.shape
x = x.view(B, -1, C)
N = H * W
q = F.linear(input=x, weight=self.q_proj.weight, bias=self.q_bias)
k = F.linear(input=x, weight=self.k_proj.weight, bias=None)
v = F.linear(input=x, weight=self.v_proj.weight, bias=self.v_bias)
q = q.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3) # B, num_heads, N, C
k = k.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
v = v.reshape(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
## rope
q = self.rope(q).type_as(v)
k = self.rope(k).type_as(v)
if self.flash_attn:
q = q.permute(0, 2, 1, 3) # B, num_heads, N, C -> B, N, num_heads, C
k = k.permute(0, 2, 1, 3)
v = v.permute(0, 2, 1, 3)
kv = torch.stack([k, v], dim=2)
x, attn_weights = self.inner_attn(q, kv, key_padding_mask=None, causal=False)
# x = xops.memory_efficient_attention(q, k, v)
x = x.reshape(B, N, -1)
x = self.inner_attn_ln(x)
else:
q = q * self.scale
attn = (q @ k.transpose(-2, -1))
attn = attn.softmax(dim=-1).type_as(x)
x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
x = self.inner_attn_ln(x)
x = self.proj(x)
x = x.view(B, H, W, C)
return x
class ResBottleneckBlock(CNNBlockBase):
"""
The standard bottleneck residual block without the last activation layer.
It contains 3 conv layers with kernels 1x1, 3x3, 1x1.
"""
def __init__(
self,
in_channels,
out_channels,
bottleneck_channels,
norm="LN",
act_layer=nn.GELU,
):
"""
Args:
in_channels (int): Number of input channels.
out_channels (int): Number of output channels.
bottleneck_channels (int): number of output channels for the 3x3
"bottleneck" conv layers.
norm (str or callable): normalization for all conv layers.
See :func:`layers.get_norm` for supported format.
act_layer (callable): activation for all conv layers.
"""
super().__init__(in_channels, out_channels, 1)
self.conv1 = Conv2d(in_channels, bottleneck_channels, 1, bias=False)
self.norm1 = get_norm(norm, bottleneck_channels)
self.act1 = act_layer()
self.conv2 = Conv2d(
bottleneck_channels,
bottleneck_channels,
3,
padding=1,
bias=False,
)
self.norm2 = get_norm(norm, bottleneck_channels)
self.act2 = act_layer()
self.conv3 = Conv2d(bottleneck_channels, out_channels, 1, bias=False)
self.norm3 = get_norm(norm, out_channels)
for layer in [self.conv1, self.conv2, self.conv3]:
weight_init.c2_msra_fill(layer)
for layer in [self.norm1, self.norm2]:
layer.weight.data.fill_(1.0)
layer.bias.data.zero_()
# zero init last norm layer.
self.norm3.weight.data.zero_()
self.norm3.bias.data.zero_()
def forward(self, x):
out = x
for layer in self.children():
out = layer(out)
out = x + out
return out
class Block(nn.Module):
"""Transformer blocks with support of window attention and residual propagation blocks"""
def __init__(
self,
dim,
num_heads,
mlp_ratio=4*2/3,
qkv_bias=True,
drop_path=0.0,
norm_layer=partial(nn.LayerNorm, eps=1e-6),
window_size=0,
use_residual_block=False,
rope=None,
flash_attn=True,
subln=False,
):
"""
Args:
dim (int): Number of input channels.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
drop_path (float): Stochastic depth rate.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks. If it equals 0, then not
use window attention.
use_residual_block (bool): If True, use a residual block after the MLP block.
input_size (int or None): Input resolution for calculating the relative positional
parameter size.
"""
super().__init__()
self.norm1 = norm_layer(dim)
self.attn = Attention(
dim,
num_heads=num_heads,
qkv_bias=qkv_bias,
rope=rope,
flash_attn=flash_attn,
subln=subln
)
from timm.models.layers import DropPath
self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
self.norm2 = norm_layer(dim)
self.mlp = SwiGLU(
in_features=dim,
hidden_features=int(dim * mlp_ratio),
subln=True,
norm_layer=norm_layer,
)
self.window_size = window_size
self.use_residual_block = use_residual_block
if use_residual_block:
# Use a residual block with bottleneck channel as dim // 2
self.residual = ResBottleneckBlock(
in_channels=dim,
out_channels=dim,
bottleneck_channels=dim // 2,
norm="LN",
)
def forward(self, x):
shortcut = x
x = self.norm1(x)
# Window partition
if self.window_size > 0:
H, W = x.shape[1], x.shape[2]
x, pad_hw = window_partition(x, self.window_size)
x = self.attn(x)
# Reverse window partition
if self.window_size > 0:
x = window_unpartition(x, self.window_size, pad_hw, (H, W))
x = shortcut + self.drop_path(x)
x = x + self.drop_path(self.mlp(self.norm2(x)))
if self.use_residual_block:
x = self.residual(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
return x
@BACKBONES.register_module()
class ViT(Backbone):
"""
This module implements Vision Transformer (ViT) backbone in :paper:`vitdet`.
"Exploring Plain Vision Transformer Backbones for Object Detection",
https://arxiv.org/abs/2203.16527
"""
def __init__(
self,
img_size=1024,
patch_size=16,
in_chans=3,
embed_dim=768,
depth=12,
num_heads=12,
mlp_ratio=4*2/3,
qkv_bias=True,
drop_path_rate=0.0,
norm_layer=partial(nn.LayerNorm, eps=1e-6),
act_layer=nn.GELU,
use_abs_pos=True,
use_rel_pos=False,
sim_fpn=None,
rope=True,
pt_hw_seq_len=16,
intp_freq=True,
window_size=0,
global_window_size=0,
window_block_indexes=(),
residual_block_indexes=(),
use_act_checkpoint=False,
pretrain_img_size=224,
pretrain_use_cls_token=True,
out_feature="last_feat",
subln=False,
flash_attn=True,
out_channels=256,
frozen=False,
):
"""
Args:
img_size (int): Input image size.
patch_size (int): Patch size.
in_chans (int): Number of input image channels.
embed_dim (int): Patch embedding dimension.
depth (int): Depth of ViT.
num_heads (int): Number of attention heads in each ViT block.
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
qkv_bias (bool): If True, add a learnable bias to query, key, value.
drop_path_rate (float): Stochastic depth rate.
norm_layer (nn.Module): Normalization layer.
act_layer (nn.Module): Activation layer.
use_abs_pos (bool): If True, use absolute positional embeddings.
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
window_size (int): Window size for window attention blocks.
window_block_indexes (list): Indexes for blocks using window attention.
residual_block_indexes (list): Indexes for blocks using conv propagation.
use_act_checkpoint (bool): If True, use activation checkpointing.
pretrain_img_size (int): input image size for pretraining models.
pretrain_use_cls_token (bool): If True, pretrainig models use class token.
out_feature (str): name of the feature from the last block.
"""
super().__init__()
self.pretrain_use_cls_token = pretrain_use_cls_token
self.patch_embed = PatchEmbed(
kernel_size=(patch_size, patch_size),
stride=(patch_size, patch_size),
in_chans=in_chans,
embed_dim=embed_dim,
)
self.frozen = frozen
if use_abs_pos:
# Initialize absolute positional embedding with pretrain image size.
num_patches = (pretrain_img_size // patch_size) * (pretrain_img_size // patch_size)
num_positions = (num_patches + 1) if pretrain_use_cls_token else num_patches
self.pos_embed = nn.Parameter(torch.zeros(1, num_positions, embed_dim))
else:
self.pos_embed = None
half_head_dim = embed_dim // num_heads // 2
hw_seq_len = img_size // patch_size
self.rope_win = VisionRotaryEmbeddingFast(
dim=half_head_dim,
pt_seq_len=pt_hw_seq_len,
ft_seq_len=window_size if intp_freq else None,
)
self.rope_glb = VisionRotaryEmbeddingFast(
dim=half_head_dim,
pt_seq_len=pt_hw_seq_len,
ft_seq_len=hw_seq_len if intp_freq else None,
)
# 新增：定义MLP降维层
self.output_proj = nn.Sequential(
nn.Linear(embed_dim, embed_dim // 2),
nn.GELU(),
nn.Linear(embed_dim // 2, out_channels),
nn.LayerNorm(out_channels, eps=1e-6)
)
# 更新输出特征通道数
self._out_feature_channels = {out_feature: out_channels}
# stochastic depth decay rule
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
self.blocks = nn.ModuleList()
for i in range(depth):
block = Block(
dim=embed_dim,
num_heads=num_heads,
mlp_ratio=mlp_ratio,
qkv_bias=qkv_bias,
drop_path=dpr[i],
norm_layer=norm_layer,
window_size=window_size if i in window_block_indexes else global_window_size,
use_residual_block=i in residual_block_indexes,
rope=self.rope_win if i in window_block_indexes else self.rope_glb,
flash_attn=flash_attn,
subln=subln,
)
if use_act_checkpoint:
# TODO: use torch.utils.checkpoint
from fairscale.nn.checkpoint import checkpoint_wrapper
block = checkpoint_wrapper(block)
self.blocks.append(block)
self._out_feature_channels = {out_feature: embed_dim}
self._out_feature_strides = {out_feature: patch_size}
self._out_features = [out_feature]
if self.pos_embed is not None:
nn.init.trunc_normal_(self.pos_embed, std=0.02)
# self.apply(self._init_weights)
self._freeze_stages()
def _freeze_stages(self):
if self.frozen:
self.eval()
for m in self.parameters():
m.requires_grad = False
def _init_weights(self, m):
if isinstance(m, nn.Linear):
nn.init.trunc_normal_(m.weight, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.LayerNorm):
nn.init.constant_(m.bias, 0)
nn.init.constant_(m.weight, 1.0)
def forward(self, x):
x = self.patch_embed(x)
if self.pos_embed is not None:
x = x + get_abs_pos(
self.pos_embed, self.pretrain_use_cls_token, (x.shape[1], x.shape[2])
)
for blk in self.blocks:
x = blk(x) # b, h, w, c
# 需要使用MLP进行降维处理
B, H, W, C = x.shape
x = x.reshape(B, H*W, C) # 展平为序列
x = self.output_proj(x) # 应用MLP降维
x = x.reshape(B, H, W, -1) # 恢复空间维度
outputs = {self._out_features[0]: x.permute(0, 3, 1, 2)}
return outputs
class SimpleFeaturePyramid(Backbone):
"""
This module implements SimpleFeaturePyramid in :paper:`vitdet`.
It creates pyramid features built on top of the input feature map.
"""
def __init__(
self,
net,
in_feature,
out_channels,
scale_factors,
top_block=None,
norm="LN",
square_pad=0,
):
"""
Args:
net (Backbone): module representing the subnetwork backbone.
Must be a subclass of :class:`Backbone`.
in_feature (str): names of the input feature maps coming
from the net.
out_channels (int): number of channels in the output feature maps.
scale_factors (list[float]): list of scaling factors to upsample or downsample
the input features for creating pyramid features.
top_block (nn.Module or None): if provided, an extra operation will
be performed on the output of the last (smallest resolution)
pyramid output, and the result will extend the result list. The top_block
further downsamples the feature map. It must have an attribute
"num_levels", meaning the number of extra pyramid levels added by
this block, and "in_feature", which is a string representing
its input feature (e.g., p5).
norm (str): the normalization to use.
square_pad (int): If > 0, require input images to be padded to specific square size.
"""
super(SimpleFeaturePyramid, self).__init__()
assert isinstance(net, Backbone)
self.scale_factors = scale_factors
input_shapes = net.output_shape()
strides = [int(input_shapes[in_feature].stride / scale) for scale in scale_factors]
_assert_strides_are_log2_contiguous(strides)
dim = input_shapes[in_feature].channels
self.stages = []
use_bias = norm == ""
for idx, scale in enumerate(scale_factors):
out_dim = dim
if scale == 4.0:
layers = [
nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2),
get_norm(norm, dim // 2),
nn.GELU(),
nn.ConvTranspose2d(dim // 2, dim // 4, kernel_size=2, stride=2),
]
out_dim = dim // 4
elif scale == 2.0:
layers = [nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2)]
out_dim = dim // 2
elif scale == 1.0:
layers = []
elif scale == 0.5:
layers = [nn.MaxPool2d(kernel_size=2, stride=2)]
else:
raise NotImplementedError(f"scale_factor={scale} is not supported yet.")
layers.extend(
[
Conv2d(
out_dim,
out_channels,
kernel_size=1,
bias=use_bias,
norm=get_norm(norm, out_channels),
),
Conv2d(
out_channels,
out_channels,
kernel_size=3,
padding=1,
bias=use_bias,
norm=get_norm(norm, out_channels),
),
]
)
layers = nn.Sequential(*layers)
stage = int(math.log2(strides[idx]))
self.add_module(f"simfp_{stage}", layers)
self.stages.append(layers)
self.net = net
self.in_feature = in_feature
self.top_block = top_block
# Return feature names are "p<stage>", like ["p2", "p3", ..., "p6"]
self._out_feature_strides = {"p{}".format(int(math.log2(s))): s for s in strides}
# top block output feature maps.
if self.top_block is not None:
for s in range(stage, stage + self.top_block.num_levels):
self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1)
self._out_features = list(self._out_feature_strides.keys())
self._out_feature_channels = {k: out_channels for k in self._out_features}
self._size_divisibility = strides[-1]
self._square_pad = square_pad
@property
def padding_constraints(self):
return {
"size_divisiblity": self._size_divisibility,
"square_size": self._square_pad,
}
def forward(self, x):
"""
Args:
x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``.
Returns:
dict[str->Tensor]:
mapping from feature map name to pyramid feature map tensor
in high to low resolution order. Returned feature names follow the FPN
convention: "p<stage>", where stage has stride = 2 ** stage e.g.,
["p2", "p3", ..., "p6"].
"""
bottom_up_features = self.net(x)
features = bottom_up_features[self.in_feature]
results = []
for stage in self.stages:
results.append(stage(features))
if self.top_block is not None:
if self.top_block.in_feature in bottom_up_features:
top_block_in_feature = bottom_up_features[self.top_block.in_feature]
else:
top_block_in_feature = results[self._out_features.index(self.top_block.in_feature)]
results.extend(self.top_block(top_block_in_feature))
assert len(self._out_features) == len(results)
return {f: res for f, res in zip(self._out_features, results)}
def get_vit_lr_decay_rate(name, lr_decay_rate=1.0, num_layers=12):
"""
Calculate lr decay rate for different ViT blocks.
Args:
name (string): parameter name.
lr_decay_rate (float): base lr decay rate.
num_layers (int): number of ViT blocks.
Returns:
lr decay rate for the given parameter.
"""
layer_id = num_layers + 1
if name.startswith("backbone"):
if ".pos_embed" in name or ".patch_embed" in name:
layer_id = 0
elif ".blocks." in name and ".residual." not in name:
layer_id = int(name[name.find(".blocks.") :].split(".")[2]) + 1
return lr_decay_rate ** (num_layers + 1 - layer_id)
LearningRateDecayOptimizerConstructor code ：
class LearningRateDecayOptimizerConstructor(DefaultOptimizerConstructor): def add_params(self, params, module, prefix='', is_dcn_module=None): """Add all parameters of module to the params list. The parameters of the given module will be added to the list of param groups, with specific rules defined by paramwise_cfg. Args: params (list[dict]): A list of param groups, it will be modified in place. module (nn.Module): The module to be added. prefix (str): The prefix of the module is_dcn_module (int|float|None): If the current module is a submodule of DCN, is_dcn_module` will be passed to
control conv_offset layer's learning rate. Defaults to None.
"""
parameter_groups = {}
print(self.paramwise_cfg)
num_layers = self.paramwise_cfg.get('num_layers') + 2
decay_rate = self.paramwise_cfg.get('decay_rate')
decay_type = self.paramwise_cfg.get('decay_type', "layer_wise")
print("Build LearningRateDecayOptimizerConstructor %s %f - %d" % (decay_type, decay_rate, num_layers))
weight_decay = self.base_wd
for name, param in module.named_parameters():
if not param.requires_grad:
continue # frozen weights
if len(param.shape) == 1 or name.endswith(".bias") or name in ('pos_embed', 'cls_token'):
group_name = "no_decay"
this_weight_decay = 0.
else:
group_name = "decay"
this_weight_decay = weight_decay
if decay_type == "layer_wise":
layer_id = get_num_layer_layer_wise(name, self.paramwise_cfg.get('num_layers'))
scale = decay_rate ** (num_layers - layer_id - 1)
elif decay_type == "vit_wise":
layer_id, scale = get_vit_lr_decay_rate(name, decay_rate, self.paramwise_cfg.get('head_decay_rate', 1.0), self.paramwise_cfg.get('num_layers'))
group_name = "layer_%d_%s" % (layer_id, group_name)
if group_name not in parameter_groups:
parameter_groups[group_name] = {
"weight_decay": this_weight_decay,
"params": [],
"param_names": [],
"lr_scale": scale,
"group_name": group_name,
"lr": scale * self.base_lr,
}
parameter_groups[group_name]["params"].append(param)
parameter_groups[group_name]["param_names"].append(name)
rank, _ = get_dist_info()
if rank == 0:
to_display = {}
for key in parameter_groups:
to_display[key] = {
"param_names": parameter_groups[key]["param_names"],
"lr_scale": parameter_groups[key]["lr_scale"],
"lr": parameter_groups[key]["lr"],
"weight_decay": parameter_groups[key]["weight_decay"],
}
print("Param groups = %s" % json.dumps(to_display, indent=2))
params.extend(parameter_groups.values())`
train parameters:
`base = [
'../../../mmdetection3d/configs/base/datasets/nus-3d.py',
'../../../mmdetection3d/configs/base/default_runtime.py'
]
backbone_norm_cfg = dict(type='LN', requires_grad=True)
plugin=True
plugin_dir='projects/mmdet3d_plugin/'
If point cloud range is changed, the models should also change their point
cloud range accordingly
point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
voxel_size = [0.2, 0.2, 8]
img_norm_cfg = dict(
mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm
For nuScenes we usually do 10-class detection
class_names = [
'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier',
'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone'
]
num_gpus = 4
batch_size = 2
num_iters_per_epoch = 28130 // (num_gpus * batch_size)
num_epochs = 24
queue_length = 1
num_frame_losses = 1
collect_keys=['lidar2img', 'intrinsics', 'extrinsics','timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv']
input_modality = dict(
use_lidar=False,
use_camera=True,
use_radar=False,
use_map=False,
use_external=True)
sim_fpn=dict(
scale_factors=[4, 2, 1, 0.5, 0.25],
in_channels=1024,
out_channels=256,
out_indices=[2, 3, 4, 5, 6],
)
model = dict(
type='Petr3D',
num_frame_head_grads=num_frame_losses,
num_frame_backbone_grads=num_frame_losses,
num_frame_losses=num_frame_losses,
use_grid_mask=True,
img_backbone=dict(
type='ViT',
img_size=320,
patch_size=16,
window_size=16,
global_window_size=20,
in_chans=3,
embed_dim=1024,
depth=24,
num_heads=16,
mlp_ratio=4*2/3,
window_block_indexes = (
list(range(0, 2)) + list(range(3, 5)) + list(range(6, 8)) + list(range(9, 11)) + list(range(12, 14)) + list(range(15, 17)) + list(range(18, 20)) + list(range(21, 23))
),
sim_fpn=None,
# sim_fpn=sim_fpn, #Only for RepDETR3D
qkv_bias=True,
drop_path_rate=0.3,
# use_act_checkpoint=False,
# xattn=False,
use_act_checkpoint=True,
flash_attn=True,
),
# img_neck=dict(
# type='CPFPN', ###remove unused parameters
# in_channels=[1024],
# out_channels=256,
# num_outs=1),
img_roi_head=dict(
type='FocalHead',
num_classes=10,
in_channels=256,
loss_cls2d=dict(
type='QualityFocalLoss',
use_sigmoid=True,
beta=2.0,
loss_weight=2.0),
loss_centerness=dict(type='GaussianFocalLoss', reduction='mean', loss_weight=1.0),
loss_bbox2d=dict(type='L1Loss', loss_weight=5.0),
loss_iou2d=dict(type='GIoULoss', loss_weight=2.0),
loss_centers2d=dict(type='L1Loss', loss_weight=10.0),
train_cfg=dict(
assigner2d=dict(
type='HungarianAssigner2D',
cls_cost=dict(type='FocalLossCost', weight=2.),
reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'),
iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0),
centers2d_cost=dict(type='BBox3DL1Cost', weight=10.0)))
),
pts_bbox_head=dict(
type='StreamPETRHead',
num_classes=10,
in_channels=256,
num_query=644,
memory_len=1024,
topk_proposals=256,
num_propagated=256,
with_ego_pos=True,
match_with_velo=False,
scalar=10, ##noise groups
noise_scale = 1.0,
dn_weight= 1.0, ##dn loss weight
split = 0.75, ###positive rate
LID=True,
with_position=True,
position_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
code_weights = [2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
transformer=dict(
type='PETRTemporalTransformer',
decoder=dict(
type='PETRTransformerDecoder',
return_intermediate=True,
num_layers=6,
transformerlayers=dict(
type='PETRTemporalDecoderLayer',
attn_cfgs=[
dict(
type='MultiheadAttention',
embed_dims=256,
num_heads=8,
dropout=0.1),
dict(
type='PETRMultiheadFlashAttention',
embed_dims=256,
num_heads=8,
dropout=0.1),
],
feedforward_channels=2048,
ffn_dropout=0.1,
with_cp=True, ###use checkpoint to save memory
operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
'ffn', 'norm')),
)),
bbox_coder=dict(
type='NMSFreeCoder',
post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
pc_range=point_cloud_range,
max_num=300,
voxel_size=voxel_size,
num_classes=10),
loss_cls=dict(
type='FocalLoss',
use_sigmoid=True,
gamma=2.0,
alpha=0.25,
loss_weight=2.0),
loss_bbox=dict(type='L1Loss', loss_weight=0.25),
loss_iou=dict(type='GIoULoss', loss_weight=0.0),),
# model training and testing settings
train_cfg=dict(pts=dict(
grid_size=[512, 512, 1],
voxel_size=voxel_size,
point_cloud_range=point_cloud_range,
out_size_factor=4,
assigner=dict(
type='HungarianAssigner3D',
cls_cost=dict(type='FocalLossCost', weight=2.0),
reg_cost=dict(type='BBox3DL1Cost', weight=0.25),
iou_cost=dict(type='IoUCost', weight=0.0), # Fake cost. This is just to make it compatible with DETR head.
pc_range=point_cloud_range),)))
dataset_type = 'CustomNuScenesDataset'
data_root = '/resource-mounts/algorithms/algorithm-0/data/nuscenes/'
file_client_args = dict(backend='disk')
ida_aug_conf = {
"resize_lim": (0.47, 0.625),
"final_dim": (320, 800),
"bot_pct_lim": (0.0, 0.0),
"rot_lim": (0.0, 0.0),
"H": 900,
"W": 1600,
"rand_flip": True,
}
train_pipeline = [
dict(type='LoadMultiViewImageFromFiles', to_float32=True),
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True,
with_label=True, with_bbox_depth=True),
dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
dict(type='ObjectNameFilter', classes=class_names),
dict(type='ResizeCropFlipRotImage', data_aug_conf = ida_aug_conf, training=True),
dict(type='GlobalRotScaleTransImage',
rot_range=[-0.3925, 0.3925],
translation_std=[0, 0, 0],
scale_ratio_range=[0.95, 1.05],
reverse_angle=True,
training=True,
),
dict(type='NormalizeMultiviewImage', **img_norm_cfg),
dict(type='PadMultiViewImage', size_divisor=32),
dict(type='PETRFormatBundle3D', class_names=class_names, collect_keys=collect_keys + ['prev_exists']),
dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys,
meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d','gt_labels_3d'))
]
test_pipeline = [
dict(type='LoadMultiViewImageFromFiles', to_float32=True),
dict(type='ResizeCropFlipRotImage', data_aug_conf = ida_aug_conf, training=False),
dict(type='NormalizeMultiviewImage', **img_norm_cfg),
dict(type='PadMultiViewImage', size_divisor=32),
dict(
type='MultiScaleFlipAug3D',
img_scale=(1333, 800),
pts_scale_ratio=1,
flip=False,
transforms=[
dict(
type='PETRFormatBundle3D',
collect_keys=collect_keys,
class_names=class_names,
with_label=False),
dict(type='Collect3D', keys=['img'] + collect_keys,
meta_keys=('filename', 'ori_shape', 'img_shape','pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token'))
])
]
data = dict(
samples_per_gpu=batch_size,
workers_per_gpu=4,
train=dict(
type=dataset_type,
data_root=data_root,
ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_train.pkl',
num_frame_losses=num_frame_losses,
seq_split_num=2, # streaming video training
seq_mode=True, # streaming video training
pipeline=train_pipeline,
classes=class_names,
modality=input_modality,
collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'],
queue_length=queue_length,
test_mode=False,
use_valid_flag=True,
filter_empty_gt=False,
box_type_3d='LiDAR'),
val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_val.pkl', classes=class_names, modality=input_modality),
test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_test.pkl', classes=class_names, modality=input_modality),
shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'),
nonshuffler_sampler=dict(type='DistributedSampler')
)
optimizer = dict(constructor='LearningRateDecayOptimizerConstructor',
type='AdamW',
lr=2e-4, betas=(0.9, 0.999), weight_decay=1e-7,
paramwise_cfg={'decay_rate': 0.9,
'head_decay_rate': 4.0,
'decay_type': 'vit_wise',
'num_layers': 24,
})
optimizer_config = dict(type='Fp16OptimizerHook', loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2))
learning policy
lr_config = dict(
policy='CosineAnnealing',
warmup='linear',
warmup_iters=500,
warmup_ratio=1.0 / 3,
min_lr_ratio=1e-3,
)
evaluation = dict(interval=num_iters_per_epoch*num_epochs, pipeline=test_pipeline)
find_unused_parameters=False #### when use checkpoint, find_unused_parameters must be False
checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=3)
runner = dict(
type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch)
load_from='/resource-mounts/algorithms/algorithm-0/ckpts/eva02_L_coco_det_sys_o365_remapped.pth'
resume_from=None
`
why has the error:
2025-06-08 16:05:05,985 - mmdet - INFO - workflow: [('train', 1)], max: 84384 iters
2025-06-08 16:05:05,987 - mmdet - INFO - Checkpoints will be saved to /resource-mounts/algorithms/algorithm-0/result_streampetr_nuscenes_real_0608 by HardDiskBackend.
2025-06-08 16:06:06,307 - mmdet - INFO - Iter [50/84384] lr: 3.189e-04, eta: 1 day, 3:39:18, time: 1.181, data_time: 0.073, memory: 9532, frame_0_loss_cls: 1.3917, frame_0_loss_bbox: 2.7703, frame_0_d0.loss_cls: 1.4073, frame_0_d0.loss_bbox: 2.7661, frame_0_d1.loss_cls: 1.3967, frame_0_d1.loss_bbox: 2.7791, frame_0_d2.loss_cls: 1.3936, frame_0_d2.loss_bbox: 2.7807, frame_0_d3.loss_cls: 1.3880, frame_0_d3.loss_bbox: 2.7810, frame_0_d4.loss_cls: 1.3791, frame_0_d4.loss_bbox: 2.7712, frame_0_dn_loss_cls: 1.1583, frame_0_dn_loss_bbox: 2.1724, frame_0_d0.dn_loss_cls: 1.1581, frame_0_d0.dn_loss_bbox: 1.9987, frame_0_d1.dn_loss_cls: 1.1510, frame_0_d1.dn_loss_bbox: 2.0393, frame_0_d2.dn_loss_cls: 1.1440, frame_0_d2.dn_loss_bbox: 2.0727, frame_0_d3.dn_loss_cls: 1.1483, frame_0_d3.dn_loss_bbox: 2.1095, frame_0_d4.dn_loss_cls: 1.1396, frame_0_d4.dn_loss_bbox: 2.1265, frame_0_enc_loss_cls: 0.7599, frame_0_enc_loss_bbox: 1.2891, frame_0_enc_loss_iou: 1.1671, frame_0_centers2d_losses: 0.4382, frame_0_centerness_losses: 1.2813, loss: 49.3589, grad_norm: nan
2025-06-08 16:07:00,056 - mmdet - INFO - Iter [100/84384] lr: 3.723e-04, eta: 1 day, 2:24:11, time: 1.075, data_time: 0.042, memory: 9532, frame_0_loss_cls: 1.2279, frame_0_loss_bbox: 2.6748, frame_0_d0.loss_cls: 1.2250, frame_0_d0.loss_bbox: 2.6757, frame_0_d1.loss_cls: 1.2257, frame_0_d1.loss_bbox: 2.6751, frame_0_d2.loss_cls: 1.2280, frame_0_d2.loss_bbox: 2.6749, frame_0_d3.loss_cls: 1.2292, frame_0_d3.loss_bbox: 2.6724, frame_0_d4.loss_cls: 1.2252, frame_0_d4.loss_bbox: 2.6732, frame_0_dn_loss_cls: 0.9124, frame_0_dn_loss_bbox: 1.8359, frame_0_d0.dn_loss_cls: 0.9026, frame_0_d0.dn_loss_bbox: 1.8466, frame_0_d1.dn_loss_cls: 0.9098, frame_0_d1.dn_loss_bbox: 1.8442, frame_0_d2.dn_loss_cls: 0.9097, frame_0_d2.dn_loss_bbox: 1.8388, frame_0_d3.dn_loss_cls: 0.9098, frame_0_d3.dn_loss_bbox: 1.8368, frame_0_d4.dn_loss_cls: 0.9119, frame_0_d4.dn_loss_bbox: 1.8413, frame_0_enc_loss_cls: 0.9047, frame_0_enc_loss_bbox: 0.3444, frame_0_enc_loss_iou: 0.8120, frame_0_centers2d_losses: 0.2805, frame_0_centerness_losses: 0.6943, loss: 42.9430, grad_norm: nan
2025-06-08 16:07:54,253 - mmdet - INFO - Iter [150/84384] lr: 4.256e-04, eta: 1 day, 2:02:44, time: 1.084, data_time: 0.048, memory: 9533, frame_0_loss_cls: 1.3132, frame_0_loss_bbox: 2.3946, frame_0_d0.loss_cls: 1.3036, frame_0_d0.loss_bbox: 2.4048, frame_0_d1.loss_cls: 1.2983, frame_0_d1.loss_bbox: 2.4000, frame_0_d2.loss_cls: 1.3031, frame_0_d2.loss_bbox: 2.4020, frame_0_d3.loss_cls: 1.3000, frame_0_d3.loss_bbox: 2.3978, frame_0_d4.loss_cls: 1.3005, frame_0_d4.loss_bbox: 2.3992, frame_0_dn_loss_cls: 0.9940, frame_0_dn_loss_bbox: 1.7972, frame_0_d0.dn_loss_cls: 0.9645, frame_0_d0.dn_loss_bbox: 1.7962, frame_0_d1.dn_loss_cls: 0.9743, frame_0_d1.dn_loss_bbox: 1.7958, frame_0_d2.dn_loss_cls: 0.9814, frame_0_d2.dn_loss_bbox: 1.7919, frame_0_d3.dn_loss_cls: 0.9831, frame_0_d3.dn_loss_bbox: 1.7961, frame_0_d4.dn_loss_cls: 0.9826, frame_0_d4.dn_loss_bbox: 1.7935, frame_0_enc_loss_cls: 0.8481, frame_0_enc_loss_bbox: 0.3590, frame_0_enc_loss_iou: 0.8317, frame_0_centers2d_losses: 0.2695, frame_0_centerness_losses: 0.7400, loss: 41.9162, grad_norm: 84.5885
2025-06-08 16:08:49,362 - mmdet - INFO - Iter [200/84384] lr: 4.789e-04, eta: 1 day, 1:57:57, time: 1.102, data_time: 0.042, memory: 9533, frame_0_loss_cls: 1.1974, frame_0_loss_bbox: 2.2007, frame_0_d0.loss_cls: 1.2035, frame_0_d0.loss_bbox: 2.2034, frame_0_d1.loss_cls: 1.1967, frame_0_d1.loss_bbox: 2.2052, frame_0_d2.loss_cls: 1.2011, frame_0_d2.loss_bbox: 2.2065, frame_0_d3.loss_cls: 1.1950, frame_0_d3.loss_bbox: 2.2099, frame_0_d4.loss_cls: 1.1942, frame_0_d4.loss_bbox: 2.2078, frame_0_dn_loss_cls: 0.8800, frame_0_dn_loss_bbox: 1.6463, frame_0_d0.dn_loss_cls: 0.8715, frame_0_d0.dn_loss_bbox: 1.6446, frame_0_d1.dn_loss_cls: 0.8787, frame_0_d1.dn_loss_bbox: 1.6472, frame_0_d2.dn_loss_cls: 0.8764, frame_0_d2.dn_loss_bbox: 1.6444, frame_0_d3.dn_loss_cls: 0.8787, frame_0_d3.dn_loss_bbox: 1.6442, frame_0_d4.dn_loss_cls: 0.8792, frame_0_d4.dn_loss_bbox: 1.6361, frame_0_enc_loss_cls: 0.8244, frame_0_enc_loss_bbox: 0.3296, frame_0_enc_loss_iou: 0.8093, frame_0_centers2d_losses: 0.2574, frame_0_centerness_losses: 0.6499, loss: 38.4196, grad_norm: 64.9579
2025-06-08 16:09:43,309 - mmdet - INFO - Iter [250/84384] lr: 5.323e-04, eta: 1 day, 1:48:10, time: 1.079, data_time: 0.044, memory: 9533, frame_0_loss_cls: 1.1885, frame_0_loss_bbox: 2.3395, frame_0_d0.loss_cls: 1.1877, frame_0_d0.loss_bbox: 2.3437, frame_0_d1.loss_cls: 1.1879, frame_0_d1.loss_bbox: 2.3360, frame_0_d2.loss_cls: 1.1927, frame_0_d2.loss_bbox: 2.3382, frame_0_d3.loss_cls: 1.1875, frame_0_d3.loss_bbox: 2.3419, frame_0_d4.loss_cls: 1.1892, frame_0_d4.loss_bbox: 2.3393, frame_0_dn_loss_cls: 0.8602, frame_0_dn_loss_bbox: 1.8234, frame_0_d0.dn_loss_cls: 0.8523, frame_0_d0.dn_loss_bbox: 1.8241, frame_0_d1.dn_loss_cls: 0.8553, frame_0_d1.dn_loss_bbox: 1.8189, frame_0_d2.dn_loss_cls: 0.8576, frame_0_d2.dn_loss_bbox: 1.8189, frame_0_d3.dn_loss_cls: 0.8585, frame_0_d3.dn_loss_bbox: 1.8187, frame_0_d4.dn_loss_cls: 0.8619, frame_0_d4.dn_loss_bbox: 1.8192, frame_0_enc_loss_cls: 0.8457, frame_0_enc_loss_bbox: 0.3578, frame_0_enc_loss_iou: 0.8405, frame_0_centers2d_losses: 0.2807, frame_0_centerness_losses: 0.7160, loss: 40.2816, grad_norm: 70.6966
2025-06-08 16:10:37,481 - mmdet - INFO - Iter [300/84384] lr: 5.856e-04, eta: 1 day, 1:42:26, time: 1.083, data_time: 0.043, memory: 9534, frame_0_loss_cls: 1.1640, frame_0_loss_bbox: 2.1763, frame_0_d0.loss_cls: 1.1642, frame_0_d0.loss_bbox: 2.1815, frame_0_d1.loss_cls: 1.1570, frame_0_d1.loss_bbox: 2.1803, frame_0_d2.loss_cls: 1.1602, frame_0_d2.loss_bbox: 2.1815, frame_0_d3.loss_cls: 1.1654, frame_0_d3.loss_bbox: 2.1793, frame_0_d4.loss_cls: 1.1620, frame_0_d4.loss_bbox: 2.1763, frame_0_dn_loss_cls: 0.8490, frame_0_dn_loss_bbox: 1.6818, frame_0_d0.dn_loss_cls: 0.8506, frame_0_d0.dn_loss_bbox: 1.6895, frame_0_d1.dn_loss_cls: 0.8487, frame_0_d1.dn_loss_bbox: 1.6858, frame_0_d2.dn_loss_cls: 0.8502, frame_0_d2.dn_loss_bbox: 1.6853, frame_0_d3.dn_loss_cls: 0.8472, frame_0_d3.dn_loss_bbox: 1.6800, frame_0_d4.dn_loss_cls: 0.8476, frame_0_d4.dn_loss_bbox: 1.6796, frame_0_enc_loss_cls: 0.8578, frame_0_enc_loss_bbox: 0.3556, frame_0_enc_loss_iou: 0.7865, frame_0_centers2d_losses: 0.2694, frame_0_centerness_losses: 0.6951, loss: 38.2078, grad_norm: 65.6546
2025-06-08 16:11:30,627 - mmdet - INFO - Iter [350/84384] lr: 6.389e-04, eta: 1 day, 1:33:59, time: 1.063, data_time: 0.045, memory: 9534, frame_0_loss_cls: 1.2191, frame_0_loss_bbox: 2.2699, frame_0_d0.loss_cls: 1.2071, frame_0_d0.loss_bbox: 2.2692, frame_0_d1.loss_cls: 1.2099, frame_0_d1.loss_bbox: 2.2685, frame_0_d2.loss_cls: 1.2113, frame_0_d2.loss_bbox: 2.2678, frame_0_d3.loss_cls: 1.2121, frame_0_d3.loss_bbox: 2.2688, frame_0_d4.loss_cls: 1.2093, frame_0_d4.loss_bbox: 2.2704, frame_0_dn_loss_cls: 0.8816, frame_0_dn_loss_bbox: 1.7664, frame_0_d0.dn_loss_cls: 0.8763, frame_0_d0.dn_loss_bbox: 1.7741, frame_0_d1.dn_loss_cls: 0.8806, frame_0_d1.dn_loss_bbox: 1.7702, frame_0_d2.dn_loss_cls: 0.8779, frame_0_d2.dn_loss_bbox: 1.7701, frame_0_d3.dn_loss_cls: 0.8818, frame_0_d3.dn_loss_bbox: 1.7690, frame_0_d4.dn_loss_cls: 0.8804, frame_0_d4.dn_loss_bbox: 1.7649, frame_0_enc_loss_cls: 0.8230, frame_0_enc_loss_bbox: 0.3285, frame_0_enc_loss_iou: 0.8250, frame_0_centers2d_losses: 0.2509, frame_0_centerness_losses: 0.6825, loss: 39.6865, grad_norm: 83.4921
2025-06-08 16:12:23,852 - mmdet - INFO - Iter [400/84384] lr: 6.922e-04, eta: 1 day, 1:27:37, time: 1.064, data_time: 0.048, memory: 9534, frame_0_loss_cls: 1.1457, frame_0_loss_bbox: 2.1336, frame_0_d0.loss_cls: 1.1500, frame_0_d0.loss_bbox: 2.1355, frame_0_d1.loss_cls: 1.1442, frame_0_d1.loss_bbox: 2.1342, frame_0_d2.loss_cls: 1.1442, frame_0_d2.loss_bbox: 2.1347, frame_0_d3.loss_cls: 1.1420, frame_0_d3.loss_bbox: 2.1375, frame_0_d4.loss_cls: 1.1469, frame_0_d4.loss_bbox: 2.1342, frame_0_dn_loss_cls: 0.8406, frame_0_dn_loss_bbox: 1.6028, frame_0_d0.dn_loss_cls: 0.8353, frame_0_d0.dn_loss_bbox: 1.6045, frame_0_d1.dn_loss_cls: 0.8379, frame_0_d1.dn_loss_bbox: 1.6009, frame_0_d2.dn_loss_cls: 0.8367, frame_0_d2.dn_loss_bbox: 1.6015, frame_0_d3.dn_loss_cls: 0.8402, frame_0_d3.dn_loss_bbox: 1.6034, frame_0_d4.dn_loss_cls: 0.8439, frame_0_d4.dn_loss_bbox: 1.6042, frame_0_enc_loss_cls: 0.7494, frame_0_enc_loss_bbox: 0.3268, frame_0_enc_loss_iou: 0.8275, frame_0_centers2d_losses: 0.2506, frame_0_centerness_losses: 0.6536, loss: 37.1425, grad_norm: 57.6195
2025-06-08 19:51:55,065 - mmdet - INFO - Iter [5350/84384] lr: 7.921e-04, eta: 2 days, 7:50:23, time: 4.311, data_time: 2.753, memory: 9791, frame_0_loss_cls: 1.2731, frame_0_loss_bbox: 2.1065, frame_0_d0.loss_cls: 1.2859, frame_0_d0.loss_bbox: 2.0993, frame_0_d1.loss_cls: 1.2785, frame_0_d1.loss_bbox: 2.1010, frame_0_d2.loss_cls: 1.3229, frame_0_d2.loss_bbox: 2.0993, frame_0_d3.loss_cls: 1.2819, frame_0_d3.loss_bbox: 2.1009, frame_0_d4.loss_cls: 1.2549, frame_0_d4.loss_bbox: 2.0821, frame_0_dn_loss_cls: 0.9599, frame_0_dn_loss_bbox: 1.7798, frame_0_d0.dn_loss_cls: 0.9579, frame_0_d0.dn_loss_bbox: 1.7751, frame_0_d1.dn_loss_cls: 0.9643, frame_0_d1.dn_loss_bbox: 1.7799, frame_0_d2.dn_loss_cls: 0.9483, frame_0_d2.dn_loss_bbox: 1.7799, frame_0_d3.dn_loss_cls: 0.9764, frame_0_d3.dn_loss_bbox: 1.7797, frame_0_d4.dn_loss_cls: 0.9548, frame_0_d4.dn_loss_bbox: 1.7799, frame_0_enc_loss_cls: 0.7812, frame_0_enc_loss_bbox: 0.2997, frame_0_enc_loss_iou: 0.7753, frame_0_centers2d_losses: 0.2020, frame_0_centerness_losses: 0.6544, loss: 39.4349, grad_norm: 5466.4919
2025-06-08 19:55:37,156 - mmdet - INFO - Iter [5400/84384] lr: 7.920e-04, eta: 2 days, 8:11:24, time: 4.442, data_time: 3.047, memory: 9791, frame_0_loss_cls: 1.4127, frame_0_loss_bbox: 2.1000, frame_0_d0.loss_cls: 1.4663, frame_0_d0.loss_bbox: 2.0927, frame_0_d1.loss_cls: 1.3890, frame_0_d1.loss_bbox: 2.0965, frame_0_d2.loss_cls: 1.4787, frame_0_d2.loss_bbox: 2.0933, frame_0_d3.loss_cls: 1.4179, frame_0_d3.loss_bbox: 2.0929, frame_0_d4.loss_cls: 1.4175, frame_0_d4.loss_bbox: 2.0674, frame_0_dn_loss_cls: 0.9936, frame_0_dn_loss_bbox: 1.7365, frame_0_d0.dn_loss_cls: 0.9595, frame_0_d0.dn_loss_bbox: 1.7317, frame_0_d1.dn_loss_cls: 1.0076, frame_0_d1.dn_loss_bbox: 1.7365, frame_0_d2.dn_loss_cls: 0.9859, frame_0_d2.dn_loss_bbox: 1.7365, frame_0_d3.dn_loss_cls: 1.0182, frame_0_d3.dn_loss_bbox: 1.7336, frame_0_d4.dn_loss_cls: 0.9837, frame_0_d4.dn_loss_bbox: 1.7365, frame_0_enc_loss_cls: 0.8939, frame_0_enc_loss_bbox: 0.3292, frame_0_enc_loss_iou: 0.7902, frame_0_centers2d_losses: 0.2234, frame_0_centerness_losses: 0.8001, loss: 40.5213, grad_norm: nan
2025-06-08 19:59:10,158 - mmdet - INFO - Iter [5450/84384] lr: 7.918e-04, eta: 2 days, 8:29:46, time: 4.260, data_time: 2.992, memory: 9791, frame_0_loss_cls: 1.3355, frame_0_loss_bbox: 1.9760, frame_0_d0.loss_cls: 1.3389, frame_0_d0.loss_bbox: 1.9659, frame_0_d1.loss_cls: 1.3302, frame_0_d1.loss_bbox: 1.9703, frame_0_d2.loss_cls: 1.3501, frame_0_d2.loss_bbox: 1.9681, frame_0_d3.loss_cls: 1.3436, frame_0_d3.loss_bbox: 1.9717, frame_0_d4.loss_cls: 1.3177, frame_0_d4.loss_bbox: 1.9612, frame_0_dn_loss_cls: 1.1040, frame_0_dn_loss_bbox: 1.6631, frame_0_d0.dn_loss_cls: 1.0870, frame_0_d0.dn_loss_bbox: 1.6617, frame_0_d1.dn_loss_cls: 1.1411, frame_0_d1.dn_loss_bbox: 1.6631, frame_0_d2.dn_loss_cls: 1.0964, frame_0_d2.dn_loss_bbox: 1.6631, frame_0_d3.dn_loss_cls: 1.1348, frame_0_d3.dn_loss_bbox: 1.6649, frame_0_d4.dn_loss_cls: 1.0992, frame_0_d4.dn_loss_bbox: 1.6631, frame_0_enc_loss_cls: 0.7449, frame_0_enc_loss_bbox: 0.2871, frame_0_enc_loss_iou: 0.7546, frame_0_centers2d_losses: 0.1996, frame_0_centerness_losses: 0.6194, loss: 39.0763, grad_norm: nan
2025-06-08 20:02:10,948 - mmdet - INFO - Iter [5500/84384] lr: 7.917e-04, eta: 2 days, 8:40:03, time: 3.616, data_time: 2.130, memory: 9791, frame_0_loss_cls: 1.2725, frame_0_loss_bbox: 2.0824, frame_0_d0.loss_cls: 1.2816, frame_0_d0.loss_bbox: 2.0715, frame_0_d1.loss_cls: 1.2776, frame_0_d1.loss_bbox: 2.0774, frame_0_d2.loss_cls: 1.3059, frame_0_d2.loss_bbox: 2.0739, frame_0_d3.loss_cls: 1.2836, frame_0_d3.loss_bbox: 2.0759, frame_0_d4.loss_cls: 1.2588, frame_0_d4.loss_bbox: 2.0346, frame_0_dn_loss_cls: 0.9835, frame_0_dn_loss_bbox: 1.7460, frame_0_d0.dn_loss_cls: 0.9603, frame_0_d0.dn_loss_bbox: 1.7409, frame_0_d1.dn_loss_cls: 0.9865, frame_0_d1.dn_loss_bbox: 1.7437, frame_0_d2.dn_loss_cls: 0.9640, frame_0_d2.dn_loss_bbox: 1.7460, frame_0_d3.dn_loss_cls: 0.9882, frame_0_d3.dn_loss_bbox: 1.7459, frame_0_d4.dn_loss_cls: 0.9804, frame_0_d4.dn_loss_bbox: 1.7460, frame_0_enc_loss_cls: 0.7139, frame_0_enc_loss_bbox: 0.3013, frame_0_enc_loss_iou: 0.7886, frame_0_centers2d_losses: 0.2037, frame_0_centerness_losses: 0.6338, loss: 39.0681, grad_norm: nan
2025-06-08 20:04:22,599 - mmdet - INFO - Iter [5550/84384] lr: 7.915e-04, eta: 2 days, 8:38:27, time: 2.633, data_time: 1.482, memory: 9791, frame_0_loss_cls: 1.2577, frame_0_loss_bbox: 1.9699, frame_0_d0.loss_cls: 1.2650, frame_0_d0.loss_bbox: 1.9596, frame_0_d1.loss_cls: 1.2725, frame_0_d1.loss_bbox: 1.9700, frame_0_d2.loss_cls: 1.3045, frame_0_d2.loss_bbox: 1.9636, frame_0_d3.loss_cls: 1.2788, frame_0_d3.loss_bbox: 1.9664, frame_0_d4.loss_cls: 1.2471, frame_0_d4.loss_bbox: 1.9460, frame_0_dn_loss_cls: 0.9034, frame_0_dn_loss_bbox: 1.6743, frame_0_d0.dn_loss_cls: 0.8873, frame_0_d0.dn_loss_bbox: 1.6777, frame_0_d1.dn_loss_cls: 0.8939, frame_0_d1.dn_loss_bbox: 1.6786, frame_0_d2.dn_loss_cls: 0.8805, frame_0_d2.dn_loss_bbox: 1.6741, frame_0_d3.dn_loss_cls: 0.8930, frame_0_d3.dn_loss_bbox: 1.6743, frame_0_d4.dn_loss_cls: 0.8991, frame_0_d4.dn_loss_bbox: 1.6742, frame_0_enc_loss_cls: 0.7060, frame_0_enc_loss_bbox: 0.3075, frame_0_enc_loss_iou: 0.7951, frame_0_centers2d_losses: 0.1988, frame_0_centerness_losses: 0.6026, loss: 37.4219, grad_norm: nan
2025-06-08 20:05:15,594 - mmdet - INFO - Iter [5600/84384] lr: 7.913e-04, eta: 2 days, 8:18:23, time: 1.060, data_time: 0.050, memory: 9791, frame_0_loss_cls: 0.2971, frame_0_loss_bbox: 0.4320, frame_0_d0.loss_cls: 0.2941, frame_0_d0.loss_bbox: 0.4303, frame_0_d1.loss_cls: 0.3021, frame_0_d1.loss_bbox: 0.4327, frame_0_d2.loss_cls: 0.3138, frame_0_d2.loss_bbox: 0.4313, frame_0_d3.loss_cls: 0.3088, frame_0_d3.loss_bbox: 0.4321, frame_0_d4.loss_cls: 0.2920, frame_0_d4.loss_bbox: 0.4230, frame_0_dn_loss_cls: 0.1897, frame_0_dn_loss_bbox: 0.3788, frame_0_d0.dn_loss_cls: 0.1871, frame_0_d0.dn_loss_bbox: 0.3778, frame_0_d1.dn_loss_cls: 0.1871, frame_0_d1.dn_loss_bbox: 0.3787, frame_0_d2.dn_loss_cls: 0.1851, frame_0_d2.dn_loss_bbox: 0.3788, frame_0_d3.dn_loss_cls: 0.1861, frame_0_d3.dn_loss_bbox: 0.3787, frame_0_d4.dn_loss_cls: 0.1889, frame_0_d4.dn_loss_bbox: 0.3787, frame_0_enc_loss_cls: nan, frame_0_enc_loss_bbox: nan, frame_0_enc_loss_iou: nan, frame_0_centers2d_losses: nan, frame_0_centerness_losses: nan, loss: nan, grad_norm: nan
2025-06-08 20:06:08,551 - mmdet - INFO - Iter [5650/84384] lr: 7.912e-04, eta: 2 days, 7:58:40, time: 1.059, data_time: 0.049, memory: 9791, frame_0_loss_cls: 0.0000, frame_0_loss_bbox: 0.0000, frame_0_d0.loss_cls: 0.0000, frame_0_d0.loss_bbox: 0.0000, frame_0_d1.loss_cls: 0.0000, frame_0_d1.loss_bbox: 0.0000, frame_0_d2.loss_cls: 0.0000, frame_0_d2.loss_bbox: 0.0000, frame_0_d3.loss_cls: 0.0000, frame_0_d3.loss_bbox: 0.0000, frame_0_d4.loss_cls: 0.0000, frame_0_d4.loss_bbox: 0.0000, frame_0_dn_loss_cls: 0.0000, frame_0_dn_loss_bbox: 0.0000, frame_0_d0.dn_loss_cls: 0.0000, frame_0_d0.dn_loss_bbox: 0.0000, frame_0_d1.dn_loss_cls: 0.0000, frame_0_d1.dn_loss_bbox: 0.0000, frame_0_d2.dn_loss_cls: 0.0000, frame_0_d2.dn_loss_bbox: 0.0000, frame_0_d3.dn_loss_cls: 0.0000, frame_0_d3.dn_loss_bbox: 0.0000, frame_0_d4.dn_loss_cls: 0.0000, frame_0_d4.dn_loss_bbox: 0.0000, frame_0_enc_loss_cls: nan, frame_0_enc_loss_bbox: nan, frame_0_enc_loss_iou: nan, frame_0_centers2d_losses: nan, frame_0_centerness_losses: nan, loss: nan, grad_norm: nan
apply the vit-large,here is the code:
LearningRateDecayOptimizerConstructor code ：
class LearningRateDecayOptimizerConstructor(DefaultOptimizerConstructor): def add_params(self, params, module, prefix='', is_dcn_module=None): """Add all parameters of module to the params list. The parameters of the given module will be added to the list of param groups, with specific rules defined by paramwise_cfg. Args: params (list[dict]): A list of param groups, it will be modified in place. module (nn.Module): The module to be added. prefix (str): The prefix of the module is_dcn_module (int|float|None): If the current module is a submodule of DCN,is_dcn_module` will be passed tocontrol conv_offset layer's learning rate. Defaults to None.
"""
parameter_groups = {}
print(self.paramwise_cfg)
num_layers = self.paramwise_cfg.get('num_layers') + 2
decay_rate = self.paramwise_cfg.get('decay_rate')
decay_type = self.paramwise_cfg.get('decay_type', "layer_wise")
print("Build LearningRateDecayOptimizerConstructor %s %f - %d" % (decay_type, decay_rate, num_layers))
weight_decay = self.base_wd
for name, param in module.named_parameters():
if not param.requires_grad:
continue # frozen weights
if len(param.shape) == 1 or name.endswith(".bias") or name in ('pos_embed', 'cls_token'):
group_name = "no_decay"
this_weight_decay = 0.
else:
group_name = "decay"
this_weight_decay = weight_decay
train parameters:
`base = [
'../../../mmdetection3d/configs/base/datasets/nus-3d.py',
'../../../mmdetection3d/configs/base/default_runtime.py'
]
backbone_norm_cfg = dict(type='LN', requires_grad=True)
plugin=True
plugin_dir='projects/mmdet3d_plugin/'
If point cloud range is changed, the models should also change their point
cloud range accordingly
point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
voxel_size = [0.2, 0.2, 8]
img_norm_cfg = dict(
mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395], to_rgb=False) # fix img_norm
For nuScenes we usually do 10-class detection
class_names = [
'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier',
'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone'
]
num_gpus = 4
batch_size = 2
num_iters_per_epoch = 28130 // (num_gpus * batch_size)
num_epochs = 24
queue_length = 1
num_frame_losses = 1
collect_keys=['lidar2img', 'intrinsics', 'extrinsics','timestamp', 'img_timestamp', 'ego_pose', 'ego_pose_inv']
input_modality = dict(
use_lidar=False,
use_camera=True,
use_radar=False,
use_map=False,
use_external=True)
sim_fpn=dict(
scale_factors=[4, 2, 1, 0.5, 0.25],
in_channels=1024,
out_channels=256,
out_indices=[2, 3, 4, 5, 6],
)
model = dict(
type='Petr3D',
num_frame_head_grads=num_frame_losses,
num_frame_backbone_grads=num_frame_losses,
num_frame_losses=num_frame_losses,
use_grid_mask=True,
img_backbone=dict(
type='ViT',
img_size=320,
patch_size=16,
window_size=16,
global_window_size=20,
in_chans=3,
embed_dim=1024,
depth=24,
num_heads=16,
mlp_ratio=4*2/3,
window_block_indexes = (
list(range(0, 2)) + list(range(3, 5)) + list(range(6, 8)) + list(range(9, 11)) + list(range(12, 14)) + list(range(15, 17)) + list(range(18, 20)) + list(range(21, 23))
),
sim_fpn=None,
# sim_fpn=sim_fpn, #Only for RepDETR3D
qkv_bias=True,
drop_path_rate=0.3,
# use_act_checkpoint=False,
# xattn=False,
use_act_checkpoint=True,
flash_attn=True,
),
# img_neck=dict(
# type='CPFPN', ###remove unused parameters
# in_channels=[1024],
# out_channels=256,
# num_outs=1),
img_roi_head=dict(
type='FocalHead',
num_classes=10,
in_channels=256,
loss_cls2d=dict(
type='QualityFocalLoss',
use_sigmoid=True,
beta=2.0,
loss_weight=2.0),
loss_centerness=dict(type='GaussianFocalLoss', reduction='mean', loss_weight=1.0),
loss_bbox2d=dict(type='L1Loss', loss_weight=5.0),
loss_iou2d=dict(type='GIoULoss', loss_weight=2.0),
loss_centers2d=dict(type='L1Loss', loss_weight=10.0),
train_cfg=dict(
assigner2d=dict(
type='HungarianAssigner2D',
cls_cost=dict(type='FocalLossCost', weight=2.),
reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'),
iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0),
centers2d_cost=dict(type='BBox3DL1Cost', weight=10.0)))
),
pts_bbox_head=dict(
type='StreamPETRHead',
num_classes=10,
in_channels=256,
num_query=644,
memory_len=1024,
topk_proposals=256,
num_propagated=256,
with_ego_pos=True,
match_with_velo=False,
scalar=10, ##noise groups
noise_scale = 1.0,
dn_weight= 1.0, ##dn loss weight
split = 0.75, ###positive rate
LID=True,
with_position=True,
position_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
code_weights = [2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
transformer=dict(
type='PETRTemporalTransformer',
decoder=dict(
type='PETRTransformerDecoder',
return_intermediate=True,
num_layers=6,
transformerlayers=dict(
type='PETRTemporalDecoderLayer',
attn_cfgs=[
dict(
type='MultiheadAttention',
embed_dims=256,
num_heads=8,
dropout=0.1),
dict(
type='PETRMultiheadFlashAttention',
embed_dims=256,
num_heads=8,
dropout=0.1),
],
feedforward_channels=2048,
ffn_dropout=0.1,
with_cp=True, ###use checkpoint to save memory
operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
'ffn', 'norm')),
)),
bbox_coder=dict(
type='NMSFreeCoder',
post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
pc_range=point_cloud_range,
max_num=300,
voxel_size=voxel_size,
num_classes=10),
loss_cls=dict(
type='FocalLoss',
use_sigmoid=True,
gamma=2.0,
alpha=0.25,
loss_weight=2.0),
loss_bbox=dict(type='L1Loss', loss_weight=0.25),
loss_iou=dict(type='GIoULoss', loss_weight=0.0),),
# model training and testing settings
train_cfg=dict(pts=dict(
grid_size=[512, 512, 1],
voxel_size=voxel_size,
point_cloud_range=point_cloud_range,
out_size_factor=4,
assigner=dict(
type='HungarianAssigner3D',
cls_cost=dict(type='FocalLossCost', weight=2.0),
reg_cost=dict(type='BBox3DL1Cost', weight=0.25),
iou_cost=dict(type='IoUCost', weight=0.0), # Fake cost. This is just to make it compatible with DETR head.
pc_range=point_cloud_range),)))
dataset_type = 'CustomNuScenesDataset'
data_root = '/resource-mounts/algorithms/algorithm-0/data/nuscenes/'
file_client_args = dict(backend='disk')
ida_aug_conf = {
"resize_lim": (0.47, 0.625),
"final_dim": (320, 800),
"bot_pct_lim": (0.0, 0.0),
"rot_lim": (0.0, 0.0),
"H": 900,
"W": 1600,
"rand_flip": True,
}
train_pipeline = [
dict(type='LoadMultiViewImageFromFiles', to_float32=True),
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_bbox=True,
with_label=True, with_bbox_depth=True),
dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
dict(type='ObjectNameFilter', classes=class_names),
dict(type='ResizeCropFlipRotImage', data_aug_conf = ida_aug_conf, training=True),
dict(type='GlobalRotScaleTransImage',
rot_range=[-0.3925, 0.3925],
translation_std=[0, 0, 0],
scale_ratio_range=[0.95, 1.05],
reverse_angle=True,
training=True,
),
dict(type='NormalizeMultiviewImage', **img_norm_cfg),
dict(type='PadMultiViewImage', size_divisor=32),
dict(type='PETRFormatBundle3D', class_names=class_names, collect_keys=collect_keys + ['prev_exists']),
dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img', 'gt_bboxes', 'gt_labels', 'centers2d', 'depths', 'prev_exists'] + collect_keys,
meta_keys=('filename', 'ori_shape', 'img_shape', 'pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token', 'gt_bboxes_3d','gt_labels_3d'))
]
test_pipeline = [
dict(type='LoadMultiViewImageFromFiles', to_float32=True),
dict(type='ResizeCropFlipRotImage', data_aug_conf = ida_aug_conf, training=False),
dict(type='NormalizeMultiviewImage', **img_norm_cfg),
dict(type='PadMultiViewImage', size_divisor=32),
dict(
type='MultiScaleFlipAug3D',
img_scale=(1333, 800),
pts_scale_ratio=1,
flip=False,
transforms=[
dict(
type='PETRFormatBundle3D',
collect_keys=collect_keys,
class_names=class_names,
with_label=False),
dict(type='Collect3D', keys=['img'] + collect_keys,
meta_keys=('filename', 'ori_shape', 'img_shape','pad_shape', 'scale_factor', 'flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'scene_token'))
])
]
data = dict(
samples_per_gpu=batch_size,
workers_per_gpu=4,
train=dict(
type=dataset_type,
data_root=data_root,
ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_train.pkl',
num_frame_losses=num_frame_losses,
seq_split_num=2, # streaming video training
seq_mode=True, # streaming video training
pipeline=train_pipeline,
classes=class_names,
modality=input_modality,
collect_keys=collect_keys + ['img', 'prev_exists', 'img_metas'],
queue_length=queue_length,
test_mode=False,
use_valid_flag=True,
filter_empty_gt=False,
box_type_3d='LiDAR'),
val=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_val.pkl', classes=class_names, modality=input_modality),
test=dict(type=dataset_type, pipeline=test_pipeline, collect_keys=collect_keys + ['img', 'img_metas'], queue_length=queue_length, ann_file=data_root + 'lhq-streampetr-nuscenes_temporal_infos_test.pkl', classes=class_names, modality=input_modality),
shuffler_sampler=dict(type='InfiniteGroupEachSampleInBatchSampler'),
nonshuffler_sampler=dict(type='DistributedSampler')
)
optimizer = dict(constructor='LearningRateDecayOptimizerConstructor',
type='AdamW',
lr=2e-4, betas=(0.9, 0.999), weight_decay=1e-7,
paramwise_cfg={'decay_rate': 0.9,
'head_decay_rate': 4.0,
'decay_type': 'vit_wise',
'num_layers': 24,
})
optimizer_config = dict(type='Fp16OptimizerHook', loss_scale='dynamic', grad_clip=dict(max_norm=35, norm_type=2))
learning policy
lr_config = dict(
policy='CosineAnnealing',
warmup='linear',
warmup_iters=500,
warmup_ratio=1.0 / 3,
min_lr_ratio=1e-3,
)
evaluation = dict(interval=num_iters_per_epoch*num_epochs, pipeline=test_pipeline)
find_unused_parameters=False #### when use checkpoint, find_unused_parameters must be False
checkpoint_config = dict(interval=num_iters_per_epoch, max_keep_ckpts=3)
runner = dict(
type='IterBasedRunner', max_iters=num_epochs * num_iters_per_epoch)
load_from='/resource-mounts/algorithms/algorithm-0/ckpts/eva02_L_coco_det_sys_o365_remapped.pth'
resume_from=None
`
why has the error:
2025-06-08 16:05:05,985 - mmdet - INFO - workflow: [('train', 1)], max: 84384 iters
2025-06-08 16:05:05,987 - mmdet - INFO - Checkpoints will be saved to /resource-mounts/algorithms/algorithm-0/result_streampetr_nuscenes_real_0608 by HardDiskBackend.
2025-06-08 16:06:06,307 - mmdet - INFO - Iter [50/84384] lr: 3.189e-04, eta: 1 day, 3:39:18, time: 1.181, data_time: 0.073, memory: 9532, frame_0_loss_cls: 1.3917, frame_0_loss_bbox: 2.7703, frame_0_d0.loss_cls: 1.4073, frame_0_d0.loss_bbox: 2.7661, frame_0_d1.loss_cls: 1.3967, frame_0_d1.loss_bbox: 2.7791, frame_0_d2.loss_cls: 1.3936, frame_0_d2.loss_bbox: 2.7807, frame_0_d3.loss_cls: 1.3880, frame_0_d3.loss_bbox: 2.7810, frame_0_d4.loss_cls: 1.3791, frame_0_d4.loss_bbox: 2.7712, frame_0_dn_loss_cls: 1.1583, frame_0_dn_loss_bbox: 2.1724, frame_0_d0.dn_loss_cls: 1.1581, frame_0_d0.dn_loss_bbox: 1.9987, frame_0_d1.dn_loss_cls: 1.1510, frame_0_d1.dn_loss_bbox: 2.0393, frame_0_d2.dn_loss_cls: 1.1440, frame_0_d2.dn_loss_bbox: 2.0727, frame_0_d3.dn_loss_cls: 1.1483, frame_0_d3.dn_loss_bbox: 2.1095, frame_0_d4.dn_loss_cls: 1.1396, frame_0_d4.dn_loss_bbox: 2.1265, frame_0_enc_loss_cls: 0.7599, frame_0_enc_loss_bbox: 1.2891, frame_0_enc_loss_iou: 1.1671, frame_0_centers2d_losses: 0.4382, frame_0_centerness_losses: 1.2813, loss: 49.3589, grad_norm: nan
2025-06-08 16:07:00,056 - mmdet - INFO - Iter [100/84384] lr: 3.723e-04, eta: 1 day, 2:24:11, time: 1.075, data_time: 0.042, memory: 9532, frame_0_loss_cls: 1.2279, frame_0_loss_bbox: 2.6748, frame_0_d0.loss_cls: 1.2250, frame_0_d0.loss_bbox: 2.6757, frame_0_d1.loss_cls: 1.2257, frame_0_d1.loss_bbox: 2.6751, frame_0_d2.loss_cls: 1.2280, frame_0_d2.loss_bbox: 2.6749, frame_0_d3.loss_cls: 1.2292, frame_0_d3.loss_bbox: 2.6724, frame_0_d4.loss_cls: 1.2252, frame_0_d4.loss_bbox: 2.6732, frame_0_dn_loss_cls: 0.9124, frame_0_dn_loss_bbox: 1.8359, frame_0_d0.dn_loss_cls: 0.9026, frame_0_d0.dn_loss_bbox: 1.8466, frame_0_d1.dn_loss_cls: 0.9098, frame_0_d1.dn_loss_bbox: 1.8442, frame_0_d2.dn_loss_cls: 0.9097, frame_0_d2.dn_loss_bbox: 1.8388, frame_0_d3.dn_loss_cls: 0.9098, frame_0_d3.dn_loss_bbox: 1.8368, frame_0_d4.dn_loss_cls: 0.9119, frame_0_d4.dn_loss_bbox: 1.8413, frame_0_enc_loss_cls: 0.9047, frame_0_enc_loss_bbox: 0.3444, frame_0_enc_loss_iou: 0.8120, frame_0_centers2d_losses: 0.2805, frame_0_centerness_losses: 0.6943, loss: 42.9430, grad_norm: nan
2025-06-08 16:07:54,253 - mmdet - INFO - Iter [150/84384] lr: 4.256e-04, eta: 1 day, 2:02:44, time: 1.084, data_time: 0.048, memory: 9533, frame_0_loss_cls: 1.3132, frame_0_loss_bbox: 2.3946, frame_0_d0.loss_cls: 1.3036, frame_0_d0.loss_bbox: 2.4048, frame_0_d1.loss_cls: 1.2983, frame_0_d1.loss_bbox: 2.4000, frame_0_d2.loss_cls: 1.3031, frame_0_d2.loss_bbox: 2.4020, frame_0_d3.loss_cls: 1.3000, frame_0_d3.loss_bbox: 2.3978, frame_0_d4.loss_cls: 1.3005, frame_0_d4.loss_bbox: 2.3992, frame_0_dn_loss_cls: 0.9940, frame_0_dn_loss_bbox: 1.7972, frame_0_d0.dn_loss_cls: 0.9645, frame_0_d0.dn_loss_bbox: 1.7962, frame_0_d1.dn_loss_cls: 0.9743, frame_0_d1.dn_loss_bbox: 1.7958, frame_0_d2.dn_loss_cls: 0.9814, frame_0_d2.dn_loss_bbox: 1.7919, frame_0_d3.dn_loss_cls: 0.9831, frame_0_d3.dn_loss_bbox: 1.7961, frame_0_d4.dn_loss_cls: 0.9826, frame_0_d4.dn_loss_bbox: 1.7935, frame_0_enc_loss_cls: 0.8481, frame_0_enc_loss_bbox: 0.3590, frame_0_enc_loss_iou: 0.8317, frame_0_centers2d_losses: 0.2695, frame_0_centerness_losses: 0.7400, loss: 41.9162, grad_norm: 84.5885
2025-06-08 16:08:49,362 - mmdet - INFO - Iter [200/84384] lr: 4.789e-04, eta: 1 day, 1:57:57, time: 1.102, data_time: 0.042, memory: 9533, frame_0_loss_cls: 1.1974, frame_0_loss_bbox: 2.2007, frame_0_d0.loss_cls: 1.2035, frame_0_d0.loss_bbox: 2.2034, frame_0_d1.loss_cls: 1.1967, frame_0_d1.loss_bbox: 2.2052, frame_0_d2.loss_cls: 1.2011, frame_0_d2.loss_bbox: 2.2065, frame_0_d3.loss_cls: 1.1950, frame_0_d3.loss_bbox: 2.2099, frame_0_d4.loss_cls: 1.1942, frame_0_d4.loss_bbox: 2.2078, frame_0_dn_loss_cls: 0.8800, frame_0_dn_loss_bbox: 1.6463, frame_0_d0.dn_loss_cls: 0.8715, frame_0_d0.dn_loss_bbox: 1.6446, frame_0_d1.dn_loss_cls: 0.8787, frame_0_d1.dn_loss_bbox: 1.6472, frame_0_d2.dn_loss_cls: 0.8764, frame_0_d2.dn_loss_bbox: 1.6444, frame_0_d3.dn_loss_cls: 0.8787, frame_0_d3.dn_loss_bbox: 1.6442, frame_0_d4.dn_loss_cls: 0.8792, frame_0_d4.dn_loss_bbox: 1.6361, frame_0_enc_loss_cls: 0.8244, frame_0_enc_loss_bbox: 0.3296, frame_0_enc_loss_iou: 0.8093, frame_0_centers2d_losses: 0.2574, frame_0_centerness_losses: 0.6499, loss: 38.4196, grad_norm: 64.9579
2025-06-08 16:09:43,309 - mmdet - INFO - Iter [250/84384] lr: 5.323e-04, eta: 1 day, 1:48:10, time: 1.079, data_time: 0.044, memory: 9533, frame_0_loss_cls: 1.1885, frame_0_loss_bbox: 2.3395, frame_0_d0.loss_cls: 1.1877, frame_0_d0.loss_bbox: 2.3437, frame_0_d1.loss_cls: 1.1879, frame_0_d1.loss_bbox: 2.3360, frame_0_d2.loss_cls: 1.1927, frame_0_d2.loss_bbox: 2.3382, frame_0_d3.loss_cls: 1.1875, frame_0_d3.loss_bbox: 2.3419, frame_0_d4.loss_cls: 1.1892, frame_0_d4.loss_bbox: 2.3393, frame_0_dn_loss_cls: 0.8602, frame_0_dn_loss_bbox: 1.8234, frame_0_d0.dn_loss_cls: 0.8523, frame_0_d0.dn_loss_bbox: 1.8241, frame_0_d1.dn_loss_cls: 0.8553, frame_0_d1.dn_loss_bbox: 1.8189, frame_0_d2.dn_loss_cls: 0.8576, frame_0_d2.dn_loss_bbox: 1.8189, frame_0_d3.dn_loss_cls: 0.8585, frame_0_d3.dn_loss_bbox: 1.8187, frame_0_d4.dn_loss_cls: 0.8619, frame_0_d4.dn_loss_bbox: 1.8192, frame_0_enc_loss_cls: 0.8457, frame_0_enc_loss_bbox: 0.3578, frame_0_enc_loss_iou: 0.8405, frame_0_centers2d_losses: 0.2807, frame_0_centerness_losses: 0.7160, loss: 40.2816, grad_norm: 70.6966
2025-06-08 16:10:37,481 - mmdet - INFO - Iter [300/84384] lr: 5.856e-04, eta: 1 day, 1:42:26, time: 1.083, data_time: 0.043, memory: 9534, frame_0_loss_cls: 1.1640, frame_0_loss_bbox: 2.1763, frame_0_d0.loss_cls: 1.1642, frame_0_d0.loss_bbox: 2.1815, frame_0_d1.loss_cls: 1.1570, frame_0_d1.loss_bbox: 2.1803, frame_0_d2.loss_cls: 1.1602, frame_0_d2.loss_bbox: 2.1815, frame_0_d3.loss_cls: 1.1654, frame_0_d3.loss_bbox: 2.1793, frame_0_d4.loss_cls: 1.1620, frame_0_d4.loss_bbox: 2.1763, frame_0_dn_loss_cls: 0.8490, frame_0_dn_loss_bbox: 1.6818, frame_0_d0.dn_loss_cls: 0.8506, frame_0_d0.dn_loss_bbox: 1.6895, frame_0_d1.dn_loss_cls: 0.8487, frame_0_d1.dn_loss_bbox: 1.6858, frame_0_d2.dn_loss_cls: 0.8502, frame_0_d2.dn_loss_bbox: 1.6853, frame_0_d3.dn_loss_cls: 0.8472, frame_0_d3.dn_loss_bbox: 1.6800, frame_0_d4.dn_loss_cls: 0.8476, frame_0_d4.dn_loss_bbox: 1.6796, frame_0_enc_loss_cls: 0.8578, frame_0_enc_loss_bbox: 0.3556, frame_0_enc_loss_iou: 0.7865, frame_0_centers2d_losses: 0.2694, frame_0_centerness_losses: 0.6951, loss: 38.2078, grad_norm: 65.6546
2025-06-08 16:11:30,627 - mmdet - INFO - Iter [350/84384] lr: 6.389e-04, eta: 1 day, 1:33:59, time: 1.063, data_time: 0.045, memory: 9534, frame_0_loss_cls: 1.2191, frame_0_loss_bbox: 2.2699, frame_0_d0.loss_cls: 1.2071, frame_0_d0.loss_bbox: 2.2692, frame_0_d1.loss_cls: 1.2099, frame_0_d1.loss_bbox: 2.2685, frame_0_d2.loss_cls: 1.2113, frame_0_d2.loss_bbox: 2.2678, frame_0_d3.loss_cls: 1.2121, frame_0_d3.loss_bbox: 2.2688, frame_0_d4.loss_cls: 1.2093, frame_0_d4.loss_bbox: 2.2704, frame_0_dn_loss_cls: 0.8816, frame_0_dn_loss_bbox: 1.7664, frame_0_d0.dn_loss_cls: 0.8763, frame_0_d0.dn_loss_bbox: 1.7741, frame_0_d1.dn_loss_cls: 0.8806, frame_0_d1.dn_loss_bbox: 1.7702, frame_0_d2.dn_loss_cls: 0.8779, frame_0_d2.dn_loss_bbox: 1.7701, frame_0_d3.dn_loss_cls: 0.8818, frame_0_d3.dn_loss_bbox: 1.7690, frame_0_d4.dn_loss_cls: 0.8804, frame_0_d4.dn_loss_bbox: 1.7649, frame_0_enc_loss_cls: 0.8230, frame_0_enc_loss_bbox: 0.3285, frame_0_enc_loss_iou: 0.8250, frame_0_centers2d_losses: 0.2509, frame_0_centerness_losses: 0.6825, loss: 39.6865, grad_norm: 83.4921
2025-06-08 16:12:23,852 - mmdet - INFO - Iter [400/84384] lr: 6.922e-04, eta: 1 day, 1:27:37, time: 1.064, data_time: 0.048, memory: 9534, frame_0_loss_cls: 1.1457, frame_0_loss_bbox: 2.1336, frame_0_d0.loss_cls: 1.1500, frame_0_d0.loss_bbox: 2.1355, frame_0_d1.loss_cls: 1.1442, frame_0_d1.loss_bbox: 2.1342, frame_0_d2.loss_cls: 1.1442, frame_0_d2.loss_bbox: 2.1347, frame_0_d3.loss_cls: 1.1420, frame_0_d3.loss_bbox: 2.1375, frame_0_d4.loss_cls: 1.1469, frame_0_d4.loss_bbox: 2.1342, frame_0_dn_loss_cls: 0.8406, frame_0_dn_loss_bbox: 1.6028, frame_0_d0.dn_loss_cls: 0.8353, frame_0_d0.dn_loss_bbox: 1.6045, frame_0_d1.dn_loss_cls: 0.8379, frame_0_d1.dn_loss_bbox: 1.6009, frame_0_d2.dn_loss_cls: 0.8367, frame_0_d2.dn_loss_bbox: 1.6015, frame_0_d3.dn_loss_cls: 0.8402, frame_0_d3.dn_loss_bbox: 1.6034, frame_0_d4.dn_loss_cls: 0.8439, frame_0_d4.dn_loss_bbox: 1.6042, frame_0_enc_loss_cls: 0.7494, frame_0_enc_loss_bbox: 0.3268, frame_0_enc_loss_iou: 0.8275, frame_0_centers2d_losses: 0.2506, frame_0_centerness_losses: 0.6536, loss: 37.1425, grad_norm: 57.6195
2025-06-08 19:51:55,065 - mmdet - INFO - Iter [5350/84384] lr: 7.921e-04, eta: 2 days, 7:50:23, time: 4.311, data_time: 2.753, memory: 9791, frame_0_loss_cls: 1.2731, frame_0_loss_bbox: 2.1065, frame_0_d0.loss_cls: 1.2859, frame_0_d0.loss_bbox: 2.0993, frame_0_d1.loss_cls: 1.2785, frame_0_d1.loss_bbox: 2.1010, frame_0_d2.loss_cls: 1.3229, frame_0_d2.loss_bbox: 2.0993, frame_0_d3.loss_cls: 1.2819, frame_0_d3.loss_bbox: 2.1009, frame_0_d4.loss_cls: 1.2549, frame_0_d4.loss_bbox: 2.0821, frame_0_dn_loss_cls: 0.9599, frame_0_dn_loss_bbox: 1.7798, frame_0_d0.dn_loss_cls: 0.9579, frame_0_d0.dn_loss_bbox: 1.7751, frame_0_d1.dn_loss_cls: 0.9643, frame_0_d1.dn_loss_bbox: 1.7799, frame_0_d2.dn_loss_cls: 0.9483, frame_0_d2.dn_loss_bbox: 1.7799, frame_0_d3.dn_loss_cls: 0.9764, frame_0_d3.dn_loss_bbox: 1.7797, frame_0_d4.dn_loss_cls: 0.9548, frame_0_d4.dn_loss_bbox: 1.7799, frame_0_enc_loss_cls: 0.7812, frame_0_enc_loss_bbox: 0.2997, frame_0_enc_loss_iou: 0.7753, frame_0_centers2d_losses: 0.2020, frame_0_centerness_losses: 0.6544, loss: 39.4349, grad_norm: 5466.4919
2025-06-08 19:55:37,156 - mmdet - INFO - Iter [5400/84384] lr: 7.920e-04, eta: 2 days, 8:11:24, time: 4.442, data_time: 3.047, memory: 9791, frame_0_loss_cls: 1.4127, frame_0_loss_bbox: 2.1000, frame_0_d0.loss_cls: 1.4663, frame_0_d0.loss_bbox: 2.0927, frame_0_d1.loss_cls: 1.3890, frame_0_d1.loss_bbox: 2.0965, frame_0_d2.loss_cls: 1.4787, frame_0_d2.loss_bbox: 2.0933, frame_0_d3.loss_cls: 1.4179, frame_0_d3.loss_bbox: 2.0929, frame_0_d4.loss_cls: 1.4175, frame_0_d4.loss_bbox: 2.0674, frame_0_dn_loss_cls: 0.9936, frame_0_dn_loss_bbox: 1.7365, frame_0_d0.dn_loss_cls: 0.9595, frame_0_d0.dn_loss_bbox: 1.7317, frame_0_d1.dn_loss_cls: 1.0076, frame_0_d1.dn_loss_bbox: 1.7365, frame_0_d2.dn_loss_cls: 0.9859, frame_0_d2.dn_loss_bbox: 1.7365, frame_0_d3.dn_loss_cls: 1.0182, frame_0_d3.dn_loss_bbox: 1.7336, frame_0_d4.dn_loss_cls: 0.9837, frame_0_d4.dn_loss_bbox: 1.7365, frame_0_enc_loss_cls: 0.8939, frame_0_enc_loss_bbox: 0.3292, frame_0_enc_loss_iou: 0.7902, frame_0_centers2d_losses: 0.2234, frame_0_centerness_losses: 0.8001, loss: 40.5213, grad_norm: nan
2025-06-08 19:59:10,158 - mmdet - INFO - Iter [5450/84384] lr: 7.918e-04, eta: 2 days, 8:29:46, time: 4.260, data_time: 2.992, memory: 9791, frame_0_loss_cls: 1.3355, frame_0_loss_bbox: 1.9760, frame_0_d0.loss_cls: 1.3389, frame_0_d0.loss_bbox: 1.9659, frame_0_d1.loss_cls: 1.3302, frame_0_d1.loss_bbox: 1.9703, frame_0_d2.loss_cls: 1.3501, frame_0_d2.loss_bbox: 1.9681, frame_0_d3.loss_cls: 1.3436, frame_0_d3.loss_bbox: 1.9717, frame_0_d4.loss_cls: 1.3177, frame_0_d4.loss_bbox: 1.9612, frame_0_dn_loss_cls: 1.1040, frame_0_dn_loss_bbox: 1.6631, frame_0_d0.dn_loss_cls: 1.0870, frame_0_d0.dn_loss_bbox: 1.6617, frame_0_d1.dn_loss_cls: 1.1411, frame_0_d1.dn_loss_bbox: 1.6631, frame_0_d2.dn_loss_cls: 1.0964, frame_0_d2.dn_loss_bbox: 1.6631, frame_0_d3.dn_loss_cls: 1.1348, frame_0_d3.dn_loss_bbox: 1.6649, frame_0_d4.dn_loss_cls: 1.0992, frame_0_d4.dn_loss_bbox: 1.6631, frame_0_enc_loss_cls: 0.7449, frame_0_enc_loss_bbox: 0.2871, frame_0_enc_loss_iou: 0.7546, frame_0_centers2d_losses: 0.1996, frame_0_centerness_losses: 0.6194, loss: 39.0763, grad_norm: nan
2025-06-08 20:02:10,948 - mmdet - INFO - Iter [5500/84384] lr: 7.917e-04, eta: 2 days, 8:40:03, time: 3.616, data_time: 2.130, memory: 9791, frame_0_loss_cls: 1.2725, frame_0_loss_bbox: 2.0824, frame_0_d0.loss_cls: 1.2816, frame_0_d0.loss_bbox: 2.0715, frame_0_d1.loss_cls: 1.2776, frame_0_d1.loss_bbox: 2.0774, frame_0_d2.loss_cls: 1.3059, frame_0_d2.loss_bbox: 2.0739, frame_0_d3.loss_cls: 1.2836, frame_0_d3.loss_bbox: 2.0759, frame_0_d4.loss_cls: 1.2588, frame_0_d4.loss_bbox: 2.0346, frame_0_dn_loss_cls: 0.9835, frame_0_dn_loss_bbox: 1.7460, frame_0_d0.dn_loss_cls: 0.9603, frame_0_d0.dn_loss_bbox: 1.7409, frame_0_d1.dn_loss_cls: 0.9865, frame_0_d1.dn_loss_bbox: 1.7437, frame_0_d2.dn_loss_cls: 0.9640, frame_0_d2.dn_loss_bbox: 1.7460, frame_0_d3.dn_loss_cls: 0.9882, frame_0_d3.dn_loss_bbox: 1.7459, frame_0_d4.dn_loss_cls: 0.9804, frame_0_d4.dn_loss_bbox: 1.7460, frame_0_enc_loss_cls: 0.7139, frame_0_enc_loss_bbox: 0.3013, frame_0_enc_loss_iou: 0.7886, frame_0_centers2d_losses: 0.2037, frame_0_centerness_losses: 0.6338, loss: 39.0681, grad_norm: nan
2025-06-08 20:04:22,599 - mmdet - INFO - Iter [5550/84384] lr: 7.915e-04, eta: 2 days, 8:38:27, time: 2.633, data_time: 1.482, memory: 9791, frame_0_loss_cls: 1.2577, frame_0_loss_bbox: 1.9699, frame_0_d0.loss_cls: 1.2650, frame_0_d0.loss_bbox: 1.9596, frame_0_d1.loss_cls: 1.2725, frame_0_d1.loss_bbox: 1.9700, frame_0_d2.loss_cls: 1.3045, frame_0_d2.loss_bbox: 1.9636, frame_0_d3.loss_cls: 1.2788, frame_0_d3.loss_bbox: 1.9664, frame_0_d4.loss_cls: 1.2471, frame_0_d4.loss_bbox: 1.9460, frame_0_dn_loss_cls: 0.9034, frame_0_dn_loss_bbox: 1.6743, frame_0_d0.dn_loss_cls: 0.8873, frame_0_d0.dn_loss_bbox: 1.6777, frame_0_d1.dn_loss_cls: 0.8939, frame_0_d1.dn_loss_bbox: 1.6786, frame_0_d2.dn_loss_cls: 0.8805, frame_0_d2.dn_loss_bbox: 1.6741, frame_0_d3.dn_loss_cls: 0.8930, frame_0_d3.dn_loss_bbox: 1.6743, frame_0_d4.dn_loss_cls: 0.8991, frame_0_d4.dn_loss_bbox: 1.6742, frame_0_enc_loss_cls: 0.7060, frame_0_enc_loss_bbox: 0.3075, frame_0_enc_loss_iou: 0.7951, frame_0_centers2d_losses: 0.1988, frame_0_centerness_losses: 0.6026, loss: 37.4219, grad_norm: nan
2025-06-08 20:05:15,594 - mmdet - INFO - Iter [5600/84384] lr: 7.913e-04, eta: 2 days, 8:18:23, time: 1.060, data_time: 0.050, memory: 9791, frame_0_loss_cls: 0.2971, frame_0_loss_bbox: 0.4320, frame_0_d0.loss_cls: 0.2941, frame_0_d0.loss_bbox: 0.4303, frame_0_d1.loss_cls: 0.3021, frame_0_d1.loss_bbox: 0.4327, frame_0_d2.loss_cls: 0.3138, frame_0_d2.loss_bbox: 0.4313, frame_0_d3.loss_cls: 0.3088, frame_0_d3.loss_bbox: 0.4321, frame_0_d4.loss_cls: 0.2920, frame_0_d4.loss_bbox: 0.4230, frame_0_dn_loss_cls: 0.1897, frame_0_dn_loss_bbox: 0.3788, frame_0_d0.dn_loss_cls: 0.1871, frame_0_d0.dn_loss_bbox: 0.3778, frame_0_d1.dn_loss_cls: 0.1871, frame_0_d1.dn_loss_bbox: 0.3787, frame_0_d2.dn_loss_cls: 0.1851, frame_0_d2.dn_loss_bbox: 0.3788, frame_0_d3.dn_loss_cls: 0.1861, frame_0_d3.dn_loss_bbox: 0.3787, frame_0_d4.dn_loss_cls: 0.1889, frame_0_d4.dn_loss_bbox: 0.3787, frame_0_enc_loss_cls: nan, frame_0_enc_loss_bbox: nan, frame_0_enc_loss_iou: nan, frame_0_centers2d_losses: nan, frame_0_centerness_losses: nan, loss: nan, grad_norm: nan
2025-06-08 20:06:08,551 - mmdet - INFO - Iter [5650/84384] lr: 7.912e-04, eta: 2 days, 7:58:40, time: 1.059, data_time: 0.049, memory: 9791, frame_0_loss_cls: 0.0000, frame_0_loss_bbox: 0.0000, frame_0_d0.loss_cls: 0.0000, frame_0_d0.loss_bbox: 0.0000, frame_0_d1.loss_cls: 0.0000, frame_0_d1.loss_bbox: 0.0000, frame_0_d2.loss_cls: 0.0000, frame_0_d2.loss_bbox: 0.0000, frame_0_d3.loss_cls: 0.0000, frame_0_d3.loss_bbox: 0.0000, frame_0_d4.loss_cls: 0.0000, frame_0_d4.loss_bbox: 0.0000, frame_0_dn_loss_cls: 0.0000, frame_0_dn_loss_bbox: 0.0000, frame_0_d0.dn_loss_cls: 0.0000, frame_0_d0.dn_loss_bbox: 0.0000, frame_0_d1.dn_loss_cls: 0.0000, frame_0_d1.dn_loss_bbox: 0.0000, frame_0_d2.dn_loss_cls: 0.0000, frame_0_d2.dn_loss_bbox: 0.0000, frame_0_d3.dn_loss_cls: 0.0000, frame_0_d3.dn_loss_bbox: 0.0000, frame_0_d4.dn_loss_cls: 0.0000, frame_0_d4.dn_loss_bbox: 0.0000, frame_0_enc_loss_cls: nan, frame_0_enc_loss_bbox: nan, frame_0_enc_loss_iou: nan, frame_0_centers2d_losses: nan, frame_0_centerness_losses: nan, loss: nan, grad_norm: nan