Image
│
▼
CNN Backbone
│
▼
Feature Map
│
▼
Flatten + Position Encoding
│
▼
Transformer Encoder
│
▼
Transformer Decoder (object queries)
│
▼
MLP Head
│
├── Class
└── Box

位置编码

class PositionEmbeddingSine(nn.Module):
    """
    This is a more standard version of the position embedding, very similar to the one
    used by the Attention is all you need paper, generalized to work on images.
    """
    def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None):
        super().__init__()
        self.num_pos_feats = num_pos_feats
        self.temperature = temperature
        self.normalize = normalize
        if scale is not None and normalize is False:
            raise ValueError("normalize should be True if scale is passed")
        if scale is None:
            scale = 2 * math.pi
        self.scale = scale

    def forward(self, tensor_list: NestedTensor):
        x = tensor_list.tensors #torch.Size([2, 2048, 27, 32])
        mask = tensor_list.mask #torch.Size([2, 27, 32])
        assert mask is not None
        not_mask = ~mask
        y_embed = not_mask.cumsum(1, dtype=torch.float32) #torch.Size([2, 27, 32])
        x_embed = not_mask.cumsum(2, dtype=torch.float32) #torch.Size([2, 27, 32])
        if self.normalize:
            eps = 1e-6
            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale

        dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) #[128]
        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) #[128]

        pos_x = x_embed[:, :, :, None] / dim_t #torch.Size([2, 27, 32, 128])  # x_embed[:, :, :, None]torch.Size([2, 27, 32, 1]) 
        pos_y = y_embed[:, :, :, None] / dim_t #torch.Size([2, 27, 32, 128])
        
        #pos_x[:, :, :, 0::2] torch.Size([2, 27, 32, 64])  
        #torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4) torch.Size([2, 27, 32, 64, 2])
        pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) #torch.Size([2, 27, 32, 128])      
        pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)#torch.Size([2, 27, 32, 128]) 
        #torch.cat((pos_y, pos_x), dim=3) torch.Size([2, 27, 32, 256])
        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) #pos torch.Size([2, 256, 27, 32])      
        return pos

这里mask是同一个batch图像大小不一样，把同一个batch里面图像统一成一样，小的图pad补充1，原图区域是0。 not_mask = ~mask是原图区域。

ENCODE

#src [h*w, b, 256]   mask[b, h*w]  pos_embed [h*w, b, 256]  
memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) #[672, b, 256]

 def with_pos_embed(self, tensor, pos: Optional[Tensor]):
        return tensor if pos is None else tensor + pos

	self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)

    def forward_post(self,
                     src,
                     src_mask: Optional[Tensor] = None,
                     src_key_padding_mask: Optional[Tensor] = None,
                     pos: Optional[Tensor] = None):
        q = k = self.with_pos_embed(src, pos) #src pos [h*w, b, 256]
        src2 = self.self_attn(q, k, value=src, attn_mask=src_mask,
                              key_padding_mask=src_key_padding_mask)[0] #src_key_padding_mask[b, h*w]
        src = src + self.dropout1(src2)
        src = self.norm1(src)
        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
        src = src + self.dropout2(src2)
        src = self.norm2(src)
        return src #[h*w, b, 256]
        
        
        
class TransformerEncoder(nn.Module):

    def __init__(self, encoder_layer, num_layers, norm=None):
        super().__init__()
        self.layers = _get_clones(encoder_layer, num_layers)
        self.num_layers = num_layers
        self.norm = norm

    def forward(self, src,
                mask: Optional[Tensor] = None,
                src_key_padding_mask: Optional[Tensor] = None,
                pos: Optional[Tensor] = None):
        output = src

        for layer in self.layers: #layer 就是 forward_post
            output = layer(output, src_mask=mask,
                           src_key_padding_mask=src_key_padding_mask, pos=pos)

        if self.norm is not None:
            output = self.norm(output)

        return output   

DECODE

self.query_embed = nn.Embedding(num_queries, hidden_dim) #100, 256
query_embed = self.query_embed.weight  #[100, 256]

query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) #[100, b, 256]     query_embed[100,256]
mask = mask.flatten(1) #[b, h*w]

tgt = torch.zeros_like(query_embed) #[100, b, 256]
memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) #[h*w, b, 256]

#tgt [100, b, 256]  memory[h*w, b, 256] mask[b, h*w]  pos_embed[h*w, b, 256]  query_embed[100, b, 256]
hs = self.decoder(tgt, memory, memory_key_padding_mask=mask,
                  pos=pos_embed, query_pos=query_embed)




def forward_post(self, tgt, memory,
                 tgt_mask: Optional[Tensor] = None,
                 memory_mask: Optional[Tensor] = None,
                 tgt_key_padding_mask: Optional[Tensor] = None,
                 memory_key_padding_mask: Optional[Tensor] = None,
                 pos: Optional[Tensor] = None,
                 query_pos: Optional[Tensor] = None):
    q = k = self.with_pos_embed(tgt, query_pos) #tgt, query_pos [100, b, 256]
    tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask,
                          key_padding_mask=tgt_key_padding_mask)[0] #[100, b, 256]
    tgt = tgt + self.dropout1(tgt2)
    tgt = self.norm1(tgt) #[100, b, 256]
    
    #tgt query_pos [100, b, 256]
    #memory  pos [h*w, b, 256]
    #memory_mask None
    #memory_key_padding_mask[b, h*w]
    #self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
    tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt, query_pos),
                               key=self.with_pos_embed(memory, pos),
                               value=memory, attn_mask=memory_mask,
                               key_padding_mask=memory_key_padding_mask)[0] # [100, b, 256]
    tgt = tgt + self.dropout2(tgt2) # [100, b, 256]
    tgt = self.norm2(tgt)
    tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
    tgt = tgt + self.dropout3(tgt2)
    tgt = self.norm3(tgt)
    return tgt

在 decoder layer 里通常是：

# 1. self-attention (query ↔ query)
tgt = self.self_attn(tgt + query_pos)

# 2. cross-attention (query ↔ image)
tgt = self.multihead_attn(tgt + query_pos, memory + pos, memory)

# 3. FFN
tgt = self.ffn(tgt)

out

#[6, b, 100, 256]
hs = self.transformer(self.input_proj(src), mask, self.query_embed.weight, pos[-1])[0] 
#[6, b, 100, 92]
outputs_class = self.class_embed(hs)
#[6, b, 100, 4]
outputs_coord = self.bbox_embed(hs).sigmoid()

LOSS

class HungarianMatcher(nn.Module):
    """This class computes an assignment between the targets and the predictions of the network

    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
    while the others are un-matched (and thus treated as non-objects).
    """

    def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1):
        """Creates the matcher

        Params:
            cost_class: This is the relative weight of the classification error in the matching cost
            cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost
            cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost
        """
        super().__init__()
        self.cost_class = cost_class
        self.cost_bbox = cost_bbox
        self.cost_giou = cost_giou
        assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0"

    @torch.no_grad()
    def forward(self, outputs, targets):
        """ Performs the matching

        Params:
            outputs: This is a dict that contains at least these entries:
                 "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits
                 "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates

            targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing:
                 "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth
                           objects in the target) containing the class labels
                 "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates
        Returns:
            A list of size batch_size, containing tuples of (index_i, index_j) where:
                - index_i is the indices of the selected predictions (in order)
                - index_j is the indices of the corresponding selected targets (in order)
            For each batch element, it holds:
                len(index_i) = len(index_j) = min(num_queries, num_target_boxes)
        """
        bs, num_queries = outputs["pred_logits"].shape[:2]
        #outputs["pred_logits"] [b, 100, 92]   flatten(0, 1) [100*b, 92]
        #outputs["pred_boxes"] [b, 100, 4] flatten(0, 1)  [100*b, 4]
        # We flatten to compute the cost matrices in a batch
        out_prob = outputs["pred_logits"].flatten(0, 1).softmax(-1)  # [batch_size * num_queries, num_classes]
        out_bbox = outputs["pred_boxes"].flatten(0, 1)  # [batch_size * num_queries, 4]

        # Also concat the target labels and boxes
        tgt_ids = torch.cat([v["labels"] for v in targets]) #[num_objs]
        tgt_bbox = torch.cat([v["boxes"] for v in targets]) #[num_objs, 4]

        # Compute the classification cost. Contrary to the loss, we don't use the NLL,
        # but approximate it in 1 - proba[target class].
        # The 1 is a constant that doesn't change the matching, it can be ommitted.
        cost_class = -out_prob[:, tgt_ids] #[100*b, num_objs]

        # Compute the L1 cost between boxes
        cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1)    #[100*b, num_objs]

        # #[100*b, num_objs] Compute the giou cost betwen boxes
        cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox))

        # Final cost matrix
        C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou
        C = C.view(bs, num_queries, -1).cpu() #[b, 100, num_objs]

        sizes = [len(v["boxes"]) for v in targets] #[2, 1,...] b个
        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] #[[[49, 62], [1, 0]], [[35], [0]]]
        #这里就代表batch0的有2个目标，用id49去匹配gt1， 用id62去匹配gt0
        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]

然后就拿匹配到的idx去计算loss

 def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs):
        loss_map = {
            'labels': self.loss_labels,
            'cardinality': self.loss_cardinality,
            'boxes': self.loss_boxes,
            'masks': self.loss_masks
        }
        assert loss in loss_map, f'do you really want to compute {loss} loss?'
        return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs)
       
   self.get_loss(loss, outputs, targets, indices, num_boxes)
   
    def loss_labels(self, outputs, targets, indices, num_boxes, log=True):
        """Classification loss (NLL)
        targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes]
        """
        assert 'pred_logits' in outputs
        src_logits = outputs['pred_logits']

        idx = self._get_src_permutation_idx(indices)
        target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)])
        target_classes = torch.full(src_logits.shape[:2], self.num_classes,
                                    dtype=torch.int64, device=src_logits.device)
        target_classes[idx] = target_classes_o

        loss_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.empty_weight)
        losses = {'loss_ce': loss_ce}

        if log:
            # TODO this should probably be a separate loss, not hacked in this one here
            losses['class_error'] = 100 - accuracy(src_logits[idx], target_classes_o)[0]
        return losses   

问题1：detr为什么不能用很大的分辨率？

detr用的下采样32倍去计算，计算量与大小成平方关系

问题2：transform如何每个元素与其他元素都有联系的