class ConvBNLayer(nn.Layer):    def __init__(self, num_channels, num_filters, filter_size, stride=1,                 groups=1, act=None, name=None, data_format="NCHW"                ):        super(ConvBNLayer, self).__init__()        self._conv = Conv2D(            in_channels=num_channels, out_channels=num_filters,            kernel_size=filter_size, stride=stride,            padding=(filter_size - 1) // 2, groups=groups,            weight_attr=ParamAttr(name=name + "_weights"), bias_attr=False,            data_format=data_format        )        if name == "conv1":            bn_name = "bn_" + name        else:            bn_name = "bn" + name[3:]        self._batch_norm = BatchNorm(            num_filters, act=act, param_attr=ParamAttr(name=bn_name + '_scale'),            bias_attr=ParamAttr(bn_name + '_offset'), moving_mean_name=bn_name + '_mean',            moving_variance_name=bn_name + '_variance', data_layout=data_format        )
    def forward(self, inputs):        y = self._conv(inputs)        y = self._batch_norm(y)        return y

class BottleneckBlock(nn.Layer):    def __init__(self, num_channels, num_filters, stride, cardinality, shortcut=True,                 name=None, data_format="NCHW"                ):        super(BottleneckBlock, self).__init__()        self.conv0 = ConvBNLayer(num_channels=num_channels, num_filters=num_filters,            filter_size=1, act='relu', name=name + "_branch2a",            data_format=data_format           )        self.conv1 = ConvBNLayer(            num_channels=num_filters, num_filters=num_filters,            filter_size=3, groups=cardinality,            stride=stride, act='relu', name=name + "_branch2b",            data_format=data_format        )
        self.conv2 = ConvBNLayer(            num_channels=num_filters,            num_filters=num_filters * 2 if cardinality == 32 else num_filters,            filter_size=1, act=None,            name=name + "_branch2c",            data_format=data_format        )
        if not shortcut:            self.short = ConvBNLayer(                num_channels=num_channels, num_filters=num_filters * 2                if cardinality == 32 else num_filters,                filter_size=1, stride=stride,                name=name + "_branch1", data_format=data_format            )
        self.shortcut = shortcut
    def forward(self, inputs):        y = self.conv0(inputs)        conv1 = self.conv1(y)        conv2 = self.conv2(conv1)
        if self.shortcut:            short = inputs        else:            short = self.short(inputs)
        y = paddle.add(x=short, y=conv2)        y = F.relu(y)        return y

class ConvBNLayer(nn.Layer):    def __init__(            self,            num_channels,            num_filters,            filter_size,            stride=1,            groups=1,            is_vd_mode=False,            act=None,            name=None, ):        super(ConvBNLayer, self).__init__()
        self.is_vd_mode = is_vd_mode        self._pool2d_avg = AvgPool2D(            kernel_size=2, stride=2, padding=0, ceil_mode=True)        self._conv = Conv2D(            in_channels=num_channels,            out_channels=num_filters,            kernel_size=filter_size,            stride=stride,            padding=(filter_size - 1) // 2,            groups=groups,            weight_attr=ParamAttr(name=name + "_weights"),            bias_attr=False)        if name == "conv1":            bn_name = "bn_" + name        else:            bn_name = "bn" + name[3:]        self._batch_norm = BatchNorm(            num_filters,            act=act,            param_attr=ParamAttr(name=bn_name + '_scale'),            bias_attr=ParamAttr(bn_name + '_offset'),            moving_mean_name=bn_name + '_mean',            moving_variance_name=bn_name + '_variance')
    def forward(self, inputs):        if self.is_vd_mode:            inputs = self._pool2d_avg(inputs)        y = self._conv(inputs)        y = self._batch_norm(y)        return y

class BottleneckBlock(nn.Layer):    def __init__(self,                 num_channels1,                 num_channels2,                 num_filters,                 stride,                 scales,                 shortcut=True,                 if_first=False,                 name=None):        super(BottleneckBlock, self).__init__()        self.stride = stride        self.scales = scales        self.conv0 = ConvBNLayer(            num_channels=num_channels1,            num_filters=num_filters,            filter_size=1,            act='relu',            name=name + "_branch2a")        self.conv1_list = []        for s in range(scales - 1):            conv1 = self.add_sublayer(                name + '_branch2b_' + str(s + 1),                ConvBNLayer(                    num_channels=num_filters // scales,                    num_filters=num_filters // scales,                    filter_size=3,                    stride=stride,                    act='relu',                    name=name + '_branch2b_' + str(s + 1)))            self.conv1_list.append(conv1)        self.pool2d_avg = AvgPool2D(kernel_size=3, stride=stride, padding=1)
        self.conv2 = ConvBNLayer(            num_channels=num_filters,            num_filters=num_channels2,            filter_size=1,            act=None,            name=name + "_branch2c")
        if not shortcut:            self.short = ConvBNLayer(                num_channels=num_channels1,                num_filters=num_channels2,                filter_size=1,                stride=1,                is_vd_mode=False if if_first else True,                name=name + "_branch1")
        self.shortcut = shortcut
    def forward(self, inputs):        y = self.conv0(inputs)        xs = paddle.split(y, self.scales, 1)        ys = []        for s, conv1 in enumerate(self.conv1_list):            if s == 0 or self.stride == 2:                ys.append(conv1(xs[s]))            else:                ys.append(conv1(xs[s] + ys[-1]))        if self.stride == 1:            ys.append(xs[-1])        else:            ys.append(self.pool2d_avg(xs[-1]))        conv1 = paddle.concat(ys, axis=1)        conv2 = self.conv2(conv1)
        if self.shortcut:            short = inputs        else:            short = self.short(inputs)        y = paddle.add(x=short, y=conv2)        y = F.relu(y)        return y

class PatchEmbed(nn.Layer):    """ Image to Patch Embedding    Args:        img_size (int): Image size.  Default: 224.        patch_size (int): Patch token size. Default: 4.        in_chans (int): Number of input image channels. Default: 3.        embed_dim (int): Number of linear projection output channels. Default: 96.        norm_layer (nn.Layer, optional): Normalization layer. Default: None    """
    def __init__(self,                 img_size=224,                 patch_size=4,                 in_chans=3,                 embed_dim=96,                 norm_layer=None):        super().__init__()        img_size = to_2tuple(img_size)        patch_size = to_2tuple(patch_size)        patches_resolution = [            img_size[0] // patch_size[0], img_size[1] // patch_size[1]        ]        self.img_size = img_size        self.patch_size = patch_size        self.patches_resolution = patches_resolution        self.num_patches = patches_resolution[0] * patches_resolution[1] #patch个数
        self.in_chans = in_chans        self.embed_dim = embed_dim
        self.proj = nn.Conv2D(            in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) #将stride和kernel_size设置为patch_size大小        if norm_layer is not None:            self.norm = norm_layer(embed_dim)        else:            self.norm = None
    def forward(self, x):        B, C, H, W = x.shape                x = self.proj(x) # B, 96, H/4, W4 
        x = x.flatten(2).transpose([0, 2, 1])  # B Ph*Pw 96        if self.norm is not None:            x = self.norm(x)        return x

class PatchMerging(nn.Layer):    r""" Patch Merging Layer.    Args:        input_resolution (tuple[int]): Resolution of input feature.        dim (int): Number of input channels.        norm_layer (nn.Layer, optional): Normalization layer.  Default: nn.LayerNorm    """
    def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm):        super().__init__()        self.input_resolution = input_resolution        self.dim = dim        self.reduction = nn.Linear(4 * dim, 2 * dim, bias_attr=False)        self.norm = norm_layer(4 * dim)        def forward(self, x):        """        x: B, H*W, C        """        H, W = self.input_resolution        B, L, C = x.shape        assert L == H * W, "input feature has wrong size"        assert H % 2 == 0 and W % 2 == 0, "x size ({}*{}) are not even.".format(            H, W)
        x = x.reshape([B, H, W, C])        # 每次降采样是两倍，因此在行方向和列方向上，间隔2选取元素。        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C        # 拼接在一起作为一整个张量，展开。通道维度会变成原先的4倍（因为H,W各缩小2倍）        x = paddle.concat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C        x = x.reshape([B, H * W // 4, 4 * C])  # B H/2*W/2 4*C 
        x = self.norm(x)        # 通过一个全连接层再调整通道维度为原来的两倍        x = self.reduction(x)
        return x

def window_partition(x, window_size):    """    Args:        x: (B, H, W, C)        window_size (int): window size
    Returns:        windows: (num_windows*B, window_size, window_size, C)    """    B, H, W, C = x.shape    x = x.reshape([B, H // window_size, window_size, W // window_size, window_size, C])    windows = x.transpose([0, 1, 3, 2, 4, 5]).reshape([-1, window_size, window_size, C])    return windows

def window_reverse(windows, window_size, H, W):    """    Args:        windows: (num_windows*B, window_size, window_size, C)        window_size (int): Window size        H (int): Height of image        W (int): Width of image
    Returns:        x: (B, H, W, C)    """    B = int(windows.shape[0] / (H * W / window_size / window_size))    x = windows.reshape([B, H // window_size, W // window_size, window_size, window_size, -1])    x = x.transpose([0, 1, 3, 2, 4, 5]).reshape([B, H, W, -1])    return x

class Mlp(nn.Layer):    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):        super().__init__()        out_features = out_features or in_features        hidden_features = hidden_features or in_features        self.fc1 = nn.Linear(in_features, hidden_features)        self.act = act_layer()        self.fc2 = nn.Linear(hidden_features, out_features)        self.drop = nn.Dropout(drop)
    def forward(self, x):        x = self.fc1(x)        x = self.act(x)        x = self.drop(x)        x = self.fc2(x)        x = self.drop(x)        return x

class WindowAttention(nn.Layer):    """ Window based multi-head self attention (W-MSA) module with relative position bias.    It supports both of shifted and non-shifted window.
    Args:        dim (int): Number of input channels.        window_size (tuple[int]): The height and width of the window.        num_heads (int): Number of attention heads.        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0        proj_drop (float, optional): Dropout ratio of output. Default: 0.0    """
    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
        super().__init__()        self.dim = dim        self.window_size = window_size  # Wh, Ww        self.num_heads = num_heads        head_dim = dim // num_heads        self.scale = qk_scale or head_dim ** -0.5
        # define a parameter table of relative position bias        relative_position_bias_table = self.create_parameter(            shape=((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads), default_initializer=nn.initializer.Constant(value=0))  # 2*Wh-1 * 2*Ww-1, nH        self.add_parameter("relative_position_bias_table", relative_position_bias_table)
        # get pair-wise relative position index for each token inside the window        coords_h = paddle.arange(self.window_size[0])        coords_w = paddle.arange(self.window_size[1])        coords = paddle.stack(paddle.meshgrid([coords_h, coords_w]))                   # 2, Wh, Ww        coords_flatten = paddle.flatten(coords, 1)                                     # 2, Wh*Ww        relative_coords = coords_flatten.unsqueeze(-1) - coords_flatten.unsqueeze(1)   # 2, Wh*Ww, Wh*Ww        relative_coords = relative_coords.transpose([1, 2, 0])                         # Wh*Ww, Wh*Ww, 2        relative_coords[:, :, 0] += self.window_size[0] - 1                            # shift to start from 0        relative_coords[:, :, 1] += self.window_size[1] - 1        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1        self.relative_position_index = relative_coords.sum(-1)                         # Wh*Ww, Wh*Ww        self.register_buffer("relative_position_index", self.relative_position_index)
        self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias)        self.attn_drop = nn.Dropout(attn_drop)        self.proj = nn.Linear(dim, dim)        self.proj_drop = nn.Dropout(proj_drop)
        self.softmax = nn.Softmax(axis=-1)
    def forward(self, x, mask=None):        """        Args:            x: input features with shape of (num_windows*B, N, C)            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None        """        B_, N, C = x.shape        qkv = self.qkv(x).reshape([B_, N, 3, self.num_heads, C // self.num_heads]).transpose([2, 0, 3, 1, 4])        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
        q = q * self.scale        attn = q @ swapdim(k ,-2, -1)
        relative_position_bias = paddle.index_select(self.relative_position_bias_table,                                                     self.relative_position_index.reshape((-1,)),axis=0).reshape((self.window_size[0] * self.window_size[1],self.window_size[0] * self.window_size[1], -1))
        relative_position_bias = relative_position_bias.transpose([2, 0, 1])  # nH, Wh*Ww, Wh*Ww        attn = attn + relative_position_bias.unsqueeze(0)
        if mask is not None:            nW = mask.shape[0]            attn = attn.reshape([B_ // nW, nW, self.num_heads, N, N]) + mask.unsqueeze(1).unsqueeze(0)            attn = attn.reshape([-1, self.num_heads, N, N])            attn = self.softmax(attn)        else:            attn = self.softmax(attn)
        attn = self.attn_drop(attn)
        x = swapdim((attn @ v),1, 2).reshape([B_, N, C])        x = self.proj(x)        x = self.proj_drop(x)        return x

#图像分块、Embeddingclass PatchEmbed(nn.Layer):    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):        super().__init__()        # 原始大小为int，转为tuple，即：img_size原始输入224，变换后为[224,224]        img_size = to_2tuple(img_size)        patch_size = to_2tuple(patch_size)        # 图像块的个数        num_patches = (img_size[1] // patch_size[1]) * \            (img_size[0] // patch_size[0])        self.img_size = img_size        self.patch_size = patch_size        self.num_patches = num_patches        # kernel_size=块大小，即每个块输出一个值，类似每个块展平后使用相同的全连接层进行处理        # 输入维度为3，输出维度为块向量长度        # 与原文中：分块、展平、全连接降维保持一致        # 输出为[B, C, H, W]        self.proj = nn.Conv2D(            in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
    def forward(self, x):        B, C, H, W = x.shape        assert H == self.img_size[0] and W == self.img_size[1], \            "Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."        # [B, C, H, W] -> [B, C, H*W] ->[B, H*W, C]        x = self.proj(x).flatten(2).transpose((0, 2, 1))        return x

#Multi-head Attentionclass Attention(nn.Layer):    def __init__(self,                 dim,                 num_heads=8,                 qkv_bias=False,                 qk_scale=None,                 attn_drop=0.,                 proj_drop=0.):        super().__init__()        self.num_heads = num_heads        head_dim = dim // num_heads        self.scale = qk_scale or head_dim**-0.5        # 计算 q,k,v 的转移矩阵        self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias)        self.attn_drop = nn.Dropout(attn_drop)        # 最终的线性层        self.proj = nn.Linear(dim, dim)        self.proj_drop = nn.Dropout(proj_drop)
    def forward(self, x):        N, C = x.shape[1:]        # 线性变换        qkv = self.qkv(x).reshape((-1, N, 3, self.num_heads, C //                                   self.num_heads)).transpose((2, 0, 3, 1, 4))        # 分割 query key value        q, k, v = qkv[0], qkv[1], qkv[2]        # Scaled Dot-Product Attention        # Matmul + Scale        attn = (q.matmul(k.transpose((0, 1, 3, 2)))) * self.scale        # SoftMax        attn = nn.functional.softmax(attn, axis=-1)        attn = self.attn_drop(attn)        # Matmul        x = (attn.matmul(v)).transpose((0, 2, 1, 3)).reshape((-1, N, C))        # 线性变换        x = self.proj(x)        x = self.proj_drop(x)        return x

class Mlp(nn.Layer):    def __init__(self,                 in_features,                 hidden_features=None,                 out_features=None,                 act_layer=nn.GELU,                 drop=0.):        super().__init__()        out_features = out_features or in_features        hidden_features = hidden_features or in_features        self.fc1 = nn.Linear(in_features, hidden_features)        self.act = act_layer()        self.fc2 = nn.Linear(hidden_features, out_features)        self.drop = nn.Dropout(drop)
    def forward(self, x):        # 输入层：线性变换        x = self.fc1(x)        # 应用激活函数        x = self.act(x)        # Dropout        x = self.drop(x)        # 输出层：线性变换        x = self.fc2(x)        # Dropout        x = self.drop(x)        return x

def drop_path(x, drop_prob=0., training=False):    if drop_prob == 0. or not training:        return x    keep_prob = paddle.to_tensor(1 - drop_prob)    shape = (paddle.shape(x)[0], ) + (1, ) * (x.ndim - 1)    random_tensor = keep_prob + paddle.rand(shape, dtype=x.dtype)    random_tensor = paddle.floor(random_tensor)    output = x.divide(keep_prob) * random_tensor    return output
class DropPath(nn.Layer):    def __init__(self, drop_prob=None):        super(DropPath, self).__init__()        self.drop_prob = drop_prob
    def forward(self, x):        return drop_path(x, self.drop_prob, self.training)

class Block(nn.Layer):    def __init__(self,                 dim,                 num_heads,                 mlp_ratio=4.,                 qkv_bias=False,                 qk_scale=None,                 drop=0.,                 attn_drop=0.,                 drop_path=0.,                 act_layer=nn.GELU,                 norm_layer='nn.LayerNorm',                 epsilon=1e-5):        super().__init__()        self.norm1 = eval(norm_layer)(dim, epsilon=epsilon)        # Multi-head Self-attention        self.attn = Attention(            dim,            num_heads=num_heads,            qkv_bias=qkv_bias,            qk_scale=qk_scale,            attn_drop=attn_drop,            proj_drop=drop)        # DropPath        self.drop_path = DropPath(drop_path) if drop_path > 0. else Identity()        self.norm2 = eval(norm_layer)(dim, epsilon=epsilon)        mlp_hidden_dim = int(dim * mlp_ratio)        self.mlp = Mlp(in_features=dim,                       hidden_features=mlp_hidden_dim,                       act_layer=act_layer,                       drop=drop)
    def forward(self, x):        # Multi-head Self-attention， Add， LayerNorm        x = x + self.drop_path(self.attn(self.norm1(x)))        # Feed Forward， Add， LayerNorm        x = x + self.drop_path(self.mlp(self.norm2(x)))        return x

#参数初始化配置trunc_normal_ = nn.initializer.TruncatedNormal(std=.02)zeros_ = nn.initializer.Constant(value=0.)ones_ = nn.initializer.Constant(value=1.)
#将输入 x 由 int 类型转为 tuple 类型def to_2tuple(x):    return tuple([x] * 2)
#定义一个什么操作都不进行的网络层class Identity(nn.Layer):    def __init__(self):        super(Identity, self).__init__()
    def forward(self, input):        return input

class VisionTransformer(nn.Layer):    def __init__(self,                 img_size=224,                 patch_size=16,                 in_chans=3,                 class_dim=1000,                 embed_dim=768,                 depth=12,                 num_heads=12,                 mlp_ratio=4,                 qkv_bias=False,                 qk_scale=None,                 drop_rate=0.,                 attn_drop_rate=0.,                 drop_path_rate=0.,                 norm_layer='nn.LayerNorm',                 epsilon=1e-5,                 **args):        super().__init__()        self.class_dim = class_dim
        self.num_features = self.embed_dim = embed_dim        # 图片分块和降维，块大小为patch_size，最终块向量维度为768        self.patch_embed = PatchEmbed(            img_size=img_size,            patch_size=patch_size,            in_chans=in_chans,            embed_dim=embed_dim)        # 分块数量        num_patches = self.patch_embed.num_patches        # 可学习的位置编码        self.pos_embed = self.create_parameter(            shape=(1, num_patches + 1, embed_dim), default_initializer=zeros_)        self.add_parameter("pos_embed", self.pos_embed)        # 人为追加class token，并使用该向量进行分类预测        self.cls_token = self.create_parameter(            shape=(1, 1, embed_dim), default_initializer=zeros_)        self.add_parameter("cls_token", self.cls_token)        self.pos_drop = nn.Dropout(p=drop_rate)
        dpr = np.linspace(0, drop_path_rate, depth)        # transformer        self.blocks = nn.LayerList([            Block(                dim=embed_dim,                num_heads=num_heads,                mlp_ratio=mlp_ratio,                qkv_bias=qkv_bias,                qk_scale=qk_scale,                drop=drop_rate,                attn_drop=attn_drop_rate,                drop_path=dpr[i],                norm_layer=norm_layer,                epsilon=epsilon) for i in range(depth)        ])
        self.norm = eval(norm_layer)(embed_dim, epsilon=epsilon)
        # Classifier head        self.head = nn.Linear(embed_dim,                              class_dim) if class_dim > 0 else Identity()
        trunc_normal_(self.pos_embed)        trunc_normal_(self.cls_token)        self.apply(self._init_weights)    # 参数初始化    def _init_weights(self, m):        if isinstance(m, nn.Linear):            trunc_normal_(m.weight)            if isinstance(m, nn.Linear) and m.bias is not None:                zeros_(m.bias)        elif isinstance(m, nn.LayerNorm):            zeros_(m.bias)            ones_(m.weight)        def forward_features(self, x):        B = paddle.shape(x)[0]        # 将图片分块，并调整每个块向量的维度        x = self.patch_embed(x)        # 将class token与前面的分块进行拼接        cls_tokens = self.cls_token.expand((B, -1, -1))        x = paddle.concat((cls_tokens, x), axis=1)        # 将编码向量中加入位置编码        x = x + self.pos_embed        x = self.pos_drop(x)        # 堆叠 transformer 结构        for blk in self.blocks:            x = blk(x)        # LayerNorm        x = self.norm(x)        # 提取分类 tokens 的输出        return x[:, 0]
    def forward(self, x):        # 获取图像特征        x = self.forward_features(x)        # 图像分类        x = self.head(x)        return x