London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import segmentation_models_pytorch as smp
class UNet_3Plus_DeepSup_CGM(nn.Module):
def __init__(self, in_channels=3, n_classes=1, encoder_name='timm-efficientnet-b3', encoder_weights='noisy-student', align_corners=True):
super(UNet_3Plus_DeepSup_CGM, self).__init__()
self.in_channels = in_channels
self.n_classes = n_classes
self.encoder_name = encoder_name
self.encoder_weights = encoder_weights
## -------------Encoder--------------
self.encoder = smp.Unet(
encoder_name = self.encoder_name,
encoder_weights = self.encoder_weights,
classes = n_classes).encoder
filters = self.encoder.out_channels[1:]
## -------------Decoder--------------
self.CatChannels = filters[0]
self.CatBlocks = 5
self.UpChannels = self.CatChannels * self.CatBlocks
'''stage 4d'''
# h1->(H, W), hd4->(H/8, W/8), Pooling 8 times
self.h1_PT_hd4 = nn.MaxPool2d(8, 8, ceil_mode=True)
self.h1_PT_hd4_conv = nn.Conv2d(filters[0], self.CatChannels, 3, padding=1)
self.h1_PT_hd4_bn = nn.BatchNorm2d(self.CatChannels)
self.h1_PT_hd4_relu = nn.ReLU(inplace=True)
# h2->(H/2, W/2), hd4->(H/8, W/8), Pooling 4 times
self.h2_PT_hd4 = nn.MaxPool2d(4, 4, ceil_mode=True)
self.h2_PT_hd4_conv = nn.Conv2d(filters[1], self.CatChannels, 3, padding=1)
self.h2_PT_hd4_bn = nn.BatchNorm2d(self.CatChannels)
self.h2_PT_hd4_relu = nn.ReLU(inplace=True)
# h3->(H/4, W/4), hd4->(H/8, W/8), Pooling 2 times
self.h3_PT_hd4 = nn.MaxPool2d(2, 2, ceil_mode=True)
self.h3_PT_hd4_conv = nn.Conv2d(filters[2], self.CatChannels, 3, padding=1)
self.h3_PT_hd4_bn = nn.BatchNorm2d(self.CatChannels)
self.h3_PT_hd4_relu = nn.ReLU(inplace=True)
# h4->(H/8, W/8), hd4->(H/8, W/8), Concatenation
self.h4_Cat_hd4_conv = nn.Conv2d(filters[3], self.CatChannels, 3, padding=1)
self.h4_Cat_hd4_bn = nn.BatchNorm2d(self.CatChannels)
self.h4_Cat_hd4_relu = nn.ReLU(inplace=True)
# hd5->(H/16, W/16), hd4->(H/8, W/8), Upsample 2 times
self.hd5_UT_hd4 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=align_corners)
self.hd5_UT_hd4_conv = nn.Conv2d(filters[4], self.CatChannels, 3, padding=1)
self.hd5_UT_hd4_bn = nn.BatchNorm2d(self.CatChannels)
self.hd5_UT_hd4_relu = nn.ReLU(inplace=True)
# fusion(h1_PT_hd4, h2_PT_hd4, h3_PT_hd4, h4_Cat_hd4, hd5_UT_hd4)
self.conv4d_1 = nn.Conv2d(self.UpChannels, self.UpChannels, 3, padding=1)
self.bn4d_1 = nn.BatchNorm2d(self.UpChannels)
self.relu4d_1 = nn.ReLU(inplace=True)
'''stage 3d'''
# h1->(H, W), hd3->(H/4, W/4), Pooling 4 times
self.h1_PT_hd3 = nn.MaxPool2d(4, 4, ceil_mode=True)
self.h1_PT_hd3_conv = nn.Conv2d(filters[0], self.CatChannels, 3, padding=1)
self.h1_PT_hd3_bn = nn.BatchNorm2d(self.CatChannels)
self.h1_PT_hd3_relu = nn.ReLU(inplace=True)
# h2->(H/2, W/2), hd3->(H/4, W/4), Pooling 2 times
self.h2_PT_hd3 = nn.MaxPool2d(2, 2, ceil_mode=True)
self.h2_PT_hd3_conv = nn.Conv2d(filters[1], self.CatChannels, 3, padding=1)
self.h2_PT_hd3_bn = nn.BatchNorm2d(self.CatChannels)
self.h2_PT_hd3_relu = nn.ReLU(inplace=True)
# h3->(H/4, W/4), hd3->(H/4, W/4), Concatenation
self.h3_Cat_hd3_conv = nn.Conv2d(filters[2], self.CatChannels, 3, padding=1)
self.h3_Cat_hd3_bn = nn.BatchNorm2d(self.CatChannels)
self.h3_Cat_hd3_relu = nn.ReLU(inplace=True)
# hd4->(H/8, W/8), hd4->(H/4, W/4), Upsample 2 times
self.hd4_UT_hd3 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=align_corners)
self.hd4_UT_hd3_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd4_UT_hd3_bn = nn.BatchNorm2d(self.CatChannels)
self.hd4_UT_hd3_relu = nn.ReLU(inplace=True)
# hd5->(H/16, W/16), hd4->(H/4, W/4), Upsample 4 times
self.hd5_UT_hd3 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=align_corners)
self.hd5_UT_hd3_conv = nn.Conv2d(filters[4], self.CatChannels, 3, padding=1)
self.hd5_UT_hd3_bn = nn.BatchNorm2d(self.CatChannels)
self.hd5_UT_hd3_relu = nn.ReLU(inplace=True)
# fusion(h1_PT_hd3, h2_PT_hd3, h3_Cat_hd3, hd4_UT_hd3, hd5_UT_hd3)
self.conv3d_1 = nn.Conv2d(self.UpChannels, self.UpChannels, 3, padding=1)
self.bn3d_1 = nn.BatchNorm2d(self.UpChannels)
self.relu3d_1 = nn.ReLU(inplace=True)
'''stage 2d '''
# h1->(H, W), hd2->(H/2, W/2), Pooling 2 times
self.h1_PT_hd2 = nn.MaxPool2d(2, 2, ceil_mode=True)
self.h1_PT_hd2_conv = nn.Conv2d(filters[0], self.CatChannels, 3, padding=1)
self.h1_PT_hd2_bn = nn.BatchNorm2d(self.CatChannels)
self.h1_PT_hd2_relu = nn.ReLU(inplace=True)
# h2->(H/2, W/2), hd2->(H/2, W/2), Concatenation
self.h2_Cat_hd2_conv = nn.Conv2d(filters[1], self.CatChannels, 3, padding=1)
self.h2_Cat_hd2_bn = nn.BatchNorm2d(self.CatChannels)
self.h2_Cat_hd2_relu = nn.ReLU(inplace=True)
# hd3->(H/4, W/4), hd2->(H/2, W/2), Upsample 2 times
self.hd3_UT_hd2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=align_corners)
self.hd3_UT_hd2_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd3_UT_hd2_bn = nn.BatchNorm2d(self.CatChannels)
self.hd3_UT_hd2_relu = nn.ReLU(inplace=True)
# hd4->(H/8, W/8), hd2->(H/2, W/2), Upsample 4 times
self.hd4_UT_hd2 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=align_corners)
self.hd4_UT_hd2_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd4_UT_hd2_bn = nn.BatchNorm2d(self.CatChannels)
self.hd4_UT_hd2_relu = nn.ReLU(inplace=True)
# hd5->(H/16, W/16), hd2->(H/2, W/2), Upsample 8 times
self.hd5_UT_hd2 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=align_corners)
self.hd5_UT_hd2_conv = nn.Conv2d(filters[4], self.CatChannels, 3, padding=1)
self.hd5_UT_hd2_bn = nn.BatchNorm2d(self.CatChannels)
self.hd5_UT_hd2_relu = nn.ReLU(inplace=True)
# fusion(h1_PT_hd2, h2_Cat_hd2, hd3_UT_hd2, hd4_UT_hd2, hd5_UT_hd2)
self.conv2d_1 = nn.Conv2d(self.UpChannels, self.UpChannels, 3, padding=1)
self.bn2d_1 = nn.BatchNorm2d(self.UpChannels)
self.relu2d_1 = nn.ReLU(inplace=True)
'''stage 1d'''
# h1->(H, W), hd1->(H, W), Concatenation
self.h1_Cat_hd1_conv = nn.Conv2d(filters[0], self.CatChannels, 3, padding=1)
self.h1_Cat_hd1_bn = nn.BatchNorm2d(self.CatChannels)
self.h1_Cat_hd1_relu = nn.ReLU(inplace=True)
# hd2->(H/2, W/2), hd1->(H, W), Upsample 2 times
self.hd2_UT_hd1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=align_corners)
self.hd2_UT_hd1_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd2_UT_hd1_bn = nn.BatchNorm2d(self.CatChannels)
self.hd2_UT_hd1_relu = nn.ReLU(inplace=True)
# hd3->(H/4, W/4), hd1->(H, W), Upsample 4 times
self.hd3_UT_hd1 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=align_corners)
self.hd3_UT_hd1_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd3_UT_hd1_bn = nn.BatchNorm2d(self.CatChannels)
self.hd3_UT_hd1_relu = nn.ReLU(inplace=True)
# hd4->(H/8, W/8), hd1->(H, W), Upsample 8 times
self.hd4_UT_hd1 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=align_corners)
self.hd4_UT_hd1_conv = nn.Conv2d(self.UpChannels, self.CatChannels, 3, padding=1)
self.hd4_UT_hd1_bn = nn.BatchNorm2d(self.CatChannels)
self.hd4_UT_hd1_relu = nn.ReLU(inplace=True)
# hd5->(H/16, W/16), hd1->(H, W), Upsample 16 times
self.hd5_UT_hd1 = nn.Upsample(scale_factor=16, mode='bilinear', align_corners=align_corners)
self.hd5_UT_hd1_conv = nn.Conv2d(filters[4], self.CatChannels, 3, padding=1)
self.hd5_UT_hd1_bn = nn.BatchNorm2d(self.CatChannels)
self.hd5_UT_hd1_relu = nn.ReLU(inplace=True)
# fusion(h1_Cat_hd1, hd2_UT_hd1, hd3_UT_hd1, hd4_UT_hd1, hd5_UT_hd1)
self.up_1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=align_corners)
self.conv1d_1 = nn.Conv2d(self.UpChannels, self.UpChannels, 3, padding=1)
self.bn1d_1 = nn.BatchNorm2d(self.UpChannels)
self.relu1d_1 = nn.ReLU(inplace=True)
# -------------Bilinear Upsampling--------------
# self.upscore6 = nn.Upsample(scale_factor=64,mode='bilinear', align_corners=align_corners)
self.upscore5 = nn.Upsample(scale_factor=32,mode='bilinear', align_corners=align_corners)
self.upscore4 = nn.Upsample(scale_factor=16,mode='bilinear', align_corners=align_corners)
self.upscore3 = nn.Upsample(scale_factor=8,mode='bilinear', align_corners=align_corners)
self.upscore2 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=align_corners)
# DeepSup
self.outconv1 = nn.Conv2d(self.UpChannels, n_classes, 3, padding=1)
self.outconv2 = nn.Conv2d(self.UpChannels, n_classes, 3, padding=1)
self.outconv3 = nn.Conv2d(self.UpChannels, n_classes, 3, padding=1)
self.outconv4 = nn.Conv2d(self.UpChannels, n_classes, 3, padding=1)
self.outconv5 = nn.Conv2d(filters[4], n_classes, 3, padding=1)
self.cls = nn.Sequential(
nn.Dropout(p=0.5),
nn.Conv2d(filters[4], n_classes, 1),
nn.AdaptiveMaxPool2d(1),
nn.Sigmoid())
def dotProduct(self,seg,cls):
B, N, H, W = seg.size()
seg = seg.view(B, N, H * W)
final = torch.einsum("ijk,ij->ijk", [seg, cls])
final = final.view(B, N, H, W)
return final
def forward(self, inputs):
## -------------Encoder-------------
features = self.encoder(inputs)[1:]
h1, h2, h3, h4, hd5 = features[0], features[1], features[2], features[3], features[4]
# -------------Classification-------------
cls_branch = self.cls(hd5).squeeze(3).squeeze(2) # (B,N,1,1)->(B,N) : attention
## -------------Decoder-------------
h1_PT_hd4 = self.h1_PT_hd4_relu(self.h1_PT_hd4_bn(self.h1_PT_hd4_conv(self.h1_PT_hd4(h1))))
h2_PT_hd4 = self.h2_PT_hd4_relu(self.h2_PT_hd4_bn(self.h2_PT_hd4_conv(self.h2_PT_hd4(h2))))
h3_PT_hd4 = self.h3_PT_hd4_relu(self.h3_PT_hd4_bn(self.h3_PT_hd4_conv(self.h3_PT_hd4(h3))))
h4_Cat_hd4 = self.h4_Cat_hd4_relu(self.h4_Cat_hd4_bn(self.h4_Cat_hd4_conv(h4)))
hd5_UT_hd4 = self.hd5_UT_hd4_relu(self.hd5_UT_hd4_bn(self.hd5_UT_hd4_conv(self.hd5_UT_hd4(hd5))))
hd4 = self.relu4d_1(self.bn4d_1(self.conv4d_1(
torch.cat((h1_PT_hd4, h2_PT_hd4, h3_PT_hd4, h4_Cat_hd4, hd5_UT_hd4), 1)))) # hd4 -> (UpChannels, H/8, W/8)
h1_PT_hd3 = self.h1_PT_hd3_relu(self.h1_PT_hd3_bn(self.h1_PT_hd3_conv(self.h1_PT_hd3(h1))))
h2_PT_hd3 = self.h2_PT_hd3_relu(self.h2_PT_hd3_bn(self.h2_PT_hd3_conv(self.h2_PT_hd3(h2))))
h3_Cat_hd3 = self.h3_Cat_hd3_relu(self.h3_Cat_hd3_bn(self.h3_Cat_hd3_conv(h3)))
hd4_UT_hd3 = self.hd4_UT_hd3_relu(self.hd4_UT_hd3_bn(self.hd4_UT_hd3_conv(self.hd4_UT_hd3(hd4))))
hd5_UT_hd3 = self.hd5_UT_hd3_relu(self.hd5_UT_hd3_bn(self.hd5_UT_hd3_conv(self.hd5_UT_hd3(hd5))))
hd3 = self.relu3d_1(self.bn3d_1(self.conv3d_1(
torch.cat((h1_PT_hd3, h2_PT_hd3, h3_Cat_hd3, hd4_UT_hd3, hd5_UT_hd3), 1)))) # hd3 -> (UpChannels, H/4, W/4)
h1_PT_hd2 = self.h1_PT_hd2_relu(self.h1_PT_hd2_bn(self.h1_PT_hd2_conv(self.h1_PT_hd2(h1))))
h2_Cat_hd2 = self.h2_Cat_hd2_relu(self.h2_Cat_hd2_bn(self.h2_Cat_hd2_conv(h2)))
hd3_UT_hd2 = self.hd3_UT_hd2_relu(self.hd3_UT_hd2_bn(self.hd3_UT_hd2_conv(self.hd3_UT_hd2(hd3))))
hd4_UT_hd2 = self.hd4_UT_hd2_relu(self.hd4_UT_hd2_bn(self.hd4_UT_hd2_conv(self.hd4_UT_hd2(hd4))))
hd5_UT_hd2 = self.hd5_UT_hd2_relu(self.hd5_UT_hd2_bn(self.hd5_UT_hd2_conv(self.hd5_UT_hd2(hd5))))
hd2 = self.relu2d_1(self.bn2d_1(self.conv2d_1(
torch.cat((h1_PT_hd2, h2_Cat_hd2, hd3_UT_hd2, hd4_UT_hd2, hd5_UT_hd2), 1)))) # hd2 -> (UpChannels, H/2, W/2)
h1_Cat_hd1 = self.h1_Cat_hd1_relu(self.h1_Cat_hd1_bn(self.h1_Cat_hd1_conv(h1)))
hd2_UT_hd1 = self.hd2_UT_hd1_relu(self.hd2_UT_hd1_bn(self.hd2_UT_hd1_conv(self.hd2_UT_hd1(hd2))))
hd3_UT_hd1 = self.hd3_UT_hd1_relu(self.hd3_UT_hd1_bn(self.hd3_UT_hd1_conv(self.hd3_UT_hd1(hd3))))
hd4_UT_hd1 = self.hd4_UT_hd1_relu(self.hd4_UT_hd1_bn(self.hd4_UT_hd1_conv(self.hd4_UT_hd1(hd4))))
hd5_UT_hd1 = self.hd5_UT_hd1_relu(self.hd5_UT_hd1_bn(self.hd5_UT_hd1_conv(self.hd5_UT_hd1(hd5))))
hd1 = self.relu1d_1(self.bn1d_1(self.conv1d_1(
self.up_1(torch.cat((h1_Cat_hd1, hd2_UT_hd1, hd3_UT_hd1, hd4_UT_hd1, hd5_UT_hd1), 1))))) # hd1 -> (UpChannels, H, W)
d5 = self.outconv5(hd5)
d5 = self.upscore5(d5) # 16 times
d4 = self.outconv4(hd4)
d4 = self.upscore4(d4) # 8 times
d3 = self.outconv3(hd3)
d3 = self.upscore3(d3) # 4 times
d2 = self.outconv2(hd2)
d2 = self.upscore2(d2) # 2 times
d1 = self.outconv1(hd1) # remaining the number of channels
d1 = self.dotProduct(d1, cls_branch)
d2 = self.dotProduct(d2, cls_branch)
d3 = self.dotProduct(d3, cls_branch)
d4 = self.dotProduct(d4, cls_branch)
d5 = self.dotProduct(d5, cls_branch)
return d1, d2, d3, d4, d5, cls_branch
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
class MultiTverskyLoss(nn.Module):
"""
Tversky Loss for segmentation adaptive with multi class segmentation
"""
def __init__(self, alpha=0.3, beta=0.7, gamma=1.5, weights=None):
"""
:param alpha (Tensor, float, optional): controls the penalty for false positives.
:param beta (Tensor, float, optional): controls the penalty for false negative.
:param gamma (Tensor, float, optional): focal coefficient
:param weights (Tensor, optional): a manual rescaling weight given to each
class. If given, it has to be a Tensor of size `C`
"""
super(MultiTverskyLoss, self).__init__()
self.alpha = alpha
self.beta = beta
self.gamma = gamma
self.weights = weights
def __name__(self):
return "MultiTverskyLoss"
def forward(self, inputs, targets):
num_class = inputs.size(1)
weight_losses = 0.0
if self.weights is not None:
assert len(self.weights) == num_class, 'number of classes should be equal to length of weights '
weights = self.weights
else:
weights = [1.0 / num_class] * num_class
input_slices = torch.split(inputs, [1] * num_class, dim=1)
target_slices = torch.split(targets, [1] * num_class, dim=1)
for idx in range(num_class):
input_idx = input_slices[idx]
target_idx = target_slices[idx]
loss_func = FocalBinaryTverskyLoss(self.alpha, self.beta, self.gamma)
loss_idx = loss_func.forward(input_idx, target_idx)
weight_losses+=loss_idx * weights[idx]
# loss = torch.Tensor(weight_losses)
# loss = loss.to(inputs.device)
# loss = torch.sum(loss)
return weight_losses London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
import torch
import torch.nn as nn
class FCN32s(nn.Module):
def __init__(self, num_classes=21):
super(FCN32s, self).__init__()
self.relu = nn.ReLU(inplace=True)
self.num_classes = num_classes
self.base_block_1 = nn.Sequential(
nn.Conv2d(3, 64, 3, 1, 1), # (?, 3, H, W) -> (?, 64, H, W)
self.relu,
nn.Conv2d(64, 64, 3, 1, 1), # (?, 64, H, W) -> (?, 64, H, W)
self.relu,
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.base_block_2 = nn.Sequential(
nn.Conv2d(64, 128, 3, 1, 1),
self.relu,
nn.Conv2d(128, 128, 3, 1, 1),
self.relu,
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.base_block_3 = nn.Sequential(
nn.Conv2d(128, 256, 3, 1, 1),
self.relu,
nn.Conv2d(256, 256, 3, 1, 1),
self.relu,
nn.Conv2d(256, 256, 3, 1, 1),
self.relu,
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.base_block_4 = nn.Sequential(
nn.Conv2d(256, 512, 3, 1, 1),
self.relu,
nn.Conv2d(512, 512, 3, 1, 1),
self.relu,
nn.Conv2d(512, 512, 3, 1, 1),
self.relu,
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.base_block_5 = nn.Sequential(
nn.Conv2d(512, 512, 3, 1, 1),
self.relu,
nn.Conv2d(512, 512, 3, 1, 1),
self.relu,
nn.Conv2d(512, 512, 3, 1, 1),
self.relu,
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.pred_block = nn.Sequential(
nn.Conv2d(512, 4096, 1),
self.relu,
nn.Conv2d(4096, 4096, 1),
self.relu,
self.Conv2d(4096, self.num_classes, 1),
self.relu,
nn.ConvTranspose2d(self.num_classes, self.num_classes, kernel_size=64, stride=32, padding=16)
)
def forward(self, x):
net = self.base_block_1(x)
net = self.base_block_2(net)
net = self.base_block_3(net)
net = self.base_block_4(net)
net = self.base_block_5(net)
return self.pred_block(net) London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.
London is the most populous city in the United Kingdom, with a metropolitan area of over 9 million inhabitants.