其他
【源头活水】Deep InfoMax损失函数小记
“问渠那得清如许,为有源头活水来”,通过前沿领域知识的学习,从其他研究领域得到启发,对研究问题的本质有更清晰的认识和理解,是自我提高的不竭源泉。为此,我们特别精选论文阅读笔记,开辟“源头活水”专栏,帮助你广泛而深入的阅读科研文献,敬请关注。
论文中的等式(8)
论文中的等式(4)
class DeepInfoMaxLoss(nn.Module):
def __init__(self, alpha=0.5, beta=1.0, gamma=0.1):
super().__init__()
self.global_d = GlobalDiscriminator()
self.local_d = LocalDiscriminator()
self.prior_d = PriorDiscriminator()
self.alpha = alpha
self.beta = beta
self.gamma = gamma
def forward(self, y, M, M_prime):
# see appendix 1A of https://arxiv.org/pdf/1808.06670.pdf
y_exp = y.unsqueeze(-1).unsqueeze(-1)
y_exp = y_exp.expand(-1, -1, 26, 26)
y_M = torch.cat((M, y_exp), dim=1)
y_M_prime = torch.cat((M_prime, y_exp), dim=1)
Ej = -F.softplus(-self.local_d(y_M)).mean()
Em = F.softplus(self.local_d(y_M_prime)).mean()
LOCAL = (Em - Ej) * self.beta
Ej = -F.softplus(-self.global_d(y, M)).mean()
Em = F.softplus(self.global_d(y, M_prime)).mean()
GLOBAL = (Em - Ej) * self.alpha
prior = torch.rand_like(y)
term_a = torch.log(self.prior_d(prior)).mean()
term_b = torch.log(1.0 - self.prior_d(y)).mean()
PRIOR = - (term_a + term_b) * self.gamma
return LOCAL + GLOBAL + PRIOR
for x, target in batch:
x = x.to(device)
optim.zero_grad()
loss_optim.zero_grad()
y, M = encoder(x)
# rotate images to create pairs for comparison
M_prime = torch.cat((M[1:], M[0].unsqueeze(0)), dim=0)
loss = loss_fn(y, M, M_prime)
train_loss.append(loss.item())
batch.set_description(str(epoch) + ' Loss: ' + str(stats.mean(train_loss[-20:])))
loss.backward()
optim.step()
loss_optim.step()
class Encoder(nn.Module):
def __init__(self):
super().__init__()
self.c0 = nn.Conv2d(3, 64, kernel_size=4, stride=1)
self.c1 = nn.Conv2d(64, 128, kernel_size=4, stride=1)
self.c2 = nn.Conv2d(128, 256, kernel_size=4, stride=1)
self.c3 = nn.Conv2d(256, 512, kernel_size=4, stride=1)
self.l1 = nn.Linear(512*20*20, 64)
self.b1 = nn.BatchNorm2d(128)
self.b2 = nn.BatchNorm2d(256)
self.b3 = nn.BatchNorm2d(512)
def forward(self, x):
h = F.relu(self.c0(x))
features = F.relu(self.b1(self.c1(h)))
h = F.relu(self.b2(self.c2(features)))
h = F.relu(self.b3(self.c3(h)))
encoded = self.l1(h.view(x.shape[0], -1))
return encoded, features
class GlobalDiscriminator(nn.Module):
def __init__(self):
super().__init__()
self.c0 = nn.Conv2d(128, 64, kernel_size=3)
self.c1 = nn.Conv2d(64, 32, kernel_size=3)
self.l0 = nn.Linear(32 * 22 * 22 + 64, 512)
self.l1 = nn.Linear(512, 512)
self.l2 = nn.Linear(512, 1)
def forward(self, y, M):
h = F.relu(self.c0(M))
h = self.c1(h)
h = h.view(y.shape[0], -1)
h = torch.cat((y, h), dim=1)
h = F.relu(self.l0(h))
h = F.relu(self.l1(h))
return self.l2(h)
class LocalDiscriminator(nn.Module):
def __init__(self):
super().__init__()
self.c0 = nn.Conv2d(192, 512, kernel_size=1)
self.c1 = nn.Conv2d(512, 512, kernel_size=1)
self.c2 = nn.Conv2d(512, 1, kernel_size=1)
def forward(self, x):
h = F.relu(self.c0(x))
h = F.relu(self.c1(h))
return self.c2(h)
class PriorDiscriminator(nn.Module):
def __init__(self):
super().__init__()
self.l0 = nn.Linear(64, 1000)
self.l1 = nn.Linear(1000, 200)
self.l2 = nn.Linear(200, 1)
def forward(self, x):
h = F.relu(self.l0(x))
h = F.relu(self.l1(h))
return torch.sigmoid(self.l2(h))
原论文中等式(7)
本文目的在于学术交流,并不代表本公众号赞同其观点或对其内容真实性负责,版权归原作者所有,如有侵权请告知删除。
“源头活水”历史文章
游走图模型-聊聊Node2Vec(论文分析、代码实践)
CVPR 2021 | 让机器想象未见的世界:反事实的零次和开集识别
解耦表示,理解用户的多样偏好和意图
REPAINT:深度强化学习中的知识迁移
Transformer 的稳健性更好吗?
读SfM-Net: Learning of Structure and Motion from Video
Conditional DETR: 通过显式寻找物体的 extremity 区域加快 DETR 的收敛
谷歌和CMU论文:使用元学习生成伪标签
Deep GNN评测,模型退化是做不深的主要原因!
CVPR2021:Spatial在左,Temporal在右,如何缓解视频动作理解中的隐式偏见之殇?
CVPR2021自监督学习论文: 理解对比损失的性质以及温度系数的作用
GAN,没错,生成对抗网络也可以被用于时间序列预测
开放世界实体分割 Open-World Entity Segmentation
ICCV‘21 | 具有样本特定触发器的隐形后门攻击
更多源头活水专栏文章,
请点击文章底部“阅读原文”查看
分享、在看,给个三连击呗!