| |
| """ |
| @inproceedings{DBLP:conf/cvpr/SmithKGCKAPFK23, |
| author = {James Seale Smith and |
| Leonid Karlinsky and |
| Vyshnavi Gutta and |
| Paola Cascante{-}Bonilla and |
| Donghyun Kim and |
| Assaf Arbelle and |
| Rameswar Panda and |
| Rog{\'{e}}rio Feris and |
| Zsolt Kira}, |
| title = {CODA-Prompt: COntinual Decomposed Attention-Based Prompting for Rehearsal-Free |
| Continual Learning}, |
| booktitle = {{IEEE/CVF} Conference on Computer Vision and Pattern Recognition, |
| {CVPR} 2023, Vancouver, BC, Canada, June 17-24, 2023}, |
| pages = {11909--11919}, |
| publisher = {{IEEE}}, |
| year = {2023} |
| } |
| |
| https://arxiv.org/abs/2211.13218 |
| |
| Adapted from https://github.com/GT-RIPL/CODA-Prompt |
| """ |
|
|
| import math |
| import copy |
| import torch |
| import torch.nn as nn |
| from torch.nn import Parameter |
| import torch.nn.functional as F |
| from .finetune import Finetune |
| from core.model.backbone.resnet import * |
| import numpy as np |
| from torch.utils.data import DataLoader |
|
|
|
|
| class Model(nn.Module): |
| |
| def __init__(self, backbone, feat_dim, num_class): |
| super().__init__() |
| self.backbone = backbone |
| self.feat_dim = feat_dim |
| self.num_class = num_class |
| self.classifier = nn.Linear(feat_dim, num_class) |
| |
| def forward(self, x, train=True): |
| if train: |
| feat, loss = self.backbone(x, train=True) |
| return self.classifier(feat), loss |
| else: |
| feat = self.backbone(x, train=False) |
| return self.classifier(feat) |
|
|
|
|
| class CodaPrompt(Finetune): |
| def __init__(self, backbone, feat_dim, num_class, **kwargs): |
| super().__init__(backbone, feat_dim, num_class, **kwargs) |
| self.kwargs = kwargs |
| self.network = Model(self.backbone, feat_dim, kwargs['init_cls_num']) |
| self.network.backbone.create_prompt('coda', n_tasks = kwargs['task_num'], prompt_param=[kwargs['pool_size'], kwargs['prompt_length'], kwargs['mu']]) |
| self.task_idx = 0 |
| self.kwargs = kwargs |
| |
| self.last_out_dim = 0 |
|
|
| def before_task(self, task_idx, buffer, train_loader, test_loaders): |
| self.task_idx = task_idx |
| self.network.backbone.task_id = task_idx |
| |
| in_features = self.network.classifier.in_features |
| out_features = self.network.classifier.out_features |
| new_out_features = self.kwargs['init_cls_num'] + task_idx * self.kwargs['inc_cls_num'] |
| new_fc = nn.Linear(in_features, new_out_features) |
| new_fc.weight.data[:out_features] = self.network.classifier.weight.data |
| new_fc.bias.data[:out_features] = self.network.classifier.bias.data |
| self.network.classifier = new_fc |
| self.network.to(self.device) |
|
|
| self.loss_fn = nn.CrossEntropyLoss(reduction='none') |
| |
| self.out_dim = new_out_features |
| self.dw_k = torch.tensor(np.ones(self.out_dim + 1, dtype=np.float32)).to(self.device) |
|
|
| def observe(self, data): |
| x, y = data['image'], data['label'] |
| x = x.to(self.device) |
| y = y.to(self.device) |
| logit, loss = self.network(x, train=True) |
|
|
| logit[:,:self.last_out_dim] = -float('inf') |
| dw_cls = self.dw_k[-1 * torch.ones(y.size()).long()] |
|
|
| loss += (self.loss_fn(logit, y) * dw_cls).mean() |
| |
| pred = torch.argmax(logit, dim=1) |
| acc = torch.sum(pred == y).item() |
|
|
| return pred, acc / x.size(0), loss |
| |
| |
|
|
| def after_task(self, task_idx, buffer, train_loader, test_loaders): |
| self.last_out_dim = self.out_dim |
|
|
| def inference(self, data): |
| x, y = data['image'], data['label'] |
| x = x.to(self.device) |
| y = y.to(self.device) |
| |
| logit = self.network(x, train=False) |
|
|
| pred = torch.argmax(logit, dim=1) |
|
|
| acc = torch.sum(pred == y).item() |
| return pred, acc / x.size(0) |
|
|
|
|
| def get_parameters(self, config): |
| return list(self.network.backbone.prompt.parameters()) + list(self.network.classifier.parameters()) |
|
|
