|
| 1 | +import time |
| 2 | +from functools import partial |
| 3 | + |
| 4 | +import dash |
| 5 | +import dash_bootstrap_components as dbc |
| 6 | +import dash_labs as dl |
| 7 | +import plotly.express as px |
| 8 | +import plotly.graph_objs as go |
| 9 | +from plotly.subplots import make_subplots |
| 10 | +import requests |
| 11 | +import torch |
| 12 | +import torch.nn as nn |
| 13 | +from torchvision import transforms as pth_transforms |
| 14 | +from PIL import Image |
| 15 | +import numpy as np |
| 16 | +from flask_caching import Cache |
| 17 | + |
| 18 | + |
| 19 | +def download_img(url, size=(500, 500)): |
| 20 | + im = Image.open(requests.get(url, stream=True).raw).convert("RGB") |
| 21 | + im.thumbnail(size) |
| 22 | + return im |
| 23 | + |
| 24 | + |
| 25 | +def compute_attentions(model, patch_size=16): |
| 26 | + def aux(img): |
| 27 | + """ |
| 28 | + Source: https://github.com/facebookresearch/dino/blob/main/visualize_attention.py |
| 29 | + """ |
| 30 | + # make the image divisible by the patch size |
| 31 | + w, h = ( |
| 32 | + img.shape[1] - img.shape[1] % patch_size, |
| 33 | + img.shape[2] - img.shape[2] % patch_size, |
| 34 | + ) |
| 35 | + img = img[:, :w, :h].unsqueeze(0) |
| 36 | + w_featmap = img.shape[-2] // patch_size |
| 37 | + h_featmap = img.shape[-1] // patch_size |
| 38 | + attentions = model.forward_selfattention(img) |
| 39 | + |
| 40 | + return attentions, w_featmap, h_featmap |
| 41 | + |
| 42 | + return aux |
| 43 | + |
| 44 | + |
| 45 | +def apply_threshold(attentions, w_featmap, h_featmap, threshold, patch_size=16): |
| 46 | + nh = attentions.shape[1] # number of head |
| 47 | + # we keep only the output patch attention |
| 48 | + attentions = attentions[0, :, 0, 1:].reshape(nh, -1) |
| 49 | + # we keep only a certain percentage of the mass |
| 50 | + val, idx = torch.sort(attentions) |
| 51 | + val /= torch.sum(val, dim=1, keepdim=True) |
| 52 | + cumval = torch.cumsum(val, dim=1) |
| 53 | + th_attn = cumval > (1 - threshold) |
| 54 | + idx2 = torch.argsort(idx) |
| 55 | + for head in range(nh): |
| 56 | + th_attn[head] = th_attn[head][idx2[head]] |
| 57 | + th_attn = th_attn.reshape(nh, w_featmap, h_featmap).float() |
| 58 | + # interpolate |
| 59 | + th_attn = ( |
| 60 | + nn.functional.interpolate( |
| 61 | + th_attn.unsqueeze(0), scale_factor=patch_size, mode="nearest" |
| 62 | + )[0] |
| 63 | + .detach() |
| 64 | + .cpu() |
| 65 | + .numpy() |
| 66 | + ) |
| 67 | + attentions = attentions.reshape(nh, w_featmap, h_featmap) |
| 68 | + attentions = nn.functional.interpolate( |
| 69 | + attentions.unsqueeze(0), scale_factor=patch_size, mode="nearest" |
| 70 | + ) |
| 71 | + attentions = attentions[0].detach().cpu().numpy() |
| 72 | + |
| 73 | + return th_attn, attentions |
| 74 | + |
| 75 | + |
| 76 | +# vars |
| 77 | +default_url = "https://dl.fbaipublicfiles.com/dino/img.png" |
| 78 | + |
| 79 | +# Load model |
| 80 | +device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") |
| 81 | +print("Running on", device) |
| 82 | +model = torch.hub.load("facebookresearch/dino:main", "dino_deits16").to(device) |
| 83 | +transform = pth_transforms.Compose( |
| 84 | + [ |
| 85 | + pth_transforms.ToTensor(), |
| 86 | + pth_transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), |
| 87 | + ] |
| 88 | +) |
| 89 | + |
| 90 | +# Initialize dash app and dash-labs template |
| 91 | +app = dash.Dash(__name__, plugins=[dl.plugins.FlexibleCallbacks()]) |
| 92 | +cache = Cache( |
| 93 | + app.server, |
| 94 | + config={ |
| 95 | + # try 'filesystem' if you don't want to setup redis |
| 96 | + "CACHE_TYPE": "filesystem", |
| 97 | + "CACHE_DIR": "flask_cache", |
| 98 | + }, |
| 99 | +) |
| 100 | +tpl = dl.templates.DbcSidebar( |
| 101 | + title="DINO Demo (using Dash Labs)", theme=dbc.themes.DARKLY, |
| 102 | +) |
| 103 | + |
| 104 | +# memoize functions |
| 105 | +predict = cache.memoize(timeout=300)(compute_attentions(model)) |
| 106 | +download_img = cache.memoize(timeout=300)(download_img) |
| 107 | + |
| 108 | +# Define callback function |
| 109 | +@app.callback( |
| 110 | + args=dict( |
| 111 | + url=tpl.textbox_input(default_url, label="Image URL", kind=dl.State), |
| 112 | + run=tpl.button_input("Run", label=""), |
| 113 | + head=tpl.dropdown_input(list(range(6)), value="0", label="Attention Head"), |
| 114 | + options=tpl.checklist_input(["use threshold", "overlay"], []), |
| 115 | + threshold=tpl.slider_input(0, 1, 0.6, 0.01), |
| 116 | + ), |
| 117 | + output=tpl.graph_output(), |
| 118 | + template=tpl, |
| 119 | +) |
| 120 | +def callback(url, run, threshold, head, options): |
| 121 | + try: |
| 122 | + im = download_img(url) |
| 123 | + except: |
| 124 | + return go.Figure().update_layout(title="Incorrect URL") |
| 125 | + |
| 126 | + ix = int(head) |
| 127 | + |
| 128 | + # Run model |
| 129 | + img = transform(im).to(device) |
| 130 | + attentions, w_featmap, h_featmap = predict(img) |
| 131 | + th_attn, scalar_attn = apply_threshold(attentions, w_featmap, h_featmap, threshold) |
| 132 | + |
| 133 | + if "use threshold" in options: |
| 134 | + attns = th_attn |
| 135 | + else: |
| 136 | + attns = scalar_attn |
| 137 | + |
| 138 | + if "overlay" in options: |
| 139 | + fig = px.imshow(im) |
| 140 | + fig.add_trace(go.Heatmap(z=attns[ix], opacity=0.55)) |
| 141 | + else: |
| 142 | + fig = make_subplots(1, 2) |
| 143 | + fig.add_trace(go.Image(z=im), 1, 1) |
| 144 | + fig.add_trace(go.Heatmap(z=attns[ix], y=np.arange(attns.shape[1], 0, -1)), 1, 2) |
| 145 | + fig.update_xaxes(matches="x") |
| 146 | + |
| 147 | + return fig |
| 148 | + |
| 149 | + |
| 150 | +app.layout = tpl.layout(app) |
| 151 | + |
| 152 | + |
| 153 | +if __name__ == "__main__": |
| 154 | + app.run_server(debug=True) |
0 commit comments