bachelor-thesis/notebooks/pmf.ipynb

In [1]:

import os
import numpy as np
import time
import random
import torch
import torchvision.transforms as transforms
#import gradio as gr
import matplotlib.pyplot as plt

from models import get_model
from dotmap import DotMap
from PIL import Image

In [2]:

args = DotMap()
args.deploy = 'finetune'
args.arch = 'dino_base_patch16'
args.no_pretrain = True
small = "https://huggingface.co/hushell/pmf_metadataset_dino/resolve/main/md_full_128x128_dinosmall_fp16_lr5e-5/best.pth?download=true"
full = 'https://huggingface.co/hushell/pmf_metadataset_dino/resolve/main/md_full_128x128_dinobase_fp16_lr5e-5/best.pth?download=true'
args.resume = full
args.api_key = 'AIzaSyAFkOGnXhy-2ZB0imDvNNqf2rHb98vR_qY'
args.cx = '06d75168141bc47f1'

args.ada_steps = 100
#args.ada_lr= 0.0001
#args.aug_prob = .95
args.ada_lr= 0.0001
args.aug_prob = .9
args.aug_types = ["color", "translation"]

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = get_model(args)
model.to(device)
checkpoint = torch.hub.load_state_dict_from_url(args.resume, map_location='cpu')
model.load_state_dict(checkpoint['model'], strict=True)

Pretrained weights found at dino_vitbase16_pretrain/dino_vitbase16_pretrain.pth

In [18]:

# image transforms
def test_transform():
    def _convert_image_to_rgb(im):
        return im.convert('RGB')

    return transforms.Compose([
        transforms.Resize(224),
        #transforms.CenterCrop(224),
        _convert_image_to_rgb,
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406],
                             std=[0.229, 0.224, 0.225]),
        ])

preprocess = test_transform()

In [42]:

def build_sup_set(shotnr, type_name, binary, good_sample_nr):
    classes = next(os.walk(f'data_custom/{type_name}/test'))[1]
    classes.remove("good")

    supp_x = []
    supp_y = []
    mapping = {
        "good" : 0
    }

    # add good manually
    x_good = [Image.open(f"data_custom/{type_name}/train/good/{x:03d}.png") for x in range(0, good_sample_nr)]
    supp_x.extend([preprocess(x) for x in x_good]) # (3, H, W))
    supp_y.extend([0] * good_sample_nr)
    
    for i,c in enumerate(classes):
        #i-=1
        x_im = [Image.open(f"data_custom/{type_name}/test/{c}/{x:03d}.png") for x in range(0, shotnr)]
        supp_x.extend([preprocess(x) for x in x_im]) # (3, H, W))
        if binary:
            supp_y.extend([1] * shotnr)
            mapping["anomaly"] = 1
        else:
            supp_y.extend([i+1] * shotnr)
            mapping[c] = i+1
    
    supp_x = torch.stack(supp_x, dim=0).unsqueeze(0).to(device) # (1, n_supp*n_labels, 3, H, W)
    supp_y = torch.tensor(supp_y).long().unsqueeze(0).to(device) # (1, n_supp*n_labels)
    return supp_x, supp_y, mapping

In [16]:

def build_test_set(shotnr, keyy, type):
    _, _, files = next(os.walk(f"data_custom/cable/test/{type}/"))
    file_count = len(files)
    print(file_count)

    queries = [preprocess(Image.open(f"data_custom/cable/test/{type}/{i:03d}.png")).unsqueeze(0).unsqueeze(0).to(device) for i in range(shotnr,file_count)]
    labels = [keyy for x in range(shotnr,file_count)]
    return queries, labels

def test(type, keyy, shotnr, folder):
    predictions = []
    _, _, files = next(os.walk(f"data_custom/{folder}/test/{type}/"))
    file_count = len(files)
    print(file_count)

    queries = [preprocess(Image.open(f"data_custom/{folder}/test/{type}/{i:03d}.png")).unsqueeze(0).unsqueeze(0).to(device) for i in range(shotnr,file_count)]
    queries = torch.cat(queries)
    with torch.cuda.amp.autocast(True):
        output = model(supp_x, supp_y, queries) # (1, 1, n_labels)

    probs = output.softmax(dim=-1).detach().cpu().numpy()
    predictions = np.argmax(probs, axis=2)
    print()
    return np.mean([x == keyy for x in predictions])
    pass
    
#def test2(folder):
#    accs = []
#    queries = []
#    labels = []
#    for t in next(os.walk(f'data_custom/cable/test'))[1]:
#        q, l = build_test_set(shots, types.get(t, 1), t)
#        queries+=q
#        labels+=l
#
#    queries = torch.cat(queries)
#    labels = np.array(labels)
#
#    with torch.cuda.amp.autocast(True):
#        output = model(supp_x, supp_y, queries) # (1, 1, n_labels)
#
#    probs = output.softmax(dim=-1).detach().cpu().numpy()
#    predictions = np.argmax(probs, axis=2)
#    print()
#    return np.mean([predictions == labels])
#    pass

#print(f"overall accuracy: {test("cable")}")

In [48]:

#bottle_accs = []
cable_accs = []

for nr in [5, 10, 15, 30]:
    folder = "cable"
    shot = 5
    supp_x, supp_y, types = build_sup_set(shot, folder, True, nr)
    accs = []
    for t in next(os.walk(f'data_custom/{folder}/test'))[1]:
        #if t == "good":
        #    continue
        accuracy = test(t, types.get(t, 1), shot, folder)
        print(f"accuracy for {t} = {accuracy}")
        accs.append(accuracy)
    print(f"overall accuracy: {np.mean(accs)}")
    cable_accs.append(np.mean(accs))

14

lr0.0001, nSupp45, nQry9: loss = 0.1475423127412796: 100%|██| 100/100 [00:29<00:00,  3.40it/s]

accuracy for cut_inner_insulation = 1.0
10

lr0.0001, nSupp45, nQry5: loss = 0.20609889924526215: 100%|█| 100/100 [00:29<00:00,  3.37it/s]

accuracy for poke_insulation = 1.0
12

lr0.0001, nSupp45, nQry7: loss = 0.12025140225887299: 100%|█| 100/100 [00:29<00:00,  3.34it/s]

accuracy for cable_swap = 0.8571428571428571
10

lr0.0001, nSupp45, nQry5: loss = 0.2130972295999527: 100%|██| 100/100 [00:30<00:00,  3.30it/s]

accuracy for cut_outer_insulation = 1.0
58

lr0.0001, nSupp45, nQry53: loss = 0.13926956057548523: 100%|█| 100/100 [00:30<00:00,  3.30it/s

accuracy for good = 0.16981132075471697
12

lr0.0001, nSupp45, nQry7: loss = 0.16337624192237854: 100%|█| 100/100 [00:30<00:00,  3.28it/s]

accuracy for missing_cable = 1.0
11

lr0.0001, nSupp45, nQry6: loss = 0.16593313217163086: 100%|█| 100/100 [00:30<00:00,  3.27it/s]

accuracy for combined = 1.0
13

lr0.0001, nSupp45, nQry8: loss = 0.16560573875904083: 100%|█| 100/100 [00:30<00:00,  3.27it/s]

accuracy for bent_wire = 1.0
10

lr0.0001, nSupp45, nQry5: loss = 0.18611018359661102: 100%|█| 100/100 [00:30<00:00,  3.28it/s]

accuracy for missing_wire = 0.8
overall accuracy: 0.8696615753219527
14

lr0.0001, nSupp50, nQry9: loss = 0.3357824385166168: 100%|██| 100/100 [00:33<00:00,  2.96it/s]

accuracy for cut_inner_insulation = 0.7777777777777778
10

lr0.0001, nSupp50, nQry5: loss = 0.3290153741836548: 100%|██| 100/100 [00:33<00:00,  2.96it/s]

accuracy for poke_insulation = 0.6
12

lr0.0001, nSupp50, nQry7: loss = 0.22177687287330627: 100%|█| 100/100 [00:33<00:00,  2.96it/s]

accuracy for cable_swap = 0.8571428571428571
10

lr0.0001, nSupp50, nQry5: loss = 0.299775630235672: 100%|███| 100/100 [00:33<00:00,  2.96it/s]

accuracy for cut_outer_insulation = 1.0
58

lr0.0001, nSupp50, nQry53: loss = 0.31954386830329895: 100%|█| 100/100 [00:33<00:00,  2.98it/s

accuracy for good = 0.32075471698113206
12

lr0.0001, nSupp50, nQry7: loss = 0.336273193359375: 100%|███| 100/100 [00:33<00:00,  2.98it/s]

accuracy for missing_cable = 0.8571428571428571
11

lr0.0001, nSupp50, nQry6: loss = 0.3643767237663269: 100%|██| 100/100 [00:33<00:00,  2.98it/s]

accuracy for combined = 1.0
13

lr0.0001, nSupp50, nQry8: loss = 0.3085792660713196: 100%|██| 100/100 [00:33<00:00,  2.98it/s]

accuracy for bent_wire = 1.0
10

lr0.0001, nSupp50, nQry5: loss = 0.34715649485588074: 100%|█| 100/100 [00:33<00:00,  2.98it/s]

accuracy for missing_wire = 0.8
overall accuracy: 0.8014242454494026
14

lr0.0001, nSupp55, nQry9: loss = 0.375447154045105: 100%|███| 100/100 [00:36<00:00,  2.76it/s]

accuracy for cut_inner_insulation = 0.6666666666666666
10

lr0.0001, nSupp55, nQry5: loss = 0.42370423674583435: 100%|█| 100/100 [00:36<00:00,  2.75it/s]

accuracy for poke_insulation = 1.0
12

lr0.0001, nSupp55, nQry7: loss = 0.3982161581516266: 100%|██| 100/100 [00:36<00:00,  2.74it/s]

accuracy for cable_swap = 0.8571428571428571
10

lr0.0001, nSupp55, nQry5: loss = 0.3903641104698181: 100%|██| 100/100 [00:36<00:00,  2.75it/s]

accuracy for cut_outer_insulation = 1.0
58

lr0.0001, nSupp55, nQry53: loss = 0.4019339382648468: 100%|█| 100/100 [00:36<00:00,  2.75it/s]

accuracy for good = 0.41509433962264153
12

lr0.0001, nSupp55, nQry7: loss = 0.4283098876476288: 100%|██| 100/100 [00:36<00:00,  2.75it/s]

accuracy for missing_cable = 0.7142857142857143
11

lr0.0001, nSupp55, nQry6: loss = 0.3741377890110016: 100%|██| 100/100 [00:36<00:00,  2.74it/s]

accuracy for combined = 0.8333333333333334
13

lr0.0001, nSupp55, nQry8: loss = 0.3858358860015869: 100%|██| 100/100 [00:36<00:00,  2.75it/s]

accuracy for bent_wire = 1.0
10

lr0.0001, nSupp55, nQry5: loss = 0.3570959270000458: 100%|██| 100/100 [00:36<00:00,  2.74it/s]

accuracy for missing_wire = 0.8
overall accuracy: 0.8096136567834681
14

lr0.0001, nSupp70, nQry9: loss = 0.5021733045578003: 100%|██| 100/100 [00:45<00:00,  2.21it/s]

accuracy for cut_inner_insulation = 0.5555555555555556
10

lr0.0001, nSupp70, nQry5: loss = 0.5203520059585571: 100%|██| 100/100 [00:45<00:00,  2.20it/s]

accuracy for poke_insulation = 0.4
12

lr0.0001, nSupp70, nQry7: loss = 0.524366021156311: 100%|███| 100/100 [00:45<00:00,  2.21it/s]

accuracy for cable_swap = 0.42857142857142855
10

lr0.0001, nSupp70, nQry5: loss = 0.5256413221359253: 100%|██| 100/100 [00:45<00:00,  2.21it/s]

accuracy for cut_outer_insulation = 1.0
58

lr0.0001, nSupp70, nQry53: loss = 0.5186663866043091: 100%|█| 100/100 [00:45<00:00,  2.21it/s]

accuracy for good = 0.7358490566037735
12

lr0.0001, nSupp70, nQry7: loss = 0.5123675465583801: 100%|██| 100/100 [00:45<00:00,  2.21it/s]

accuracy for missing_cable = 0.7142857142857143
11

lr0.0001, nSupp70, nQry6: loss = 0.5076506733894348: 100%|██| 100/100 [00:45<00:00,  2.21it/s]

accuracy for combined = 0.8333333333333334
13

lr0.0001, nSupp70, nQry8: loss = 0.490247517824173: 100%|███| 100/100 [00:45<00:00,  2.21it/s]

accuracy for bent_wire = 0.875
10

lr0.0001, nSupp70, nQry5: loss = 0.3723257780075073: 100%|██| 100/100 [00:45<00:00,  2.21it/s]

accuracy for missing_wire = 0.4
overall accuracy: 0.6602883431499785

In [39]:

print(np.array(bottle_accs))

[0.57380952 0.76705653 0.84191176]

In [49]:

print(np.array(cable_accs))

[0.86966158 0.80142425 0.80961366 0.66028834]

P>M>F: Resulsts:

bottle: jeweils 1,3,5 shots normal [0.67910401 0.71710526 0.78860294]

inbalanced - mehr good shots 5,10,15,30 -> alle anderen nur 5 [0.78768382 0.78860294 0.75827206 0.74356618]

2 ways nur detektieren ob fehlerhaft oder nicht 1,3,5 shots [0.86422306 0.93201754 0.93933824]

inbalance 2 way 5,10,15,30 -> rest 5 [0.92371324 0.87867647 0.86397059 0.87775735]

nur fehlerklasse erkennen 1,3,5 [0.57380952 0.76705653 0.84191176]

cable: jeweils 1,3,5 shots normal [0.25199021 0.44388328 0.46975059]

inbalanced - mehr good shots 5,10,15,30 -> alle anderen nur 5 [0.50425859 0.48023277 0.43118282 0.41842534]

2 ways nur detektieren ob fehlerhaft oder nicht 1,3,5 shots [0.79263485 0.8707712 0.86756514]

inbalance 2 way 5,10,15,30 -> rest 5 [0.86966158 0.80142425 0.80961366 0.66028834]

nur fehlerklasse erkennen 1,3,5 [0.24383256 0.43800505 0.51304563]