Results of CMSC498v Project

.gitignore

-__pycache__/
-.ipynb_checkpoints/
-*.log
-*.pth
-!models/*.pth
-!models_scaled/*.pth
-data/
+.idea/*
+examples/data/

examples/cifar.py

@@ -36,12 +36,19 @@ def select_model(m):
 if __name__ == "__main__": 
     args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50, 
                 opt='sgd', lr=0.05)
+    args.method = "madry"
+    # args.batch_size = 20
+    # args.test_batch_size = 20
+    args.prefix = "test_cifar/prefix"
+    # args.epochs = 35
+    print(args)
 
     print("saving file to {}".format(args.prefix))
     setproctitle.setproctitle(args.prefix)
 
     train_log = open(args.prefix + "_train.log", "w")
     test_log = open(args.prefix + "_test.log", "w")
+    full_test_log = open(args.prefix + "_full_test.log", "w")
 
     train_loader, _ = pblm.cifar_loaders(args.batch_size)
     _, test_loader = pblm.cifar_loaders(args.test_batch_size)
@@ -129,7 +136,9 @@ if __name__ == "__main__":
                    test_log, args.verbose, args.real_time,
                    norm_type=args.norm_test, bounded_input=False, 
                    **kwargs)
-            
+
+            print("Test PGD Error:", err, file=full_test_log)
+
             if err < best_err: 
                 best_err = err
                 torch.save({

examples/cifar_test3.py

+import waitGPU
+# import setGPU
+# waitGPU.wait(utilization=20, available_memory=10000, interval=60)
+# waitGPU.wait(gpu_ids=[1,3], utilization=20, available_memory=10000, interval=60)
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from torch.autograd import Variable
+import torch.backends.cudnn as cudnn
+cudnn.benchmark = True
+
+import torchvision.transforms as transforms
+import torchvision.datasets as datasets
+import random
+    
+import setproctitle
+
+import problems as pblm
+from trainer import *
+
+import math
+import numpy
+
+import datetime
+
+def select_model(m): 
+    if m == 'large': 
+        # raise ValueError
+        model = pblm.cifar_model_large().cuda()
+    elif m == 'resnet': 
+        model = pblm.cifar_model_resnet(N=args.resnet_N, factor=args.resnet_factor).cuda()
+    else: 
+        model = pblm.cifar_model().cuda() 
+    return model
+
+if __name__ == "__main__":
+
+    for method in ["mix", "robust", "madry", "baseline"]:
+        waitGPU.wait(utilization=20, available_memory=6000, interval=2)
+
+        args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50,
+                    opt='sgd', lr=0.05)
+        args.method = method
+        args.batch_size = 50
+        args.test_batch_size = 20
+        args.prefix = "test3/" + args.method + ""
+        args.epochs = 50
+        print(args)
+        print("saving file to {}".format(args.prefix))
+        setproctitle.setproctitle(args.prefix)
+        train_log = open(args.prefix + "_train.log", "w")
+        baseline_test_log = open(args.prefix + "_baseline_test.log", "w")
+        madry_test_log = open(args.prefix + "_madry_test.log", "w")
+        robust_test_log = open(args.prefix + "_robust_test.log", "w")
+        full_test_log = open(args.prefix + "_full_test.log", "w")
+
+
+        train_loader, _ = pblm.cifar_loaders(args.batch_size)
+        # For robust training since it requires more memory.
+        # This may be unfair
+        train_loader2, _ = pblm.cifar_loaders(20)
+        _, test_loader = pblm.cifar_loaders(args.test_batch_size)
+
+        torch.manual_seed(args.seed)
+        torch.cuda.manual_seed_all(args.seed)
+        random.seed(0)
+        numpy.random.seed(0)
+
+        sampler_indices = []
+        model = [select_model(args.model)]
+
+        kwargs = pblm.args2kwargs(args)
+        best_err = 1
+
+        start_time = datetime.datetime.now()
+
+        for _ in range(0,args.cascade):
+            # if _ > 0:
+            #     # reduce dataset to just uncertified examples
+            #     print("Reducing dataset...")
+            #     train_loader = sampler_robust_cascade(train_loader, model, args.epsilon,
+            #                                           args.test_batch_size,
+            #                                           norm_type=args.norm_test, **kwargs)
+            #     if train_loader is None:
+            #         print('No more examples, terminating')
+            #         break
+            #     sampler_indices.append(train_loader.sampler.indices)
+            #
+            #     print("Adding a new model")
+            #     model.append(select_model(args.model))
+
+            if args.opt == 'adam':
+                opt = optim.Adam(model[-1].parameters(), lr=args.lr)
+            elif args.opt == 'sgd':
+                opt = optim.SGD(model[-1].parameters(), lr=args.lr,
+                                momentum=args.momentum,
+                                weight_decay=args.weight_decay)
+            else:
+                raise ValueError("Unknown optimizer")
+            lr_scheduler = optim.lr_scheduler.StepLR(opt, step_size=10, gamma=0.5)
+            eps_schedule = np.linspace(args.starting_epsilon,
+                                       args.epsilon,
+                                       args.schedule_length)
+
+            for t in range(args.epochs):
+                lr_scheduler.step(epoch=max(t-len(eps_schedule), 0))
+                if t < len(eps_schedule) and args.starting_epsilon is not None:
+                    epsilon = float(eps_schedule[t])
+                else:
+                    epsilon = args.epsilon
+
+
+                # standard training
+                if args.method == 'baseline':
+                    train_baseline(train_loader, model[0], opt, t, train_log,
+                                    args.verbose)
+                # madry training
+                elif args.method=='madry':
+                    train_madry(train_loader, model[0], args.epsilon,
+                                opt, t, train_log, args.verbose)
+                # robust training
+                elif args.method=='robust':
+                    train_robust(train_loader2, model[0], opt, epsilon, t,
+                       train_log, args.verbose, args.real_time,
+                       norm_type=args.norm_train, bounded_input=False, clip_grad=1,
+                       **kwargs)
+                elif args.method == "mix":
+                    if t < 20:
+                        train_madry(train_loader, model[0], args.epsilon,
+                                    opt, t, train_log, args.verbose)
+                    else:
+                        train_robust(train_loader2, model[0], opt, epsilon, t,
+                                     train_log, args.verbose, args.real_time,
+                                     norm_type=args.norm_train, bounded_input=False, clip_grad=1,
+                                     **kwargs)
+
+
+                time_diff = datetime.datetime.now() - start_time
+                print("Train Time Diff")
+                print(time_diff.total_seconds())
+
+                baseline_err = evaluate_baseline(test_loader, model[0], t, baseline_test_log,
+                                                 args.verbose)
+                madry_err = evaluate_madry(test_loader, model[0], args.epsilon,
+                                           t, madry_test_log, args.verbose)
+                robust_err = evaluate_robust(test_loader, model[0], args.epsilon,
+                                             t, robust_test_log, args.verbose, args.real_time,
+                                             norm_type=args.norm_test, bounded_input=False, **kwargs)
+                err = robust_err
+
+                start_time = datetime.datetime.now() - time_diff
+
+                print(time_diff.total_seconds(), baseline_err, madry_err, robust_err, file=full_test_log)
+
+                if err < best_err:
+                    best_err = err
+                    torch.save({
+                        'state_dict' : [m.state_dict() for m in model],
+                        'err' : best_err,
+                        'epoch' : t,
+                        'sampler_indices' : sampler_indices
+                        }, args.prefix + "_best.pth")
+
+                torch.save({
+                    'state_dict': [m.state_dict() for m in model],
+                    'err' : err,
+                    'epoch' : t,
+                    'sampler_indices' : sampler_indices
+                    }, args.prefix + "_checkpoint.pth")
+
+                if time_diff.total_seconds() > 8000:
+                    break

examples/cifar_test6.py

+import waitGPU
+# import setGPU
+# waitGPU.wait(utilization=20, available_memory=10000, interval=60)
+# waitGPU.wait(gpu_ids=[1,3], utilization=20, available_memory=10000, interval=60)
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from torch.autograd import Variable
+import torch.backends.cudnn as cudnn
+cudnn.benchmark = True
+
+import torchvision.transforms as transforms
+import torchvision.datasets as datasets
+import random
+    
+import setproctitle
+
+import problems as pblm
+from trainer import *
+
+import math
+import numpy
+
+import datetime
+
+def select_model(m): 
+    if m == 'large': 
+        # raise ValueError
+        model = pblm.cifar_model_large().cuda()
+    elif m == 'resnet': 
+        model = pblm.cifar_model_resnet(N=args.resnet_N, factor=args.resnet_factor).cuda()
+    else: 
+        model = pblm.cifar_model().cuda() 
+    return model
+
+if __name__ == "__main__":
+
+    for method in ["mix"]:
+        waitGPU.wait(utilization=20, available_memory=6000, interval=2)
+
+        args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50,
+                    opt='sgd', lr=0.05)
+        args.method = method
+        args.batch_size = 50
+        args.test_batch_size = 20
+        args.prefix = "test6/" + args.method + ""
+        args.epochs = 50
+        print(args)
+        print("saving file to {}".format(args.prefix))
+        setproctitle.setproctitle(args.prefix)
+        train_log = open(args.prefix + "_train.log", "w")
+        baseline_test_log = open(args.prefix + "_baseline_test.log", "w")
+        madry_test_log = open(args.prefix + "_madry_test.log", "w")
+        robust_test_log = open(args.prefix + "_robust_test.log", "w")
+        full_test_log = open(args.prefix + "_full_test.log", "w")
+
+
+        train_loader, _ = pblm.cifar_loaders(args.batch_size)
+        # For robust training since it requires more memory.
+        # This may be unfair
+        train_loader2, _ = pblm.cifar_loaders(20)
+        _, test_loader = pblm.cifar_loaders(args.test_batch_size)
+
+        torch.manual_seed(args.seed)
+        torch.cuda.manual_seed_all(args.seed)
+        random.seed(0)
+        numpy.random.seed(0)
+
+        sampler_indices = []
+        model = [select_model(args.model)]
+
+        kwargs = pblm.args2kwargs(args)
+        best_err = 1
+
+        start_time = datetime.datetime.now()
+
+        for _ in range(0,args.cascade):
+            # if _ > 0:
+            #     # reduce dataset to just uncertified examples
+            #     print("Reducing dataset...")
+            #     train_loader = sampler_robust_cascade(train_loader, model, args.epsilon,
+            #                                           args.test_batch_size,
+            #                                           norm_type=args.norm_test, **kwargs)
+            #     if train_loader is None:
+            #         print('No more examples, terminating')
+            #         break
+            #     sampler_indices.append(train_loader.sampler.indices)
+            #
+            #     print("Adding a new model")
+            #     model.append(select_model(args.model))
+
+            if args.opt == 'adam':
+                opt = optim.Adam(model[-1].parameters(), lr=args.lr)
+            elif args.opt == 'sgd':
+                opt = optim.SGD(model[-1].parameters(), lr=args.lr,
+                                momentum=args.momentum,
+                                weight_decay=args.weight_decay)
+            else:
+                raise ValueError("Unknown optimizer")
+            lr_scheduler = optim.lr_scheduler.StepLR(opt, step_size=10, gamma=0.5)
+            eps_schedule = np.linspace(args.starting_epsilon,
+                                       args.epsilon,
+                                       args.schedule_length)
+
+            for t in range(args.epochs):
+                lr_scheduler.step(epoch=max(t-len(eps_schedule), 0))
+                if t < len(eps_schedule) and args.starting_epsilon is not None:
+                    epsilon = float(eps_schedule[t])
+                else:
+                    epsilon = args.epsilon
+
+
+                # standard training
+                if args.method == 'baseline':
+                    train_baseline(train_loader, model[0], opt, t, train_log,
+                                    args.verbose)
+                # madry training
+                elif args.method=='madry':
+                    train_madry(train_loader, model[0], args.epsilon,
+                                opt, t, train_log, args.verbose)
+                # robust training
+                elif args.method=='robust':
+                    train_robust(train_loader2, model[0], opt, epsilon, t,
+                       train_log, args.verbose, args.real_time,
+                       norm_type=args.norm_train, bounded_input=False, clip_grad=1,
+                       **kwargs)
+                elif args.method == "mix":
+                    if t < 20:
+                        train_madry(train_loader, model[0], 0.2,
+                                    opt, t, train_log, args.verbose)
+                    else:
+                        train_robust(train_loader2, model[0], opt, args.epsilon, t,
+                                     train_log, args.verbose, args.real_time,
+                                     norm_type=args.norm_train, bounded_input=False, clip_grad=1,
+                                     **kwargs)
+
+
+                time_diff = datetime.datetime.now() - start_time
+                print("Train Time Diff")
+                print(time_diff.total_seconds())
+
+                baseline_err = evaluate_baseline(test_loader, model[0], t, baseline_test_log,
+                                                 args.verbose)
+                madry_err = evaluate_madry(test_loader, model[0], args.epsilon,
+                                           t, madry_test_log, args.verbose)
+                robust_err = evaluate_robust(test_loader, model[0], args.epsilon,
+                                             t, robust_test_log, args.verbose, args.real_time,
+                                             norm_type=args.norm_test, bounded_input=False, **kwargs)
+                err = robust_err
+
+                start_time = datetime.datetime.now() - time_diff
+
+                print(time_diff.total_seconds(), baseline_err, madry_err, robust_err, file=full_test_log)
+
+                if err < best_err:
+                    best_err = err
+                    torch.save({
+                        'state_dict' : [m.state_dict() for m in model],
+                        'err' : best_err,
+                        'epoch' : t,
+                        'sampler_indices' : sampler_indices
+                        }, args.prefix + "_best.pth")
+
+                torch.save({
+                    'state_dict': [m.state_dict() for m in model],
+                    'err' : err,
+                    'epoch' : t,
+                    'sampler_indices' : sampler_indices
+                    }, args.prefix + "_checkpoint.pth")
+
+                if time_diff.total_seconds() > 8000:
+                    break

examples/figures.py

+import re
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+import json
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+from plotnine import ggplot, aes,\
+    geom_point, geom_line,\
+    scale_color_discrete, scale_shape_discrete, scale_color_brewer,\
+    xlab, ylab, theme, element_rect, facet_grid, ggtitle
+
+
+to_digit = re.compile("tensor\(([\d\.]*),")
+remove_nl = re.compile("\n")
+
+# Test 1
+
+baseline_file = open("test1/baseline_full_test.log", "r")
+baseline_arr = baseline_file.readlines()
+baseline_arr2 = []
+for i, val in enumerate(baseline_arr):
+    filteredstr = re.sub("\n", "", val)
+    filteredstr = re.sub("\,\s", ",", filteredstr)
+    baseline_arr2.append(filteredstr.split(" "))
+    baseline_arr2[i][1] = re.match("tensor\(([\d\.]*),", baseline_arr2[i][1]).group(1)
+    baseline_arr2[i][2] = re.match("tensor\(([\d\.]*),", baseline_arr2[i][2]).group(1)
+
+# [time, baseline_err, madry_err, robust_err]
+baseline_arr2 = np.array(baseline_arr2, dtype=float)
+
+madry_file = open("test1/madry_full_test.log", "r")
+madry_arr = madry_file.readlines()
+madry_arr2 = []
+for i, val in enumerate(madry_arr):
+    filteredstr = re.sub("\n", "", val)
+    filteredstr = re.sub("\,\s", ",", filteredstr)
+    madry_arr2.append(filteredstr.split(" "))
+    madry_arr2[i][1] = re.match("tensor\(([\d\.]*),", madry_arr2[i][1]).group(1)
+    madry_arr2[i][2] = re.match("tensor\(([\d\.]*),", madry_arr2[i][2]).group(1)
+
+# [time, madry_err, madry_err, robust_err]
+madry_arr2 = np.array(madry_arr2, dtype=float)
+
+
+robust_file = open("test1/robust_full_test.log", "r")
+robust_arr = robust_file.readlines()
+robust_arr2 = []
+for i, val in enumerate(robust_arr):
+    filteredstr = re.sub("\n", "", val)
+    filteredstr = re.sub("\,\s", ",", filteredstr)
+    robust_arr2.append(filteredstr.split(" "))
+    robust_arr2[i][1] = re.match("tensor\(([\d\.]*),", robust_arr2[i][1]).group(1)
+    robust_arr2[i][2] = re.match("tensor\(([\d\.]*),", robust_arr2[i][2]).group(1)
+
+# [time, robust_err, madry_err, robust_err]
+robust_arr2 = np.array(robust_arr2, dtype=float)
+
+
+mix_file = open("test1/mix_full_test.log", "r")
+mix_arr = mix_file.readlines()
+mix_arr2 = []
+for i, val in enumerate(mix_arr):
+    filteredstr = re.sub("\n", "", val)
+    filteredstr = re.sub("\,\s", ",", filteredstr)
+    mix_arr2.append(filteredstr.split(" "))
+    mix_arr2[i][1] = re.match("tensor\(([\d\.]*),", mix_arr2[i][1]).group(1)
+    mix_arr2[i][2] = re.match("tensor\(([\d\.]*),", mix_arr2[i][2]).group(1)
+
+# [time, mix_err, madry_err, robust_err]
+mix_arr2 = np.array(mix_arr2, dtype=float)
+
+# plt.plot(baseline_arr2[:,0], baseline_arr2[:,1], label='baseline')
+# plt.plot(madry_arr2[:,0], madry_arr2[:,1], label='madry')
+# plt.plot(robust_arr2[:,0], robust_arr2[:,1], label='robust')
+# plt.plot(mix_arr2[:,0], mix_arr2[:,1], label='mix')
+# print(baseline_arr2[:,0])
+# print(madry_arr2[:,0])
+
+# plt.xlabel('Time (seconds)')
+# plt.ylabel('Error')
+# plt.title("Baseline Error")
+# plt.legend()
+# plt.savefig("test1/baselineErr.png")
+# plt.clf()
+
+
+# plt.plot(baseline_arr2[:,0], baseline_arr2[:,2], label='baseline')
+# plt.plot(madry_arr2[:,0], madry_arr2[:,2], label='madry')
+# plt.plot(robust_arr2[:,0], robust_arr2[:,2], label='robust')
+# plt.plot(mix_arr2[:,0], mix_arr2[:,2], label='mix')
+# print(baseline_arr2[:,0])
+# print(madry_arr2[:,0])
+
+# plt.xlabel('Time (seconds)')
+# plt.ylabel('Error')
+# plt.title("Madry (PGD) Error")
+# plt.legend()
+# plt.savefig("test1/madryErr.png")
+# plt.clf()
+
+
+# plt.plot(baseline_arr2[:,0], baseline_arr2[:,3], label='baseline')
+# plt.plot(madry_arr2[:,0], madry_arr2[:,3], label='madry')
+# plt.plot(robust_arr2[:,0], robust_arr2[:,3], label='robust')
+# plt.plot(mix_arr2[:,0], mix_arr2[:,3], label='mix')
+# print(baseline_arr2[:,0])
+# print(madry_arr2[:,0])
+
+# plt.xlabel('Time (seconds)')
+# plt.ylabel('Error')
+# plt.title("Robust Error")
+# plt.legend()
+# plt.savefig("test1/robustErr.png")
+# plt.clf()
+
+x = {}
+y = {}
+method = {}
+
+index = 0
+for element1, element2 in zip(baseline_arr2[:,0], baseline_arr2[:,1]):    
+    x[str(index)] = element1    
+    y[str(index)] = element2   
+    method[str(index)] = "Baseline"
+    index = index + 1
+
+for element1, element2 in zip(madry_arr2[:,0], madry_arr2[:,1]):    
+    x[str(index)] = element1    
+    y[str(index)] = element2    
+    method[str(index)] = "PGD"
+    index = index + 1
+
+for element1, element2 in zip(robust_arr2[:,0], robust_arr2[:,1]):    
+    x[str(index)] = element1    
+    y[str(index)] = element2    
+    method[str(index)] = "Provable"
+    index = index + 1
+
+for element1, element2 in zip(mix_arr2[:,0], mix_arr2[:,1]):    
+    x[str(index)] = element1    
+    y[str(index)] = element2    
+    method[str(index)] = "Mix"
+    index = index + 1
+
+jsonFile = {}
+jsonFile['method'] = method
+jsonFile['x'] = x
+jsonFile['y'] = y
+
+def baseline_error():    
+    al_baselines_len = len(jsonFile['x'])    
+    df = pd.DataFrame(jsonFile)
+
+    method_type = CategoricalDtype(
+        categories=['Baseline', 'PGD', 'Provable', 'Mix'], ordered=True)
+    df['method'] = df['method'].astype(method_type)
+
+    p = (
+            ggplot(df)
+            + aes(x='x', y='y', shape='method', color='method')
+            + geom_point(size=4, stroke=0)
+            + geom_line(size=1)
+            + scale_shape_discrete(name='Method')
+            + scale_color_brewer(
+                type='qual',
+                palette=2,
+                name='Method')
+            + xlab('Training Time (seconds)')
+            + ylab('Adversarial Error')
+            + theme(
+                # manually position legend, not used anymore
+                # legend_position=(0.70, 0.65),
+                # legend_background=element_rect(fill=(0, 0, 0, 0)),
+                aspect_ratio=0.8,
+            )         
+            + ggtitle("Baseline Error")   
+    )
+    fig_dir = '.'
+    p.save(os.path.join(fig_dir, 'baselineErr.pdf'))
+
+if __name__ == '__main__':
+    baseline_error()
+    # madry_error()
+    # robust_error()    
\ No newline at end of file