Commit 5578b8f0 authored by David Li's avatar David Li
Browse files

Results of CMSC498v Project

parent 93010927
__pycache__/
.ipynb_checkpoints/
*.log
*.pth
!models/*.pth
!models_scaled/*.pth
data/
.idea/*
examples/data/
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
......@@ -36,12 +36,19 @@ def select_model(m):
if __name__ == "__main__":
args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50,
opt='sgd', lr=0.05)
args.method = "madry"
# args.batch_size = 20
# args.test_batch_size = 20
args.prefix = "test_cifar/prefix"
# args.epochs = 35
print(args)
print("saving file to {}".format(args.prefix))
setproctitle.setproctitle(args.prefix)
train_log = open(args.prefix + "_train.log", "w")
test_log = open(args.prefix + "_test.log", "w")
full_test_log = open(args.prefix + "_full_test.log", "w")
train_loader, _ = pblm.cifar_loaders(args.batch_size)
_, test_loader = pblm.cifar_loaders(args.test_batch_size)
......@@ -129,7 +136,9 @@ if __name__ == "__main__":
test_log, args.verbose, args.real_time,
norm_type=args.norm_test, bounded_input=False,
**kwargs)
print("Test PGD Error:", err, file=full_test_log)
if err < best_err:
best_err = err
torch.save({
......
import waitGPU
# import setGPU
# waitGPU.wait(utilization=20, available_memory=10000, interval=60)
# waitGPU.wait(gpu_ids=[1,3], utilization=20, available_memory=10000, interval=60)
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import random
import setproctitle
import problems as pblm
from trainer import *
import math
import numpy
import datetime
def select_model(m):
if m == 'large':
# raise ValueError
model = pblm.cifar_model_large().cuda()
elif m == 'resnet':
model = pblm.cifar_model_resnet(N=args.resnet_N, factor=args.resnet_factor).cuda()
else:
model = pblm.cifar_model().cuda()
return model
if __name__ == "__main__":
for method in ["mix", "robust", "madry", "baseline"]:
waitGPU.wait(utilization=20, available_memory=6000, interval=2)
args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50,
opt='sgd', lr=0.05)
args.method = method
args.batch_size = 50
args.test_batch_size = 20
args.prefix = "test3/" + args.method + ""
args.epochs = 50
print(args)
print("saving file to {}".format(args.prefix))
setproctitle.setproctitle(args.prefix)
train_log = open(args.prefix + "_train.log", "w")
baseline_test_log = open(args.prefix + "_baseline_test.log", "w")
madry_test_log = open(args.prefix + "_madry_test.log", "w")
robust_test_log = open(args.prefix + "_robust_test.log", "w")
full_test_log = open(args.prefix + "_full_test.log", "w")
train_loader, _ = pblm.cifar_loaders(args.batch_size)
# For robust training since it requires more memory.
# This may be unfair
train_loader2, _ = pblm.cifar_loaders(20)
_, test_loader = pblm.cifar_loaders(args.test_batch_size)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
random.seed(0)
numpy.random.seed(0)
sampler_indices = []
model = [select_model(args.model)]
kwargs = pblm.args2kwargs(args)
best_err = 1
start_time = datetime.datetime.now()
for _ in range(0,args.cascade):
# if _ > 0:
# # reduce dataset to just uncertified examples
# print("Reducing dataset...")
# train_loader = sampler_robust_cascade(train_loader, model, args.epsilon,
# args.test_batch_size,
# norm_type=args.norm_test, **kwargs)
# if train_loader is None:
# print('No more examples, terminating')
# break
# sampler_indices.append(train_loader.sampler.indices)
#
# print("Adding a new model")
# model.append(select_model(args.model))
if args.opt == 'adam':
opt = optim.Adam(model[-1].parameters(), lr=args.lr)
elif args.opt == 'sgd':
opt = optim.SGD(model[-1].parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
raise ValueError("Unknown optimizer")
lr_scheduler = optim.lr_scheduler.StepLR(opt, step_size=10, gamma=0.5)
eps_schedule = np.linspace(args.starting_epsilon,
args.epsilon,
args.schedule_length)
for t in range(args.epochs):
lr_scheduler.step(epoch=max(t-len(eps_schedule), 0))
if t < len(eps_schedule) and args.starting_epsilon is not None:
epsilon = float(eps_schedule[t])
else:
epsilon = args.epsilon
# standard training
if args.method == 'baseline':
train_baseline(train_loader, model[0], opt, t, train_log,
args.verbose)
# madry training
elif args.method=='madry':
train_madry(train_loader, model[0], args.epsilon,
opt, t, train_log, args.verbose)
# robust training
elif args.method=='robust':
train_robust(train_loader2, model[0], opt, epsilon, t,
train_log, args.verbose, args.real_time,
norm_type=args.norm_train, bounded_input=False, clip_grad=1,
**kwargs)
elif args.method == "mix":
if t < 20:
train_madry(train_loader, model[0], args.epsilon,
opt, t, train_log, args.verbose)
else:
train_robust(train_loader2, model[0], opt, epsilon, t,
train_log, args.verbose, args.real_time,
norm_type=args.norm_train, bounded_input=False, clip_grad=1,
**kwargs)
time_diff = datetime.datetime.now() - start_time
print("Train Time Diff")
print(time_diff.total_seconds())
baseline_err = evaluate_baseline(test_loader, model[0], t, baseline_test_log,
args.verbose)
madry_err = evaluate_madry(test_loader, model[0], args.epsilon,
t, madry_test_log, args.verbose)
robust_err = evaluate_robust(test_loader, model[0], args.epsilon,
t, robust_test_log, args.verbose, args.real_time,
norm_type=args.norm_test, bounded_input=False, **kwargs)
err = robust_err
start_time = datetime.datetime.now() - time_diff
print(time_diff.total_seconds(), baseline_err, madry_err, robust_err, file=full_test_log)
if err < best_err:
best_err = err
torch.save({
'state_dict' : [m.state_dict() for m in model],
'err' : best_err,
'epoch' : t,
'sampler_indices' : sampler_indices
}, args.prefix + "_best.pth")
torch.save({
'state_dict': [m.state_dict() for m in model],
'err' : err,
'epoch' : t,
'sampler_indices' : sampler_indices
}, args.prefix + "_checkpoint.pth")
if time_diff.total_seconds() > 8000:
break
import waitGPU
# import setGPU
# waitGPU.wait(utilization=20, available_memory=10000, interval=60)
# waitGPU.wait(gpu_ids=[1,3], utilization=20, available_memory=10000, interval=60)
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import random
import setproctitle
import problems as pblm
from trainer import *
import math
import numpy
import datetime
def select_model(m):
if m == 'large':
# raise ValueError
model = pblm.cifar_model_large().cuda()
elif m == 'resnet':
model = pblm.cifar_model_resnet(N=args.resnet_N, factor=args.resnet_factor).cuda()
else:
model = pblm.cifar_model().cuda()
return model
if __name__ == "__main__":
for method in ["mix"]:
waitGPU.wait(utilization=20, available_memory=6000, interval=2)
args = pblm.argparser(epsilon = 0.0347, starting_epsilon=0.001, batch_size = 50,
opt='sgd', lr=0.05)
args.method = method
args.batch_size = 50
args.test_batch_size = 20
args.prefix = "test6/" + args.method + ""
args.epochs = 50
print(args)
print("saving file to {}".format(args.prefix))
setproctitle.setproctitle(args.prefix)
train_log = open(args.prefix + "_train.log", "w")
baseline_test_log = open(args.prefix + "_baseline_test.log", "w")
madry_test_log = open(args.prefix + "_madry_test.log", "w")
robust_test_log = open(args.prefix + "_robust_test.log", "w")
full_test_log = open(args.prefix + "_full_test.log", "w")
train_loader, _ = pblm.cifar_loaders(args.batch_size)
# For robust training since it requires more memory.
# This may be unfair
train_loader2, _ = pblm.cifar_loaders(20)
_, test_loader = pblm.cifar_loaders(args.test_batch_size)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
random.seed(0)
numpy.random.seed(0)
sampler_indices = []
model = [select_model(args.model)]
kwargs = pblm.args2kwargs(args)
best_err = 1
start_time = datetime.datetime.now()
for _ in range(0,args.cascade):
# if _ > 0:
# # reduce dataset to just uncertified examples
# print("Reducing dataset...")
# train_loader = sampler_robust_cascade(train_loader, model, args.epsilon,
# args.test_batch_size,
# norm_type=args.norm_test, **kwargs)
# if train_loader is None:
# print('No more examples, terminating')
# break
# sampler_indices.append(train_loader.sampler.indices)
#
# print("Adding a new model")
# model.append(select_model(args.model))
if args.opt == 'adam':
opt = optim.Adam(model[-1].parameters(), lr=args.lr)
elif args.opt == 'sgd':
opt = optim.SGD(model[-1].parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
raise ValueError("Unknown optimizer")
lr_scheduler = optim.lr_scheduler.StepLR(opt, step_size=10, gamma=0.5)
eps_schedule = np.linspace(args.starting_epsilon,
args.epsilon,
args.schedule_length)
for t in range(args.epochs):
lr_scheduler.step(epoch=max(t-len(eps_schedule), 0))
if t < len(eps_schedule) and args.starting_epsilon is not None:
epsilon = float(eps_schedule[t])
else:
epsilon = args.epsilon
# standard training
if args.method == 'baseline':
train_baseline(train_loader, model[0], opt, t, train_log,
args.verbose)
# madry training
elif args.method=='madry':
train_madry(train_loader, model[0], args.epsilon,
opt, t, train_log, args.verbose)
# robust training
elif args.method=='robust':
train_robust(train_loader2, model[0], opt, epsilon, t,
train_log, args.verbose, args.real_time,
norm_type=args.norm_train, bounded_input=False, clip_grad=1,
**kwargs)
elif args.method == "mix":
if t < 20:
train_madry(train_loader, model[0], 0.2,
opt, t, train_log, args.verbose)
else:
train_robust(train_loader2, model[0], opt, args.epsilon, t,
train_log, args.verbose, args.real_time,
norm_type=args.norm_train, bounded_input=False, clip_grad=1,
**kwargs)
time_diff = datetime.datetime.now() - start_time
print("Train Time Diff")
print(time_diff.total_seconds())
baseline_err = evaluate_baseline(test_loader, model[0], t, baseline_test_log,
args.verbose)
madry_err = evaluate_madry(test_loader, model[0], args.epsilon,
t, madry_test_log, args.verbose)
robust_err = evaluate_robust(test_loader, model[0], args.epsilon,
t, robust_test_log, args.verbose, args.real_time,
norm_type=args.norm_test, bounded_input=False, **kwargs)
err = robust_err
start_time = datetime.datetime.now() - time_diff
print(time_diff.total_seconds(), baseline_err, madry_err, robust_err, file=full_test_log)
if err < best_err:
best_err = err
torch.save({
'state_dict' : [m.state_dict() for m in model],
'err' : best_err,
'epoch' : t,
'sampler_indices' : sampler_indices
}, args.prefix + "_best.pth")
torch.save({
'state_dict': [m.state_dict() for m in model],
'err' : err,
'epoch' : t,
'sampler_indices' : sampler_indices
}, args.prefix + "_checkpoint.pth")
if time_diff.total_seconds() > 8000:
break
import re
import numpy as np
import matplotlib.pyplot as plt
import os
import json
import pandas as pd
from pandas.api.types import CategoricalDtype
from plotnine import ggplot, aes,\
geom_point, geom_line,\
scale_color_discrete, scale_shape_discrete, scale_color_brewer,\
xlab, ylab, theme, element_rect, facet_grid, ggtitle
to_digit = re.compile("tensor\(([\d\.]*),")
remove_nl = re.compile("\n")
# Test 1
baseline_file = open("test1/baseline_full_test.log", "r")
baseline_arr = baseline_file.readlines()
baseline_arr2 = []
for i, val in enumerate(baseline_arr):
filteredstr = re.sub("\n", "", val)
filteredstr = re.sub("\,\s", ",", filteredstr)
baseline_arr2.append(filteredstr.split(" "))
baseline_arr2[i][1] = re.match("tensor\(([\d\.]*),", baseline_arr2[i][1]).group(1)
baseline_arr2[i][2] = re.match("tensor\(([\d\.]*),", baseline_arr2[i][2]).group(1)
# [time, baseline_err, madry_err, robust_err]
baseline_arr2 = np.array(baseline_arr2, dtype=float)
madry_file = open("test1/madry_full_test.log", "r")
madry_arr = madry_file.readlines()
madry_arr2 = []
for i, val in enumerate(madry_arr):
filteredstr = re.sub("\n", "", val)
filteredstr = re.sub("\,\s", ",", filteredstr)
madry_arr2.append(filteredstr.split(" "))
madry_arr2[i][1] = re.match("tensor\(([\d\.]*),", madry_arr2[i][1]).group(1)
madry_arr2[i][2] = re.match("tensor\(([\d\.]*),", madry_arr2[i][2]).group(1)
# [time, madry_err, madry_err, robust_err]
madry_arr2 = np.array(madry_arr2, dtype=float)
robust_file = open("test1/robust_full_test.log", "r")
robust_arr = robust_file.readlines()
robust_arr2 = []
for i, val in enumerate(robust_arr):
filteredstr = re.sub("\n", "", val)
filteredstr = re.sub("\,\s", ",", filteredstr)
robust_arr2.append(filteredstr.split(" "))
robust_arr2[i][1] = re.match("tensor\(([\d\.]*),", robust_arr2[i][1]).group(1)
robust_arr2[i][2] = re.match("tensor\(([\d\.]*),", robust_arr2[i][2]).group(1)
# [time, robust_err, madry_err, robust_err]
robust_arr2 = np.array(robust_arr2, dtype=float)
mix_file = open("test1/mix_full_test.log", "r")
mix_arr = mix_file.readlines()
mix_arr2 = []
for i, val in enumerate(mix_arr):
filteredstr = re.sub("\n", "", val)
filteredstr = re.sub("\,\s", ",", filteredstr)
mix_arr2.append(filteredstr.split(" "))
mix_arr2[i][1] = re.match("tensor\(([\d\.]*),", mix_arr2[i][1]).group(1)
mix_arr2[i][2] = re.match("tensor\(([\d\.]*),", mix_arr2[i][2]).group(1)
# [time, mix_err, madry_err, robust_err]
mix_arr2 = np.array(mix_arr2, dtype=float)
# plt.plot(baseline_arr2[:,0], baseline_arr2[:,1], label='baseline')
# plt.plot(madry_arr2[:,0], madry_arr2[:,1], label='madry')
# plt.plot(robust_arr2[:,0], robust_arr2[:,1], label='robust')
# plt.plot(mix_arr2[:,0], mix_arr2[:,1], label='mix')
# print(baseline_arr2[:,0])
# print(madry_arr2[:,0])
# plt.xlabel('Time (seconds)')
# plt.ylabel('Error')
# plt.title("Baseline Error")
# plt.legend()
# plt.savefig("test1/baselineErr.png")
# plt.clf()
# plt.plot(baseline_arr2[:,0], baseline_arr2[:,2], label='baseline')
# plt.plot(madry_arr2[:,0], madry_arr2[:,2], label='madry')
# plt.plot(robust_arr2[:,0], robust_arr2[:,2], label='robust')
# plt.plot(mix_arr2[:,0], mix_arr2[:,2], label='mix')
# print(baseline_arr2[:,0])
# print(madry_arr2[:,0])
# plt.xlabel('Time (seconds)')
# plt.ylabel('Error')
# plt.title("Madry (PGD) Error")
# plt.legend()
# plt.savefig("test1/madryErr.png")
# plt.clf()
# plt.plot(baseline_arr2[:,0], baseline_arr2[:,3], label='baseline')
# plt.plot(madry_arr2[:,0], madry_arr2[:,3], label='madry')
# plt.plot(robust_arr2[:,0], robust_arr2[:,3], label='robust')
# plt.plot(mix_arr2[:,0], mix_arr2[:,3], label='mix')
# print(baseline_arr2[:,0])
# print(madry_arr2[:,0])
# plt.xlabel('Time (seconds)')
# plt.ylabel('Error')
# plt.title("Robust Error")
# plt.legend()
# plt.savefig("test1/robustErr.png")
# plt.clf()
x = {}
y = {}
method = {}
index = 0
for element1, element2 in zip(baseline_arr2[:,0], baseline_arr2[:,1]):
x[str(index)] = element1
y[str(index)] = element2
method[str(index)] = "Baseline"
index = index + 1
for element1, element2 in zip(madry_arr2[:,0], madry_arr2[:,1]):
x[str(index)] = element1
y[str(index)] = element2
method[str(index)] = "PGD"
index = index + 1
for element1, element2 in zip(robust_arr2[:,0], robust_arr2[:,1]):
x[str(index)] = element1
y[str(index)] = element2
method[str(index)] = "Provable"
index = index + 1
for element1, element2 in zip(mix_arr2[:,0], mix_arr2[:,1]):
x[str(index)] = element1
y[str(index)] = element2
method[str(index)] = "Mix"
index = index + 1
jsonFile = {}
jsonFile['method'] = method
jsonFile['x'] = x
jsonFile['y'] = y
def baseline_error():
al_baselines_len = len(jsonFile['x'])
df = pd.DataFrame(jsonFile)
method_type = CategoricalDtype(
categories=['Baseline', 'PGD', 'Provable', 'Mix'], ordered=True)
df['method'] = df['method'].astype(method_type)
p = (
ggplot(df)
+ aes(x='x', y='y', shape='method', color='method')
+ geom_point(size=4, stroke=0)
+ geom_line(size=1)
+ scale_shape_discrete(name='Method')
+ scale_color_brewer(
type='qual',
palette=2,
name='Method')
+ xlab('Training Time (seconds)')
+ ylab('Adversarial Error')
+ theme(
# manually position legend, not used anymore
# legend_position=(0.70, 0.65),
# legend_background=element_rect(fill=(0, 0, 0, 0)),
aspect_ratio=0.8,
)
+ ggtitle("Baseline Error")
)
fig_dir = '.'
p.save(os.path.join(fig_dir, 'baselineErr.pdf'))
if __name__ == '__main__':
baseline_error()
# madry_error()
# robust_error()
\ No newline at end of file