Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit f0abda4

Browse files
Big ugly commit, may Linus forgive me.
1 parent f260e1a commit f0abda4

29 files changed

+1401
-287
lines changed

‎hyperfind.py‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import argparse
22
import copy
33
import os
4+
import statistics
45

56
import ray
67
import yaml
@@ -22,6 +23,9 @@ def parse_args():
2223
parser.add_argument("-per", "--gpu-percent", type=float, default=0.5)
2324
parser.add_argument("-topn", "--topn", default=5, type=int)
2425
parser.add_argument("-earlystop", default="ucir", type=str)
26+
parser.add_argument("-options", "--options", default=None, nargs="+")
27+
parser.add_argument("-threads", default=2, type=int)
28+
parser.add_argument("-resume", default=False, action="store_true")
2529

2630
return parser.parse_args()
2731

@@ -31,15 +35,17 @@ def train_func(config, reporter):
3135
train_args.update(config)
3236

3337
train_args["device"] = [0]
34-
train_args["threads"] = 2
35-
train_args["logging"] = "critical"
38+
train_args["logging"] = "warning"
3639
train_args["no_progressbar"] = True
3740

38-
for i, (avg_inc_acc, last_acc, _) in enumerate(inclearn.train.train(train_args)):
39-
last_acc = last_acc * 100
41+
all_acc = []
42+
for i, (avg_inc_acc, last_acc, _, is_last) in enumerate(inclearn.train.train(train_args)):
43+
if is_last:
44+
all_acc.append(avg_inc_acc)
4045

41-
reporter(avg_inc_acc=avg_inc_acc)
42-
return avg_inc_acc
46+
total_avg_inc_acc = statistics.mean(all_acc)
47+
reporter(avg_inc_acc=total_avg_inc_acc)
48+
return total_avg_inc_acc
4349

4450

4551
def _get_abs_path(path):
@@ -61,6 +67,7 @@ def analyse_ray_dump(ray_directory, topn):
6167
elif col == "avg_inc_acc":
6268
result_index = index
6369

70+
print("Ray config: {}".format(ray_directory))
6471
print("Best Config:")
6572
print(
6673
"avg_inc_acc: {} with {}.".format(
@@ -90,7 +97,6 @@ def _get_line_results(df, row_index, mapping):
9097
for col, index in mapping.items():
9198
if col.startswith("var:"):
9299
col = col[4:]
93-
94100
results[col] = df.iloc[row_index][index]
95101
return results
96102

@@ -109,7 +115,7 @@ def set_seen_gpus(gpus):
109115
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(gpus)
110116

111117

112-
def get_tune_config(tune_options):
118+
def get_tune_config(tune_options, options_files):
113119
with open(tune_options) as f:
114120
options = yaml.load(f, Loader=yaml.FullLoader)
115121

@@ -120,6 +126,10 @@ def get_tune_config(tune_options):
120126
else:
121127
config[k.replace("var:", "")] = tune.grid_search(v)
122128

129+
if options_files is not None:
130+
print("Options files: {}".format(options_files))
131+
config["options"] = [os.path.realpath(op) for op in options_files]
132+
123133
return config
124134

125135

@@ -129,7 +139,8 @@ def main():
129139
set_seen_gpus(args.gpus)
130140

131141
if args.tune is not None:
132-
config = get_tune_config(args.tune)
142+
config = get_tune_config(args.tune, args.options)
143+
config["threads"] = args.threads
133144
ray.init()
134145
tune.run(
135146
train_func,
@@ -140,11 +151,15 @@ def main():
140151
"cpu": 2,
141152
"gpu": args.gpu_percent
142153
},
143-
local_dir=args.ray_directory
154+
local_dir=args.ray_directory,
155+
resume=args.resume
144156
)
145157

146158
args.ray_directory = os.path.join(args.ray_directory, args.tune.rstrip("/").split("/")[-1])
147159

160+
if args.tune is not None:
161+
print("\n\n", args.tune, "\n\n")
162+
148163
if args.ray_directory is not None:
149164
best_config = analyse_ray_dump(_get_abs_path(args.ray_directory), args.topn)
150165

‎inclearn/convnet/my_resnet.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,17 @@
22
33
https://github.com/srebuffi/iCaRL/blob/master/iCaRL-TheanoLasagne/utils_cifar100.py
44
"""
5+
import logging
6+
57
import torch
68
import torch.nn as nn
79
import torch.nn.functional as F
810
from torch.nn import init
911

1012
from inclearn.lib import pooling
1113

14+
logger = logging.getLogger(__name__)
15+
1216

1317
class DownsampleStride(nn.Module):
1418

@@ -189,7 +193,7 @@ def __init__(
189193
raise ValueError("Unused kwargs: {}.".format(kwargs))
190194

191195
self.all_attentions = all_attentions
192-
print("Downsampling type", downsampling)
196+
logger.info("Downsampling type {}".format(downsampling))
193197
self._downsampling_type = downsampling
194198
self.last_relu = last_relu
195199

@@ -255,9 +259,7 @@ def _make_layer(self, Block, planes, increase_dim=False, n=None):
255259
planes = 2 * planes
256260

257261
for i in range(n):
258-
layers.append(
259-
Block(planes, last_relu=False, downsampling=self._downsampling_type)
260-
)
262+
layers.append(Block(planes, last_relu=False, downsampling=self._downsampling_type))
261263

262264
return Stage(layers, block_relu=self.last_relu)
263265

@@ -282,11 +284,7 @@ def forward(self, x):
282284
else:
283285
attentions = [feats_s1[-1], feats_s2[-1], feats_s3[-1], x]
284286

285-
return {
286-
"raw_features": raw_features,
287-
"features": features,
288-
"attention": attentions
289-
}
287+
return {"raw_features": raw_features, "features": features, "attention": attentions}
290288

291289
def end_features(self, x):
292290
x = self.pool(x)

‎inclearn/lib/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# flake8: noqa
22
from . import (
3-
calibration, callbacks, data, factory, herding, loops, losses, metrics, network, pooling,
4-
results_utils, schedulers, utils, vizualization
3+
calibration, callbacks, data, distance, factory, herding, loops, losses, metrics, network,
4+
pooling, results_utils, schedulers, utils, vizualization
55
)

‎inclearn/lib/data/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# flake8: noqa
22
from .datasets import *
3+
from .download import *
34
from .incdataset import *
45
from .samplers import *
56
from .weights import *

‎inclearn/lib/data/download.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import gzip
2+
import io
3+
import logging
4+
import os
5+
import urllib.request
6+
7+
logger = logging.getLogger(__name__)
8+
9+
URLS = {
10+
"googlenews":
11+
"https://github.com/eyaler/word2vec-slim/raw/master/GoogleNews-vectors-negative300-SLIM.bin.gz"
12+
}
13+
14+
15+
def fetch_word_embeddings(folder, name="googlenews"):
16+
if name == "googlenews":
17+
return _fetch_googlenews_word2vec(folder)
18+
raise ValueError("Unknown embedding type {}.".format(name))
19+
20+
21+
def _fetch_googlenews_word2vec(folder):
22+
output_file = os.path.join(folder, "googlenews.bin")
23+
24+
if os.path.exists(output_file):
25+
logger.info("googlenews.bin already exist! Skipping.")
26+
return output_file
27+
28+
response = urllib.request.urlopen(URLS["googlenews"])
29+
30+
logger.info("Downloading googlenews...")
31+
compressed_file = io.BytesIO(response.read())
32+
33+
logger.info("Decompressing googlenews...")
34+
decompressed_file = gzip.GzipFile(fileobj=compressed_file, mode='rb')
35+
36+
with open(output_file, 'wb+') as f:
37+
f.write(decompressed_file.read())
38+
39+
return output_file

‎inclearn/lib/data/incdataset.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
from torch.utils.data import DataLoader
88
from torchvision import transforms
99

10-
from .datasets import (ImageNet100, ImageNet100UCIR, ImageNet1000,
11-
TinyImageNet200, iCIFAR10, iCIFAR100)
10+
from .datasets import (
11+
ImageNet100, ImageNet100UCIR, ImageNet1000, TinyImageNet200, iCIFAR10, iCIFAR100
12+
)
1213

1314
logger = logging.getLogger(__name__)
1415

@@ -49,7 +50,8 @@ def __init__(
4950
sampler_config=None,
5051
data_path="data",
5152
class_order=None,
52-
dataset_transforms=None
53+
dataset_transforms=None,
54+
all_test_classes=False,
5355
):
5456
datasets = _get_datasets(dataset_name)
5557
self._setup_data(
@@ -80,6 +82,7 @@ def __init__(
8082
self._onehot = onehot
8183
self._sampler = sampler
8284
self._sampler_config = sampler_config
85+
self._all_test_classes = all_test_classes
8386

8487
@property
8588
def n_tasks(self):
@@ -97,7 +100,13 @@ def new_task(self, memory=None, memory_val=None):
97100
x_val, y_val = self._select(
98101
self.data_val, self.targets_val, low_range=min_class, high_range=max_class
99102
)
100-
x_test, y_test = self._select(self.data_test, self.targets_test, high_range=max_class)
103+
if self._all_test_classes:
104+
logger.info("Testing on all classes!")
105+
x_test, y_test = self._select(
106+
self.data_test, self.targets_test, high_range=sum(self.increments)
107+
)
108+
else:
109+
x_test, y_test = self._select(self.data_test, self.targets_test, high_range=max_class)
101110

102111
if self._onehot:
103112

@@ -108,12 +117,12 @@ def to_onehot(x):
108117
y_train = to_onehot(y_train)
109118

110119
if memory is not None:
111-
print("Set memory of size: {}.".format(memory[0].shape[0]))
120+
logger.info("Set memory of size: {}.".format(memory[0].shape[0]))
112121
x_train, y_train, train_memory_flags = self._add_memory(x_train, y_train, *memory)
113122
else:
114123
train_memory_flags = np.zeros((x_train.shape[0],))
115124
if memory_val is not None:
116-
print("Set validation memory of size: {}.".format(memory_val[0].shape[0]))
125+
logger.info("Set validation memory of size: {}.".format(memory_val[0].shape[0]))
117126
x_val, y_val, val_memory_flags = self._add_memory(x_val, y_val, *memory_val)
118127
else:
119128
val_memory_flags = np.zeros((x_val.shape[0],))
@@ -191,7 +200,7 @@ def get_custom_loader(
191200
data = np.concatenate(data)
192201
targets = np.concatenate(targets)
193202

194-
if memory is not None:
203+
if memory is not Noneor (isinstance(memory, tuple) andmemory[0] isNone):
195204
if len(data) > 0:
196205
data, targets, memory_flags = self._add_memory(data, targets, *memory)
197206
else:
@@ -283,7 +292,7 @@ def _setup_data(
283292
elif dataset.class_order is not None:
284293
order = dataset.class_order
285294

286-
print("Dataset {}: class ordering: {}.".format(dataset, order))
295+
logger.info("Dataset {}: class ordering: {}.".format(dataset, order))
287296

288297
self.class_order.append(order)
289298

‎inclearn/lib/distance.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import torch
2+
from torch.nn import functional as F
3+
4+
5+
def squared_euclidian_distance(a, b):
6+
return torch.cdist(a, b)**2
7+
8+
9+
def cosine_similarity(a, b):
10+
return torch.mm(F.normalize(a, p=2, dim=-1), F.normalize(b, p=2, dim=-1).T)
11+
12+
13+
def stable_cosine_distance(a, b, squared=True):
14+
"""Computes the pairwise distance matrix with numerical stability."""
15+
mat = torch.cat([a, b])
16+
17+
pairwise_distances_squared = torch.add(
18+
mat.pow(2).sum(dim=1, keepdim=True).expand(mat.size(0), -1),
19+
torch.t(mat).pow(2).sum(dim=0, keepdim=True).expand(mat.size(0), -1)
20+
) - 2 * (torch.mm(mat, torch.t(mat)))
21+
22+
# Deal with numerical inaccuracies. Set small negatives to zero.
23+
pairwise_distances_squared = torch.clamp(pairwise_distances_squared, min=0.0)
24+
25+
# Get the mask where the zero distances are at.
26+
error_mask = torch.le(pairwise_distances_squared, 0.0)
27+
28+
# Optionally take the sqrt.
29+
if squared:
30+
pairwise_distances = pairwise_distances_squared
31+
else:
32+
pairwise_distances = torch.sqrt(pairwise_distances_squared + error_mask.float() * 1e-16)
33+
34+
# Undo conditionally adding 1e-16.
35+
pairwise_distances = torch.mul(pairwise_distances, (error_mask == False).float())
36+
37+
# Explicitly set diagonals to zero.
38+
mask_offdiagonals = 1 - torch.eye(*pairwise_distances.size(), device=pairwise_distances.device)
39+
pairwise_distances = torch.mul(pairwise_distances, mask_offdiagonals)
40+
41+
return pairwise_distances[:a.shape[0], a.shape[0]:]

‎inclearn/lib/factory.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ def get_model(args):
5656
"bic": models.BiC,
5757
"ucir": models.UCIR,
5858
"still": models.STILL,
59-
"lwm": models.LwM
59+
"lwm": models.LwM,
60+
"zil": models.ZIL
6061
}
6162

6263
model = args["model"].lower()
@@ -85,7 +86,8 @@ def get_data(args, class_order=None):
8586
data_path=args["data_path"],
8687
class_order=class_order,
8788
seed=args["seed"],
88-
dataset_transforms=args.get("dataset_transforms", {})
89+
dataset_transforms=args.get("dataset_transforms", {}),
90+
all_test_classes=args.get("all_test_classes", False)
8991
)
9092

9193

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /