Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit b396bc8

Browse files
committed
added autoencoders, recurrent networks, MTL,etc
1 parent 6693c8c commit b396bc8

File tree

5 files changed

+1631
-23
lines changed

5 files changed

+1631
-23
lines changed

‎MultiTaskLearning.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from torch import optim
1616
from torchvision import datasets, transforms, models
1717
import matplotlib.pyplot as plt
18-
from sklearn import metrics
18+
# from sklearn import metrics
1919
%matplotlib inline
2020

2121
#%%
@@ -336,20 +336,23 @@ def forward(self, input_imgs):
336336
return prd_color, prd_gender, prd_region, prd_fighting, prd_alingment
337337

338338
def _set_freeze_(self, status):
339-
for p in self.features:
339+
for n,p in self.features.named_parameters():
340340
p.requires_grad = status
341+
# for m in self.features.children():
342+
# for p in m.parameters():
343+
# p.requires_grad=status
344+
341345

342346
def freeze_feature_layers(self):
343-
self._set_freeze_(True)
347+
self._set_freeze_(False)
344348

345349
def unfreeze_feature_layers(self):
346-
self._set_freeze_(False)
350+
self._set_freeze_(True)
347351

348352

349353
model = Resnet18_multiTaskNet(True, True)
350354
print(model)
351355

352-
353356
#%%
354357
# now lets train our model
355358
# we can have different optimizers for each head or a single one for the whole model

‎Pytorch_basic_introduction.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,9 @@
129129
# we have other options such as new_tensor, new_empty, new_full, new_zeros
130130
new_tensor_full = tensor_special.new_full(size=(2,2), fill_value=.3)
131131
print(new_tensor_full)
132-
133-
132+
# we create a new tensor with the same dtype and device
133+
new_tensor_newtensor = tensor_special.new_tensor(np.random.uniform(-1,1,size=(2,2)))
134+
print(new_tensor_newtensor)
134135

135136
#%%
136137
# now that we learnt how to create a new tensor, initialize a tensor, specify different dtypes, device, etc

‎autoencoders.py

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -880,7 +880,7 @@ def visualize_grid2(imgs, label, normalize=True):
880880
# https://jaan.io/what-is-variational-autoencoder-vae-tutorial/
881881
# https://www.youtube.com/watch?v=uaaqyVS9-rM
882882
# http://blog.shakirm.com/2015/10/machine-learning-trick-of-the-day-4-reparameterisation-tricks/
883-
883+
# https://www.reddit.com/r/MLQuestions/comments/dl7mya/a_few_more_questions_about_vaes/
884884

885885
# we'll also have an example concerning words(in NLP domain) and see how we can
886886
# leverage VAEs in that domain as well. for now lets see how we can implement this
@@ -895,7 +895,20 @@ def visualize_grid2(imgs, label, normalize=True):
895895
# will help you grasp one aspect very good!
896896
#
897897
# now lets define our VAE model .
898+
899+
898900
class VAE(nn.Module):
901+
902+
903+
def conv(self, in_dim, out_dim, k_size=3, stride=2, padding=1, batch_norm=True, bias=False):
904+
return nn.Sequential(nn.Conv2d(in_dim, out_dim, k_size, stride, padding, bias=bias),
905+
nn.BatchNorm2d(out_dim) if batch_norm else nn.Identity(),
906+
nn.ReLU())
907+
908+
def deconv(self, in_dim, out_dim, k_size=3, stride=2, padding=1, batch_norm=True, bias=False):
909+
return nn.Sequential(nn.ConvTranspose2d(in_dim, out_dim, k_size, stride, padding, bias=bias),
910+
nn.BatchNorm2d(out_dim) if batch_norm else nn.Identity(),
911+
nn.ReLU())
899912
def __init__(self, embedding_size=100):
900913
super().__init__()
901914

@@ -913,11 +926,26 @@ def __init__(self, embedding_size=100):
913926
# density.
914927
# We can sample from this distribution to get noisy values of the
915928
# representations z .
916-
929+
917930
self.fc1 = nn.Linear(28*28, 512)
918-
self.fc1_mu = nn.Linear(512, self.embedding_size) # mean
931+
self.encoder = nn.Sequential(self.conv(3,768),
932+
self.conv(768,512),
933+
self.conv(512,256),
934+
nn.MaxPool2d(2,2),#16
935+
self.conv(256,128),
936+
self.conv(128,64),
937+
nn.MaxPool2d(2,2),#8
938+
self.conv(64, 32),
939+
nn.MaxPool2d(2,2),#4
940+
self.conv(32, 16),
941+
nn.MaxPool2d(2,2),#2x2
942+
self.conv(16, 8),
943+
nn.MaxPool2d(2,2),#1x1
944+
)
945+
946+
self.fc1_mu = nn.Linear(8, self.embedding_size) # mean
919947
# we use log since we want to prevent getting negative variance
920-
self.fc1_std = nn.Linear(512, self.embedding_size) #logvariance
948+
self.fc1_std = nn.Linear(8, self.embedding_size) #logvariance
921949

922950
# our decoder will accept a randomly sampled vector using
923951
# our mu and std.
@@ -939,14 +967,23 @@ def __init__(self, embedding_size=100):
939967
# log-likelihood logpφ(x∣z) whose units are nats. This measure tells us how
940968
# effectively the decoder has learned to reconstruct an input image x given
941969
# its latent representation z.
942-
self.decoder = nn.Sequential( nn.Linear(self.embedding_size, 512),
943-
nn.ReLU(),
944-
nn.Linear(512, 28*28),
945-
# in normal situations we wouldnt use sigmoid
946-
# but since we want our values to be in [0,1]
947-
# we use sigmoid. for loss we will then have
948-
# to use, plain BCE (and specifically not BCEWithLogits)
949-
nn.Sigmoid())
970+
self.decoder = nn.Sequential(nn.Linear(self.embedding_size, 8*1*1),
971+
deconv(8, 768,kernel_size=4,stride=2),
972+
deconv(768,512,kernel_size=4,stride=2),
973+
deconv(512, 256 ,kernel_size=4,stride=2),
974+
deconv(256,128,kernel_size=4,stride=2),
975+
deconv(128,3,kernel_size=4,stride=2),
976+
# deconv(64,32,kernel_size=4,stride=2),
977+
# deconv(32,3,kernel_size=4,stride=2),
978+
nn.Sigmoid())
979+
# self.decoder = nn.Sequential( nn.Linear(self.embedding_size, 512),
980+
# nn.ReLU(),
981+
# nn.Linear(512, 28*28),
982+
# # in normal situations we wouldnt use sigmoid
983+
# # but since we want our values to be in [0,1]
984+
# # we use sigmoid. for loss we will then have
985+
# # to use, plain BCE (and specifically not BCEWithLogits)
986+
# nn.Sigmoid())
950987

951988

952989

@@ -1029,8 +1066,9 @@ def reparamtrization_trick(self, mu, logvar):
10291066
return mu + eps*std
10301067
#
10311068
def encode(self, input):
1032-
input = input.view(input.size(0), -1)
1033-
output = F.relu(self.fc1(input))
1069+
# input = input.view(input.size(0), -1)
1070+
# output = F.relu(self.fc1(input))
1071+
output = self.encoder(input)
10341072
# we dont use activations here
10351073
mu = self.fc1_mu(output)
10361074
log_var = self.fc1_std(output)
@@ -1186,6 +1224,11 @@ def loss_function(outputs, inputs, mu, logvar, reduction ='mean', use_mse = Fals
11861224
#%%
11871225
# now lets train :
11881226
epochs = 50
1227+
dataset_train = datasets.CIFAR10('cifar10', train=True, download=True,transform=transforms.ToTensor())
1228+
dataset_test = datasets.CIFAR10('cifar10', train=False, download=True,transform=transforms.ToTensor())
1229+
1230+
dataloader_train = torch.utils.data.DataLoader(dataset_train,batch_size=128,shuffle=True)
1231+
dataloader_test = torch.utils.data.DataLoader(dataset_test,batch_size=128,shuffle=False)
11891232

11901233
embeddingsize = 2
11911234
interval = 2000

‎basic_Pytorch_introduction_NeuralNetworks.py

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -918,6 +918,10 @@ def forward(self, inputs):
918918
# Note that torch.nn.ModuleDict.update with other unordered mapping types
919919
# (e.g., Python's plain dict) does not preserve the order of the merged mapping.
920920

921+
# important note: the forward pass for this wont happen!
922+
# becasue there is not a forwardpass implemented for moduledict
923+
# its just a container. Seqeuntial however, does support forward
924+
#
921925
class sequential_net6(nn.Module):
922926
def __init__(self):
923927
super().__init__()
@@ -932,7 +936,17 @@ def __init__(self):
932936
self.model['maxpool1'] = nn.MaxPool2d(2, 2)
933937

934938
def forward(self, inputs):
935-
return self.model(inputs)
939+
# simply doing :
940+
# return self.model(inputs)
941+
# wont work as moduledict dosnt implemet forward()
942+
# so you must manually forward all layers here. i.e do
943+
out = inputs
944+
# named_children() iterates over all higherlevel blocks/modules only
945+
# we dont use modules as biases and weights are also modules which dont
946+
# implement forward!!
947+
for n,m in self.model.named_children():
948+
out = m(out)
949+
return out
936950

937951
# as you can see, our foward function has become a one liner!
938952
# whats the benifit of doing this ?
@@ -1042,7 +1056,7 @@ def forward(self, x):
10421056

10431057
print(sequentialnet5)
10441058
print(sequentialnet6)
1045-
1059+
print(sequentialnet6(torch.rand(2,3,24,24)))
10461060
print(sequentialnet7)
10471061
print(sequentialnet8)
10481062
print(sequentialnet9)
@@ -1235,6 +1249,39 @@ def forward(self, input):
12351249
print(e)
12361250
print(acc_val)
12371251
print(acc_train)
1252+
#%%
1253+
# from torchvision import models
1254+
# import torch.nn as nn
1255+
1256+
# print(dir(models))
1257+
# # lets use resnet18! by setting pretrained = True, we'll also be down-
1258+
# #loading the imagenet weights.
1259+
# resnet18 = models.resnet18(pretrained=False)
1260+
# # lets print the model
1261+
# print(f'\nORIGINAL MODEL : \n{resnet18}\n')
1262+
# resnet18.fc = nn.Linear(resnet18.fc.in_features,10)
1263+
# # we can freeze all layers except the one(s) we want
1264+
# # in one go using named_children like this:
1265+
# for n,m in resnet18.named_children():
1266+
# if n!='fc':
1267+
# print(n)
1268+
# for param in m.parameters():
1269+
# param.requires_grad=False
1270+
1271+
# for n,m in resnet18.named_children():
1272+
# for param in m.parameters():
1273+
# print(f'{n} : {param.requires_grad}')
1274+
1275+
1276+
# #using named parameters:
1277+
# for name, param in resnet18.named_parameters():
1278+
# param.requires_grad = False
1279+
# print(f'name: {name} requires_grad : {param.requires_grad}')
1280+
1281+
# print('\nUnfreezing the new FC layer')
1282+
# for name, param in resnet18.fc.named_parameters():
1283+
# param.requires_grad = True
1284+
# print(f'name: {name} requires_grad : {param.requires_grad}')
12381285

12391286
#%%
12401287
# OK we are ready to see how fine-tuning works in Pytorch

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /