merging code to this guy since it's getting linked alot

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Alec Radford
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

faces/README.md

@@ -0,0 +1,15 @@
+Modify data_dir in lib/config.py to point to directory with faces hdf5.
+
+*Currently this data file is not released due to size/data restrictions.* 
+
+Run train_uncond_dcgan.py to train face model from paper. It will create a few folders and save training info, model parameters, and samples periodically. Should be ~ 12 hours/overnight.
+
+Libs you'll need installed/configured to run it:
+- theano
+- cudnn
+- fuel/h5py
+- sklearn
+- numpy
+- scipy
+- matplotlib
+- tqdm

faces/load.py

@@ -0,0 +1,31 @@
+import sys
+sys.path.append('..')
+
+import os
+from fuel.datasets.hdf5 import H5PYDataset
+from fuel.schemes import ShuffledScheme, SequentialScheme
+from fuel.streams import DataStream
+
+from lib.config import data_dir
+
+def faces(ntrain=None, nval=None, ntest=None, batch_size=128):
+    path = os.path.join(data_dir, 'faces_364293_128px.hdf5')
+    tr_data = H5PYDataset(path, which_sets=('train',))
+    te_data = H5PYDataset(path, which_sets=('test',))
+
+    if ntrain is None:
+        ntrain = tr_data.num_examples
+    if ntest is None:
+        ntest = te_data.num_examples
+    if nval is None:
+        nval = te_data.num_examples
+
+    tr_scheme = ShuffledScheme(examples=ntrain, batch_size=batch_size)
+    tr_stream = DataStream(tr_data, iteration_scheme=tr_scheme)
+
+    te_scheme = SequentialScheme(examples=ntest, batch_size=batch_size)
+    te_stream = DataStream(te_data, iteration_scheme=te_scheme)
+
+    val_scheme = SequentialScheme(examples=nval, batch_size=batch_size)
+    val_stream = DataStream(tr_data, iteration_scheme=val_scheme)
+    return tr_data, te_data, tr_stream, val_stream, te_stream

faces/train_uncond_dcgan.py

@@ -0,0 +1,228 @@
+import sys
+sys.path.append('..')
+
+import os
+import json
+from time import time
+import numpy as np
+from tqdm import tqdm
+from matplotlib import pyplot as plt
+from sklearn.externals import joblib
+
+import theano
+import theano.tensor as T
+from theano.sandbox.cuda.dnn import dnn_conv
+
+from lib import activations
+from lib import updates
+from lib import inits
+from lib.vis import color_grid_vis
+from lib.rng import py_rng, np_rng
+from lib.ops import batchnorm, conv_cond_concat, deconv, dropout, l2normalize
+from lib.metrics import nnc_score, nnd_score
+from lib.theano_utils import floatX, sharedX
+from lib.data_utils import OneHot, shuffle, iter_data, center_crop, patch
+
+from load import faces
+
+def transform(X):
+    X = [center_crop(x, npx) for x in X]
+    return floatX(X).transpose(0, 3, 1, 2)/127.5 - 1.
+
+def inverse_transform(X):
+    X = (X.reshape(-1, nc, npx, npx).transpose(0, 2, 3, 1)+1.)/2.
+    return X
+
+k = 1             # # of discrim updates for each gen update
+l2 = 1e-5         # l2 weight decay
+nvis = 196        # # of samples to visualize during training
+b1 = 0.5          # momentum term of adam
+nc = 3            # # of channels in image
+nbatch = 128      # # of examples in batch
+npx = 64          # # of pixels width/height of images
+nz = 100          # # of dim for Z
+ngf = 128         # # of gen filters in first conv layer
+ndf = 128         # # of discrim filters in first conv layer
+nx = npx*npx*nc   # # of dimensions in X
+niter = 25        # # of iter at starting learning rate
+niter_decay = 0   # # of iter to linearly decay learning rate to zero
+lr = 0.0002       # initial learning rate for adam
+ntrain = 350000   # # of examples to train on
+
+tr_data, te_data, tr_stream, val_stream, te_stream = faces(ntrain=ntrain)
+
+tr_handle = tr_data.open()
+vaX, = tr_data.get_data(tr_handle, slice(0, 10000))
+vaX = transform(vaX)
+
+desc = 'uncond_dcgan'
+model_dir = 'models/%s'%desc
+samples_dir = 'samples/%s'%desc
+if not os.path.exists('logs/'):
+    os.makedirs('logs/')
+if not os.path.exists(model_dir):
+    os.makedirs(model_dir)
+if not os.path.exists(samples_dir):
+    os.makedirs(samples_dir)
+
+relu = activations.Rectify()
+sigmoid = activations.Sigmoid()
+lrelu = activations.LeakyRectify()
+tanh = activations.Tanh()
+bce = T.nnet.binary_crossentropy
+
+gifn = inits.Normal(scale=0.02)
+difn = inits.Normal(scale=0.02)
+gain_ifn = inits.Normal(loc=1., scale=0.02)
+bias_ifn = inits.Constant(c=0.)
+
+gw  = gifn((nz, ngf*8*4*4), 'gw')
+gg = gain_ifn((ngf*8*4*4), 'gg')
+gb = bias_ifn((ngf*8*4*4), 'gb')
+gw2 = gifn((ngf*8, ngf*4, 5, 5), 'gw2')
+gg2 = gain_ifn((ngf*4), 'gg2')
+gb2 = bias_ifn((ngf*4), 'gb2')
+gw3 = gifn((ngf*4, ngf*2, 5, 5), 'gw3')
+gg3 = gain_ifn((ngf*2), 'gg3')
+gb3 = bias_ifn((ngf*2), 'gb3')
+gw4 = gifn((ngf*2, ngf, 5, 5), 'gw4')
+gg4 = gain_ifn((ngf), 'gg4')
+gb4 = bias_ifn((ngf), 'gb4')
+gwx = gifn((ngf, nc, 5, 5), 'gwx')
+
+dw  = difn((ndf, nc, 5, 5), 'dw')
+dw2 = difn((ndf*2, ndf, 5, 5), 'dw2')
+dg2 = gain_ifn((ndf*2), 'dg2')
+db2 = bias_ifn((ndf*2), 'db2')
+dw3 = difn((ndf*4, ndf*2, 5, 5), 'dw3')
+dg3 = gain_ifn((ndf*4), 'dg3')
+db3 = bias_ifn((ndf*4), 'db3')
+dw4 = difn((ndf*8, ndf*4, 5, 5), 'dw4')
+dg4 = gain_ifn((ndf*8), 'dg4')
+db4 = bias_ifn((ndf*8), 'db4')
+dwy = difn((ndf*8*4*4, 1), 'dwy')
+
+gen_params = [gw, gg, gb, gw2, gg2, gb2, gw3, gg3, gb3, gw4, gg4, gb4, gwx]
+discrim_params = [dw, dw2, dg2, db2, dw3, dg3, db3, dw4, dg4, db4, dwy]
+
+def gen(Z, w, g, b, w2, g2, b2, w3, g3, b3, w4, g4, b4, wx):
+    h = relu(batchnorm(T.dot(Z, w), g=g, b=b))
+    h = h.reshape((h.shape[0], ngf*8, 4, 4))
+    h2 = relu(batchnorm(deconv(h, w2, subsample=(2, 2), border_mode=(2, 2)), g=g2, b=b2))
+    h3 = relu(batchnorm(deconv(h2, w3, subsample=(2, 2), border_mode=(2, 2)), g=g3, b=b3))
+    h4 = relu(batchnorm(deconv(h3, w4, subsample=(2, 2), border_mode=(2, 2)), g=g4, b=b4))
+    x = tanh(deconv(h4, wx, subsample=(2, 2), border_mode=(2, 2)))
+    return x
+
+def discrim(X, w, w2, g2, b2, w3, g3, b3, w4, g4, b4, wy):
+    h = lrelu(dnn_conv(X, w, subsample=(2, 2), border_mode=(2, 2)))
+    h2 = lrelu(batchnorm(dnn_conv(h, w2, subsample=(2, 2), border_mode=(2, 2)), g=g2, b=b2))
+    h3 = lrelu(batchnorm(dnn_conv(h2, w3, subsample=(2, 2), border_mode=(2, 2)), g=g3, b=b3))
+    h4 = lrelu(batchnorm(dnn_conv(h3, w4, subsample=(2, 2), border_mode=(2, 2)), g=g4, b=b4))
+    h4 = T.flatten(h4, 2)
+    y = sigmoid(T.dot(h4, wy))
+    return y
+
+X = T.tensor4()
+Z = T.matrix()
+
+gX = gen(Z, *gen_params)
+
+p_real = discrim(X, *discrim_params)
+p_gen = discrim(gX, *discrim_params)
+
+d_cost_real = bce(p_real, T.ones(p_real.shape)).mean()
+d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean()
+g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean()
+
+d_cost = d_cost_real + d_cost_gen
+g_cost = g_cost_d
+
+cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen]
+
+lrt = sharedX(lr)
+d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2))
+g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2))
+d_updates = d_updater(discrim_params, d_cost)
+g_updates = g_updater(gen_params, g_cost)
+updates = d_updates + g_updates
+
+print 'COMPILING'
+t = time()
+_train_g = theano.function([X, Z], cost, updates=g_updates)
+_train_d = theano.function([X, Z], cost, updates=d_updates)
+_gen = theano.function([Z], gX)
+print '%.2f seconds to compile theano functions'%(time()-t)
+
+vis_idxs = py_rng.sample(np.arange(len(vaX)), nvis)
+vaX_vis = inverse_transform(vaX[vis_idxs])
+color_grid_vis(vaX_vis, (14, 14), 'samples/%s_etl_test.png'%desc)
+
+sample_zmb = floatX(np_rng.uniform(-1., 1., size=(nvis, nz)))
+
+def gen_samples(n, nbatch=128):
+    samples = []
+    n_gen = 0
+    for i in range(n/nbatch):
+        zmb = floatX(np_rng.uniform(-1., 1., size=(nbatch, nz)))
+        xmb = _gen(zmb)
+        samples.append(xmb)
+        n_gen += len(xmb)
+    n_left = n-n_gen
+    zmb = floatX(np_rng.uniform(-1., 1., size=(n_left, nz)))
+    xmb = _gen(zmb)
+    samples.append(xmb)    
+    return np.concatenate(samples, axis=0)
+
+f_log = open('logs/%s.ndjson'%desc, 'wb')
+log_fields = [
+    'n_epochs', 
+    'n_updates', 
+    'n_examples', 
+    'n_seconds',
+    '1k_va_nnd',
+    '10k_va_nnd',
+    '100k_va_nnd',
+    'g_cost',
+    'd_cost',
+]
+
+vaX = vaX.reshape(len(vaX), -1)
+
+print desc.upper()
+n_updates = 0
+n_check = 0
+n_epochs = 0
+n_updates = 0
+n_examples = 0
+t = time()
+for epoch in range(niter):
+    for imb, in tqdm(tr_stream.get_epoch_iterator(), total=ntrain/nbatch):
+        imb = transform(imb)
+        zmb = floatX(np_rng.uniform(-1., 1., size=(len(imb), nz)))
+        if n_updates % (k+1) == 0:
+            cost = _train_g(imb, zmb)
+        else:
+            cost = _train_d(imb, zmb)
+        n_updates += 1
+        n_examples += len(imb)
+    g_cost = float(cost[0])
+    d_cost = float(cost[1])
+    gX = gen_samples(100000)
+    gX = gX.reshape(len(gX), -1)
+    va_nnd_1k = nnd_score(gX[:1000], vaX, metric='euclidean')
+    va_nnd_10k = nnd_score(gX[:10000], vaX, metric='euclidean')
+    va_nnd_100k = nnd_score(gX[:100000], vaX, metric='euclidean')
+    log = [n_epochs, n_updates, n_examples, time()-t, va_nnd_1k, va_nnd_10k, va_nnd_100k, g_cost, d_cost]
+    print '%.0f %.2f %.2f %.2f %.4f %.4f'%(epoch, va_nnd_1k, va_nnd_10k, va_nnd_100k, g_cost, d_cost)
+    f_log.write(json.dumps(dict(zip(log_fields, log)))+'\n')
+    f_log.flush()
+
+    samples = np.asarray(_gen(sample_zmb))
+    color_grid_vis(inverse_transform(samples), (14, 14), 'samples/%s/%d.png'%(desc, n_epochs))
+    n_epochs += 1
+    if n_epochs > niter:
+        lrt.set_value(floatX(lrt.get_value() - lr/niter_decay))
+    if n_epochs in [1, 2, 3, 4, 5, 10, 15, 20, 25]:
+        joblib.dump([p.get_value() for p in gen_params], 'models/%s/%d_gen_params.jl'%(desc, n_epochs))
+        joblib.dump([p.get_value() for p in discrim_params], 'models/%s/%d_discrim_params.jl'%(desc, n_epochs))

mnist/README.md

@@ -3,10 +3,10 @@ Modify data_dir in lib/config.py to point to directory with mnist files.
 Run train_cond_dcgan.py to train mnist model from appendix. It will create a few folders and save training info, model parameters, and samples periodically. Should take ~ an hour to run on a good GPU. 
 
 Libs you'll need installed/configured to run it:
-theano
-cudnn
-sklearn
-numpy
-scipy
-matplotlib
-tqdm
+- theano
+- cudnn
+- sklearn
+- numpy
+- scipy
+- matplotlib
+- tqdm