diff --git a/.gitignore b/.gitignore index 6dd9d944e495c5b56a838218e14a4c4fab31fad6..1ebd42eec2b64ef300e981c363a51ae314a6cdfb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ data/checkpoints/* data/sequences/* data/train/* data/test/* +data/c3d/* diff --git a/models.py b/models.py index 379c96961ffb58c42e20c058c0cc0602c1a03c58..aae6d0af78a53d0b3658ab4f49c45b07e0c80b86 100644 --- a/models.py +++ b/models.py @@ -1,7 +1,7 @@ """ A collection of models we'll use to attempt to classify videos. """ -from keras.layers import Dense, Flatten, Dropout +from keras.layers import Dense, Flatten, Dropout, ZeroPadding3D from keras.layers.recurrent import LSTM from keras.models import Sequential, load_model from keras.optimizers import Adam @@ -57,12 +57,16 @@ class ResearchModels(): print("Loading Conv3D") self.input_shape = (seq_length, 80, 80, 3) self.model = self.conv_3d() + elif model == 'c3d': + print("Loading C3D") + self.input_shape = (seq_length, 80, 80, 3) + self.model = self.c3d() else: print("Unknown network.") sys.exit() # Now compile the network. - optimizer = Adam(lr=1e-6) # aggressively small learning rate + optimizer = Adam(lr=1e-5) self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics) @@ -159,3 +163,65 @@ class ResearchModels(): model.add(Dense(self.nb_classes, activation='softmax')) return model + + def c3d(self): + """ + Build a 3D convolutional network, aka C3D. + https://arxiv.org/pdf/1412.0767.pdf + + With thanks: + https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2 + """ + model = Sequential() + # 1st layer group + model.add(Conv3D(64, 3, 3, 3, activation='relu', + border_mode='same', name='conv1', + subsample=(1, 1, 1), + input_shape=self.input_shape)) + model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), + border_mode='valid', name='pool1')) + # 2nd layer group + model.add(Conv3D(128, 3, 3, 3, activation='relu', + border_mode='same', name='conv2', + subsample=(1, 1, 1))) + model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), + border_mode='valid', name='pool2')) + # 3rd layer group + model.add(Conv3D(256, 3, 3, 3, activation='relu', + border_mode='same', name='conv3a', + subsample=(1, 1, 1))) + model.add(Conv3D(256, 3, 3, 3, activation='relu', + border_mode='same', name='conv3b', + subsample=(1, 1, 1))) + model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), + border_mode='valid', name='pool3')) + # 4th layer group + model.add(Conv3D(512, 3, 3, 3, activation='relu', + border_mode='same', name='conv4a', + subsample=(1, 1, 1))) + model.add(Conv3D(512, 3, 3, 3, activation='relu', + border_mode='same', name='conv4b', + subsample=(1, 1, 1))) + model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), + border_mode='valid', name='pool4')) + + # 5th layer group + model.add(Conv3D(512, 3, 3, 3, activation='relu', + border_mode='same', name='conv5a', + subsample=(1, 1, 1))) + model.add(Conv3D(512, 3, 3, 3, activation='relu', + border_mode='same', name='conv5b', + subsample=(1, 1, 1))) + model.add(ZeroPadding3D(padding=(0, 1, 1))) + model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), + border_mode='valid', name='pool5')) + model.add(Flatten()) + + # FC layers group + model.add(Dense(4096, activation='relu', name='fc6')) + model.add(Dropout(0.5)) + model.add(Dense(4096, activation='relu', name='fc7')) + model.add(Dropout(0.5)) + model.add(Dense(self.nb_classes, activation='softmax')) + + return model