Jean-Marc Valin | 3ff7e1a | 2018-11-03 02:39:15 -0400 | [diff] [blame] | 1 | #!/usr/bin/python3 |
| 2 | |
| 3 | from __future__ import print_function |
| 4 | |
| 5 | from keras.models import Sequential |
| 6 | from keras.models import Model |
| 7 | from keras.layers import Input |
| 8 | from keras.layers import Dense |
| 9 | from keras.layers import LSTM |
| 10 | from keras.layers import GRU |
| 11 | from keras.layers import CuDNNGRU |
| 12 | from keras.layers import SimpleRNN |
| 13 | from keras.layers import Dropout |
| 14 | from keras import losses |
| 15 | import h5py |
| 16 | from keras.optimizers import Adam |
| 17 | |
| 18 | from keras.constraints import Constraint |
| 19 | from keras import backend as K |
| 20 | import numpy as np |
| 21 | |
| 22 | import tensorflow as tf |
| 23 | from keras.backend.tensorflow_backend import set_session |
| 24 | config = tf.ConfigProto() |
| 25 | config.gpu_options.per_process_gpu_memory_fraction = 0.44 |
| 26 | set_session(tf.Session(config=config)) |
| 27 | |
| 28 | def binary_crossentrop2(y_true, y_pred): |
| 29 | return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_true, y_pred), axis=-1) |
| 30 | |
| 31 | def binary_accuracy2(y_true, y_pred): |
| 32 | return K.mean(K.cast(K.equal(y_true, K.round(y_pred)), 'float32') + K.cast(K.equal(y_true, 0.5), 'float32'), axis=-1) |
| 33 | |
| 34 | def quant_model(model): |
| 35 | weights = model.get_weights() |
| 36 | for k in range(len(weights)): |
| 37 | weights[k] = np.maximum(-128, np.minimum(127, np.round(128*weights[k])*0.0078125)) |
| 38 | model.set_weights(weights) |
| 39 | |
| 40 | class WeightClip(Constraint): |
| 41 | '''Clips the weights incident to each hidden unit to be inside a range |
| 42 | ''' |
| 43 | def __init__(self, c=2): |
| 44 | self.c = c |
| 45 | |
| 46 | def __call__(self, p): |
| 47 | return K.clip(p, -self.c, self.c) |
| 48 | |
| 49 | def get_config(self): |
| 50 | return {'name': self.__class__.__name__, |
| 51 | 'c': self.c} |
| 52 | |
| 53 | reg = 0.000001 |
| 54 | constraint = WeightClip(.998) |
| 55 | |
| 56 | print('Build model...') |
| 57 | |
| 58 | main_input = Input(shape=(None, 25), name='main_input') |
| 59 | x = Dense(32, activation='tanh', kernel_constraint=constraint, bias_constraint=constraint)(main_input) |
| 60 | #x = CuDNNGRU(24, return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) |
| 61 | x = GRU(24, recurrent_activation='sigmoid', activation='tanh', return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) |
| 62 | x = Dense(2, activation='sigmoid', kernel_constraint=constraint, bias_constraint=constraint)(x) |
| 63 | model = Model(inputs=main_input, outputs=x) |
| 64 | |
| 65 | batch_size = 2048 |
| 66 | |
| 67 | print('Loading data...') |
| 68 | with h5py.File('features10b.h5', 'r') as hf: |
| 69 | all_data = hf['data'][:] |
| 70 | print('done.') |
| 71 | |
| 72 | window_size = 1500 |
| 73 | |
| 74 | nb_sequences = len(all_data)//window_size |
| 75 | print(nb_sequences, ' sequences') |
| 76 | x_train = all_data[:nb_sequences*window_size, :-2] |
| 77 | x_train = np.reshape(x_train, (nb_sequences, window_size, 25)) |
| 78 | |
| 79 | y_train = np.copy(all_data[:nb_sequences*window_size, -2:]) |
| 80 | y_train = np.reshape(y_train, (nb_sequences, window_size, 2)) |
| 81 | |
| 82 | print("Marking ignores") |
| 83 | for s in y_train: |
| 84 | for e in s: |
| 85 | if (e[1] >= 1): |
| 86 | break |
| 87 | e[0] = 0.5 |
| 88 | |
| 89 | all_data = 0; |
| 90 | x_train = x_train.astype('float32') |
| 91 | y_train = y_train.astype('float32') |
| 92 | |
| 93 | print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape) |
| 94 | |
| 95 | model.load_weights('newweights10a1b_ep206.hdf5') |
| 96 | |
| 97 | #weights = model.get_weights() |
| 98 | #for k in range(len(weights)): |
| 99 | # weights[k] = np.round(128*weights[k])*0.0078125 |
| 100 | #model.set_weights(weights) |
| 101 | |
| 102 | # try using different optimizers and different optimizer configs |
| 103 | model.compile(loss=binary_crossentrop2, |
| 104 | optimizer=Adam(0.0001), |
| 105 | metrics=[binary_accuracy2]) |
| 106 | |
| 107 | print('Train...') |
| 108 | quant_model(model) |
| 109 | model.fit(x_train, y_train, |
| 110 | batch_size=batch_size, |
| 111 | epochs=10, validation_data=(x_train, y_train)) |
| 112 | model.save("newweights10a1c_ep10.hdf5") |
| 113 | |
| 114 | quant_model(model) |
| 115 | model.fit(x_train, y_train, |
| 116 | batch_size=batch_size, |
| 117 | epochs=50, initial_epoch=10) |
| 118 | model.save("newweights10a1c_ep50.hdf5") |
| 119 | |
| 120 | model.compile(loss=binary_crossentrop2, |
| 121 | optimizer=Adam(0.0001), |
| 122 | metrics=[binary_accuracy2]) |
| 123 | |
| 124 | quant_model(model) |
| 125 | model.fit(x_train, y_train, |
| 126 | batch_size=batch_size, |
| 127 | epochs=100, initial_epoch=50) |
| 128 | model.save("newweights10a1c_ep100.hdf5") |
| 129 | |
| 130 | quant_model(model) |
| 131 | model.fit(x_train, y_train, |
| 132 | batch_size=batch_size, |
| 133 | epochs=150, initial_epoch=100) |
| 134 | model.save("newweights10a1c_ep150.hdf5") |
| 135 | |
| 136 | quant_model(model) |
| 137 | model.fit(x_train, y_train, |
| 138 | batch_size=batch_size, |
| 139 | epochs=200, initial_epoch=150) |
| 140 | model.save("newweights10a1c_ep200.hdf5") |
| 141 | |
| 142 | quant_model(model) |
| 143 | model.fit(x_train, y_train, |
| 144 | batch_size=batch_size, |
| 145 | epochs=201, initial_epoch=200) |
| 146 | model.save("newweights10a1c_ep201.hdf5") |
| 147 | |
| 148 | quant_model(model) |
| 149 | model.fit(x_train, y_train, |
| 150 | batch_size=batch_size, |
| 151 | epochs=202, initial_epoch=201, validation_data=(x_train, y_train)) |
| 152 | model.save("newweights10a1c_ep202.hdf5") |
| 153 | |
| 154 | quant_model(model) |
| 155 | model.fit(x_train, y_train, |
| 156 | batch_size=batch_size, |
| 157 | epochs=203, initial_epoch=202, validation_data=(x_train, y_train)) |
| 158 | model.save("newweights10a1c_ep203.hdf5") |
| 159 | |
| 160 | quant_model(model) |
| 161 | model.fit(x_train, y_train, |
| 162 | batch_size=batch_size, |
| 163 | epochs=204, initial_epoch=203, validation_data=(x_train, y_train)) |
| 164 | model.save("newweights10a1c_ep204.hdf5") |
| 165 | |
| 166 | quant_model(model) |
| 167 | model.fit(x_train, y_train, |
| 168 | batch_size=batch_size, |
| 169 | epochs=205, initial_epoch=204, validation_data=(x_train, y_train)) |
| 170 | model.save("newweights10a1c_ep205.hdf5") |
| 171 | |
| 172 | quant_model(model) |
| 173 | model.fit(x_train, y_train, |
| 174 | batch_size=batch_size, |
| 175 | epochs=206, initial_epoch=205, validation_data=(x_train, y_train)) |
| 176 | model.save("newweights10a1c_ep206.hdf5") |
| 177 | |