Getting error at generator function

[mahmuduzzaman]

I am getting error running this code

> for i in range(epochs):
>     print("Epoch {} start at time ".format(i),datetime.now())
>     generator = generator_function(train_length,batch_size)
>     cv_generator = generator_function(cv_length,32,"cv")
>     model.fit_generator(generator, epochs=1, steps_per_epoch=train_steps,verbose=0,callbacks=[tensorboard],validation_data=cv_generator,validation_steps=cv_steps)
>     #model.save_weights("model_epoch_{}.h5".format(i))

Error :

Epoch 0 start at time 2020-05-14 19:09:42.070648
WARNING:tensorflow:Model failed to serialize as JSON. Ignoring...

---

ResourceExhaustedError Traceback (most recent call last)
in
3 generator = generator_function(train_length,batch_size)
4 cv_generator = generator_function(cv_length,32,"cv")
----> 5 model.fit_generator(generator, epochs=1, steps_per_epoch=train_steps,verbose=0,callbacks=[tensorboard],validation_data=cv_generator,validation_steps=cv_steps)
6 #model.save_weights("model_epoch_{}.h5".format(i))

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1295 shuffle=shuffle,
1296 initial_epoch=initial_epoch,
-> 1297 steps_name='steps_per_epoch')
1298
1299 def evaluate_generator(self,

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_generator.py in model_iteration(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)
263
264 is_deferred = not model._is_compiled
--> 265 batch_outs = batch_function(*batch_data)
266 if not isinstance(batch_outs, list):
267 batch_outs = [batch_outs]

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight, reset_metrics)
971 outputs = training_v2_utils.train_on_batch(
972 self, x, y=y, sample_weight=sample_weight,
--> 973 class_weight=class_weight, reset_metrics=reset_metrics)
974 outputs = (outputs['total_loss'] + outputs['output_losses'] +
975 outputs['metrics'])

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py in train_on_batch(model, x, y, sample_weight, class_weight, reset_metrics)
262 y,
263 sample_weights=sample_weights,
--> 264 output_loss_metrics=model._output_loss_metrics)
265
266 if reset_metrics:

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in train_on_batch(model, inputs, targets, sample_weights, output_loss_metrics)
309 sample_weights=sample_weights,
310 training=True,
--> 311 output_loss_metrics=output_loss_metrics))
312 if not isinstance(outs, list):
313 outs = [outs]

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in _process_single_batch(model, inputs, targets, output_loss_metrics, sample_weights, training)
250 output_loss_metrics=output_loss_metrics,
251 sample_weights=sample_weights,
--> 252 training=training))
253 if total_loss is None:
254 raise ValueError('The model cannot be run '

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in _model_loss(model, inputs, targets, output_loss_metrics, sample_weights, training)
125 inputs = nest.map_structure(ops.convert_to_tensor, inputs)
126
--> 127 outs = model(inputs, **kwargs)
128 outs = nest.flatten(outs)
129

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in call(self, inputs, *args, **kwargs)
889 with base_layer_utils.autocast_context_manager(
890 self._compute_dtype):
--> 891 outputs = self.call(cast_inputs, *args, **kwargs)
892 self._handle_activity_regularization(inputs, outputs)
893 self._set_mask_metadata(inputs, outputs, input_masks)

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in call(self, inputs, training, mask)
706 return self._run_internal_graph(
707 inputs, training=training, mask=mask,
--> 708 convert_kwargs_to_constants=base_layer_utils.call_context().saving)
709
710 def compute_output_shape(self, input_shape):

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in _run_internal_graph(self, inputs, training, mask, convert_kwargs_to_constants)
858
859 # Compute outputs.
--> 860 output_tensors = layer(computed_tensors, **kwargs)
861
862 # Update tensor_dict.

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in call(self, inputs, *args, **kwargs)
889 with base_layer_utils.autocast_context_manager(
890 self._compute_dtype):
--> 891 outputs = self.call(cast_inputs, *args, **kwargs)
892 self._handle_activity_regularization(inputs, outputs)
893 self._set_mask_metadata(inputs, outputs, input_masks)

in call(self, prob)
25
26 #outer = tf.matrix_band_part(outer, 0, self.token_span)
---> 27 outer = tf.compat.v1.matrix_band_part(outer, 0, self.token_span)
28 #print(outer.shape)
29

~/miniconda2/envs/py3/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py in matrix_band_part(input, num_lower, num_upper, name)
5033 else:
5034 message = e.message
-> 5035 _six.raise_from(_core._status_to_exception(e.code, message), None)
5036 # Add nodes to the TensorFlow graph.
5037 try:

~/miniconda2/envs/py3/lib/python3.7/site-packages/six.py in raise_from(value, from_value)

ResourceExhaustedError: OOM when allocating tensor with shape[64,680,680] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu [Op:MatrixBandPart]