Reading .h5 file is extremely slow

My data is stored in .h5 format. I use a data generator to fit the model and it is extremely slow. A snippet of my code is provided below.

def open_data_file(filename, readwrite="r"):
    return tables.open_file(filename, readwrite)

data_file_opened = open_data_file(os.path.abspath("../data/data.h5"))

train_generator, validation_generator, n_train_steps, n_validation_steps = get_training_and_validation_generators(
        data_file_opened,
        ......)

where:

def get_training_and_validation_generators(data_file, batch_size, ...):
    training_generator = data_generator(data_file, training_list,....)

data_generator function is as follows:

def data_generator(data_file, index_list,....):
      orig_index_list = index_list
    while True:
        x_list = list()
        y_list = list()
        if patch_shape:
            index_list = create_patch_index_list(orig_index_list, data_file, patch_shape,
                                                 patch_overlap, patch_start_offset,pred_specific=pred_specific)
        else:
            index_list = copy.copy(orig_index_list)

        while len(index_list) > 0:
            index = index_list.pop()
            add_data(x_list, y_list, data_file, index, augment=augment, augment_flip=augment_flip,
                     augment_distortion_factor=augment_distortion_factor, patch_shape=patch_shape,
                     skip_blank=skip_blank, permute=permute)
            if len(x_list) == batch_size or (len(index_list) == 0 and len(x_list) > 0):
                yield convert_data(x_list, y_list, n_labels=n_labels, labels=labels, num_model=num_model,overlap_label=overlap_label)
                x_list = list()
                y_list = list()

add_data() is as follows:

def add_data(x_list, y_list, data_file, index, augment=False, augment_flip=False, augment_distortion_factor=0.25,
             patch_shape=False, skip_blank=True, permute=False):
    '''
    add qualified x,y to the generator list
    '''
#     pdb.set_trace()
    data, truth = get_data_from_file(data_file, index, patch_shape=patch_shape)
    
    if np.sum(truth) == 0:
        return
    if augment:
        affine = np.load('affine.npy')
        data, truth = augment_data(data, truth, affine, flip=augment_flip, scale_deviation=augment_distortion_factor)

    if permute:
        if data.shape[-3] != data.shape[-2] or data.shape[-2] != data.shape[-1]:
            raise ValueError("To utilize permutations, data array must be in 3D cube shape with all dimensions having "
                             "the same length.")
        data, truth = random_permutation_x_y(data, truth[np.newaxis])
    else:
        truth = truth[np.newaxis]

    if not skip_blank or np.any(truth != 0):
        x_list.append(data)
        y_list.append(truth)

Model training:

def train_model(model, model_file,....):
    model.fit(training_generator,
                        steps_per_epoch=steps_per_epoch,
                        epochs=n_epochs,
                        verbose = 2,
                        validation_data=validation_generator,
                        validation_steps=validation_steps)

My dataset is large: data.h5 is 55GB. It takes around 7000s to complete one epoch. And I get a segmentation fault error after like 6 epochs. The batch size is set to 1, because otherwise, I get a resource exhausted error. Is there an efficient way to read data.h5 in the generator so that training is faster and doesn't lead to out-of-memory errors?

from Reading .h5 file is extremely slow