Recognizing Handwritten Digits - tutorial - training model step - fails

Herve

Traceback (most recent call last):
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply
    return dic[self.func_name](self.file_reader.read(x))
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/preprocessing.py", line 37, in read
    return folder.get_download_stream(file_id)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/managed_folder.py", line 277, in get_download_stream
    "path" : path
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 422, in jek_or_backend_stream_call
    return backend_stream_call(path, data, err_msg, **kwargs)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 413, in backend_stream_call
    return _handle_stream_resp(backend_api_post_call(path, data, stream=True, **kwargs), err_msg = err_msg)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 469, in _handle_stream_resp
    raise Exception("%s: %s" % (err_msg, _get_error_message(err_data).encode("utf8")))
Exception: None: b'No such file in this managed folder.'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/server.py", line 46, in serve
    ret = api_command(arg)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/dkuapi.py", line 45, in aux
    return api(**kwargs)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/commands.py", line 456, in train_prediction_keras
    pipeline.generated_features_mapping)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/prediction_entrypoints.py", line 441, in prediction_train_model_keras
    generated_features_mapping)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/keras_support.py", line 126, in get_keras_model
    shuffle=keras_params["shuffleData"])
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
    return func(*args, **kwargs)
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/engine/training.py", line 1732, in fit_generator
    initial_epoch=initial_epoch)
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/engine/training_generator.py", line 185, in fit_generator
    generator_output = next(output_generator)
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 625, in get
    six.reraise(*sys.exc_info())
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/six.py", line 703, in reraise
    raise value
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 610, in get
    inputs = future.get(timeout=30)
  File "/usr/lib/python3.6/multiprocessing/pool.py", line 644, in get
    raise self._value
  File "/usr/lib/python3.6/multiprocessing/pool.py", line 119, in worker
    result = (True, func(*args, **kwds))
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 406, in get_index
    return _SHARED_SEQUENCES[uid][i]
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/sequences.py", line 131, in __getitem__
    batch = self.__get_batch_from_array(index_array)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/sequences.py", line 160, in __get_batch_from_array
    batch_transformed = self.pipeline.process(batch_input_df)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1958, in process
    new_mf = step.process(input_df, cur_mf, result, self.generated_features_mapping)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1597, in process
    blk = np.array(serie.apply(self.__apply_user_defined_func(dic)).tolist())
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/pandas/core/series.py", line 3848, in apply
    mapped = lib.map_infer(values, f, convert=convert_dtype)
  File "pandas/_libs/lib.pyx", line 2329, in pandas._libs.lib.map_infer
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply
    return dic[self.func_name](self.file_reader.read(x))
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/base/utils.py", line 176, in __exit__
    exc_tb)
  File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/six.py", line 702, in reraise
    raise value.with_traceback(tb)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply
    return dic[self.func_name](self.file_reader.read(x))
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/preprocessing.py", line 37, in read
    return folder.get_download_stream(file_id)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/managed_folder.py", line 277, in get_download_stream
    "path" : path
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 422, in jek_or_backend_stream_call
    return backend_stream_call(path, data, err_msg, **kwargs)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 413, in backend_stream_call
    return _handle_stream_resp(backend_api_post_call(path, data, stream=True, **kwargs), err_msg = err_msg)
  File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 469, in _handle_stream_resp
    raise Exception("%s: %s" % (err_msg, _get_error_message(err_data).encode("utf8")))
Exception: Failed to preprocess the following file: 'training/2/57890.png', Error: None: b'No such file in this managed folder.'

Alex_Reutter

Hi @Herve
, that file is in the managed folder, but the path from the top of the folder (from this image) appears to be mnist_png/mnist_png/training/2/57890.png and not training/2/57890.png.

This is a little weird because the Python code recipe that creates the dataset from the folder (e.g., http://gallery.dataiku.com/projects/image_classification/recipes/compute_mnist/) should extract the full path.

Herve

from keras.layers import Conv2D, MaxPooling2D, Input
from keras.layers import Dense, BatchNormalization, Activation
from keras.layers import Flatten, Dropout
from keras.models import Model
from keras.optimizers import Adam

def build_model(input_shapes, n_classes=None):

#### DEFINING INPUT AND BASE ARCHITECTURE
# You need to modify the name and shape of the "image_input"
# according to the preprocessing and name of your
# initial feature.
# This feature should to be preprocessed as an "Image", with a
# custom preprocessing.
image_shape = (28, 28, 3)
image_input_name = "path_preprocessed"

image_input = Input(shape=image_shape, name=image_input_name)

#### DEFINING THE ARCHITECTURE
x = Conv2D(32, kernel_size=3, padding='same', activation='relu')(image_input)
x = Conv2D(32, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(n_classes, activation='softmax')(x)

model = Model(inputs=image_input, outputs=predictions)
return model

def compile_model(model):
model.compile(
optimizer="adam",
loss="categorical_crossentropy"
)
return model

Herve

from keras.preprocessing.image import img_to_array, load_img

# Custom image preprocessing function.
# Must return a numpy ndarray representing the image.
# - image_file is a file like object
def preprocess_image(image_file):
img = load_img(image_file,target_size=(28, 28, 3))
array = img_to_array(img)

# Define the actual preprocessing here
array = array / 255
return array

Alex_Reutter

Hi @Herve
, the line

Exception: Failed to preprocess the following file: 'training/2/57890.png', Error: None: b'No such file in this managed folder.'

...suggests that DSS can't find the file during preprocessing. Can you confirm that the file is in the folder selected in the Features handling panel of the design of the deep learning model?

Thanks,

Alex

Herve

File is there,

 

yes

Herve