Recognizing Handwritten Digits - tutorial - training model step - fails
Traceback (most recent call last): File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply return dic[self.func_name](self.file_reader.read(x)) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/preprocessing.py", line 37, in read return folder.get_download_stream(file_id) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/managed_folder.py", line 277, in get_download_stream "path" : path File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 422, in jek_or_backend_stream_call return backend_stream_call(path, data, err_msg, **kwargs) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 413, in backend_stream_call return _handle_stream_resp(backend_api_post_call(path, data, stream=True, **kwargs), err_msg = err_msg) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 469, in _handle_stream_resp raise Exception("%s: %s" % (err_msg, _get_error_message(err_data).encode("utf8"))) Exception: None: b'No such file in this managed folder.' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/server.py", line 46, in serve ret = api_command(arg) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/dkuapi.py", line 45, in aux return api(**kwargs) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/commands.py", line 456, in train_prediction_keras pipeline.generated_features_mapping) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/prediction_entrypoints.py", line 441, in prediction_train_model_keras generated_features_mapping) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/keras_support.py", line 126, in get_keras_model shuffle=keras_params["shuffleData"]) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper return func(*args, **kwargs) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/engine/training.py", line 1732, in fit_generator initial_epoch=initial_epoch) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/engine/training_generator.py", line 185, in fit_generator generator_output = next(output_generator) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 625, in get six.reraise(*sys.exc_info()) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/six.py", line 703, in reraise raise value File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 610, in get inputs = future.get(timeout=30) File "/usr/lib/python3.6/multiprocessing/pool.py", line 644, in get raise self._value File "/usr/lib/python3.6/multiprocessing/pool.py", line 119, in worker result = (True, func(*args, **kwds)) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/keras/utils/data_utils.py", line 406, in get_index return _SHARED_SEQUENCES[uid][i] File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/sequences.py", line 131, in __getitem__ batch = self.__get_batch_from_array(index_array) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/sequences.py", line 160, in __get_batch_from_array batch_transformed = self.pipeline.process(batch_input_df) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1958, in process new_mf = step.process(input_df, cur_mf, result, self.generated_features_mapping) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1597, in process blk = np.array(serie.apply(self.__apply_user_defined_func(dic)).tolist()) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/pandas/core/series.py", line 3848, in apply mapped = lib.map_infer(values, f, convert=convert_dtype) File "pandas/_libs/lib.pyx", line 2329, in pandas._libs.lib.map_infer File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply return dic[self.func_name](self.file_reader.read(x)) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/base/utils.py", line 176, in __exit__ exc_tb) File "/home/hblanc/DATA_DIR/code-envs/python/PYTHON36/lib/python3.6/site-packages/six.py", line 702, in reraise raise value.with_traceback(tb) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/preprocessing/dataframe_preprocessing.py", line 1609, in func_to_apply return dic[self.func_name](self.file_reader.read(x)) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/doctor/deep_learning/preprocessing.py", line 37, in read return folder.get_download_stream(file_id) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/managed_folder.py", line 277, in get_download_stream "path" : path File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 422, in jek_or_backend_stream_call return backend_stream_call(path, data, err_msg, **kwargs) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 413, in backend_stream_call return _handle_stream_resp(backend_api_post_call(path, data, stream=True, **kwargs), err_msg = err_msg) File "/home/hblanc/dataiku-dss-8.0.5/python/dataiku/core/intercom.py", line 469, in _handle_stream_resp raise Exception("%s: %s" % (err_msg, _get_error_message(err_data).encode("utf8"))) Exception: Failed to preprocess the following file: 'training/2/57890.png', Error: None: b'No such file in this managed folder.'
Best Answer
-
Hi @Herve
, that file is in the managed folder, but the path from the top of the folder (from this image) appears to be mnist_png/mnist_png/training/2/57890.png and not training/2/57890.png.This is a little weird because the Python code recipe that creates the dataset from the folder (e.g., http://gallery.dataiku.com/projects/image_classification/recipes/compute_mnist/) should extract the full path.
Answers
-
Herve Partner, Dataiku DSS Core Designer, Dataiku DSS & SQL, Dataiku DSS ML Practitioner, Dataiku DSS Core Concepts, Dataiku DSS Adv Designer, Registered Posts: 58 Partner
from keras.layers import Conv2D, MaxPooling2D, Input
from keras.layers import Dense, BatchNormalization, Activation
from keras.layers import Flatten, Dropout
from keras.models import Model
from keras.optimizers import Adamdef build_model(input_shapes, n_classes=None):
#### DEFINING INPUT AND BASE ARCHITECTURE
# You need to modify the name and shape of the "image_input"
# according to the preprocessing and name of your
# initial feature.
# This feature should to be preprocessed as an "Image", with a
# custom preprocessing.
image_shape = (28, 28, 3)
image_input_name = "path_preprocessed"image_input = Input(shape=image_shape, name=image_input_name)
#### DEFINING THE ARCHITECTURE
x = Conv2D(32, kernel_size=3, padding='same', activation='relu')(image_input)
x = Conv2D(32, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.2)(x)
predictions = Dense(n_classes, activation='softmax')(x)model = Model(inputs=image_input, outputs=predictions)
return modeldef compile_model(model):
model.compile(
optimizer="adam",
loss="categorical_crossentropy"
)
return model -
Herve Partner, Dataiku DSS Core Designer, Dataiku DSS & SQL, Dataiku DSS ML Practitioner, Dataiku DSS Core Concepts, Dataiku DSS Adv Designer, Registered Posts: 58 Partner
from keras.preprocessing.image import img_to_array, load_img
# Custom image preprocessing function.
# Must return a numpy ndarray representing the image.
# - image_file is a file like object
def preprocess_image(image_file):
img = load_img(image_file,target_size=(28, 28, 3))
array = img_to_array(img)# Define the actual preprocessing here
array = array / 255
return array -
Hi @Herve
, the lineException: Failed to preprocess the following file: 'training/2/57890.png', Error: None: b'No such file in this managed folder.'
...suggests that DSS can't find the file during preprocessing. Can you confirm that the file is in the folder selected in the Features handling panel of the design of the deep learning model?
Thanks,
Alex
-
Herve Partner, Dataiku DSS Core Designer, Dataiku DSS & SQL, Dataiku DSS ML Practitioner, Dataiku DSS Core Concepts, Dataiku DSS Adv Designer, Registered Posts: 58 Partner
File is there,
yes
-
Herve Partner, Dataiku DSS Core Designer, Dataiku DSS & SQL, Dataiku DSS ML Practitioner, Dataiku DSS Core Concepts, Dataiku DSS Adv Designer, Registered Posts: 58 Partner