Create Evaluation Recipe with API in Dataiku 9

dongari
dongari Registered Posts: 6 ✭✭✭✭
edited July 16 in Using Dataiku

Dear Dataiku Community,

my organization will still use Dataiku 9.0.4 for a while, before updating to Dataiku 10.

In this version, it seems like I can only create an Evaluation Recipe using the Graphical User Interface.
Indeed, the API code:

erc = p.new_recipe(type='evaluation', name=f"evaluate_on_mytest") 
erc.with_input(f"mytest")
erc.with_input_model(f"mymodel")
erc.with_output(f"myscores")
erc.with_output_metrics(f"mymetrics")
er = erc.create()
er.run(wait=True)

should work well in Dataiku 10 but it is not working in Dataiku 9 because there is not such a thing as type='evaluation' for the RecipeCreator, i.e., an EvaluationRecipeCreator in https://github.com/dataiku/dataiku-api-client-python/blob/release/9.0/dataikuapi/dss/recipe.py.

I'm wondering if there is an alternative way to create an Evaluation Recipe in Dataiku 9.0.4 via API.

Thanks for your attention,

Daniele


Operating system used: Unix

Tagged:

Best Answer

  • pmasiphelps
    pmasiphelps Dataiker, Dataiku DSS Core Designer, Registered Posts: 33 Dataiker
    edited July 17 Answer ✓

    Hi,

    Indeed, this will be much simpler in v10+ using the code you pasted

    Below is a workaround that you can try for your v9 instance. Note that the raw_payload at the bottom works for classification models. If you're doing a regression model, you'll need to change this (as noted in the comments).

    import dataiku
    
    def create_managed_dataset(project_key, dataset_name, connection_name):
        """
        Creates a managed dataset in a given project, in a given connection
        """
        client = dataiku.api_client()
        project = client.get_project(project_key)
        
        ds_helper = project.new_managed_dataset_creation_helper(dataset_name).with_store_into(connection_name)
        
        ds_helper.create()
    
    def create_datasets_for_evaluation_recipe(project_key, main_output_dataset, metrics_output_dataset, connection_name):
        """
        Creates two managed datasets (for an evaluation recipe) in a given project, in a given connection
        """
        create_managed_dataset(project_key, main_output_dataset, connection_name)
        create_managed_dataset(project_key, metrics_output_dataset, connection_name)
    
    def create_evaluation_recipe_shell(project_key, input_dataset_name, model_id, main_output_dataset, metrics_output_dataset, connection_name):
        """
        Creates a new 'shell' evaluation recipe in a given project
        Returns the recipe name
        """
        client = dataiku.api_client()
        project = client.get_project(project_key)
        
        create_datasets_for_evaluation_recipe(project_key, main_output_dataset, metrics_output_dataset, connection_name)
        
        recipe_name = 'evaluate_on_' + input_dataset_name
        
        recipe_proto = {"type": 'evaluation',
                        "name": recipe_name,
                        "inputs": {'main': {'items': [{'deps': [],
                                      'ref': input_dataset_name}]},
                                   'model': {'items': [{'deps': [], 'ref': model_id}]}},
                        "outputs": {'main': {'items': [{'appendMode': False,
                                      'ref': main_output_dataset}]},
                                    'metrics': {'items': [{'appendMode': True,
                                      'ref': metrics_output_dataset}]}}
                        }
        
        creation_settings = {}
        creation_settings["rawCreation"] = True
        
        new_recipe = project.create_recipe(recipe_proto, creation_settings)
        
        return new_recipe.name
    
    def update_evaluation_recipe_shell(project_key, recipe_name, raw_payload):
        """
        Updates a 'shell' evaluation recipe with a proper raw payload
        """
        client = dataiku.api_client()
        project = client.get_project(project_key)
        
        recipe = project.get_recipe(recipe_name)
        recipe_settings = recipe.get_settings()
        recipe_settings.set_payload(raw_payload)
        recipe_settings.save()
        
    
    def create_new_evaluation_recipe(project_key, input_dataset_name, model_id, main_output_dataset, metrics_output_dataset, connection_name, raw_payload):
        """
        Creates a new evaluation recipe
        """
        new_recipe_name = create_evaluation_recipe_shell(project_key, input_dataset_name, model_id, main_output_dataset, metrics_output_dataset, connection_name)
        update_evaluation_recipe_shell(project_key, new_recipe_name, raw_payload)
    
    project_key = "PROJECT_KEY"
    input_dataset_name = "INPUT_DATASET_NAME"
    model_id = "MODEL_ID"
    main_output_dataset = "NEW_MAIN_OUTPUT_DATASET_NAME"
    metrics_output_dataset = "NEW_METRICS_OUTPUT_DATASET_NAME"
    connection_name = "CONNECTION_NAME" #e.g. "filesystem_managed
    
    # This raw payload works for classification models. To get the payload for Regression models, you'll need to get the settings of another evaluate recipe (project.get_recipe().get_settings().get_payload()) and copy over
    # You may want to change some of these values if you'd like to see different metrics in your output dataset, or enable GPUs for scoring, for example
    raw_payload = '{\n  "outputs": [\n    "prediction_correct"\n  ],\n  "perGPUMemoryFraction": 0.5,\n  "sqlPipelineParams": {\n    "pipelineAllowMerge": true,\n    "pipelineAllowStart": true\n  },\n  "backendType": "PY_MEMORY",\n  "filterInputColumns": false,\n  "useGPU": false,\n  "keptInputColumns": [],\n  "pythonBatchSize": 100000,\n  "gpuAllowGrowth": false,\n  "outputProbaPercentiles": false,\n  "outputProbabilities": true,\n  "forceOriginalEngine": false,\n  "metrics": [\n    "precision",\n    "recall",\n    "auc",\n    "f1",\n    "accuracy",\n    "mcc",\n    "hammingLoss",\n    "logLoss",\n    "lift",\n    "calibrationLoss",\n    "customScore"\n  ],\n  "individualExplanationParams": {\n    "subChunkSize": 5000,\n    "method": "ICE",\n    "shapleyBackgroundSize": 100,\n    "nbExplanations": 3\n  },\n  "forcedClassifierThreshold": 0,\n  "batchSize": 100,\n  "outputExplanations": false,\n  "overrideModelSpecifiedThreshold": false,\n  "sparkParams": {\n    "pipelineAllowMerge": true,\n    "sparkPreparedDFStorageLevel": "MEMORY_AND_DISK",\n    "pipelineAllowStart": true,\n    "sparkExecutionEngine": "SPARK_SUBMIT",\n    "sparkConf": {\n      "inheritConf": "default",\n      "conf": []\n    },\n    "sparkRepartitionNonHDFS": 1,\n    "sparkUseGlobalMetastore": false\n  },\n  "gpuList": [\n    0\n  ]\n}'
    
    create_new_evaluation_recipe(project_key, input_dataset_name, model_id, main_output_dataset, metrics_output_dataset, connection_name, raw_payload)

    Best,

    Pat

Setup Info
    Tags
      Help me…