[12:15:41] [INFO] [dku] running compute_ScoredImagesPrepared_NP - ---------------------------------------- [12:15:41] [INFO] [dku] running compute_ScoredImagesPrepared_NP - DSS startup: jek version:8.0.2 [12:15:41] [INFO] [dku] running compute_ScoredImagesPrepared_NP - DSS home: /home/dataiku/dss [12:15:41] [INFO] [dku] running compute_ScoredImagesPrepared_NP - OS: Linux 4.19.0-13-cloud-amd64 amd64 - Java: Oracle Corporation 1.8.0_262 [12:15:41] [INFO] [dku.flow.jobrunner] running compute_ScoredImagesPrepared_NP - Allocated a slot for this activity! [12:15:41] [INFO] [dku.flow.jobrunner] running compute_ScoredImagesPrepared_NP - Run activity [12:15:41] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Executing default pre-activity lifecycle hook [12:15:41] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Checking if sources are ready [12:15:41] [DEBUG] [dku.dataset.hash] running compute_ScoredImagesPrepared_NP - Readiness cache miss for dataset__admin__EMOTIONANALYSIS.ScoredImages__NP [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImages","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Enumerating Filesystem dataset prefix= [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumerating local filesystem prefix=/ [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumeration done nb_paths=1 size=12640 [12:15:41] [INFO] [dku.dataset.hash] running compute_ScoredImagesPrepared_NP - Caching readiness for dataset__admin__EMOTIONANALYSIS.ScoredImages__NP s=READY h=bb9whCn2yMe7100iVQUv4A [12:15:41] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Checked source readiness EMOTIONANALYSIS.ScoredImages -> true [12:15:41] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Computing hashes to propagate BEFORE activity [12:15:41] [DEBUG] [dku.dataset.hash] running compute_ScoredImagesPrepared_NP - Readiness cache miss for dataset__admin__EMOTIONANALYSIS.ScoredImages__NP [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImages","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Enumerating Filesystem dataset prefix= [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumerating local filesystem prefix=/ [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumeration done nb_paths=1 size=12640 [12:15:41] [INFO] [dku.dataset.hash] running compute_ScoredImagesPrepared_NP - Caching readiness for dataset__admin__EMOTIONANALYSIS.ScoredImages__NP s=READY h=bb9whCn2yMe7100iVQUv4A [12:15:41] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Recorded 1 hashes before activity run [12:15:41] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Building recipe runner of type [12:15:41] [INFO] [com.dataiku.dip.hive.HiveConfigurator] running compute_ScoredImagesPrepared_NP - Hive support is disabled (no hadoop) [12:15:41] [INFO] [com.dataiku.dip.impala.ImpalaConfigurator] running compute_ScoredImagesPrepared_NP - Impala support is disabled (no hadoop) [12:15:41] [INFO] [dku.recipes.shaker] running compute_ScoredImagesPrepared_NP - User-selected engine: null - used engine: DSS [12:15:41] [INFO] [dku.flow.shaker] running compute_ScoredImagesPrepared_NP - SET PAYLOAD { "columnsSelection": { "mode": "ALL" }, "explorationSampling": { "_refreshTrigger": 0, "selection": { "filter": { "distinct": false, "enabled": false }, "latestPartitionsN": 1, "maxRecords": 10000, "ordering": { "rules": [], "enabled": false }, "withinFirstN": -1, "partitionSelectionMethod": "ALL", "maxStoredBytes": 104857600, "targetRatio": 0.02, "maxReadUncompressedBytes": -1, "samplingMethod": "HEAD_SEQUENTIAL", "timeout": -1 }, "autoRefreshSample": false }, "explorationFilters": [], "origin": "PREPARE_RECIPE", "exploreUIParams": { "autoRefresh": true }, "steps": [ { "preview": false, "metaType": "PROCESSOR", "disabled": false, "type": "PythonUDF", "params": { "mode": "ROW", "envSelection": { "envName": "py27opencv", "envMode": "EXPLICIT_ENV" }, "stopOnError": true, "pythonSourceCode": "import json\ndef process(row):\n max_val \u003d 0\n max_label \u003d None\n\n emotions \u003d [\u0027calm\u0027, \u0027sad\u0027, \u0027surprised\u0027, \u0027neutral\u0027, \u0027fearful\u0027, \u0027angry\u0027, \u0027happy\u0027, \u0027disgust\u0027]\n\n for e in emotions:\n p \u003d float(json.loads(row[\u0027prediction\u0027])[\u0027prediction_{}\u0027.format(e)])\n if p \u003e max_val:\n max_val \u003d p\n max_label \u003d e\n\n row[\u0027max_prediction\u0027] \u003d max_val\n row[\u0027max_label\u0027] \u003d max_label\n row[\u0027label\u0027] \u003d row[\u0027images\u0027].split(\u0027_\u0027)[1]\n row[\u0027correct\u0027] \u003d 1 if row[\u0027label\u0027] \u003d\u003d row[\u0027max_label\u0027] else 0\n row[\u0027video_path\u0027] \u003d \u0027/{}.mp4\u0027.format(row[\u0027images\u0027].split(\u0027_\u0027)[0])\n\n return row", "useKernel": true, "sourceColumnsList": [], "usePythonUnicode": true, "vectorSize": 256, "vectorize": false }, "alwaysShowComment": false } ], "maxProcessedMemTableBytes": -1, "previewMode": "ALL_ROWS", "vizSampling": { "_refreshTrigger": 0, "autoRefreshSample": false }, "analysisColumnData": {}, "columnWidthsByName": { "static_value": 105, "images": 274, "rowLength": 100, "prediction": 300, "error": 100 }, "sorting": [], "globalSearchQuery": "", "coloring": { "scheme": "MEANING_AND_STATUS", "individualColumns": [], "valueColoringMode": "HASH" } } [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImages","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.flow.shaker] running compute_ScoredImagesPrepared_NP - Shaker recipe, from ScoredImages of type Filesystem [12:15:41] [DEBUG] [dku.job.activity] running compute_ScoredImagesPrepared_NP - Filling source sizes [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImages","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Enumerating Filesystem dataset prefix= [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumerating local filesystem prefix=/ [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumeration done nb_paths=1 size=12640 [12:15:41] [DEBUG] [dku.job.activity] running compute_ScoredImagesPrepared_NP - Done filling source sizes [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImagesPrepared","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Clear partitions [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Clearing partition as a folder : 'NP' [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Done clearing partition 'NP' [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImagesPrepared","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [INFO] [dku.datasets.ftplike] running compute_ScoredImagesPrepared_NP - Enumerating Filesystem dataset prefix= [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumerating local filesystem prefix=/ [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumeration done nb_paths=1 size=12640 [12:15:41] [INFO] [dku.flow.shaker] running compute_ScoredImagesPrepared_NP - Using parallel runner [12:15:41] [DEBUG] [dku.flow.stream] running compute_ScoredImagesPrepared_NP - ADD PARTITION files: 1 [12:15:41] [INFO] [dku.datasets.file] running compute_ScoredImagesPrepared_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"EMOTIONANALYSIS/ScoredImagesPrepared","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [12:15:41] [WARN] [dku.fs.local] running compute_ScoredImagesPrepared_NP - File does not exist: /home/dataiku/dss/managed_datasets/EMOTIONANALYSIS/ScoredImagesPrepared [12:15:41] [DEBUG] [dku.fs.local] running compute_ScoredImagesPrepared_NP - Enumerating local filesystem prefix=/ [12:15:41] [INFO] [dku.output.file] running compute_ScoredImagesPrepared_NP - INIT Resplittable : {"successful":true,"prefixDoesntExists":true} [12:15:41] [WARN] [dku.fs.local] running compute_ScoredImagesPrepared_NP - File does not exist: /home/dataiku/dss/managed_datasets/EMOTIONANALYSIS/ScoredImagesPrepared/out-s0.csv.gz [12:15:41] [INFO] [dku.output.file] running compute_ScoredImagesPrepared_NP - Writing base=/ split=0 chunk=0 -> target = out-s0.csv.gz [12:15:41] [INFO] [dku.shaker.python] running compute_ScoredImagesPrepared_NP - CREATING A Python kernel, with pythonlibs=null [12:15:41] [INFO] [dku.block.link] running compute_ScoredImagesPrepared_NP - Started a socket on port 45225 [12:15:41] [INFO] [dku.shaker.python.param] running compute_ScoredImagesPrepared_NP - Pick code env for python udf with {"envMode":"EXPLICIT_ENV","envName":"py27opencv"} [12:15:41] [INFO] [dku.venv.selector] running compute_ScoredImagesPrepared_NP - Select code env lang=PYTHON projectSelection={"mode":"INHERIT","preventOverride":false} globalDefault=null [12:15:41] [INFO] [dku.shaker.python] running compute_ScoredImagesPrepared_NP - Use code env py27opencv [12:15:41] [INFO] [dku.code.envs.resolution] running compute_ScoredImagesPrepared_NP - Executing Python activity in env: py27opencv [12:15:41] [INFO] [dku.python.single_command.kernel] running compute_ScoredImagesPrepared_NP - Starting Python process for kernel python-single-command-kernel [12:15:41] [INFO] [dku.security.process] running compute_ScoredImagesPrepared_NP - Starting process (regular) [12:15:41] [INFO] [dku.security.process] running compute_ScoredImagesPrepared_NP - Process started with pid=538 [12:15:41] [INFO] [dku.processes.cgroups] running compute_ScoredImagesPrepared_NP - Will use cgroups [] [12:15:41] [INFO] [dku.processes.cgroups] running compute_ScoredImagesPrepared_NP - Applying rules to used cgroups: [] [12:15:41] [INFO] [dku.resourceusage] - Reporting start of CRU:{"context":{"type":"JOB_ACTIVITY","authIdentifier":"admin","projectKey":"EMOTIONANALYSIS","jobId":"Build_ScoredImagesPrepared_2021-01-08T12-15-40.514","activityId":"compute_ScoredImagesPrepared_NP","activityType":"recipe","recipeType":"shaker","recipeName":"compute_ScoredImagesPrepared"},"type":"LOCAL_PROCESS","id":"TTDALp3coSQuX9o7","startTime":1610108141332,"localProcess":{"cpuCurrent":0.0}} [12:15:41] [INFO] [dku.usage.computeresource.jek] - Reporting start of resource usage: {"context":{"type":"JOB_ACTIVITY","authIdentifier":"admin","projectKey":"EMOTIONANALYSIS","jobId":"Build_ScoredImagesPrepared_2021-01-08T12-15-40.514","activityId":"compute_ScoredImagesPrepared_NP","activityType":"recipe","recipeType":"shaker","recipeName":"compute_ScoredImagesPrepared"},"type":"LOCAL_PROCESS","id":"TTDALp3coSQuX9o7","startTime":1610108141332,"localProcess":{"cpuCurrent":0.0}} [12:15:41] [INFO] [dku.usage.computeresource.jek] - Reporting update of resource usage: {"context":{"type":"JOB_ACTIVITY","authIdentifier":"admin","projectKey":"EMOTIONANALYSIS","jobId":"Build_ScoredImagesPrepared_2021-01-08T12-15-40.514","activityId":"compute_ScoredImagesPrepared_NP","activityType":"recipe","recipeType":"shaker","recipeName":"compute_ScoredImagesPrepared"},"type":"LOCAL_PROCESS","id":"TTDALp3coSQuX9o7","startTime":1610108141332,"localProcess":{"pid":538,"commandName":"/home/dataiku/dss/code-envs/python/py27opencv/bin/python","cpuUserTimeMS":40,"cpuSystemTimeMS":10,"cpuChildrenUserTimeMS":0,"cpuChildrenSystemTimeMS":0,"cpuTotalMS":50,"cpuCurrent":0.0,"vmSizeMB":209,"vmRSSMB":11,"vmHWMMB":11,"vmRSSAnonMB":4,"vmDataMB":125,"vmSizePeakMB":209,"vmRSSPeakMB":11,"vmRSSTotalMBS":0,"majorFaults":0,"childrenMajorFaults":0}} [12:15:41] [DEBUG] [dku.resource] - Process stats for pid 538: {"pid":538,"commandName":"/home/dataiku/dss/code-envs/python/py27opencv/bin/python","cpuUserTimeMS":40,"cpuSystemTimeMS":10,"cpuChildrenUserTimeMS":0,"cpuChildrenSystemTimeMS":0,"cpuTotalMS":50,"cpuCurrent":0.0,"vmSizeMB":209,"vmRSSMB":11,"vmHWMMB":11,"vmRSSAnonMB":4,"vmDataMB":125,"vmSizePeakMB":209,"vmRSSPeakMB":11,"vmRSSTotalMBS":0,"majorFaults":0,"childrenMajorFaults":0} [12:15:42] [INFO] [dku.link.secret_protected] running compute_ScoredImagesPrepared_NP - Connected to kernel [12:15:42] [INFO] [dku.flow.shaker] running compute_ScoredImagesPrepared_NP - Shaker recipe generated, 2 runnables [12:15:42] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Recipe runner built, will use 2 thread(s) [12:15:42] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Starting execution thread: com.dataiku.dip.dataflow.exec.stream.ParallelStreamSlaveRunnable@f3ea78a [12:15:42] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Starting execution thread: com.dataiku.dip.dataflow.exec.stream.ParallelStreamControlRunnable@e6e0806 [12:15:42] [DEBUG] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Execution threads started, waiting for activity end [12:15:42] [INFO] [dku.flow.activity] - Run thread for activity compute_ScoredImagesPrepared_NP starting [12:15:42] [INFO] [dku.pipeline] - start running [12:15:42] [INFO] [dku.flow.activity] - Run thread for activity compute_ScoredImagesPrepared_NP starting [12:15:42] [DEBUG] [dku.flow.stream] - Offering /out-s0.csv.gz for runners to take [12:15:42] [INFO] [dku.flow.stream] - Offering poison [12:15:42] [INFO] [dku.flow.stream] - Done offering, waiting for all runners to finish [12:15:42] [INFO] [dku.flow.stream] - Have 0 done - nthreads 1 [12:15:42] [INFO] [dku.format] - Extractor run: limit=null totalRecords=0 [12:15:42] [INFO] [dku.flow.stream] - Split runner has work to do : /out-s0.csv.gz [12:15:42] [INFO] [dku] - getCompression filename=**out-s0.csv.gz** [12:15:42] [INFO] [dku] - getCompression filename=**out-s0.csv.gz** [12:15:42] [INFO] [dku.format] - Start compressed [GZIP] stream: /home/dataiku/dss/managed_datasets/EMOTIONANALYSIS/ScoredImages/out-s0.csv.gz / totalRecsBefore=0 [12:15:42] [INFO] [dku] - getCompression filename=**out-s0.csv.gz** [12:15:42] [INFO] [dku] - getCompression filename=**out-s0.csv.gz** [12:15:42] [ERROR] [dku.pipeline] - Parallel stream worker failed java.lang.IllegalArgumentException: Python runtime error/ : prediction_calm at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processWithKernel(AbstractPythonUDF.java:219) at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processRow(AbstractPythonUDF.java:120) at com.dataiku.dip.recipes.shaker.ShakerRecipeRunner$SplitRunner$MainInputHandler.processRow(ShakerRecipeRunner.java:155) at com.dataiku.dip.datalayer.ProcessorOutputToSIP.emitRow(ProcessorOutputToSIP.java:24) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:406) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:186) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.extractSimple(ArchiveCapableFormatExtractor.java:154) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.run(ArchiveCapableFormatExtractor.java:59) at com.dataiku.dip.dataflow.exec.stream.ParallelStreamSlaveRunnable.run(ParallelStreamSlaveRunnable.java:61) at com.dataiku.dip.dataflow.jobrunner.ActivityRunner$FlowRunnableThread.run(ActivityRunner.java:374) [12:15:42] [INFO] [dku.pipeline] - done running [12:15:42] [INFO] [dku.flow.stream] - Parallel streamer done [12:15:42] [INFO] [dku.flow.activity] - Run thread failed for activity compute_ScoredImagesPrepared_NP java.lang.IllegalArgumentException: Python runtime error/ : prediction_calm at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processWithKernel(AbstractPythonUDF.java:219) at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processRow(AbstractPythonUDF.java:120) at com.dataiku.dip.recipes.shaker.ShakerRecipeRunner$SplitRunner$MainInputHandler.processRow(ShakerRecipeRunner.java:155) at com.dataiku.dip.datalayer.ProcessorOutputToSIP.emitRow(ProcessorOutputToSIP.java:24) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:406) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:186) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.extractSimple(ArchiveCapableFormatExtractor.java:154) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.run(ArchiveCapableFormatExtractor.java:59) at com.dataiku.dip.dataflow.exec.stream.ParallelStreamSlaveRunnable.run(ParallelStreamSlaveRunnable.java:61) at com.dataiku.dip.dataflow.jobrunner.ActivityRunner$FlowRunnableThread.run(ActivityRunner.java:374) [12:15:42] [INFO] [dku.flow.activity] - Run thread done for activity compute_ScoredImagesPrepared_NP [12:15:42] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - activity is finished [12:15:42] [ERROR] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Activity failed java.lang.IllegalArgumentException: Python runtime error/ : prediction_calm at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processWithKernel(AbstractPythonUDF.java:219) at com.dataiku.dip.shaker.processors.udf.AbstractPythonUDF.processRow(AbstractPythonUDF.java:120) at com.dataiku.dip.recipes.shaker.ShakerRecipeRunner$SplitRunner$MainInputHandler.processRow(ShakerRecipeRunner.java:155) at com.dataiku.dip.datalayer.ProcessorOutputToSIP.emitRow(ProcessorOutputToSIP.java:24) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:406) at com.dataiku.dip.input.formats.csv.CSVFormatExtractor.doExtractStream(CSVFormatExtractor.java:186) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.extractSimple(ArchiveCapableFormatExtractor.java:154) at com.dataiku.dip.input.formats.ArchiveCapableFormatExtractor.run(ArchiveCapableFormatExtractor.java:59) at com.dataiku.dip.dataflow.exec.stream.ParallelStreamSlaveRunnable.run(ParallelStreamSlaveRunnable.java:61) at com.dataiku.dip.dataflow.jobrunner.ActivityRunner$FlowRunnableThread.run(ActivityRunner.java:374) [12:15:42] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Executing default post-activity lifecycle hook [12:15:42] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Removing samples for EMOTIONANALYSIS.ScoredImagesPrepared [12:15:42] [INFO] [dku.flow.activity] running compute_ScoredImagesPrepared_NP - Done post-activity tasks