Status: Initializing [10:01:24] [INFO] [dku] running compute_Delta_initial_value_NP - ---------------------------------------- [10:01:24] [INFO] [dku] running compute_Delta_initial_value_NP - DSS startup: jek version:5.1.5 [10:01:24] [INFO] [dku] running compute_Delta_initial_value_NP - DSS home: /data/dataiku/dss_design [10:01:24] [INFO] [dku] running compute_Delta_initial_value_NP - OS: Linux 3.10.0-957.21.3.el7.x86_64 amd64 - Java: Oracle Corporation 1.8.0_121 [10:01:24] [INFO] [dku.flow.jobrunner] running compute_Delta_initial_value_NP - Allocated a slot for this activity! [10:01:24] [INFO] [dku.flow.jobrunner] running compute_Delta_initial_value_NP - Run activity [10:01:24] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Executing default pre-activity lifecycle hook [10:01:24] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Checking if sources are ready [10:01:24] [DEBUG] [dku.db.internal] running compute_Delta_initial_value_NP - Borrowing a connection. Read-only: false [10:01:24] [DEBUG] [dku.dataset.hash] running compute_Delta_initial_value_NP - Readiness cache miss for dataset__s562850__TEST_SQL.test_SQL__NP [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"path":"/data/dataiku/dss_design/uploads/TEST_SQL/datasets/test_SQL","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [INFO] [dku.datasets.ftplike] running compute_Delta_initial_value_NP - Enumerating Filesystem dataset prefix= [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumerating local filesystem prefix=/ [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumeration done nb_paths=1 size=10465 [10:01:24] [INFO] [dku.dataset.hash] running compute_Delta_initial_value_NP - Caching readiness for dataset__s562850__TEST_SQL.test_SQL__NP s=READY h=g162UwdXlvrQu9Woe7ONqg [10:01:24] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Checked source readiness TEST_SQL.test_SQL -> true [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Computing hashes to propagate BEFORE activity [10:01:24] [DEBUG] [dku.db.internal] running compute_Delta_initial_value_NP - Borrowing a connection. Read-only: false [10:01:24] [DEBUG] [dku.dataset.hash] running compute_Delta_initial_value_NP - Readiness cache miss for dataset__s562850__TEST_SQL.test_SQL__NP [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"path":"/data/dataiku/dss_design/uploads/TEST_SQL/datasets/test_SQL","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [INFO] [dku.datasets.ftplike] running compute_Delta_initial_value_NP - Enumerating Filesystem dataset prefix= [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumerating local filesystem prefix=/ [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumeration done nb_paths=1 size=10465 [10:01:24] [INFO] [dku.dataset.hash] running compute_Delta_initial_value_NP - Caching readiness for dataset__s562850__TEST_SQL.test_SQL__NP s=READY h=g162UwdXlvrQu9Woe7ONqg [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Recorded 1 hashes before activity run [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Building recipe runner of type [10:01:24] [DEBUG] [dku.job.activity] running compute_Delta_initial_value_NP - Filling source sizes [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"path":"/data/dataiku/dss_design/uploads/TEST_SQL/datasets/test_SQL","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [INFO] [dku.datasets.ftplike] running compute_Delta_initial_value_NP - Enumerating Filesystem dataset prefix= [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumerating local filesystem prefix=/ [10:01:24] [DEBUG] [dku.fs.local] running compute_Delta_initial_value_NP - Enumeration done nb_paths=1 size=10465 [10:01:24] [DEBUG] [dku.job.activity] running compute_Delta_initial_value_NP - Done filling source sizes [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"connection":"filesystem_managed","path":"TEST_SQL/Delta_initial_value","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [WARN] [dku.fs.local] running compute_Delta_initial_value_NP - File does not exist: /data/dataiku/dss_design/managed_datasets/TEST_SQL/Delta_initial_value [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"path":"/data/dataiku/dss_design/uploads/TEST_SQL/datasets/test_SQL","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [INFO] [dku.datasets.file] running compute_Delta_initial_value_NP - Building Filesystem handler config: {"path":"/data/dataiku/dss_design/uploads/TEST_SQL/datasets/test_SQL","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Recipe runner built, will use 1 thread(s) [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Starting execution thread: com.dataiku.dip.recipes.code.sparksql.SparkSQLQueryRecipeRunner@68647fb7 [10:01:24] [DEBUG] [dku.flow.activity] running compute_Delta_initial_value_NP - Execution threads started, waiting for activity end [10:01:24] [INFO] [dku.flow.activity] - Run thread for activity compute_Delta_initial_value_NP starting [10:01:24] [INFO] [dku.spark.sparksql] - Found 1 parts and the index of the select is 0 [10:01:24] [INFO] [dku.datasets.file] - Building Filesystem handler config: {"connection":"filesystem_managed","path":"TEST_SQL/Delta_initial_value","notReadyIfEmpty":false,"filesSelectionRules":{"mode":"ALL","excludeRules":[],"includeRules":[],"explicitFiles":[]}} [10:01:24] [INFO] [dku.datasets.ftplike] - Clear partitions [10:01:24] [WARN] [dku.fs.local] - File does not exist: /data/dataiku/dss_design/managed_datasets/TEST_SQL/Delta_initial_value [10:01:24] [INFO] [dku.datasets.ftplike] - Clearing partition as a folder : 'NP' [10:01:24] [WARN] [dku.fs.local] - File does not exist: /data/dataiku/dss_design/managed_datasets/TEST_SQL/Delta_initial_value [10:01:24] [INFO] [dku.datasets.ftplike] - Done clearing partition 'NP' [10:01:24] [INFO] [dku.recipes.sparksql] - Executing SparkSQL with sparkConfig: {"inheritConf":"default","conf":[]} [10:01:24] [INFO] [dku.spark.activity] - configure activity helper activityProjectKey=TEST_SQL activity={ "activityId": "compute_Delta_initial_value_NP", "recipes": [ { "projectKey": "TEST_SQL", "name": "compute_Delta_initial_value", "type": "spark_sql_query" } ], "sources": [ { "projectKey": "TEST_SQL", "datasetName": "test_SQL", "partitionIds": [ "NP" ] } ], "targets": [ { "projectKey": "TEST_SQL", "datasetName": "Delta_initial_value", "partitionId": "NP" } ], "requiredReason": { "type": "NON_RECURSIVE_FORCED_BUILD", "previousRecordTime": 0 }, "dependencies": [], "recipeProjectKey": "TEST_SQL", "recipeName": "compute_Delta_initial_value", "recipeType": "spark_sql_query" } type=class com.dataiku.dip.dataflow.JobActivity [10:01:24] [INFO] [dku.spark.activity] - Resulting relevant projects: ["TEST_SQL"] [10:01:24] [DEBUG] [dku.security.fsacl] - exec chown: sudo -n /etc/dataiku-security/8S16CJR5Rasdr43E7Q506EWg/execwrapper.sh acls chown --path /data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5 --owner s562850 [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,100 19506 INFO Reading security conf from /etc/dataiku-security/8S16CJR5Rasdr43E7Q506EWg [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,101 19506 INFO Allowed path: /data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5 in /data/dataiku/dss_design [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,101 19506 INFO Reading security conf from /etc/dataiku-security/8S16CJR5Rasdr43E7Q506EWg [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,107 19506 INFO Allowed group ids: [60814, 60815] [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,113 19506 INFO User s562850 belongs to groups: [48957, 60814, 48910, 60815, 59045, 48908, 48912, 60395, 59056] [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,113 19506 INFO User s562850 belongs to group 60814 (a0big500) -> allowed [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,113 19506 INFO Executing chown -Rh: /data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5 -> s562850 [10:01:25] [INFO] [dku.utils] - Done waiting for return value, got 0 [10:01:25] [DEBUG] [dku.security.fsacl] - exec setfacl: sudo -n /etc/dataiku-security/8S16CJR5Rasdr43E7Q506EWg/execwrapper.sh acls setfacl --path /data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5 --perm u::rwx --perm g::--- --perm o:--- --perm d:u:sp0iku00:rwx --perm u:sp0iku00:rwx --recursive [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,208 19514 INFO Reading security conf from /etc/dataiku-security/8S16CJR5Rasdr43E7Q506EWg [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,209 19514 INFO Allowed path: /data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5 in /data/dataiku/dss_design [10:01:25] [INFO] [dku.utils] - 2020-09-23 10:01:25,209 19514 INFO Executing: ['setfacl', '-RP', '-m', 'u::rwx,g::---,o:---,d:u:sp0iku00:rwx,u:sp0iku00:rwx', '/data/dataiku/dss_design/jobs/TEST_SQL/Build_Delta_initial_value_2020-09-23T08-01-19.070/compute_Delta_initial_value_NP/sparkbased-recipe/outu2ORCFu3M7c5'] [10:01:25] [INFO] [dku.utils] - Done waiting for return value, got 0 [10:01:25] [INFO] [dku.flow.activity] - Run thread failed for activity compute_Delta_initial_value_NP java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V at org.apache.hadoop.conf.Configuration.set(Configuration.java:1358) at org.apache.hadoop.conf.Configuration.set(Configuration.java:1339) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getYarnConfWithRmHaId(YarnClientUtils.java:104) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getRmPrincipal(YarnClientUtils.java:71) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getRmPrincipal(YarnClientUtils.java:49) at org.apache.hadoop.mapred.Master.getMasterPrincipal(Master.java:58) at com.dataiku.dip.security.impersonation.HadoopDelegationTokensGenerator.generateSparkTokenFile(HadoopDelegationTokensGenerator.java:56) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:176) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:136) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:132) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.doRunSpark(AbstractSparkBasedRecipeRunner.java:111) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.runSpark(AbstractSparkBasedRecipeRunner.java:93) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.runSpark(AbstractSparkBasedRecipeRunner.java:81) at com.dataiku.dip.recipes.code.sparksql.SparkSQLQueryRecipeRunnerBase.executeJobDef(SparkSQLQueryRecipeRunnerBase.java:37) at com.dataiku.dip.recipes.code.sparksql.SparkSQLQueryRecipeRunner.run(SparkSQLQueryRecipeRunner.java:38) at com.dataiku.dip.dataflow.jobrunner.ActivityRunner$FlowRunnableThread.run(ActivityRunner.java:366) [10:01:25] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - activity is finished [10:01:25] [ERROR] [dku.flow.activity] running compute_Delta_initial_value_NP - Activity failed java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V at org.apache.hadoop.conf.Configuration.set(Configuration.java:1358) at org.apache.hadoop.conf.Configuration.set(Configuration.java:1339) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getYarnConfWithRmHaId(YarnClientUtils.java:104) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getRmPrincipal(YarnClientUtils.java:71) at org.apache.hadoop.yarn.client.util.YarnClientUtils.getRmPrincipal(YarnClientUtils.java:49) at org.apache.hadoop.mapred.Master.getMasterPrincipal(Master.java:58) at com.dataiku.dip.security.impersonation.HadoopDelegationTokensGenerator.generateSparkTokenFile(HadoopDelegationTokensGenerator.java:56) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:176) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:136) at com.dataiku.dip.recipes.code.spark.SparkBasedActivityHelper.configure(SparkBasedActivityHelper.java:132) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.doRunSpark(AbstractSparkBasedRecipeRunner.java:111) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.runSpark(AbstractSparkBasedRecipeRunner.java:93) at com.dataiku.dip.dataflow.exec.AbstractSparkBasedRecipeRunner.runSpark(AbstractSparkBasedRecipeRunner.java:81) at com.dataiku.dip.recipes.code.sparksql.SparkSQLQueryRecipeRunnerBase.executeJobDef(SparkSQLQueryRecipeRunnerBase.java:37) at com.dataiku.dip.recipes.code.sparksql.SparkSQLQueryRecipeRunner.run(SparkSQLQueryRecipeRunner.java:38) at com.dataiku.dip.dataflow.jobrunner.ActivityRunner$FlowRunnableThread.run(ActivityRunner.java:366) [10:01:25] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Executing default post-activity lifecycle hook [10:01:25] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Removing samples for TEST_SQL.Delta_initial_value [10:01:25] [INFO] [dku.flow.activity] running compute_Delta_initial_value_NP - Done post-activity tasks