payload: {"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/282956395","pull_request_review_id":236197050,"id":282956395,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI4Mjk1NjM5NQ==","diff_hunk":"@@ -71,6 +82,99 @@ private[spark] class CoarseGrainedExecutorBackend(\n     }(ThreadUtils.sameThread)\n   }\n \n+  // Check that the executor resources at startup will satisfy the user specified task\n+  // requirements (spark.task.resource.*) and that they match the executor configs\n+  // specified by the user (spark.executor.resource.*) to catch mismatches between what\n+  // the user requested and what resource manager gave or what the discovery script found.\n+  private def checkExecResourcesMeetTaskRequirements(\n+      taskResourceConfigs: Array[(String, String)],\n+      actualExecResources: Map[String, ResourceInformation]): Unit = {\n+\n+    // get just the map of resource name to count\n+    val taskResourcesAndCounts = taskResourceConfigs.\n+      withFilter { case (k, v) => k.endsWith(SPARK_RESOURCE_COUNT_POSTFIX)}.\n+      map { case (k, v) => (k.dropRight(SPARK_RESOURCE_COUNT_POSTFIX.size), v)}\n+\n+    taskResourcesAndCounts.foreach { case (rName, taskReqCount) =>\n+      if (actualExecResources.contains(rName)) {\n+        val execResourceInfo = actualExecResources(rName)\n+\n+        if (execResourceInfo.addresses.size < taskReqCount.toLong) {\n+          throw new SparkException(s\"Executor resource: $rName with addresses: \" +\n+            s\"${execResourceInfo.addresses.mkString(\",\")} doesn't meet the task \" +\n+            s\"requirements of needing $taskReqCount of them\")\n+        }\n+        // also make sure the executor resource count on start matches the\n+        // spark.executor.resource configs specified by user\n+        val userExecCountConfigName =\n+          SPARK_EXECUTOR_RESOURCE_PREFIX + rName + SPARK_RESOURCE_COUNT_POSTFIX\n+        val userExecConfigCount = env.conf.getOption(userExecCountConfigName).\n+          getOrElse(throw new SparkException(s\"Executor resource: $rName not specified \" +\n+            s\"via config: $userExecCountConfigName, but required \" +\n+            s\"by the task, please fix your configuration\"))\n+\n+        if (userExecConfigCount.toLong != execResourceInfo.addresses.size) {","path":"core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala","position":85,"original_position":85,"commit_id":"3d8e5cd3d846176d8d049f9b4511ffa6149ef837","original_commit_id":"3d8e5cd3d846176d8d049f9b4511ffa6149ef837","user":{"login":"mengxr","id":829644,"node_id":"MDQ6VXNlcjgyOTY0NA==","avatar_url":"https://avatars2.githubusercontent.com/u/829644?v=4","gravatar_id":"","url":"https://api.github.com/users/mengxr","html_url":"https://github.com/mengxr","followers_url":"https://api.github.com/users/mengxr/followers","following_url":"https://api.github.com/users/mengxr/following{/other_user}","gists_url":"https://api.github.com/users/mengxr/gists{/gist_id}","starred_url":"https://api.github.com/users/mengxr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mengxr/subscriptions","organizations_url":"https://api.github.com/users/mengxr/orgs","repos_url":"https://api.github.com/users/mengxr/repos","events_url":"https://api.github.com/users/mengxr/events{/privacy}","received_events_url":"https://api.github.com/users/mengxr/received_events","type":"User","site_admin":false},"body":"Should we allow `userExecConfigCount.toLong < execResourceInfo.addresses.size`? The discovery script might return more.","created_at":"2019-05-10T16:36:57Z","updated_at":"2019-05-10T16:57:19Z","html_url":"https://github.com/apache/spark/pull/24406#discussion_r282956395","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/24406","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/282956395"},"html":{"href":"https://github.com/apache/spark/pull/24406#discussion_r282956395"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/24406"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/24406","id":271742769,"node_id":"MDExOlB1bGxSZXF1ZXN0MjcxNzQyNzY5","html_url":"https://github.com/apache/spark/pull/24406","diff_url":"https://github.com/apache/spark/pull/24406.diff","patch_url":"https://github.com/apache/spark/pull/24406.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/24406","number":24406,"state":"open","locked":false,"title":"[SPARK-27024] Executor interface for cluster managers to support GPU and other resources","user":{"login":"tgravescs","id":4563792,"node_id":"MDQ6VXNlcjQ1NjM3OTI=","avatar_url":"https://avatars2.githubusercontent.com/u/4563792?v=4","gravatar_id":"","url":"https://api.github.com/users/tgravescs","html_url":"https://github.com/tgravescs","followers_url":"https://api.github.com/users/tgravescs/followers","following_url":"https://api.github.com/users/tgravescs/following{/other_user}","gists_url":"https://api.github.com/users/tgravescs/gists{/gist_id}","starred_url":"https://api.github.com/users/tgravescs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tgravescs/subscriptions","organizations_url":"https://api.github.com/users/tgravescs/orgs","repos_url":"https://api.github.com/users/tgravescs/repos","events_url":"https://api.github.com/users/tgravescs/events{/privacy}","received_events_url":"https://api.github.com/users/tgravescs/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nAdd in GPU and generic resource type allocation to the executors.\r\n\r\nNote this is part of a bigger feature for gpu-aware scheduling and is just how the executor find the resources. The general flow :\r\n\r\n   - users ask for a certain set of resources, for instance number of gpus - each cluster manager has a specific way to do this.\r\n  -  cluster manager allocates a container or set of resources (standalone mode)\r\n-    When spark launches the executor in that container, the executor either has to be told what resources it has or it has to auto discover them.\r\n  -  Executor has to register with Driver and tell the driver the set of resources it has so the scheduler can use that to schedule tasks that requires a certain amount of each of those resources\r\n\r\nIn this pr I added configs and arguments to the executor to be able discover resources. The argument to the executor is intended to be used by standalone mode or other cluster managers that don't have isolation so that it can assign specific resources to specific executors in case there are multiple executors on a node. The argument is a file contains JSON Array of ResourceInformation objects.\r\n\r\nThe discovery script is meant to be used in an isolated environment where the executor only sees the resources it should use.\r\n\r\nNote that there will be follow on PRs to add other parts like the scheduler part. See the epic high level jira: https://issues.apache.org/jira/browse/SPARK-24615\r\n\r\n## How was this patch tested?\r\n\r\nAdded unit tests and manually tested.\r\n\r\nPlease review http://spark.apache.org/contributing.html before opening a pull request.\r\n","created_at":"2019-04-18T15:48:24Z","updated_at":"2019-05-10T16:57:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"07d2b5c338907dd5bdc8da8fa2a91580d54e516d","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/24406/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/24406/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/24406/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/3d8e5cd3d846176d8d049f9b4511ffa6149ef837","head":{"label":"tgravescs:gpu-sched-executor-clean","ref":"gpu-sched-executor-clean","sha":"3d8e5cd3d846176d8d049f9b4511ffa6149ef837","user":{"login":"tgravescs","id":4563792,"node_id":"MDQ6VXNlcjQ1NjM3OTI=","avatar_url":"https://avatars2.githubusercontent.com/u/4563792?v=4","gravatar_id":"","url":"https://api.github.com/users/tgravescs","html_url":"https://github.com/tgravescs","followers_url":"https://api.github.com/users/tgravescs/followers","following_url":"https://api.github.com/users/tgravescs/following{/other_user}","gists_url":"https://api.github.com/users/tgravescs/gists{/gist_id}","starred_url":"https://api.github.com/users/tgravescs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tgravescs/subscriptions","organizations_url":"https://api.github.com/users/tgravescs/orgs","repos_url":"https://api.github.com/users/tgravescs/repos","events_url":"https://api.github.com/users/tgravescs/events{/privacy}","received_events_url":"https://api.github.com/users/tgravescs/received_events","type":"User","site_admin":false},"repo":{"id":17249522,"node_id":"MDEwOlJlcG9zaXRvcnkxNzI0OTUyMg==","name":"spark","full_name":"tgravescs/spark","private":false,"owner":{"login":"tgravescs","id":4563792,"node_id":"MDQ6VXNlcjQ1NjM3OTI=","avatar_url":"https://avatars2.githubusercontent.com/u/4563792?v=4","gravatar_id":"","url":"https://api.github.com/users/tgravescs","html_url":"https://github.com/tgravescs","followers_url":"https://api.github.com/users/tgravescs/followers","following_url":"https://api.github.com/users/tgravescs/following{/other_user}","gists_url":"https://api.github.com/users/tgravescs/gists{/gist_id}","starred_url":"https://api.github.com/users/tgravescs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tgravescs/subscriptions","organizations_url":"https://api.github.com/users/tgravescs/orgs","repos_url":"https://api.github.com/users/tgravescs/repos","events_url":"https://api.github.com/users/tgravescs/events{/privacy}","received_events_url":"https://api.github.com/users/tgravescs/received_events","type":"User","site_admin":false},"html_url":"https://github.com/tgravescs/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/tgravescs/spark","forks_url":"https://api.github.com/repos/tgravescs/spark/forks","keys_url":"https://api.github.com/repos/tgravescs/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tgravescs/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tgravescs/spark/teams","hooks_url":"https://api.github.com/repos/tgravescs/spark/hooks","issue_events_url":"https://api.github.com/repos/tgravescs/spark/issues/events{/number}","events_url":"https://api.github.com/repos/tgravescs/spark/events","assignees_url":"https://api.github.com/repos/tgravescs/spark/assignees{/user}","branches_url":"https://api.github.com/repos/tgravescs/spark/branches{/branch}","tags_url":"https://api.github.com/repos/tgravescs/spark/tags","blobs_url":"https://api.github.com/repos/tgravescs/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tgravescs/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tgravescs/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/tgravescs/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tgravescs/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/tgravescs/spark/languages","stargazers_url":"https://api.github.com/repos/tgravescs/spark/stargazers","contributors_url":"https://api.github.com/repos/tgravescs/spark/contributors","subscribers_url":"https://api.github.com/repos/tgravescs/spark/subscribers","subscription_url":"https://api.github.com/repos/tgravescs/spark/subscription","commits_url":"https://api.github.com/repos/tgravescs/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/tgravescs/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/tgravescs/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/tgravescs/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/tgravescs/spark/contents/{+path}","compare_url":"https://api.github.com/repos/tgravescs/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tgravescs/spark/merges","archive_url":"https://api.github.com/repos/tgravescs/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tgravescs/spark/downloads","issues_url":"https://api.github.com/repos/tgravescs/spark/issues{/number}","pulls_url":"https://api.github.com/repos/tgravescs/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/tgravescs/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/tgravescs/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tgravescs/spark/labels{/name}","releases_url":"https://api.github.com/repos/tgravescs/spark/releases{/id}","deployments_url":"https://api.github.com/repos/tgravescs/spark/deployments","created_at":"2014-02-27T13:26:24Z","updated_at":"2019-04-17T16:23:47Z","pushed_at":"2019-05-09T18:30:49Z","git_url":"git://github.com/tgravescs/spark.git","ssh_url":"git@github.com:tgravescs/spark.git","clone_url":"https://github.com/tgravescs/spark.git","svn_url":"https://github.com/tgravescs/spark","homepage":null,"size":273991,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":2,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"6eca435ac9d068a9500dac526b825c0ca574f3fd","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"node_id":"MDEwOlJlcG9zaXRvcnkxNzE2NTY1OA==","name":"spark","full_name":"apache/spark","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/spark","description":"Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2019-05-10T16:27:04Z","pushed_at":"2019-05-10T16:43:34Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":"","size":296161,"stargazers_count":21765,"watchers_count":21765,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":18846,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":469,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":18846,"open_issues":469,"watchers":21765,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/24406"},"html":{"href":"https://github.com/apache/spark/pull/24406"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/24406"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/24406/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/24406/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/24406/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/3d8e5cd3d846176d8d049f9b4511ffa6149ef837"}},"author_association":"CONTRIBUTOR"}}
other: {"actor":{"display_login":"mengxr"}}