payload: {"action":"opened","number":2545,"pull_request":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/2545","id":256110193,"node_id":"MDExOlB1bGxSZXF1ZXN0MjU2MTEwMTkz","html_url":"https://github.com/allenai/allennlp/pull/2545","diff_url":"https://github.com/allenai/allennlp/pull/2545.diff","patch_url":"https://github.com/allenai/allennlp/pull/2545.patch","issue_url":"https://api.github.com/repos/allenai/allennlp/issues/2545","number":2545,"state":"open","locked":false,"title":"Extend evaluate to work with untrained models.","user":{"login":"brendan-ai2","id":16342367,"node_id":"MDQ6VXNlcjE2MzQyMzY3","avatar_url":"https://avatars2.githubusercontent.com/u/16342367?v=4","gravatar_id":"","url":"https://api.github.com/users/brendan-ai2","html_url":"https://github.com/brendan-ai2","followers_url":"https://api.github.com/users/brendan-ai2/followers","following_url":"https://api.github.com/users/brendan-ai2/following{/other_user}","gists_url":"https://api.github.com/users/brendan-ai2/gists{/gist_id}","starred_url":"https://api.github.com/users/brendan-ai2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brendan-ai2/subscriptions","organizations_url":"https://api.github.com/users/brendan-ai2/orgs","repos_url":"https://api.github.com/users/brendan-ai2/repos","events_url":"https://api.github.com/users/brendan-ai2/events{/privacy}","received_events_url":"https://api.github.com/users/brendan-ai2/received_events","type":"User","site_admin":false},"body":"- Allow `allennlp evaluate` to operate on untrained models, i.e. `.jsonnet` config.\r\n- Motivation: For the S2 salience work we're comparing against a number of untrained baselines like tf-idf. It's convenient to do this in AllenNLP to access the dataset readers and metrics.","created_at":"2019-02-26T01:36:43Z","updated_at":"2019-02-26T01:36:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/allenai/allennlp/pulls/2545/commits","review_comments_url":"https://api.github.com/repos/allenai/allennlp/pulls/2545/comments","review_comment_url":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/allennlp/issues/2545/comments","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/84980a800a5440eb70ad111dafffd338b29848ac","head":{"label":"brendan-ai2:untrained-eval","ref":"untrained-eval","sha":"84980a800a5440eb70ad111dafffd338b29848ac","user":{"login":"brendan-ai2","id":16342367,"node_id":"MDQ6VXNlcjE2MzQyMzY3","avatar_url":"https://avatars2.githubusercontent.com/u/16342367?v=4","gravatar_id":"","url":"https://api.github.com/users/brendan-ai2","html_url":"https://github.com/brendan-ai2","followers_url":"https://api.github.com/users/brendan-ai2/followers","following_url":"https://api.github.com/users/brendan-ai2/following{/other_user}","gists_url":"https://api.github.com/users/brendan-ai2/gists{/gist_id}","starred_url":"https://api.github.com/users/brendan-ai2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brendan-ai2/subscriptions","organizations_url":"https://api.github.com/users/brendan-ai2/orgs","repos_url":"https://api.github.com/users/brendan-ai2/repos","events_url":"https://api.github.com/users/brendan-ai2/events{/privacy}","received_events_url":"https://api.github.com/users/brendan-ai2/received_events","type":"User","site_admin":false},"repo":{"id":138935946,"node_id":"MDEwOlJlcG9zaXRvcnkxMzg5MzU5NDY=","name":"allennlp","full_name":"brendan-ai2/allennlp","private":false,"owner":{"login":"brendan-ai2","id":16342367,"node_id":"MDQ6VXNlcjE2MzQyMzY3","avatar_url":"https://avatars2.githubusercontent.com/u/16342367?v=4","gravatar_id":"","url":"https://api.github.com/users/brendan-ai2","html_url":"https://github.com/brendan-ai2","followers_url":"https://api.github.com/users/brendan-ai2/followers","following_url":"https://api.github.com/users/brendan-ai2/following{/other_user}","gists_url":"https://api.github.com/users/brendan-ai2/gists{/gist_id}","starred_url":"https://api.github.com/users/brendan-ai2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brendan-ai2/subscriptions","organizations_url":"https://api.github.com/users/brendan-ai2/orgs","repos_url":"https://api.github.com/users/brendan-ai2/repos","events_url":"https://api.github.com/users/brendan-ai2/events{/privacy}","received_events_url":"https://api.github.com/users/brendan-ai2/received_events","type":"User","site_admin":false},"html_url":"https://github.com/brendan-ai2/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":true,"url":"https://api.github.com/repos/brendan-ai2/allennlp","forks_url":"https://api.github.com/repos/brendan-ai2/allennlp/forks","keys_url":"https://api.github.com/repos/brendan-ai2/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/brendan-ai2/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/brendan-ai2/allennlp/teams","hooks_url":"https://api.github.com/repos/brendan-ai2/allennlp/hooks","issue_events_url":"https://api.github.com/repos/brendan-ai2/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/brendan-ai2/allennlp/events","assignees_url":"https://api.github.com/repos/brendan-ai2/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/brendan-ai2/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/brendan-ai2/allennlp/tags","blobs_url":"https://api.github.com/repos/brendan-ai2/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/brendan-ai2/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/brendan-ai2/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/brendan-ai2/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/brendan-ai2/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/brendan-ai2/allennlp/languages","stargazers_url":"https://api.github.com/repos/brendan-ai2/allennlp/stargazers","contributors_url":"https://api.github.com/repos/brendan-ai2/allennlp/contributors","subscribers_url":"https://api.github.com/repos/brendan-ai2/allennlp/subscribers","subscription_url":"https://api.github.com/repos/brendan-ai2/allennlp/subscription","commits_url":"https://api.github.com/repos/brendan-ai2/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/brendan-ai2/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/brendan-ai2/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/brendan-ai2/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/brendan-ai2/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/brendan-ai2/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/brendan-ai2/allennlp/merges","archive_url":"https://api.github.com/repos/brendan-ai2/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/brendan-ai2/allennlp/downloads","issues_url":"https://api.github.com/repos/brendan-ai2/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/brendan-ai2/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/brendan-ai2/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/brendan-ai2/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/brendan-ai2/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/brendan-ai2/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/brendan-ai2/allennlp/deployments","created_at":"2018-06-27T21:42:43Z","updated_at":"2018-07-18T22:32:31Z","pushed_at":"2019-02-26T01:31:01Z","git_url":"git://github.com/brendan-ai2/allennlp.git","ssh_url":"git@github.com:brendan-ai2/allennlp.git","clone_url":"https://github.com/brendan-ai2/allennlp.git","svn_url":"https://github.com/brendan-ai2/allennlp","homepage":"http://www.allennlp.org/","size":59149,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"6d8da97312bfbde05a41558668ff63d92a9928e9","user":{"login":"allenai","id":5667695,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2Njc2OTU=","avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":91356408,"node_id":"MDEwOlJlcG9zaXRvcnk5MTM1NjQwOA==","name":"allennlp","full_name":"allenai/allennlp","private":false,"owner":{"login":"allenai","id":5667695,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2Njc2OTU=","avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/allenai/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":false,"url":"https://api.github.com/repos/allenai/allennlp","forks_url":"https://api.github.com/repos/allenai/allennlp/forks","keys_url":"https://api.github.com/repos/allenai/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/allennlp/teams","hooks_url":"https://api.github.com/repos/allenai/allennlp/hooks","issue_events_url":"https://api.github.com/repos/allenai/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/allennlp/events","assignees_url":"https://api.github.com/repos/allenai/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/allennlp/tags","blobs_url":"https://api.github.com/repos/allenai/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/allennlp/languages","stargazers_url":"https://api.github.com/repos/allenai/allennlp/stargazers","contributors_url":"https://api.github.com/repos/allenai/allennlp/contributors","subscribers_url":"https://api.github.com/repos/allenai/allennlp/subscribers","subscription_url":"https://api.github.com/repos/allenai/allennlp/subscription","commits_url":"https://api.github.com/repos/allenai/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/allennlp/merges","archive_url":"https://api.github.com/repos/allenai/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/allennlp/downloads","issues_url":"https://api.github.com/repos/allenai/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/allenai/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/allenai/allennlp/deployments","created_at":"2017-05-15T15:52:41Z","updated_at":"2019-02-26T00:48:25Z","pushed_at":"2019-02-25T05:54:48Z","git_url":"git://github.com/allenai/allennlp.git","ssh_url":"git@github.com:allenai/allennlp.git","clone_url":"https://github.com/allenai/allennlp.git","svn_url":"https://github.com/allenai/allennlp","homepage":"http://www.allennlp.org/","size":58383,"stargazers_count":5425,"watchers_count":5425,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1024,"mirror_url":null,"archived":false,"open_issues_count":172,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":1024,"open_issues":172,"watchers":5425,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2545"},"html":{"href":"https://github.com/allenai/allennlp/pull/2545"},"issue":{"href":"https://api.github.com/repos/allenai/allennlp/issues/2545"},"comments":{"href":"https://api.github.com/repos/allenai/allennlp/issues/2545/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2545/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2545/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/allennlp/statuses/84980a800a5440eb70ad111dafffd338b29848ac"}},"author_association":"MEMBER","merged":false,"mergeable":null,"rebaseable":null,"mergeable_state":"unknown","merged_by":null,"comments":0,"review_comments":0,"maintainer_can_modify":true,"commits":1,"additions":36,"deletions":5,"changed_files":2}}
other: {"actor":{"display_login":"brendan-ai2"}}