payload: {"action":"created","comment":{"url":"https://api.github.com/repos/pravega/pravega/pulls/comments/265079663","pull_request_review_id":213657830,"id":265079663,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDI2NTA3OTY2Mw==","diff_hunk":"@@ -87,29 +86,38 @@ public CommitRequestHandler(final StreamMetadataStore streamMetadataStore,\n     public CompletableFuture<Void> execute(CommitEvent event) {\n         String scope = event.getScope();\n         String stream = event.getStream();\n-        int epoch = event.getEpoch();\n         OperationContext context = streamMetadataStore.createContext(scope, stream);\n-        log.debug(\"Attempting to commit available transactions on epoch {} on stream {}/{}\", event.getEpoch(), event.getScope(), event.getStream());\n+        log.debug(\"Attempting to commit available transactions on stream {}/{}\", event.getScope(), event.getStream());\n \n         CompletableFuture<Void> future = new CompletableFuture<>();\n \n-        tryCommitTransactions(scope, stream, epoch, context)\n+        // Note: we will ignore the epoch in the event. It has been deprecated. \n+        // The logic now finds the smallest epoch with transactions and commits them.\n+        tryCommitTransactions(scope, stream, context)\n                 .whenComplete((r, e) -> {\n                     if (e != null) {\n                         Throwable cause = Exceptions.unwrap(e);\n                         // for operation not allowed, we will report the event\n                         if (cause instanceof StoreException.OperationNotAllowedException) {\n-                            log.debug(\"Cannot commit transaction on epoch {} on stream {}/{}. Postponing\", epoch, scope, stream);\n+                            log.debug(\"Cannot commit transaction on stream {}/{}. Postponing\", scope, stream);\n                         } else {\n-                            log.error(\"Exception while attempting to commit transaction on epoch {} on stream {}/{}\", epoch, scope, stream, e);\n+                            log.error(\"Exception while attempting to commit transaction on stream {}/{}\", scope, stream, e);\n                         }\n                         future.completeExceptionally(cause);\n                     } else {\n-                        log.debug(\"Successfully committed transactions on epoch {} on stream {}/{}\", epoch, scope, stream);\n+                        if (r >= 0) {\n+                            log.debug(\"Successfully committed transactions on epoch {} on stream {}/{}\", r, scope, stream);\n+                        } else {\n+                            log.debug(\"No transactions found in committing state on stream {}/{}\", r, scope, stream);\n+                        }\n                         if (processedEvents != null) {\n-                            processedEvents.offer(event);\n+                            try {\n+                                processedEvents.offer(event);\n+                            } catch (Exception ex) {\n+                                // ignore, this processed events is only added for enabling unit testing this class","path":"controller/src/main/java/io/pravega/controller/server/eventProcessor/requesthandlers/CommitRequestHandler.java","position":47,"original_position":47,"commit_id":"11aa905fb7ba2558b943673c33ca4215305895bd","original_commit_id":"11aa905fb7ba2558b943673c33ca4215305895bd","user":{"login":"fpj","id":572920,"node_id":"MDQ6VXNlcjU3MjkyMA==","avatar_url":"https://avatars3.githubusercontent.com/u/572920?v=4","gravatar_id":"","url":"https://api.github.com/users/fpj","html_url":"https://github.com/fpj","followers_url":"https://api.github.com/users/fpj/followers","following_url":"https://api.github.com/users/fpj/following{/other_user}","gists_url":"https://api.github.com/users/fpj/gists{/gist_id}","starred_url":"https://api.github.com/users/fpj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fpj/subscriptions","organizations_url":"https://api.github.com/users/fpj/orgs","repos_url":"https://api.github.com/users/fpj/repos","events_url":"https://api.github.com/users/fpj/events{/privacy}","received_events_url":"https://api.github.com/users/fpj/received_events","type":"User","site_admin":false},"body":"What's `processedEvents`, is that state of the object only used for testing?","created_at":"2019-03-13T11:20:07Z","updated_at":"2019-03-13T11:45:20Z","html_url":"https://github.com/pravega/pravega/pull/3419#discussion_r265079663","pull_request_url":"https://api.github.com/repos/pravega/pravega/pulls/3419","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/pravega/pravega/pulls/comments/265079663"},"html":{"href":"https://github.com/pravega/pravega/pull/3419#discussion_r265079663"},"pull_request":{"href":"https://api.github.com/repos/pravega/pravega/pulls/3419"}}},"pull_request":{"url":"https://api.github.com/repos/pravega/pravega/pulls/3419","id":258182076,"node_id":"MDExOlB1bGxSZXF1ZXN0MjU4MTgyMDc2","html_url":"https://github.com/pravega/pravega/pull/3419","diff_url":"https://github.com/pravega/pravega/pull/3419.diff","patch_url":"https://github.com/pravega/pravega/pull/3419.patch","issue_url":"https://api.github.com/repos/pravega/pravega/issues/3419","number":3419,"state":"open","locked":false,"title":"Issue 3420: Commit order guarantee for Sequential Transactions","user":{"login":"shiveshr","id":13659022,"node_id":"MDQ6VXNlcjEzNjU5MDIy","avatar_url":"https://avatars2.githubusercontent.com/u/13659022?v=4","gravatar_id":"","url":"https://api.github.com/users/shiveshr","html_url":"https://github.com/shiveshr","followers_url":"https://api.github.com/users/shiveshr/followers","following_url":"https://api.github.com/users/shiveshr/following{/other_user}","gists_url":"https://api.github.com/users/shiveshr/gists{/gist_id}","starred_url":"https://api.github.com/users/shiveshr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shiveshr/subscriptions","organizations_url":"https://api.github.com/users/shiveshr/orgs","repos_url":"https://api.github.com/users/shiveshr/repos","events_url":"https://api.github.com/users/shiveshr/events{/privacy}","received_events_url":"https://api.github.com/users/shiveshr/received_events","type":"User","site_admin":false},"body":"**Change log description**  \r\n\r\nSince we consolidated multiple transactions in the same epoch at the time of commit, this had a drawback that the ordering of transactions commit did not match the order in which the commits were issued. \r\nThis is unintuitive for developers and detrimental to watermarking. \r\nWith this change, while keeping the benefits of consolidation, we have guaranteed strong ordering for \"sequential\" transactions committed on a per writer basis. \r\n\r\nWe say that a set of transactions are sequential when no two transactions in the set are executed concurrently and each transaction begins and commits sequentially.  For such transactions, we ensure that the transactions are committed in the order in which their commit was requested. \r\n\r\n**Purpose of the change**  \r\nFixes #3420 \r\n\r\n**What the code does**  \r\nThere are two parts to this change:\r\n1. introduction of writerId and commitTime in transactions.\r\n2. introduction of ordering positions for transactions. \r\n\r\nAs transaction commits are issued on a stream, a reference to the transaction id is optimistically added to the ordered set. This ordered set is implemented as a Zookeeper based scheme (ref `ZkOrderedStore`) which uses PersistentSequential znodes to assign order to transactions as their commit is attempted. \r\n\r\nThe transaction commit workflow (which works against a commitEvent) used to take requests per epoch and consolidate transactions marked for committing in the epoch and committed them. This could break the order. \r\nSo instead, now we always commit transactions in smallest available epoch first before moving forward. \r\nSo the CommitRequestHandler calls into store to startCommittingTransactions without supplying the epoch. \r\nThe store fetches ordered positions of transactions from the commit order set. It then filters transactions in smallest epoch that are in committing state and orders them by their positions in the ordered set. \r\n\r\nNow by doing this we automatically guarantee order within the epoch. And since we also start with smallest epoch, this means it writer did sequential transactions, their older transactions would be on lower epochs and hence order is not broken. \r\nNote: for non sequential transactions, we are not guaranteeing the order. \r\n\r\n**How to verify it**  \r\nunit tests added to verify transaction ordering as per expectation. \r\nI am also adding a new unit for ZkOrderedStore. ","created_at":"2019-03-05T04:43:31Z","updated_at":"2019-03-13T11:45:20Z","closed_at":null,"merged_at":null,"merge_commit_sha":"fb1725b8310df73deb5264e9ccfc8b4d4d8ab7b6","assignee":{"login":"tkaitchuck","id":4756752,"node_id":"MDQ6VXNlcjQ3NTY3NTI=","avatar_url":"https://avatars1.githubusercontent.com/u/4756752?v=4","gravatar_id":"","url":"https://api.github.com/users/tkaitchuck","html_url":"https://github.com/tkaitchuck","followers_url":"https://api.github.com/users/tkaitchuck/followers","following_url":"https://api.github.com/users/tkaitchuck/following{/other_user}","gists_url":"https://api.github.com/users/tkaitchuck/gists{/gist_id}","starred_url":"https://api.github.com/users/tkaitchuck/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tkaitchuck/subscriptions","organizations_url":"https://api.github.com/users/tkaitchuck/orgs","repos_url":"https://api.github.com/users/tkaitchuck/repos","events_url":"https://api.github.com/users/tkaitchuck/events{/privacy}","received_events_url":"https://api.github.com/users/tkaitchuck/received_events","type":"User","site_admin":false},"assignees":[{"login":"tkaitchuck","id":4756752,"node_id":"MDQ6VXNlcjQ3NTY3NTI=","avatar_url":"https://avatars1.githubusercontent.com/u/4756752?v=4","gravatar_id":"","url":"https://api.github.com/users/tkaitchuck","html_url":"https://github.com/tkaitchuck","followers_url":"https://api.github.com/users/tkaitchuck/followers","following_url":"https://api.github.com/users/tkaitchuck/following{/other_user}","gists_url":"https://api.github.com/users/tkaitchuck/gists{/gist_id}","starred_url":"https://api.github.com/users/tkaitchuck/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tkaitchuck/subscriptions","organizations_url":"https://api.github.com/users/tkaitchuck/orgs","repos_url":"https://api.github.com/users/tkaitchuck/repos","events_url":"https://api.github.com/users/tkaitchuck/events{/privacy}","received_events_url":"https://api.github.com/users/tkaitchuck/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/pravega/pravega/pulls/3419/commits","review_comments_url":"https://api.github.com/repos/pravega/pravega/pulls/3419/comments","review_comment_url":"https://api.github.com/repos/pravega/pravega/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pravega/pravega/issues/3419/comments","statuses_url":"https://api.github.com/repos/pravega/pravega/statuses/11aa905fb7ba2558b943673c33ca4215305895bd","head":{"label":"shiveshr:transaction-order-with-separate-queue","ref":"transaction-order-with-separate-queue","sha":"11aa905fb7ba2558b943673c33ca4215305895bd","user":{"login":"shiveshr","id":13659022,"node_id":"MDQ6VXNlcjEzNjU5MDIy","avatar_url":"https://avatars2.githubusercontent.com/u/13659022?v=4","gravatar_id":"","url":"https://api.github.com/users/shiveshr","html_url":"https://github.com/shiveshr","followers_url":"https://api.github.com/users/shiveshr/followers","following_url":"https://api.github.com/users/shiveshr/following{/other_user}","gists_url":"https://api.github.com/users/shiveshr/gists{/gist_id}","starred_url":"https://api.github.com/users/shiveshr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shiveshr/subscriptions","organizations_url":"https://api.github.com/users/shiveshr/orgs","repos_url":"https://api.github.com/users/shiveshr/repos","events_url":"https://api.github.com/users/shiveshr/events{/privacy}","received_events_url":"https://api.github.com/users/shiveshr/received_events","type":"User","site_admin":false},"repo":{"id":92372740,"node_id":"MDEwOlJlcG9zaXRvcnk5MjM3Mjc0MA==","name":"pravega-1","full_name":"shiveshr/pravega-1","private":false,"owner":{"login":"shiveshr","id":13659022,"node_id":"MDQ6VXNlcjEzNjU5MDIy","avatar_url":"https://avatars2.githubusercontent.com/u/13659022?v=4","gravatar_id":"","url":"https://api.github.com/users/shiveshr","html_url":"https://github.com/shiveshr","followers_url":"https://api.github.com/users/shiveshr/followers","following_url":"https://api.github.com/users/shiveshr/following{/other_user}","gists_url":"https://api.github.com/users/shiveshr/gists{/gist_id}","starred_url":"https://api.github.com/users/shiveshr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shiveshr/subscriptions","organizations_url":"https://api.github.com/users/shiveshr/orgs","repos_url":"https://api.github.com/users/shiveshr/repos","events_url":"https://api.github.com/users/shiveshr/events{/privacy}","received_events_url":"https://api.github.com/users/shiveshr/received_events","type":"User","site_admin":false},"html_url":"https://github.com/shiveshr/pravega-1","description":"Pravega - Streaming as a new software defined storage primitive","fork":true,"url":"https://api.github.com/repos/shiveshr/pravega-1","forks_url":"https://api.github.com/repos/shiveshr/pravega-1/forks","keys_url":"https://api.github.com/repos/shiveshr/pravega-1/keys{/key_id}","collaborators_url":"https://api.github.com/repos/shiveshr/pravega-1/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/shiveshr/pravega-1/teams","hooks_url":"https://api.github.com/repos/shiveshr/pravega-1/hooks","issue_events_url":"https://api.github.com/repos/shiveshr/pravega-1/issues/events{/number}","events_url":"https://api.github.com/repos/shiveshr/pravega-1/events","assignees_url":"https://api.github.com/repos/shiveshr/pravega-1/assignees{/user}","branches_url":"https://api.github.com/repos/shiveshr/pravega-1/branches{/branch}","tags_url":"https://api.github.com/repos/shiveshr/pravega-1/tags","blobs_url":"https://api.github.com/repos/shiveshr/pravega-1/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/shiveshr/pravega-1/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/shiveshr/pravega-1/git/refs{/sha}","trees_url":"https://api.github.com/repos/shiveshr/pravega-1/git/trees{/sha}","statuses_url":"https://api.github.com/repos/shiveshr/pravega-1/statuses/{sha}","languages_url":"https://api.github.com/repos/shiveshr/pravega-1/languages","stargazers_url":"https://api.github.com/repos/shiveshr/pravega-1/stargazers","contributors_url":"https://api.github.com/repos/shiveshr/pravega-1/contributors","subscribers_url":"https://api.github.com/repos/shiveshr/pravega-1/subscribers","subscription_url":"https://api.github.com/repos/shiveshr/pravega-1/subscription","commits_url":"https://api.github.com/repos/shiveshr/pravega-1/commits{/sha}","git_commits_url":"https://api.github.com/repos/shiveshr/pravega-1/git/commits{/sha}","comments_url":"https://api.github.com/repos/shiveshr/pravega-1/comments{/number}","issue_comment_url":"https://api.github.com/repos/shiveshr/pravega-1/issues/comments{/number}","contents_url":"https://api.github.com/repos/shiveshr/pravega-1/contents/{+path}","compare_url":"https://api.github.com/repos/shiveshr/pravega-1/compare/{base}...{head}","merges_url":"https://api.github.com/repos/shiveshr/pravega-1/merges","archive_url":"https://api.github.com/repos/shiveshr/pravega-1/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/shiveshr/pravega-1/downloads","issues_url":"https://api.github.com/repos/shiveshr/pravega-1/issues{/number}","pulls_url":"https://api.github.com/repos/shiveshr/pravega-1/pulls{/number}","milestones_url":"https://api.github.com/repos/shiveshr/pravega-1/milestones{/number}","notifications_url":"https://api.github.com/repos/shiveshr/pravega-1/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/shiveshr/pravega-1/labels{/name}","releases_url":"https://api.github.com/repos/shiveshr/pravega-1/releases{/id}","deployments_url":"https://api.github.com/repos/shiveshr/pravega-1/deployments","created_at":"2017-05-25T06:31:49Z","updated_at":"2019-03-13T03:23:41Z","pushed_at":"2019-03-13T03:23:39Z","git_url":"git://github.com/shiveshr/pravega-1.git","ssh_url":"git@github.com:shiveshr/pravega-1.git","clone_url":"https://github.com/shiveshr/pravega-1.git","svn_url":"https://github.com/shiveshr/pravega-1","homepage":"http://pravega.io","size":31277,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pravega:master","ref":"master","sha":"7b9fce657cbfba54e9c6bdee24d915f9c731c954","user":{"login":"pravega","id":25698199,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI1Njk4MTk5","avatar_url":"https://avatars1.githubusercontent.com/u/25698199?v=4","gravatar_id":"","url":"https://api.github.com/users/pravega","html_url":"https://github.com/pravega","followers_url":"https://api.github.com/users/pravega/followers","following_url":"https://api.github.com/users/pravega/following{/other_user}","gists_url":"https://api.github.com/users/pravega/gists{/gist_id}","starred_url":"https://api.github.com/users/pravega/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pravega/subscriptions","organizations_url":"https://api.github.com/users/pravega/orgs","repos_url":"https://api.github.com/users/pravega/repos","events_url":"https://api.github.com/users/pravega/events{/privacy}","received_events_url":"https://api.github.com/users/pravega/received_events","type":"Organization","site_admin":false},"repo":{"id":63096822,"node_id":"MDEwOlJlcG9zaXRvcnk2MzA5NjgyMg==","name":"pravega","full_name":"pravega/pravega","private":false,"owner":{"login":"pravega","id":25698199,"node_id":"MDEyOk9yZ2FuaXphdGlvbjI1Njk4MTk5","avatar_url":"https://avatars1.githubusercontent.com/u/25698199?v=4","gravatar_id":"","url":"https://api.github.com/users/pravega","html_url":"https://github.com/pravega","followers_url":"https://api.github.com/users/pravega/followers","following_url":"https://api.github.com/users/pravega/following{/other_user}","gists_url":"https://api.github.com/users/pravega/gists{/gist_id}","starred_url":"https://api.github.com/users/pravega/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pravega/subscriptions","organizations_url":"https://api.github.com/users/pravega/orgs","repos_url":"https://api.github.com/users/pravega/repos","events_url":"https://api.github.com/users/pravega/events{/privacy}","received_events_url":"https://api.github.com/users/pravega/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/pravega/pravega","description":"Pravega - Streaming as a new software defined storage primitive","fork":false,"url":"https://api.github.com/repos/pravega/pravega","forks_url":"https://api.github.com/repos/pravega/pravega/forks","keys_url":"https://api.github.com/repos/pravega/pravega/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pravega/pravega/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pravega/pravega/teams","hooks_url":"https://api.github.com/repos/pravega/pravega/hooks","issue_events_url":"https://api.github.com/repos/pravega/pravega/issues/events{/number}","events_url":"https://api.github.com/repos/pravega/pravega/events","assignees_url":"https://api.github.com/repos/pravega/pravega/assignees{/user}","branches_url":"https://api.github.com/repos/pravega/pravega/branches{/branch}","tags_url":"https://api.github.com/repos/pravega/pravega/tags","blobs_url":"https://api.github.com/repos/pravega/pravega/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pravega/pravega/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pravega/pravega/git/refs{/sha}","trees_url":"https://api.github.com/repos/pravega/pravega/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pravega/pravega/statuses/{sha}","languages_url":"https://api.github.com/repos/pravega/pravega/languages","stargazers_url":"https://api.github.com/repos/pravega/pravega/stargazers","contributors_url":"https://api.github.com/repos/pravega/pravega/contributors","subscribers_url":"https://api.github.com/repos/pravega/pravega/subscribers","subscription_url":"https://api.github.com/repos/pravega/pravega/subscription","commits_url":"https://api.github.com/repos/pravega/pravega/commits{/sha}","git_commits_url":"https://api.github.com/repos/pravega/pravega/git/commits{/sha}","comments_url":"https://api.github.com/repos/pravega/pravega/comments{/number}","issue_comment_url":"https://api.github.com/repos/pravega/pravega/issues/comments{/number}","contents_url":"https://api.github.com/repos/pravega/pravega/contents/{+path}","compare_url":"https://api.github.com/repos/pravega/pravega/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pravega/pravega/merges","archive_url":"https://api.github.com/repos/pravega/pravega/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pravega/pravega/downloads","issues_url":"https://api.github.com/repos/pravega/pravega/issues{/number}","pulls_url":"https://api.github.com/repos/pravega/pravega/pulls{/number}","milestones_url":"https://api.github.com/repos/pravega/pravega/milestones{/number}","notifications_url":"https://api.github.com/repos/pravega/pravega/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pravega/pravega/labels{/name}","releases_url":"https://api.github.com/repos/pravega/pravega/releases{/id}","deployments_url":"https://api.github.com/repos/pravega/pravega/deployments","created_at":"2016-07-11T19:41:03Z","updated_at":"2019-03-13T11:45:00Z","pushed_at":"2019-03-13T10:20:51Z","git_url":"git://github.com/pravega/pravega.git","ssh_url":"git@github.com:pravega/pravega.git","clone_url":"https://github.com/pravega/pravega.git","svn_url":"https://github.com/pravega/pravega","homepage":"http://pravega.io","size":28810,"stargazers_count":469,"watchers_count":469,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":127,"mirror_url":null,"archived":false,"open_issues_count":307,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":127,"open_issues":307,"watchers":469,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pravega/pravega/pulls/3419"},"html":{"href":"https://github.com/pravega/pravega/pull/3419"},"issue":{"href":"https://api.github.com/repos/pravega/pravega/issues/3419"},"comments":{"href":"https://api.github.com/repos/pravega/pravega/issues/3419/comments"},"review_comments":{"href":"https://api.github.com/repos/pravega/pravega/pulls/3419/comments"},"review_comment":{"href":"https://api.github.com/repos/pravega/pravega/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pravega/pravega/pulls/3419/commits"},"statuses":{"href":"https://api.github.com/repos/pravega/pravega/statuses/11aa905fb7ba2558b943673c33ca4215305895bd"}},"author_association":"CONTRIBUTOR"}}
other: {"actor":{"display_login":"fpj"}}