From adedf103c46fa38f7781e0a7140175441047fd3e Mon Sep 17 00:00:00 2001 From: mnoah1 Date: Mon, 12 Oct 2026 20:23:03 +0000 Subject: [PATCH] feat(stovepipe): include current summaries in request history Summary: ## Why? History callers need the current request status alongside retained lifecycle events. ## What? Add `request_summary` to both request-history response shapes, preserving existing event fields and field numbers. Both selectors share a reader for retained logs and the stored current summary; wire mapping reuses the existing summary mapper. Successful lookups require a summary. A missing summary fails with a retryable infrastructure error because initial log persistence precedes summary creation. Summary identity or URI mismatches fail as consistency errors. Summaries are not reconstructed from retained event subsets. Use a generated controller mock in handler tests and update the history contract documentation. ## Test Plan - Bazel controller, mapper, and handler unit tests passed. - Docker integration suite passed, including failed lookups before summary creation and successful responses afterward, without an operational Request row. - `make lint check-tidy check-gazelle check-mocks` passed. ## Issue None. --- Makefile | 1 + api/stovepipe/proto/stovepipe.proto | 8 +- api/stovepipe/protopb/stovepipe.pb.go | 86 ++++++---- api/stovepipe/protopb/stovepipe.pb.yarpc.go | 136 +++++++-------- doc/rfc/stovepipe/request-history-api.md | 157 +++--------------- service/stovepipe/server/handler/BUILD.bazel | 2 + service/stovepipe/server/handler/server.go | 4 +- .../stovepipe/server/handler/server_test.go | 53 ++---- .../server/mapper/request_history.go | 13 +- .../server/mapper/request_history_test.go | 28 +++- stovepipe/controller/mock/BUILD.bazel | 12 ++ .../controller/mock/request_history_mock.go | 72 ++++++++ stovepipe/controller/read_errors.go | 14 ++ stovepipe/controller/request_history.go | 60 +++++-- stovepipe/controller/request_history_test.go | 117 ++++++++++++- stovepipe/entity/request_history.go | 4 +- test/integration/stovepipe/BUILD.bazel | 1 + test/integration/stovepipe/suite_test.go | 29 +++- 18 files changed, 496 insertions(+), 301 deletions(-) create mode 100644 stovepipe/controller/mock/BUILD.bazel create mode 100644 stovepipe/controller/mock/request_history_mock.go diff --git a/Makefile b/Makefile index f40320156..418f46ff8 100644 --- a/Makefile +++ b/Makefile @@ -613,6 +613,7 @@ GO_GENERATE_PACKAGES := \ ./platform/extension/hook/... \ ./platform/extension/messagequeue/... \ ./runway/extension/merger/... \ + ./stovepipe/controller/... \ ./stovepipe/core/requestlog/... \ ./stovepipe/extension/buildrunner/... \ ./stovepipe/extension/projectstatus/... \ diff --git a/api/stovepipe/proto/stovepipe.proto b/api/stovepipe/proto/stovepipe.proto index 24b657d79..482067238 100644 --- a/api/stovepipe/proto/stovepipe.proto +++ b/api/stovepipe/proto/stovepipe.proto @@ -86,18 +86,22 @@ message HistoryEvent { string outcome_reason = 5; } -// RequestHistory identifies one request and contains its ordered events. +// RequestHistory identifies one request and contains its ordered events and current summary. message RequestHistory { // Queue-scoped decimal request identifier. string request_id = 1; // Events ordered by occurrence time and stable event identity. repeated HistoryEvent events = 2; + // Current materialized lifecycle summary. Always present in successful responses. + RequestSummary request_summary = 3; } -// GetRequestHistoryByIDResponse contains all retained events for one request. +// GetRequestHistoryByIDResponse contains retained events and the current summary for one request. message GetRequestHistoryByIDResponse { // Events ordered by occurrence time and stable event identity. repeated HistoryEvent events = 1; + // Current materialized lifecycle summary. Always present in successful responses. + RequestSummary request_summary = 2; } // GetRequestHistoryByURIResponse contains all retained request histories for an exact URI. diff --git a/api/stovepipe/protopb/stovepipe.pb.go b/api/stovepipe/protopb/stovepipe.pb.go index 48dc97754..ff121a9fa 100644 --- a/api/stovepipe/protopb/stovepipe.pb.go +++ b/api/stovepipe/protopb/stovepipe.pb.go @@ -473,15 +473,17 @@ func (*HistoryEvent_RequestState) isHistoryEvent_Occurrence() {} func (*HistoryEvent_Event) isHistoryEvent_Occurrence() {} -// RequestHistory identifies one request and contains its ordered events. +// RequestHistory identifies one request and contains its ordered events and current summary. type RequestHistory struct { state protoimpl.MessageState `protogen:"open.v1"` // Queue-scoped decimal request identifier. RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // Events ordered by occurrence time and stable event identity. - Events []*HistoryEvent `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Events []*HistoryEvent `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` + // Current materialized lifecycle summary. Always present in successful responses. + RequestSummary *RequestSummary `protobuf:"bytes,3,opt,name=request_summary,json=requestSummary,proto3" json:"request_summary,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RequestHistory) Reset() { @@ -528,13 +530,22 @@ func (x *RequestHistory) GetEvents() []*HistoryEvent { return nil } -// GetRequestHistoryByIDResponse contains all retained events for one request. +func (x *RequestHistory) GetRequestSummary() *RequestSummary { + if x != nil { + return x.RequestSummary + } + return nil +} + +// GetRequestHistoryByIDResponse contains retained events and the current summary for one request. type GetRequestHistoryByIDResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Events ordered by occurrence time and stable event identity. - Events []*HistoryEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Events []*HistoryEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + // Current materialized lifecycle summary. Always present in successful responses. + RequestSummary *RequestSummary `protobuf:"bytes,2,opt,name=request_summary,json=requestSummary,proto3" json:"request_summary,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetRequestHistoryByIDResponse) Reset() { @@ -574,6 +585,13 @@ func (x *GetRequestHistoryByIDResponse) GetEvents() []*HistoryEvent { return nil } +func (x *GetRequestHistoryByIDResponse) GetRequestSummary() *RequestSummary { + if x != nil { + return x.RequestSummary + } + return nil +} + // GetRequestHistoryByURIResponse contains all retained request histories for an exact URI. type GetRequestHistoryByURIResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1252,13 +1270,15 @@ const file_stovepipe_proto_rawDesc = "" + "\x05event\x18\x04 \x01(\tH\x00R\x05event\x12%\n" + "\x0eoutcome_reason\x18\x05 \x01(\tR\routcomeReasonB\f\n" + "\n" + - "occurrence\"q\n" + + "occurrence\"\xc6\x01\n" + "\x0eRequestHistory\x12\x1d\n" + "\n" + "request_id\x18\x01 \x01(\tR\trequestId\x12@\n" + - "\x06events\x18\x02 \x03(\v2(.uber.submitqueue.stovepipe.HistoryEventR\x06events\"a\n" + + "\x06events\x18\x02 \x03(\v2(.uber.submitqueue.stovepipe.HistoryEventR\x06events\x12S\n" + + "\x0frequest_summary\x18\x03 \x01(\v2*.uber.submitqueue.stovepipe.RequestSummaryR\x0erequestSummary\"\xb6\x01\n" + "\x1dGetRequestHistoryByIDResponse\x12@\n" + - "\x06events\x18\x01 \x03(\v2(.uber.submitqueue.stovepipe.HistoryEventR\x06events\"j\n" + + "\x06events\x18\x01 \x03(\v2(.uber.submitqueue.stovepipe.HistoryEventR\x06events\x12S\n" + + "\x0frequest_summary\x18\x02 \x01(\v2*.uber.submitqueue.stovepipe.RequestSummaryR\x0erequestSummary\"j\n" + "\x1eGetRequestHistoryByURIResponse\x12H\n" + "\thistories\x18\x01 \x03(\v2*.uber.submitqueue.stovepipe.RequestHistoryR\thistories\"\xab\x01\n" + "\x1cGetProjectStatusByURIRequest\x12\x14\n" + @@ -1353,27 +1373,29 @@ var file_stovepipe_proto_goTypes = []any{ } var file_stovepipe_proto_depIdxs = []int32{ 6, // 0: uber.submitqueue.stovepipe.RequestHistory.events:type_name -> uber.submitqueue.stovepipe.HistoryEvent - 6, // 1: uber.submitqueue.stovepipe.GetRequestHistoryByIDResponse.events:type_name -> uber.submitqueue.stovepipe.HistoryEvent - 7, // 2: uber.submitqueue.stovepipe.GetRequestHistoryByURIResponse.histories:type_name -> uber.submitqueue.stovepipe.RequestHistory - 11, // 3: uber.submitqueue.stovepipe.GetProjectStatusByURIResponse.projects:type_name -> uber.submitqueue.stovepipe.ProjectValidation - 13, // 4: uber.submitqueue.stovepipe.ListResponse.requests:type_name -> uber.submitqueue.stovepipe.RequestSummary - 0, // 5: uber.submitqueue.stovepipe.Stovepipe.Ping:input_type -> uber.submitqueue.stovepipe.PingRequest - 2, // 6: uber.submitqueue.stovepipe.Stovepipe.Ingest:input_type -> uber.submitqueue.stovepipe.IngestRequest - 4, // 7: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByID:input_type -> uber.submitqueue.stovepipe.GetRequestHistoryByIDRequest - 5, // 8: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByURI:input_type -> uber.submitqueue.stovepipe.GetRequestHistoryByURIRequest - 10, // 9: uber.submitqueue.stovepipe.Stovepipe.GetProjectStatusByURI:input_type -> uber.submitqueue.stovepipe.GetProjectStatusByURIRequest - 14, // 10: uber.submitqueue.stovepipe.Stovepipe.List:input_type -> uber.submitqueue.stovepipe.ListRequest - 1, // 11: uber.submitqueue.stovepipe.Stovepipe.Ping:output_type -> uber.submitqueue.stovepipe.PingResponse - 3, // 12: uber.submitqueue.stovepipe.Stovepipe.Ingest:output_type -> uber.submitqueue.stovepipe.IngestResponse - 8, // 13: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByID:output_type -> uber.submitqueue.stovepipe.GetRequestHistoryByIDResponse - 9, // 14: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByURI:output_type -> uber.submitqueue.stovepipe.GetRequestHistoryByURIResponse - 12, // 15: uber.submitqueue.stovepipe.Stovepipe.GetProjectStatusByURI:output_type -> uber.submitqueue.stovepipe.GetProjectStatusByURIResponse - 15, // 16: uber.submitqueue.stovepipe.Stovepipe.List:output_type -> uber.submitqueue.stovepipe.ListResponse - 11, // [11:17] is the sub-list for method output_type - 5, // [5:11] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 13, // 1: uber.submitqueue.stovepipe.RequestHistory.request_summary:type_name -> uber.submitqueue.stovepipe.RequestSummary + 6, // 2: uber.submitqueue.stovepipe.GetRequestHistoryByIDResponse.events:type_name -> uber.submitqueue.stovepipe.HistoryEvent + 13, // 3: uber.submitqueue.stovepipe.GetRequestHistoryByIDResponse.request_summary:type_name -> uber.submitqueue.stovepipe.RequestSummary + 7, // 4: uber.submitqueue.stovepipe.GetRequestHistoryByURIResponse.histories:type_name -> uber.submitqueue.stovepipe.RequestHistory + 11, // 5: uber.submitqueue.stovepipe.GetProjectStatusByURIResponse.projects:type_name -> uber.submitqueue.stovepipe.ProjectValidation + 13, // 6: uber.submitqueue.stovepipe.ListResponse.requests:type_name -> uber.submitqueue.stovepipe.RequestSummary + 0, // 7: uber.submitqueue.stovepipe.Stovepipe.Ping:input_type -> uber.submitqueue.stovepipe.PingRequest + 2, // 8: uber.submitqueue.stovepipe.Stovepipe.Ingest:input_type -> uber.submitqueue.stovepipe.IngestRequest + 4, // 9: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByID:input_type -> uber.submitqueue.stovepipe.GetRequestHistoryByIDRequest + 5, // 10: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByURI:input_type -> uber.submitqueue.stovepipe.GetRequestHistoryByURIRequest + 10, // 11: uber.submitqueue.stovepipe.Stovepipe.GetProjectStatusByURI:input_type -> uber.submitqueue.stovepipe.GetProjectStatusByURIRequest + 14, // 12: uber.submitqueue.stovepipe.Stovepipe.List:input_type -> uber.submitqueue.stovepipe.ListRequest + 1, // 13: uber.submitqueue.stovepipe.Stovepipe.Ping:output_type -> uber.submitqueue.stovepipe.PingResponse + 3, // 14: uber.submitqueue.stovepipe.Stovepipe.Ingest:output_type -> uber.submitqueue.stovepipe.IngestResponse + 8, // 15: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByID:output_type -> uber.submitqueue.stovepipe.GetRequestHistoryByIDResponse + 9, // 16: uber.submitqueue.stovepipe.Stovepipe.GetRequestHistoryByURI:output_type -> uber.submitqueue.stovepipe.GetRequestHistoryByURIResponse + 12, // 17: uber.submitqueue.stovepipe.Stovepipe.GetProjectStatusByURI:output_type -> uber.submitqueue.stovepipe.GetProjectStatusByURIResponse + 15, // 18: uber.submitqueue.stovepipe.Stovepipe.List:output_type -> uber.submitqueue.stovepipe.ListResponse + 13, // [13:19] is the sub-list for method output_type + 7, // [7:13] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_stovepipe_proto_init() } diff --git a/api/stovepipe/protopb/stovepipe.pb.yarpc.go b/api/stovepipe/protopb/stovepipe.pb.yarpc.go index b55bcac99..178dcff5e 100644 --- a/api/stovepipe/protopb/stovepipe.pb.yarpc.go +++ b/api/stovepipe/protopb/stovepipe.pb.yarpc.go @@ -476,73 +476,75 @@ var ( var yarpcFileDescriptorClosurefabdb6b3c0b09022 = [][]byte{ // stovepipe.proto []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x57, 0x4f, 0x73, 0xdb, 0x44, - 0x14, 0xb7, 0xec, 0xc6, 0xb1, 0x5f, 0x1c, 0x27, 0xdd, 0x94, 0xe0, 0x9a, 0xa6, 0x13, 0x04, 0xa1, - 0x6e, 0x67, 0xea, 0x0c, 0x65, 0x86, 0x29, 0x1c, 0x18, 0x6a, 0x0a, 0xb5, 0x67, 0x08, 0x64, 0x64, - 0xc2, 0x01, 0x0e, 0x1a, 0x59, 0x7e, 0x75, 0xb6, 0x8d, 0xb4, 0xca, 0xee, 0x2a, 0xd0, 0x1e, 0xb8, - 0x71, 0x65, 0x86, 0x6f, 0xc0, 0x9d, 0x2f, 0xc1, 0xd7, 0xe0, 0xc2, 0xd7, 0xe0, 0xca, 0xec, 0x6a, - 0x25, 0xcb, 0x8e, 0x2d, 0xb7, 0xe1, 0xd0, 0x9b, 0xf6, 0xfd, 0xdd, 0xfd, 0xbd, 0xdf, 0x7b, 0xbb, - 0x82, 0x2d, 0x21, 0xd9, 0x05, 0x46, 0x34, 0xc2, 0x6e, 0xc4, 0x99, 0x64, 0xa4, 0x1d, 0x8f, 0x90, - 0x77, 0x45, 0x3c, 0x0a, 0xa8, 0x3c, 0x8f, 0x31, 0xc6, 0x6e, 0x66, 0x61, 0xdf, 0x81, 0x8d, 0x63, - 0x1a, 0x4e, 0x1c, 0x3c, 0x8f, 0x51, 0x48, 0xd2, 0x82, 0xf5, 0x00, 0x85, 0xf0, 0x26, 0xd8, 0xb2, - 0xf6, 0xad, 0x4e, 0xdd, 0x49, 0x97, 0xf6, 0xaf, 0x16, 0x34, 0x12, 0x4b, 0x11, 0xb1, 0x50, 0xe0, - 0x72, 0x53, 0xf2, 0x2e, 0x34, 0x04, 0xf2, 0x0b, 0xea, 0xa3, 0x1b, 0x7a, 0x01, 0xb6, 0xca, 0x5a, - 0xbd, 0x61, 0x64, 0xdf, 0x78, 0x01, 0x92, 0x5b, 0x50, 0x97, 0x34, 0x40, 0x21, 0xbd, 0x20, 0x6a, - 0x55, 0xf6, 0xad, 0x4e, 0xc5, 0x99, 0x0a, 0x48, 0x1b, 0x6a, 0xa7, 0x4c, 0x48, 0xed, 0x7c, 0x4d, - 0x3b, 0x67, 0x6b, 0xfb, 0x00, 0x36, 0x07, 0xe1, 0x04, 0x85, 0x4c, 0xb7, 0x7c, 0x03, 0xd6, 0xf4, - 0xa1, 0xcc, 0x2e, 0x92, 0x85, 0xbd, 0x0f, 0xcd, 0xd4, 0xcc, 0xec, 0xb7, 0x09, 0x65, 0x3a, 0x36, - 0x46, 0x65, 0x3a, 0xb6, 0x87, 0x70, 0xeb, 0x09, 0xa6, 0x51, 0xfa, 0x54, 0x48, 0xc6, 0x5f, 0xf4, - 0x5e, 0x0c, 0x1e, 0x17, 0xc6, 0x25, 0x7b, 0x00, 0x3c, 0x31, 0x70, 0xe9, 0xd8, 0x9c, 0xac, 0x6e, - 0x24, 0x83, 0xb1, 0xfd, 0x04, 0xf6, 0x16, 0x04, 0x3d, 0x71, 0x06, 0xc5, 0x51, 0xb7, 0xa1, 0x12, - 0x73, 0x6a, 0xc2, 0xa9, 0x4f, 0xfb, 0x2f, 0x0b, 0x1a, 0xc6, 0xff, 0xcb, 0x0b, 0x0c, 0x25, 0xb9, - 0x09, 0x35, 0x54, 0x1f, 0x6e, 0x76, 0x88, 0x75, 0xbd, 0x1e, 0x8c, 0x15, 0xde, 0x19, 0x76, 0x6e, - 0x20, 0x74, 0x98, 0x8a, 0xb3, 0x91, 0xc9, 0x8e, 0x04, 0x39, 0x80, 0xcd, 0x74, 0xdb, 0x42, 0x7a, - 0x12, 0x35, 0xe6, 0xf5, 0x7e, 0xc9, 0x69, 0x18, 0xf1, 0x50, 0x49, 0xc9, 0x2e, 0xac, 0xe9, 0xa0, - 0x09, 0xea, 0xfd, 0x92, 0x93, 0x2c, 0xc9, 0x01, 0x34, 0x59, 0x2c, 0x7d, 0x16, 0xa0, 0xcb, 0xd1, - 0x13, 0x2c, 0x6c, 0xad, 0xe9, 0x2d, 0x6c, 0x1a, 0xa9, 0xa3, 0x85, 0xbd, 0x06, 0x00, 0xf3, 0xfd, - 0x98, 0x73, 0x0c, 0x7d, 0xb4, 0xcf, 0xa1, 0x39, 0x0b, 0xc4, 0x1c, 0x78, 0xd6, 0x1c, 0x78, 0xe4, - 0x73, 0xa8, 0xea, 0x74, 0xea, 0x04, 0x95, 0xce, 0xc6, 0x83, 0x4e, 0x77, 0x39, 0x71, 0xbb, 0x79, - 0x70, 0x1c, 0xe3, 0x67, 0x7b, 0x0b, 0xe1, 0x57, 0x35, 0x35, 0x24, 0x98, 0xa6, 0xb0, 0xae, 0x98, - 0xe2, 0x19, 0xdc, 0x5e, 0x56, 0x61, 0x93, 0xa3, 0x0f, 0xf5, 0x53, 0x2d, 0xa7, 0x98, 0xa6, 0xb9, - 0x57, 0x94, 0x66, 0x36, 0x96, 0x33, 0x75, 0xb6, 0xff, 0xb4, 0x34, 0x47, 0x8f, 0x39, 0x7b, 0x86, - 0xbe, 0x2e, 0x51, 0x2c, 0x5e, 0x81, 0x4d, 0x7b, 0x00, 0xfe, 0xa9, 0x17, 0x4e, 0xd0, 0x9d, 0x92, - 0xaa, 0x9e, 0x48, 0x4e, 0x38, 0x55, 0xdd, 0x15, 0x25, 0x11, 0x45, 0xab, 0xb2, 0x5f, 0x51, 0xdd, - 0x95, 0xae, 0xc9, 0x3b, 0x50, 0x8f, 0xbc, 0x09, 0xba, 0x82, 0xbe, 0x4c, 0x5a, 0x6f, 0xcd, 0xa9, - 0x29, 0xc1, 0x90, 0xbe, 0xd4, 0x71, 0xb5, 0x52, 0xb2, 0xe7, 0x98, 0x32, 0x40, 0x9b, 0x7f, 0xa7, - 0x04, 0xf6, 0x08, 0xae, 0x9b, 0x9d, 0x7e, 0xef, 0x9d, 0xd1, 0xb1, 0x27, 0x29, 0x0b, 0xd5, 0x94, - 0x30, 0xc1, 0x53, 0xd6, 0x9a, 0x25, 0xb9, 0x0b, 0x5b, 0x23, 0x8e, 0xde, 0x73, 0x15, 0x71, 0x8c, - 0x13, 0x8e, 0xc9, 0xa0, 0xb0, 0xfa, 0x25, 0xa7, 0x99, 0x2a, 0x1e, 0x6b, 0x79, 0xaf, 0x06, 0x55, - 0x8e, 0x22, 0x3e, 0x93, 0xf6, 0xdf, 0x15, 0x5d, 0xe1, 0x45, 0x88, 0x18, 0xf4, 0x57, 0x70, 0x2c, - 0x43, 0xac, 0xbc, 0x1c, 0xb1, 0xca, 0x3c, 0x62, 0x37, 0xa1, 0x36, 0xf2, 0x44, 0xa2, 0x4c, 0xe6, - 0xd1, 0xba, 0x5a, 0x2b, 0xd5, 0x7b, 0xf3, 0x8d, 0x95, 0xc0, 0x32, 0xdb, 0x56, 0x36, 0x6c, 0xc6, - 0xd1, 0xd8, 0x93, 0x38, 0x76, 0x3d, 0xa9, 0x3a, 0x14, 0x92, 0x0e, 0x35, 0xc2, 0x47, 0xf2, 0x48, - 0x90, 0xcf, 0xa0, 0xcd, 0x31, 0x62, 0x82, 0x2a, 0x12, 0xb8, 0xf3, 0xc8, 0x54, 0x0d, 0x32, 0xad, - 0xa9, 0x4d, 0x6f, 0x06, 0x23, 0xf2, 0x10, 0x5a, 0x06, 0x59, 0x37, 0xc1, 0x4a, 0xb8, 0x3e, 0x0b, - 0xa2, 0x33, 0x94, 0xd8, 0x5a, 0xdf, 0xb7, 0x3a, 0x35, 0x67, 0xd7, 0xe8, 0x9d, 0x44, 0xfd, 0x85, - 0xd1, 0x92, 0x41, 0x8e, 0x0f, 0x35, 0x4d, 0xd7, 0xfb, 0x45, 0x74, 0xbd, 0x54, 0xe3, 0x1c, 0x7d, - 0x3e, 0x80, 0xad, 0x10, 0x7f, 0x96, 0x6e, 0x8e, 0x26, 0xf5, 0x64, 0x50, 0x28, 0xf1, 0x71, 0x4a, - 0x95, 0xde, 0x0e, 0x5c, 0xcf, 0x1d, 0xd6, 0xd4, 0xf6, 0x8f, 0x72, 0x36, 0x30, 0x86, 0x71, 0x10, - 0x78, 0xab, 0x07, 0xc6, 0x1b, 0x29, 0xe6, 0x7d, 0xd8, 0xd1, 0x4a, 0x77, 0xb6, 0xa4, 0x55, 0x5d, - 0xd2, 0x6d, 0xad, 0x3a, 0xc9, 0xd5, 0xf5, 0x7d, 0x68, 0x7a, 0xbe, 0x8f, 0xd1, 0xd4, 0x72, 0x5d, - 0x5b, 0x36, 0x52, 0xa9, 0xb6, 0xba, 0x3c, 0x60, 0x6b, 0x0b, 0x06, 0xac, 0xfd, 0xaf, 0x05, 0x1b, - 0x5f, 0xd3, 0x15, 0x77, 0x1f, 0xf9, 0x18, 0xde, 0xce, 0xa7, 0x64, 0xdc, 0xf5, 0x9e, 0x4a, 0xe4, - 0xd9, 0xd5, 0xd0, 0x2f, 0x39, 0x3b, 0xd3, 0xec, 0xdf, 0xf2, 0x47, 0x4a, 0x7b, 0x24, 0x48, 0x17, - 0x48, 0xe6, 0x37, 0xc2, 0xa7, 0x8c, 0xa3, 0x72, 0xd1, 0xb7, 0x73, 0xdf, 0x72, 0xb6, 0x53, 0x5d, - 0x4f, 0xab, 0x8e, 0xfe, 0xd7, 0xb0, 0xe8, 0xed, 0xc2, 0x8d, 0x2c, 0xd7, 0x19, 0xfb, 0x09, 0xb9, - 0x3b, 0x62, 0x71, 0x38, 0x9e, 0x91, 0xc7, 0x51, 0x94, 0xca, 0xed, 0x5f, 0xa0, 0x91, 0x1c, 0xdc, - 0xb4, 0xf9, 0x57, 0x50, 0x33, 0x55, 0x79, 0x9d, 0x19, 0x6b, 0x78, 0xe5, 0x64, 0xbe, 0x8b, 0x18, - 0x5b, 0x5e, 0xc0, 0xd8, 0x07, 0xff, 0xac, 0x41, 0x7d, 0x98, 0x86, 0x23, 0x3f, 0xc2, 0x35, 0xf5, - 0x16, 0x22, 0x77, 0x0a, 0x1b, 0x65, 0xfa, 0xae, 0x6a, 0x77, 0x56, 0x1b, 0x26, 0x07, 0xb3, 0x4b, - 0xc4, 0x83, 0x6a, 0xf2, 0x74, 0x21, 0x77, 0x8b, 0xbc, 0x66, 0x5e, 0x41, 0xed, 0x7b, 0xaf, 0x62, - 0x9a, 0xa5, 0xf8, 0xcd, 0x82, 0xb7, 0x16, 0x5e, 0x94, 0xe4, 0x61, 0x51, 0x9c, 0xa2, 0xf7, 0x52, - 0xfb, 0x93, 0x2b, 0x78, 0x66, 0x1b, 0xfa, 0xdd, 0x82, 0xdd, 0xc5, 0xd7, 0x2a, 0x79, 0xdd, 0xb8, - 0xd3, 0xeb, 0xb1, 0xfd, 0xe9, 0x55, 0x5c, 0xe7, 0x41, 0xba, 0x7c, 0xd7, 0xac, 0x04, 0x69, 0xe9, - 0x85, 0xbd, 0x12, 0xa4, 0xe5, 0x17, 0x9b, 0x5d, 0x52, 0xac, 0x53, 0x3d, 0x50, 0xcc, 0xba, 0xdc, - 0x78, 0x28, 0x66, 0x5d, 0xbe, 0x9d, 0xec, 0x52, 0x0f, 0xe1, 0xb6, 0xcf, 0x82, 0x02, 0x87, 0x5e, - 0x33, 0xe3, 0xff, 0xb1, 0xfa, 0xad, 0x38, 0xb6, 0x7e, 0xf8, 0x70, 0x42, 0xe5, 0x69, 0x3c, 0xea, - 0xfa, 0x2c, 0x38, 0x54, 0x8e, 0x87, 0x39, 0xc7, 0x43, 0x2f, 0xa2, 0x87, 0x99, 0xf3, 0xa1, 0xfe, - 0x13, 0x89, 0x46, 0xa3, 0xaa, 0xfe, 0xf8, 0xe8, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xd3, - 0x4f, 0x39, 0xa5, 0x0c, 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x57, 0x4d, 0x73, 0x1b, 0x45, + 0x13, 0xd6, 0x4a, 0xb1, 0x2c, 0xb5, 0x64, 0xd9, 0x19, 0xe7, 0xf5, 0xab, 0x88, 0x38, 0x65, 0x16, + 0x4c, 0x94, 0x54, 0x45, 0x2e, 0x4c, 0x15, 0x15, 0x38, 0x50, 0x44, 0x04, 0x22, 0x55, 0x61, 0x70, + 0xad, 0x30, 0x07, 0x38, 0x6c, 0xad, 0x56, 0x1d, 0x79, 0x13, 0xef, 0xce, 0x66, 0x66, 0xd6, 0xe0, + 0x1c, 0xb8, 0x71, 0xa5, 0x8a, 0x7f, 0xc0, 0x9d, 0x3b, 0x67, 0x4e, 0xfc, 0x07, 0x2e, 0xfc, 0x0d, + 0xae, 0xd4, 0x7c, 0xec, 0xea, 0xc3, 0xd2, 0xca, 0x49, 0x38, 0x70, 0xdb, 0xe9, 0xaf, 0xe9, 0x79, + 0xe6, 0xe9, 0xee, 0x59, 0xd8, 0xe4, 0x82, 0x9e, 0x63, 0x1c, 0xc4, 0xd8, 0x89, 0x19, 0x15, 0x94, + 0xb4, 0x92, 0x21, 0xb2, 0x0e, 0x4f, 0x86, 0x61, 0x20, 0x9e, 0x27, 0x98, 0x60, 0x27, 0xb3, 0xb0, + 0xef, 0x40, 0xed, 0x38, 0x88, 0xc6, 0x0e, 0x3e, 0x4f, 0x90, 0x0b, 0xd2, 0x84, 0xf5, 0x10, 0x39, + 0xf7, 0xc6, 0xd8, 0xb4, 0xf6, 0xac, 0x76, 0xd5, 0x49, 0x97, 0xf6, 0x8f, 0x16, 0xd4, 0xb5, 0x25, + 0x8f, 0x69, 0xc4, 0x71, 0xb9, 0x29, 0x79, 0x13, 0xea, 0x1c, 0xd9, 0x79, 0xe0, 0xa3, 0x1b, 0x79, + 0x21, 0x36, 0x8b, 0x4a, 0x5d, 0x33, 0xb2, 0x2f, 0xbc, 0x10, 0xc9, 0x2d, 0xa8, 0x8a, 0x20, 0x44, + 0x2e, 0xbc, 0x30, 0x6e, 0x96, 0xf6, 0xac, 0x76, 0xc9, 0x99, 0x08, 0x48, 0x0b, 0x2a, 0xa7, 0x94, + 0x0b, 0xe5, 0x7c, 0x4d, 0x39, 0x67, 0x6b, 0x7b, 0x1f, 0x36, 0xfa, 0xd1, 0x18, 0xb9, 0x48, 0x53, + 0xbe, 0x01, 0x6b, 0xea, 0x50, 0x26, 0x0b, 0xbd, 0xb0, 0xf7, 0xa0, 0x91, 0x9a, 0x99, 0x7c, 0x1b, + 0x50, 0x0c, 0x46, 0xc6, 0xa8, 0x18, 0x8c, 0xec, 0x01, 0xdc, 0x7a, 0x8c, 0x69, 0x94, 0x5e, 0xc0, + 0x05, 0x65, 0x17, 0xdd, 0x8b, 0xfe, 0xa3, 0xdc, 0xb8, 0x64, 0x17, 0x80, 0x69, 0x03, 0x37, 0x18, + 0x99, 0x93, 0x55, 0x8d, 0xa4, 0x3f, 0xb2, 0x1f, 0xc3, 0xee, 0x82, 0xa0, 0x27, 0x4e, 0x3f, 0x3f, + 0xea, 0x16, 0x94, 0x12, 0x16, 0x98, 0x70, 0xf2, 0xd3, 0xfe, 0xdd, 0x82, 0xba, 0xf1, 0xff, 0xf4, + 0x1c, 0x23, 0x41, 0x6e, 0x42, 0x05, 0xe5, 0x87, 0x9b, 0x1d, 0x62, 0x5d, 0xad, 0xfb, 0x23, 0x89, + 0x77, 0x86, 0x9d, 0x1b, 0x72, 0x15, 0xa6, 0xe4, 0xd4, 0x32, 0xd9, 0x11, 0x27, 0xfb, 0xb0, 0x91, + 0xa6, 0xcd, 0x85, 0x27, 0x50, 0x61, 0x5e, 0xed, 0x15, 0x9c, 0xba, 0x11, 0x0f, 0xa4, 0x94, 0xec, + 0xc0, 0x9a, 0x0a, 0xaa, 0x51, 0xef, 0x15, 0x1c, 0xbd, 0x24, 0xfb, 0xd0, 0xa0, 0x89, 0xf0, 0x69, + 0x88, 0x2e, 0x43, 0x8f, 0xd3, 0xa8, 0xb9, 0xa6, 0x52, 0xd8, 0x30, 0x52, 0x47, 0x09, 0xbb, 0x75, + 0x00, 0xea, 0xfb, 0x09, 0x63, 0x18, 0xf9, 0x68, 0xff, 0x61, 0x41, 0x63, 0x16, 0x89, 0x39, 0xf4, + 0xac, 0x39, 0xf4, 0xc8, 0xc7, 0x50, 0x56, 0xfb, 0xc9, 0x23, 0x94, 0xda, 0xb5, 0xc3, 0x76, 0x67, + 0x39, 0x73, 0x3b, 0xd3, 0xe8, 0x38, 0xc6, 0x8f, 0x0c, 0x60, 0x33, 0x3b, 0x67, 0x12, 0x86, 0x1e, + 0xbb, 0x50, 0x27, 0xad, 0x1d, 0xde, 0xcb, 0x0b, 0x65, 0xb2, 0x1c, 0x68, 0x0f, 0xa7, 0xc1, 0x66, + 0xd6, 0xf6, 0x6f, 0xd6, 0xc2, 0x5b, 0x95, 0x54, 0x31, 0xdc, 0x9a, 0x24, 0x6e, 0xfd, 0x7b, 0x89, + 0x17, 0x5f, 0x3b, 0xf1, 0xa7, 0x70, 0x7b, 0x19, 0x1b, 0x4d, 0xe2, 0x3d, 0xa8, 0x9e, 0x2a, 0x79, + 0x80, 0x69, 0xee, 0x57, 0xd9, 0xd0, 0xc4, 0x72, 0x26, 0xce, 0xf6, 0xaf, 0x96, 0xaa, 0xa7, 0x63, + 0x46, 0x9f, 0xa2, 0xaf, 0xe8, 0x94, 0xf0, 0x2b, 0x30, 0x7f, 0x17, 0xc0, 0x3f, 0xf5, 0xa2, 0x31, + 0xba, 0x93, 0x02, 0xa8, 0x6a, 0xc9, 0x09, 0x0b, 0x64, 0x27, 0x88, 0x75, 0x44, 0xde, 0x2c, 0xed, + 0x95, 0x64, 0x27, 0x48, 0xd7, 0xe4, 0x0d, 0xa8, 0xc6, 0xde, 0x18, 0x5d, 0x1e, 0xbc, 0xd0, 0x6d, + 0x62, 0xcd, 0xa9, 0x48, 0xc1, 0x20, 0x78, 0xa1, 0xe2, 0x2a, 0xa5, 0xa0, 0xcf, 0x30, 0x65, 0xab, + 0x32, 0xff, 0x4a, 0x0a, 0xec, 0x21, 0x5c, 0x37, 0x99, 0x7e, 0xed, 0x9d, 0x05, 0x23, 0x4f, 0x04, + 0x34, 0x92, 0x1d, 0xcd, 0x04, 0x4f, 0x2b, 0xcc, 0x2c, 0xc9, 0x5d, 0xd8, 0x1c, 0x32, 0xf4, 0x9e, + 0xc9, 0x88, 0x23, 0x1c, 0x33, 0xd4, 0x4d, 0xcd, 0xea, 0x15, 0x9c, 0x46, 0xaa, 0x78, 0xa4, 0xe4, + 0xdd, 0x0a, 0x94, 0x19, 0xf2, 0xe4, 0x4c, 0xd8, 0x7f, 0x96, 0x14, 0x6d, 0x16, 0x21, 0x62, 0xd0, + 0x5f, 0x51, 0x0e, 0x19, 0x62, 0xc5, 0xe5, 0x88, 0x95, 0xe6, 0x11, 0xbb, 0x09, 0x95, 0xa1, 0xc7, + 0xb5, 0x52, 0xf7, 0xce, 0x75, 0xb9, 0x96, 0xaa, 0xb7, 0xe6, 0x9b, 0x80, 0x86, 0x65, 0xb6, 0x05, + 0xd8, 0xb0, 0x91, 0xc4, 0x23, 0x4f, 0xe0, 0xc8, 0xf5, 0x84, 0xec, 0x26, 0xa0, 0xbb, 0x89, 0x11, + 0x3e, 0x14, 0x47, 0x9c, 0x7c, 0x04, 0x2d, 0x86, 0x31, 0xe5, 0x81, 0x24, 0x81, 0x3b, 0x8f, 0x4c, + 0xd9, 0x20, 0xd3, 0x9c, 0xd8, 0x74, 0x67, 0x30, 0x22, 0x0f, 0xa0, 0x69, 0x90, 0x75, 0x35, 0x56, + 0xdc, 0xf5, 0x69, 0x18, 0x9f, 0xa1, 0xc0, 0xe6, 0xfa, 0x9e, 0xd5, 0xae, 0x38, 0x3b, 0x46, 0xef, + 0x68, 0xf5, 0x27, 0x46, 0x4b, 0xfa, 0x53, 0x7c, 0xa8, 0x28, 0xba, 0xde, 0xcf, 0xa3, 0xeb, 0xa5, + 0x3b, 0x9e, 0xa2, 0xcf, 0x3b, 0xb0, 0x19, 0xe1, 0xf7, 0xc2, 0x9d, 0xa2, 0x49, 0x55, 0x37, 0x35, + 0x29, 0x3e, 0x4e, 0xa9, 0xd2, 0xdd, 0x86, 0xeb, 0x53, 0x87, 0x35, 0x77, 0xfb, 0x4b, 0x31, 0xeb, + 0x6d, 0xa6, 0xd8, 0xfe, 0x93, 0x97, 0x79, 0x1f, 0xb6, 0x95, 0xd2, 0x9d, 0xbd, 0xd2, 0xb2, 0xba, + 0xd2, 0x2d, 0xa5, 0x3a, 0x99, 0xba, 0xd7, 0xb7, 0xa1, 0xe1, 0xf9, 0x3e, 0xc6, 0x13, 0xcb, 0x75, + 0x65, 0x59, 0x4f, 0xa5, 0xca, 0xea, 0xf2, 0x30, 0xa8, 0x2c, 0x18, 0x06, 0xf6, 0xdf, 0x16, 0xd4, + 0x3e, 0x0f, 0x56, 0xcc, 0x69, 0xf2, 0x3e, 0xfc, 0x7f, 0x7a, 0x4b, 0xca, 0x5c, 0xef, 0x89, 0x40, + 0x96, 0x8d, 0xb1, 0x5e, 0xc1, 0xd9, 0x9e, 0xec, 0xfe, 0x25, 0x7b, 0x28, 0xb5, 0x47, 0x9c, 0x74, + 0x80, 0x64, 0x7e, 0x43, 0x7c, 0x42, 0x19, 0x4a, 0x17, 0xf5, 0x92, 0xe8, 0x59, 0xce, 0x56, 0xaa, + 0xeb, 0x2a, 0xd5, 0xd1, 0x6b, 0x35, 0x8b, 0xee, 0x0e, 0xdc, 0xc8, 0xf6, 0x3a, 0xa3, 0xdf, 0x21, + 0x73, 0x87, 0x34, 0x89, 0x46, 0x33, 0xf2, 0x24, 0x8e, 0x53, 0xb9, 0xfd, 0x03, 0xd4, 0xf5, 0xc1, + 0x4d, 0x99, 0x7f, 0x06, 0x15, 0x73, 0x2b, 0x2f, 0xd3, 0x63, 0xd3, 0xa6, 0x9e, 0xf9, 0x2e, 0x62, + 0x6c, 0x71, 0x01, 0x63, 0x0f, 0xff, 0x5a, 0x83, 0xea, 0x20, 0x0d, 0x47, 0xbe, 0x85, 0x6b, 0xf2, + 0xdd, 0x46, 0xee, 0xe4, 0x16, 0xca, 0xe4, 0x0d, 0xd8, 0x6a, 0xaf, 0x36, 0xd4, 0x07, 0xb3, 0x0b, + 0xc4, 0x83, 0xb2, 0x7e, 0x66, 0x91, 0xbb, 0x79, 0x5e, 0x33, 0x2f, 0xb6, 0xd6, 0xbd, 0xab, 0x98, + 0x66, 0x5b, 0xfc, 0x64, 0xc1, 0xff, 0x16, 0x4e, 0x5f, 0xf2, 0x20, 0x2f, 0x4e, 0xde, 0xdb, 0xae, + 0xf5, 0xc1, 0x2b, 0x78, 0x66, 0x09, 0xfd, 0x6c, 0xc1, 0xce, 0xe2, 0xb1, 0x4a, 0x5e, 0x36, 0xee, + 0x64, 0x3c, 0xb6, 0x3e, 0x7c, 0x15, 0xd7, 0x79, 0x90, 0x2e, 0xcf, 0x9a, 0x95, 0x20, 0x2d, 0x1d, + 0xd8, 0x2b, 0x41, 0x5a, 0x3e, 0xd8, 0xec, 0x82, 0x64, 0x9d, 0xac, 0x81, 0x7c, 0xd6, 0x4d, 0xb5, + 0x87, 0x7c, 0xd6, 0x4d, 0x97, 0x93, 0x5d, 0xe8, 0x22, 0xdc, 0xf6, 0x69, 0x98, 0xe3, 0xd0, 0x6d, + 0x64, 0xfc, 0x3f, 0x96, 0xbf, 0x40, 0xc7, 0xd6, 0x37, 0xef, 0x8e, 0x03, 0x71, 0x9a, 0x0c, 0x3b, + 0x3e, 0x0d, 0x0f, 0xa4, 0xe3, 0xc1, 0x94, 0xe3, 0x81, 0x17, 0x07, 0x07, 0x99, 0xf3, 0x81, 0xfa, + 0x6b, 0x8a, 0x87, 0xc3, 0xb2, 0xfa, 0x78, 0xef, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x06, + 0x3a, 0x24, 0x51, 0x0d, 0x00, 0x00, }, } diff --git a/doc/rfc/stovepipe/request-history-api.md b/doc/rfc/stovepipe/request-history-api.md index bc00fae44..e2a6a9d09 100644 --- a/doc/rfc/stovepipe/request-history-api.md +++ b/doc/rfc/stovepipe/request-history-api.md @@ -1,159 +1,48 @@ # Stovepipe Request History API -## Summary +## Contract -Stovepipe exposes retained request history through queue-scoped point lookups by request ID and exact URI. URI lookup returns every retained request history associated with that commit; the current insert-once mapping produces exactly one. Both selectors return complete ordered events, and URI lookup groups them by request ID. +Stovepipe exposes retained request history through queue-scoped point lookups by request ID and exact URI. Both selectors return complete ordered events and the current materialized `RequestSummary`. The published [protobuf contract](../../../api/stovepipe/proto/stovepipe.proto) defines the fields. -The API reads the append-only model defined by [Stovepipe Request Log](request-log.md) directly. It does not replay history into current state or introduce a second persisted history projection. Current commit status remains a separate read concern derived from operational entities rather than request history. +`GetRequestHistoryByIDResponse` contains `events` and `request_summary`. `GetRequestHistoryByURIResponse` contains repeated histories, each with its request ID, events, and `request_summary`. These summary fields are additive; existing event fields and field numbers remain unchanged. -## API Coverage +Both methods require a queue. A selector belonging to another queue is not resolved across shards. URI lookup retains its plural response; the insert-once `RequestURIStore` resolves exactly one request per `(queue, URI)`. -The API supports the same selectors as [SubmitQueue Gateway Request History APIs](../submitqueue/history-api.md): +## Selection and Current State -1. request ID, for callers retaining the receipt returned by `Ingest`; -2. exact URI, for callers starting from a commit identity. +Request-ID lookup validates the queue and ID, then reads that queue's `RequestLogStore` and `RequestSummaryStore` by request ID. URI lookup validates the queue and URI, resolves the existing request-URI primary-key mapping, and uses the same history reader. It also verifies that the summary's URI matches the selector. -Both methods require a queue because Stovepipe storage and authorization are queue-scoped. A selector belonging to another queue is not found rather than resolved across shards. +At least one retained log record defines the existence of a history. A summary alone does not establish a retained history, and the operational `Request` entity is not required for either read. A returned summary must match the selected queue and request ID; disagreement is an infrastructure consistency error. -SubmitQueue's URI method returns several histories because the same change may be submitted repeatedly. Stovepipe uses the same plural response shape. Its current insert-once `RequestURIStore` permits only one request per `(queue, URI)`, so the initial implementation returns one history. If revalidation later retains multiple attempts, the URI index must enumerate their request IDs, but the public contract does not need to change. +The summary is the existing current lifecycle projection maintained by the request-log materializer. It includes request identity, queue, commit URI, baseline URI, current state, state timestamp, acceptance timestamp, and outcome reason. The mapper reuses the public summary representation exposed by List, omitting internal storage versions. -## Representative Contract +History is read before its summary. The two reads are eventually consistent, not a transactional snapshot: the current summary can represent newer state than the retained events returned by the earlier read, or lag a newly appended state event. The API does not replay the returned events to derive a summary or fabricate missing occurrences from current state. -The protobuf contract is: +A successful response always includes the summary. Initial state-log persistence precedes summary creation, so a concurrent read or a failed projection write can temporarily expose history without its summary. That gap fails the lookup with a retryable infrastructure error; ingestion retries complete the projection. Other summary-read failures also fail the lookup. -```proto -message GetRequestHistoryByIDRequest { - string queue = 1; - string request_id = 2; -} +## Events and Ordering -message GetRequestHistoryByURIRequest { - string queue = 1; - string uri = 2; -} +The append-only history model is defined by [Stovepipe Request Log](request-log.md). Event IDs are opaque; clients may compare them but never parse them. Request-state, outcome-reason, and event strings tolerate additive vocabulary changes. -message HistoryEvent { - string event_id = 1; - int64 timestamp_ms = 2; - oneof occurrence { - string request_state = 3; - string event = 4; - } - string outcome_reason = 7; -} +Events are returned by `(timestamp_ms ASC, event_id ASC)`. URI histories are ordered by numeric request ID ascending. Timestamps determine display order; persisted request versions determine causal ordering internally and are not exposed as history-event fields. Later repair may insert an older occurrence into its chronological position. -message RequestHistory { - string request_id = 1; - repeated HistoryEvent events = 2; -} +## Retention and Errors -message GetRequestHistoryByIDResponse { - repeated HistoryEvent events = 1; -} +History, request-summary, and request-URI mapping retention must support the same advertised lookup period. Neither endpoint promises a lifetime longer than the records required by its selector. -message GetRequestHistoryByURIResponse { - repeated RequestHistory histories = 1; -} - -service Stovepipe { - rpc GetRequestHistoryByID(GetRequestHistoryByIDRequest) returns (GetRequestHistoryByIDResponse) {} - rpc GetRequestHistoryByURI(GetRequestHistoryByURIRequest) returns (GetRequestHistoryByURIResponse) {} -} -``` - -The response shapes and `HistoryEvent` name match their SubmitQueue equivalents: request-ID lookup returns events directly, while URI lookup groups each request's events under a `RequestHistory` in a repeated field. Stovepipe retains `GetRequestHistoryByURI` because URI is its VCS-agnostic commit identity; SubmitQueue's corresponding selector is specifically named `GetRequestHistoryByChangeURI` after its change model. - -Event IDs are opaque. Clients may compare them but never parse their format. Request-state and event values are strings so clients can tolerate additive vocabulary changes. Request version remains persisted for internal transition ordering but is not part of the public event, matching SubmitQueue's API boundary. - -## Selection Flow - -Request-ID lookup validates the queue and ID and lists the queue's `RequestLogStore` records. At least one retained log record defines the existence of a history; the API does not consult the operational `Request` entity. Its response projects only the ordered events, matching SubmitQueue's equivalent API. - -URI lookup currently resolves one request ID from the existing `RequestURIStore` primary key and delegates to the same retained-log read. Supporting multiple retained attempts later requires the URI index to enumerate their request IDs; each history still uses a primary-key log read. The API does not scan the request log by URI. A missing mapping or a mapped request ID with no retained log is not found. - -Every request-URI mapping must be retained for the same advertised period as its history. Otherwise URI lookup could lose coverage while request-ID lookup still succeeds. Each `RequestHistory` contains only the selected request ID and its events; operational request context is not duplicated in the history response. - -## Public Projection - -Each stored `RequestLog` maps to exactly one public `HistoryEvent`. The API does not deduplicate, coalesce, or infer missing occurrences. - -History events representing state changes set `request_state` and preserve each durable transition. The public vocabulary is: - -| Public request state | Meaning | -|---|---| -| `accepted` | The request was durably admitted. | -| `processing` | The request was admitted to validation with a selected strategy. | -| `superseded` | A newer request replaced this request before validation completed. | -| `succeeded` | The request's validation work completed successfully. | -| `failed` | The request's validation work failed or could not continue. | -| `cancelled` | The request's validation work was cancelled. | - -These strings intentionally match the current domain states one-to-one, but they are a stable public history vocabulary: an internal refactor cannot rename or reinterpret an existing wire value. In particular, `succeeded` and `failed` remain distinct rather than collapsing into a derived snapshot phase such as `finalizing`. - -Build and validation-fact occurrences set `event`. The `occurrence` oneof makes state and event mutually exclusive without a redundant type field. A terminal Request state entry never substitutes for the fact event. - -The raw `RequestLog.Metadata` map, request version, dependency errors, credentials, and stack traces are not exposed. Typed metadata projections can be added later when a concrete client need defines their contract. `outcome_reason` uses a bounded public vocabulary. - -## Materialization - -SubmitQueue's `PersistLog` operation performs two jobs after receiving a log message: - -1. append the `RequestLog` row that is itself returned as history; -2. consider status rows for a gateway-owned `RequestSummary`, resolve out-of-order candidates using request version, terminal precedence, and timestamp, then update URI and queue-list projections. - -Event rows remain in SubmitQueue history but never participate in current-status materialization. The materializer exists because the gateway owns public reads but cannot read the orchestrator's mutable Request store. - -Stovepipe has no equivalent ownership gap. The same service owns the queue-scoped `Request`, `ValidationFact`, request-URI mapping, and request-log store. Operational reads use their owning entities, while request history reads retained log records directly. - -Stovepipe calls `requestlog.Materializer.PersistLog` directly, without an intermediate topic. Its initial materializer appends only the request log: it does not add `RequestSummary`, replay history to determine current state, or materialize another history table. The controller performs only an in-memory wire projection from stored log records to protobuf messages. This avoids a second winner-selection algorithm competing with Request CAS state. - -`PersistLog` receives the whole queue-scoped storage aggregate so a future current-status or queue-list API can add SubmitQueue-style summary and index projections behind the same write boundary without changing request-log producers. Such projections remain deferred until a concrete read path needs them; `Request` remains authoritative in the initial implementation. - -## Ordering and Consistency - -Events are returned by `(timestamp_ms ASC, event_id ASC)`. URI histories are ordered by request ID ascending, compared numerically (see [Resource IDs](../submitqueue/workflow.md#resource-ids)). Timestamps are display order, not conflict resolution. Persisted Request versions provide internal causal ordering for state transitions, while Build identity and write-once terminal status ensure one triggered and one finished occurrence per build. - -History may briefly lag a source entity between the source write and history creation. The pipeline blocks its dependent handoff during that window, and retry or reconciliation repairs the missing entry. The read API never fabricates an occurrence from current state. - -Request-ID lookup and the corresponding history within URI lookup return the same stored records and event ordering. A later repair may insert an older occurrence into its correct chronological position. - -## Retention - -Request-ID lookup returns all currently retained events for the selected request. URI lookup returns every retained request history associated with the exact URI. This matches SubmitQueue's request-history response shapes. The bounded lifecycle vocabulary and reasonable per-request build limits keep each history modest. - -History and request-URI mapping retention must support the same advertised lookup period. The API does not promise a lifetime longer than every record required by its selector. The initial rollout exposes only requests accepted after all history writers and repair paths are active; older requests without retained logs are outside the lookup period and are not backfilled. - -## Errors and Authorization - -- Empty queue or selector is invalid. -- An unknown request ID or URI, including one scoped to the wrong queue, is not found. -- A request ID or URI with no retained history is not found, including a URI mapping whose request ID has no retained log rows. -- Retryable storage failures are unavailable; context cancellation and deadline errors retain their canonical codes. +- An empty queue or selector is invalid. +- A request ID without retained logs is not found, including when its summary exists. +- An unknown queue-scoped URI mapping is not found. +- A URI mapping without retained logs is an infrastructure inconsistency. +- An absent summary is a retryable infrastructure error; summary identity or URI mismatches are infrastructure consistency errors. +- Storage failures propagate with their underlying error classifications. Authorization follows the same queue policy as other Stovepipe reads. Possession of a request ID alone does not bypass queue authorization. ## Testing -Contract and controller tests cover: - -- matching request-ID events and the corresponding history inside the plural URI response; -- deterministic equal-timestamp ordering; -- state, build-event, and fact-event public mapping; -- queue isolation and cross-queue not found; -- unknown selectors and URI mappings without retained logs; -- a repaired older occurrence appearing in chronological position; -- unknown future request-state and event strings remaining readable. +Tests cover matching events and summaries across selectors, required summaries and retryable materialization gaps, current summaries independent of retained event subsets, queue and URI consistency, storage-read failures, histories absent despite a stored summary, stable event ordering, and unknown future state and outcome-reason strings. ## Alternatives Considered -### Materialize current status from history - -Rejected because `Request` and `ValidationFact` already own current operational state and verdicts. Replaying history would add another reconciliation path without enabling either selector. - -### Query history directly by URI - -Rejected because it would add query-by-attribute capability to `RequestLogStore`. The existing exact request-URI mapping resolves the primary request key for any backend. - -### Return only the authoritative history by URI - -Rejected because a singular response would require another API or a breaking shape change if Stovepipe later retains multiple validation attempts for one URI. The repeated response accurately contains one history under the current invariant and can grow without changing existing clients. +Replaying returned history into current state is rejected because retention can remove events and the existing materialized summary already owns that projection. Querying history directly by URI is rejected because it would require an attribute query in `RequestLogStore`; the existing mapping resolves the primary request key for any backend. Returning events without a summary is rejected because a successful lookup must include current request status; callers can retry a temporary projection gap. diff --git a/service/stovepipe/server/handler/BUILD.bazel b/service/stovepipe/server/handler/BUILD.bazel index ce8429cc7..d25dbfbdd 100644 --- a/service/stovepipe/server/handler/BUILD.bazel +++ b/service/stovepipe/server/handler/BUILD.bazel @@ -22,9 +22,11 @@ go_test( deps = [ "//api/stovepipe/protopb:go_default_library", "//stovepipe/controller:go_default_library", + "//stovepipe/controller/mock:go_default_library", "//stovepipe/entity:go_default_library", "//stovepipe/extension/storage:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", + "@org_uber_go_mock//gomock:go_default_library", ], ) diff --git a/service/stovepipe/server/handler/server.go b/service/stovepipe/server/handler/server.go index 37f31608c..c64a61261 100644 --- a/service/stovepipe/server/handler/server.go +++ b/service/stovepipe/server/handler/server.go @@ -67,11 +67,11 @@ func (s *StovepipeServer) Ingest(ctx context.Context, req *pb.IngestRequest) (*p // GetRequestHistoryByID returns retained history for one request ID. func (s *StovepipeServer) GetRequestHistoryByID(ctx context.Context, req *pb.GetRequestHistoryByIDRequest) (*pb.GetRequestHistoryByIDResponse, error) { - events, err := s.requestHistoryController.GetRequestHistoryByID(ctx, mapper.ProtoToGetRequestHistoryByIDRequest(req)) + history, err := s.requestHistoryController.GetRequestHistoryByID(ctx, mapper.ProtoToGetRequestHistoryByIDRequest(req)) if err != nil { return nil, err } - return &pb.GetRequestHistoryByIDResponse{Events: mapper.HistoryEventsToProto(events)}, nil + return mapper.RequestHistoryByIDToProto(history), nil } // GetRequestHistoryByURI returns retained histories for one commit URI. diff --git a/service/stovepipe/server/handler/server_test.go b/service/stovepipe/server/handler/server_test.go index 42c035e95..e0c74ea42 100644 --- a/service/stovepipe/server/handler/server_test.go +++ b/service/stovepipe/server/handler/server_test.go @@ -22,25 +22,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "github.com/uber/submitqueue/api/stovepipe/protopb" - "github.com/uber/submitqueue/stovepipe/controller" + controllermock "github.com/uber/submitqueue/stovepipe/controller/mock" "github.com/uber/submitqueue/stovepipe/entity" + "go.uber.org/mock/gomock" ) -type fakeRequestHistoryController struct { - getByID func(context.Context, entity.GetRequestHistoryByIDRequest) ([]entity.RequestLog, error) - getByURI func(context.Context, entity.GetRequestHistoryByURIRequest) ([]entity.RequestHistory, error) -} - -var _ controller.RequestHistoryController = (*fakeRequestHistoryController)(nil) - -func (f *fakeRequestHistoryController) GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) ([]entity.RequestLog, error) { - return f.getByID(ctx, req) -} - -func (f *fakeRequestHistoryController) GetRequestHistoryByURI(ctx context.Context, req entity.GetRequestHistoryByURIRequest) ([]entity.RequestHistory, error) { - return f.getByURI(ctx, req) -} - func TestGetRequestHistoryByID(t *testing.T) { controllerErr := errors.New("controller failed") logs := []entity.RequestLog{ @@ -60,20 +46,18 @@ func TestGetRequestHistoryByID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var gotReq entity.GetRequestHistoryByIDRequest - fake := &fakeRequestHistoryController{ - getByID: func(_ context.Context, req entity.GetRequestHistoryByIDRequest) ([]entity.RequestLog, error) { - gotReq = req - return tt.logs, tt.err - }, + history := entity.RequestHistory{ + RequestID: "1", Events: tt.logs, + RequestSummary: entity.RequestSummary{RequestID: "1", Queue: "monorepo/main", URI: "uri", State: entity.RequestStateSucceeded}, } - srv := &StovepipeServer{requestHistoryController: fake} + historyController := controllermock.NewMockRequestHistoryController(gomock.NewController(t)) + historyController.EXPECT().GetRequestHistoryByID(gomock.Any(), entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "1"}).Return(history, tt.err) + srv := &StovepipeServer{requestHistoryController: historyController} resp, err := srv.GetRequestHistoryByID(context.Background(), &pb.GetRequestHistoryByIDRequest{ Queue: "monorepo/main", RequestId: "1", }) - assert.Equal(t, entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "1"}, gotReq) if tt.err != nil { require.ErrorIs(t, err, tt.err) assert.Nil(t, resp) @@ -81,6 +65,8 @@ func TestGetRequestHistoryByID(t *testing.T) { } require.NoError(t, err) require.Len(t, resp.Events, tt.wantLogs) + require.NotNil(t, resp.RequestSummary) + assert.Equal(t, "succeeded", resp.RequestSummary.GetRequestState()) if tt.wantLogs > 0 { assert.Equal(t, "accepted", resp.Events[0].GetRequestState()) assert.Equal(t, "build_triggered", resp.Events[1].GetEvent()) @@ -92,8 +78,9 @@ func TestGetRequestHistoryByID(t *testing.T) { func TestGetRequestHistoryByURI(t *testing.T) { controllerErr := errors.New("controller failed") histories := []entity.RequestHistory{{ - RequestID: "1", - Events: []entity.RequestLog{{ID: "occurrence/1", State: entity.RequestStateAccepted}}, + RequestID: "1", + Events: []entity.RequestLog{{ID: "occurrence/1", State: entity.RequestStateAccepted}}, + RequestSummary: entity.RequestSummary{RequestID: "1", Queue: "monorepo/main", URI: "git://monorepo/abc", State: entity.RequestStateProcessing}, }} tests := []struct { name string @@ -108,20 +95,14 @@ func TestGetRequestHistoryByURI(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var gotReq entity.GetRequestHistoryByURIRequest - fake := &fakeRequestHistoryController{ - getByURI: func(_ context.Context, req entity.GetRequestHistoryByURIRequest) ([]entity.RequestHistory, error) { - gotReq = req - return tt.histories, tt.err - }, - } - srv := &StovepipeServer{requestHistoryController: fake} + historyController := controllermock.NewMockRequestHistoryController(gomock.NewController(t)) + historyController.EXPECT().GetRequestHistoryByURI(gomock.Any(), entity.GetRequestHistoryByURIRequest{Queue: "monorepo/main", URI: "git://monorepo/abc"}).Return(tt.histories, tt.err) + srv := &StovepipeServer{requestHistoryController: historyController} resp, err := srv.GetRequestHistoryByURI(context.Background(), &pb.GetRequestHistoryByURIRequest{ Queue: "monorepo/main", Uri: "git://monorepo/abc", }) - assert.Equal(t, entity.GetRequestHistoryByURIRequest{Queue: "monorepo/main", URI: "git://monorepo/abc"}, gotReq) if tt.err != nil { require.ErrorIs(t, err, tt.err) assert.Nil(t, resp) @@ -130,6 +111,8 @@ func TestGetRequestHistoryByURI(t *testing.T) { require.NoError(t, err) require.Len(t, resp.Histories, tt.wantHistories) if tt.wantHistories > 0 { + require.NotNil(t, resp.Histories[0].RequestSummary) + assert.Equal(t, "processing", resp.Histories[0].RequestSummary.GetRequestState()) assert.Equal(t, "1", resp.Histories[0].RequestId) require.Len(t, resp.Histories[0].Events, 1) assert.Equal(t, "accepted", resp.Histories[0].Events[0].GetRequestState()) diff --git a/service/stovepipe/server/mapper/request_history.go b/service/stovepipe/server/mapper/request_history.go index 551648d35..4e421b182 100644 --- a/service/stovepipe/server/mapper/request_history.go +++ b/service/stovepipe/server/mapper/request_history.go @@ -29,6 +29,14 @@ func ProtoToGetRequestHistoryByURIRequest(req *pb.GetRequestHistoryByURIRequest) return entity.GetRequestHistoryByURIRequest{URI: req.GetUri(), Queue: req.GetQueue()} } +// RequestHistoryByIDToProto maps retained events and the current summary to their wire representation. +func RequestHistoryByIDToProto(history entity.RequestHistory) *pb.GetRequestHistoryByIDResponse { + return &pb.GetRequestHistoryByIDResponse{ + Events: HistoryEventsToProto(history.Events), + RequestSummary: RequestSummaryToProto(history.RequestSummary), + } +} + // HistoryEventsToProto maps retained request-log events to wire history events. func HistoryEventsToProto(logs []entity.RequestLog) []*pb.HistoryEvent { events := make([]*pb.HistoryEvent, len(logs)) @@ -53,8 +61,9 @@ func RequestHistoriesToProto(histories []entity.RequestHistory) []*pb.RequestHis result := make([]*pb.RequestHistory, len(histories)) for i, history := range histories { result[i] = &pb.RequestHistory{ - RequestId: history.RequestID, - Events: HistoryEventsToProto(history.Events), + RequestId: history.RequestID, + Events: HistoryEventsToProto(history.Events), + RequestSummary: RequestSummaryToProto(history.RequestSummary), } } return result diff --git a/service/stovepipe/server/mapper/request_history_test.go b/service/stovepipe/server/mapper/request_history_test.go index 61add03cb..0d7a83f6c 100644 --- a/service/stovepipe/server/mapper/request_history_test.go +++ b/service/stovepipe/server/mapper/request_history_test.go @@ -147,19 +147,20 @@ func TestRequestHistoriesToProto(t *testing.T) { { name: "maps groups in order", histories: []entity.RequestHistory{ - {RequestID: "1", Events: logs}, - {RequestID: "2", Events: []entity.RequestLog{}}, + {RequestID: "1", Events: logs, RequestSummary: entity.RequestSummary{RequestID: "1"}}, + {RequestID: "2", Events: []entity.RequestLog{}, RequestSummary: entity.RequestSummary{RequestID: "2"}}, }, want: []*pb.RequestHistory{ { - RequestId: "1", + RequestId: "1", + RequestSummary: &pb.RequestSummary{RequestId: "1"}, Events: []*pb.HistoryEvent{{ EventId: "state/1", TimestampMs: 10, Occurrence: &pb.HistoryEvent_RequestState{RequestState: "accepted"}, }}, }, - {RequestId: "2", Events: []*pb.HistoryEvent{}}, + {RequestId: "2", Events: []*pb.HistoryEvent{}, RequestSummary: &pb.RequestSummary{RequestId: "2"}}, }, }, {name: "nil input", histories: nil, want: []*pb.RequestHistory{}}, @@ -172,3 +173,22 @@ func TestRequestHistoriesToProto(t *testing.T) { }) } } + +func TestRequestHistorySummaryMapping(t *testing.T) { + history := entity.RequestHistory{ + RequestID: "7", Events: []entity.RequestLog{{ID: "accepted", State: entity.RequestStateAccepted}}, + RequestSummary: entity.RequestSummary{ + RequestID: "7", Queue: "queue", URI: "uri", BaseURI: "base", State: "future-state", + StateTimestampMs: 200, AcceptedAtMs: 100, OutcomeReason: "future-reason", + }, + } + byID := RequestHistoryByIDToProto(history) + byURI := RequestHistoriesToProto([]entity.RequestHistory{history})[0] + assert.Equal(t, byID.Events, byURI.Events) + want := &pb.RequestSummary{ + RequestId: "7", Queue: "queue", ChangeUri: "uri", BaseUri: "base", RequestState: "future-state", + StateUpdatedAtMs: 200, AcceptedAtMs: 100, OutcomeReason: "future-reason", + } + assert.Equal(t, want, byID.RequestSummary) + assert.Equal(t, want, byURI.RequestSummary) +} diff --git a/stovepipe/controller/mock/BUILD.bazel b/stovepipe/controller/mock/BUILD.bazel new file mode 100644 index 000000000..45dd6b743 --- /dev/null +++ b/stovepipe/controller/mock/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["request_history_mock.go"], + importpath = "github.com/uber/submitqueue/stovepipe/controller/mock", + visibility = ["//visibility:public"], + deps = [ + "//stovepipe/entity:go_default_library", + "@org_uber_go_mock//gomock:go_default_library", + ], +) diff --git a/stovepipe/controller/mock/request_history_mock.go b/stovepipe/controller/mock/request_history_mock.go new file mode 100644 index 000000000..2d9f443ae --- /dev/null +++ b/stovepipe/controller/mock/request_history_mock.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: request_history.go +// +// Generated by this command: +// +// mockgen -source=request_history.go -destination=mock/request_history_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/stovepipe/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestHistoryController is a mock of RequestHistoryController interface. +type MockRequestHistoryController struct { + ctrl *gomock.Controller + recorder *MockRequestHistoryControllerMockRecorder + isgomock struct{} +} + +// MockRequestHistoryControllerMockRecorder is the mock recorder for MockRequestHistoryController. +type MockRequestHistoryControllerMockRecorder struct { + mock *MockRequestHistoryController +} + +// NewMockRequestHistoryController creates a new mock instance. +func NewMockRequestHistoryController(ctrl *gomock.Controller) *MockRequestHistoryController { + mock := &MockRequestHistoryController{ctrl: ctrl} + mock.recorder = &MockRequestHistoryControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestHistoryController) EXPECT() *MockRequestHistoryControllerMockRecorder { + return m.recorder +} + +// GetRequestHistoryByID mocks base method. +func (m *MockRequestHistoryController) GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) (entity.RequestHistory, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRequestHistoryByID", ctx, req) + ret0, _ := ret[0].(entity.RequestHistory) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRequestHistoryByID indicates an expected call of GetRequestHistoryByID. +func (mr *MockRequestHistoryControllerMockRecorder) GetRequestHistoryByID(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestHistoryByID", reflect.TypeOf((*MockRequestHistoryController)(nil).GetRequestHistoryByID), ctx, req) +} + +// GetRequestHistoryByURI mocks base method. +func (m *MockRequestHistoryController) GetRequestHistoryByURI(ctx context.Context, req entity.GetRequestHistoryByURIRequest) ([]entity.RequestHistory, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRequestHistoryByURI", ctx, req) + ret0, _ := ret[0].([]entity.RequestHistory) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRequestHistoryByURI indicates an expected call of GetRequestHistoryByURI. +func (mr *MockRequestHistoryControllerMockRecorder) GetRequestHistoryByURI(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestHistoryByURI", reflect.TypeOf((*MockRequestHistoryController)(nil).GetRequestHistoryByURI), ctx, req) +} diff --git a/stovepipe/controller/read_errors.go b/stovepipe/controller/read_errors.go index 2555f6a02..398dfe4b2 100644 --- a/stovepipe/controller/read_errors.go +++ b/stovepipe/controller/read_errors.go @@ -81,6 +81,20 @@ func IsRequestHistoryNotFound(err error) bool { return errors.As(err, &byURI) } +// RequestHistoryConsistencyError indicates that a summary disagrees with the selected history. +type RequestHistoryConsistencyError struct { + // Queue is the selected queue. + Queue string + // RequestID identifies the selected request. + RequestID string + // Reason describes the disagreement with the persisted summary. + Reason string +} + +func (e *RequestHistoryConsistencyError) Error() string { + return fmt.Sprintf("request history %s queue=%q request_id=%q", e.Reason, e.Queue, e.RequestID) +} + // ListConsistencyError indicates an invalid acceptance mapping, a missing summary, // or disagreement between the two. It represents an infrastructure failure, not a user lookup miss. type ListConsistencyError struct { diff --git a/stovepipe/controller/request_history.go b/stovepipe/controller/request_history.go index 4f320b968..73d361f70 100644 --- a/stovepipe/controller/request_history.go +++ b/stovepipe/controller/request_history.go @@ -14,6 +14,8 @@ package controller +//go:generate mockgen -source=request_history.go -destination=mock/request_history_mock.go -package=mock + import ( "context" "fmt" @@ -28,7 +30,7 @@ import ( // RequestHistoryController handles retained request-history lookups. type RequestHistoryController interface { - GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) ([]entity.RequestLog, error) + GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) (entity.RequestHistory, error) GetRequestHistoryByURI(ctx context.Context, req entity.GetRequestHistoryByURIRequest) ([]entity.RequestHistory, error) } @@ -49,45 +51,45 @@ func NewRequestHistoryController(logger *zap.SugaredLogger, scope tally.Scope, s } } -// GetRequestHistoryByID returns every retained log event for one request ID. -func (c *requestHistoryController) GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) (logs []entity.RequestLog, retErr error) { +// GetRequestHistoryByID returns retained events and the current summary for one request ID. +func (c *requestHistoryController) GetRequestHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) (history entity.RequestHistory, retErr error) { op := metrics.Begin(c.metricsScope, "get_by_id", metrics.StorageLatencyBuckets, metrics.TagsFromContext(ctx)...) defer func() { op.Complete(retErr) }() - logs, retErr = c.readHistoryByID(ctx, req) + history, retErr = c.readHistoryByID(ctx, req) if retErr != nil { - return nil, retErr + return entity.RequestHistory{}, retErr } c.logger.Debugw("request history retrieved", "request_id", req.ID, "queue", req.Queue, - "event_count", len(logs), + "event_count", len(history.Events), ) - return logs, nil + return history, nil } -func (c *requestHistoryController) readHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) ([]entity.RequestLog, error) { +func (c *requestHistoryController) readHistoryByID(ctx context.Context, req entity.GetRequestHistoryByIDRequest) (entity.RequestHistory, error) { if err := validateHistoryIdentifier("queue", req.Queue); err != nil { - return nil, fmt.Errorf("GetRequestHistoryByID invalid queue=%q: %w", req.Queue, err) + return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByID invalid queue=%q: %w", req.Queue, err) } if err := validateRequestID(req.ID); err != nil { - return nil, fmt.Errorf("GetRequestHistoryByID invalid request_id=%q queue=%q: %w", req.ID, req.Queue, err) + return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByID invalid request_id=%q queue=%q: %w", req.ID, req.Queue, err) } stores, err := c.stores.For(storage.Config{QueueName: req.Queue}) if err != nil { - return nil, fmt.Errorf("GetRequestHistoryByID failed to resolve storage for queue %q: %w", req.Queue, err) + return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByID failed to resolve storage for queue %q: %w", req.Queue, err) } - logs, err := stores.GetRequestLogStore().List(ctx, req.ID) + history, err := readRequestHistory(ctx, stores, req.Queue, req.ID) if err != nil { if storage.IsNotFound(err) { - return nil, errs.NewUserError(&RequestHistoryByIDNotFoundError{RequestID: req.ID}) + return entity.RequestHistory{}, errs.NewUserError(&RequestHistoryByIDNotFoundError{RequestID: req.ID}) } - return nil, fmt.Errorf("GetRequestHistoryByID failed to list request logs request_id=%s: %w", req.ID, err) + return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByID failed to read history request_id=%s: %w", req.ID, err) } - return logs, nil + return history, nil } // GetRequestHistoryByURI returns the retained history mapped to an exact commit URI. @@ -129,13 +131,35 @@ func (c *requestHistoryController) readHistoryByURI(ctx context.Context, req ent return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByURI failed to resolve request URI %s: %w", req.URI, err) } - logs, err := stores.GetRequestLogStore().List(ctx, requestID) + history, err := readRequestHistory(ctx, stores, req.Queue, requestID) if err != nil { if storage.IsNotFound(err) { return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByURI found URI mapping without retained request logs uri=%q request_id=%q: %w", req.URI, requestID, err) } - return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByURI failed to list request logs uri=%s request_id=%s: %w", req.URI, requestID, err) + return entity.RequestHistory{}, fmt.Errorf("GetRequestHistoryByURI failed to read history uri=%s request_id=%s: %w", req.URI, requestID, err) } - return entity.RequestHistory{RequestID: requestID, Events: logs}, nil + if history.RequestSummary.URI != req.URI { + return entity.RequestHistory{}, &RequestHistoryConsistencyError{Queue: req.Queue, RequestID: requestID, Reason: "URI mapping disagrees with request summary"} + } + return history, nil +} + +func readRequestHistory(ctx context.Context, stores storage.Storage, queue, requestID string) (entity.RequestHistory, error) { + logs, err := stores.GetRequestLogStore().List(ctx, requestID) + if err != nil { + return entity.RequestHistory{}, fmt.Errorf("failed to list request logs: %w", err) + } + summary, err := stores.GetRequestSummaryStore().Get(ctx, requestID) + if err != nil { + if storage.IsNotFound(err) { + // The initial log is persisted before its summary; this visibility gap is retryable. + return entity.RequestHistory{}, errs.NewRetryableError(fmt.Errorf("request summary is not visible queue=%q request_id=%q", queue, requestID)) + } + return entity.RequestHistory{}, fmt.Errorf("failed to read request summary: %w", err) + } + if summary.RequestID != requestID || summary.Queue != queue { + return entity.RequestHistory{}, &RequestHistoryConsistencyError{Queue: queue, RequestID: requestID, Reason: "request summary disagrees with history selector"} + } + return entity.RequestHistory{RequestID: requestID, Events: logs, RequestSummary: summary}, nil } diff --git a/stovepipe/controller/request_history_test.go b/stovepipe/controller/request_history_test.go index 3ad919674..b8fc3a3d6 100644 --- a/stovepipe/controller/request_history_test.go +++ b/stovepipe/controller/request_history_test.go @@ -82,11 +82,16 @@ func TestGetRequestHistoryByID(t *testing.T) { factory := storagemock.NewMockFactory(mockCtrl) stores := storagemock.NewMockStorage(mockCtrl) logStore := storagemock.NewMockRequestLogStore(mockCtrl) + summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl) if !tt.wantInvalid { factory.EXPECT().For(storage.Config{QueueName: tt.req.Queue}).Return(stores, tt.factoryErr) if tt.factoryErr == nil { stores.EXPECT().GetRequestLogStore().Return(logStore) logStore.EXPECT().List(gomock.Any(), tt.req.ID).Return(tt.logs, tt.listErr) + if tt.listErr == nil { + stores.EXPECT().GetRequestSummaryStore().Return(summaryStore) + summaryStore.EXPECT().Get(gomock.Any(), tt.req.ID).Return(entity.RequestSummary{RequestID: requestID, Queue: queue}, nil) + } } } @@ -97,7 +102,7 @@ func TestGetRequestHistoryByID(t *testing.T) { got, err := controller.GetRequestHistoryByID(ctx, tt.req) - assert.Equal(t, tt.wantLogs, got) + assert.Equal(t, tt.wantLogs, got.Events) if tt.wantInvalid { assert.True(t, IsInvalidRequest(err)) } @@ -148,7 +153,8 @@ func TestGetRequestHistoryByURI(t *testing.T) { {ID: "event/a", RequestID: requestID, TimestampMs: 20, Event: entity.RequestEventBuildTriggered}, {ID: "event/a", RequestID: requestID, TimestampMs: 20, Event: entity.RequestEventBuildTriggered}, } - wantHistory := []entity.RequestHistory{{RequestID: requestID, Events: logs}} + summary := entity.RequestSummary{RequestID: requestID, Queue: queue, URI: uri} + wantHistory := []entity.RequestHistory{{RequestID: requestID, Events: logs, RequestSummary: summary}} tests := []struct { name string @@ -182,6 +188,7 @@ func TestGetRequestHistoryByURI(t *testing.T) { stores := storagemock.NewMockStorage(mockCtrl) uriStore := storagemock.NewMockRequestURIStore(mockCtrl) logStore := storagemock.NewMockRequestLogStore(mockCtrl) + summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl) if !tt.wantInvalid { factory.EXPECT().For(storage.Config{QueueName: tt.req.Queue}).Return(stores, tt.factoryErr) if tt.factoryErr == nil { @@ -190,6 +197,10 @@ func TestGetRequestHistoryByURI(t *testing.T) { if tt.mappingErr == nil { stores.EXPECT().GetRequestLogStore().Return(logStore) logStore.EXPECT().List(gomock.Any(), tt.mappedID).Return(logs, tt.listErr) + if tt.listErr == nil { + stores.EXPECT().GetRequestSummaryStore().Return(summaryStore) + summaryStore.EXPECT().Get(gomock.Any(), tt.mappedID).Return(summary, nil) + } } } } @@ -276,6 +287,108 @@ func TestRequestHistoryNotFoundErrors(t *testing.T) { assert.False(t, IsRequestHistoryNotFound(errors.New("other"))) } +func TestRequestHistoryCurrentSummary(t *testing.T) { + const queue, requestID, uri = "monorepo/main", "42", "git://monorepo/main/commit" + summary := entity.RequestSummary{ + RequestID: requestID, Queue: queue, URI: uri, BaseURI: "git://monorepo/main/base", State: entity.RequestStateSucceeded, + StateTimestampMs: 200, AcceptedAtMs: 100, OutcomeReason: entity.RequestOutcomeReasonBuildSucceeded, + } + logs := []entity.RequestLog{{ID: "accepted", RequestID: requestID, TimestampMs: 100, State: entity.RequestStateAccepted}} + readErr := errors.New("summary read failed") + tests := []struct { + name string + modify func(*entity.RequestSummary) + readErr error + consistency bool + retryable bool + }{ + {name: "current summary is independent of retained events"}, + {name: "missing summary is retryable infrastructure failure", readErr: fmt.Errorf("summary: %w", storage.ErrNotFound), retryable: true}, + {name: "summary read failure", readErr: readErr}, + {name: "wrong request ID", modify: func(s *entity.RequestSummary) { s.RequestID = "43" }, consistency: true}, + {name: "wrong queue", modify: func(s *entity.RequestSummary) { s.Queue = "other/main" }, consistency: true}, + } + for _, selector := range []string{"ID", "URI"} { + for _, tt := range tests { + t.Run(selector+"/"+tt.name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + factory := storagemock.NewMockFactory(mockCtrl) + stores := storagemock.NewMockStorage(mockCtrl) + logStore := storagemock.NewMockRequestLogStore(mockCtrl) + summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl) + factory.EXPECT().For(storage.Config{QueueName: queue}).Return(stores, nil) + if selector == "URI" { + uriStore := storagemock.NewMockRequestURIStore(mockCtrl) + stores.EXPECT().GetRequestURIStore().Return(uriStore) + uriStore.EXPECT().GetIDByURI(gomock.Any(), uri).Return(requestID, nil) + } + stores.EXPECT().GetRequestLogStore().Return(logStore) + logStore.EXPECT().List(gomock.Any(), requestID).Return(logs, nil) + stores.EXPECT().GetRequestSummaryStore().Return(summaryStore) + storedSummary := summary + if tt.modify != nil { + tt.modify(&storedSummary) + } + summaryStore.EXPECT().Get(gomock.Any(), requestID).Return(storedSummary, tt.readErr) + controller := NewRequestHistoryController(zap.NewNop().Sugar(), tally.NoopScope, factory) + var history entity.RequestHistory + var err error + if selector == "ID" { + history, err = controller.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{Queue: queue, ID: requestID}) + } else { + var histories []entity.RequestHistory + histories, err = controller.GetRequestHistoryByURI(context.Background(), entity.GetRequestHistoryByURIRequest{Queue: queue, URI: uri}) + if err == nil { + require.Len(t, histories, 1) + history = histories[0] + } + } + assert.False(t, errs.IsUserError(err)) + assert.False(t, IsRequestHistoryNotFound(err)) + assert.Equal(t, tt.retryable, errs.IsRetryable(err)) + if tt.retryable { + require.Error(t, err) + assert.Equal(t, entity.RequestHistory{}, history) + return + } + if tt.consistency { + var consistency *RequestHistoryConsistencyError + require.ErrorAs(t, err, &consistency) + return + } + if tt.readErr != nil { + require.ErrorIs(t, err, tt.readErr) + assert.False(t, IsRequestHistoryNotFound(err)) + return + } + require.NoError(t, err) + assert.Equal(t, requestID, history.RequestID) + assert.Equal(t, logs, history.Events) + assert.Equal(t, summary, history.RequestSummary) + }) + } + } +} + +func TestRequestHistoryRejectsMismatchedSummaryURI(t *testing.T) { + mockCtrl := gomock.NewController(t) + factory := storagemock.NewMockFactory(mockCtrl) + stores := storagemock.NewMockStorage(mockCtrl) + uriStore := storagemock.NewMockRequestURIStore(mockCtrl) + logStore := storagemock.NewMockRequestLogStore(mockCtrl) + summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl) + factory.EXPECT().For(storage.Config{QueueName: "queue"}).Return(stores, nil) + stores.EXPECT().GetRequestURIStore().Return(uriStore) + uriStore.EXPECT().GetIDByURI(gomock.Any(), "uri").Return("1", nil) + stores.EXPECT().GetRequestLogStore().Return(logStore) + logStore.EXPECT().List(gomock.Any(), "1").Return([]entity.RequestLog{{ID: "accepted"}}, nil) + stores.EXPECT().GetRequestSummaryStore().Return(summaryStore) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{RequestID: "1", Queue: "queue", URI: "other-uri"}, nil) + _, err := NewRequestHistoryController(zap.NewNop().Sugar(), tally.NoopScope, factory).GetRequestHistoryByURI(context.Background(), entity.GetRequestHistoryByURIRequest{Queue: "queue", URI: "uri"}) + var consistency *RequestHistoryConsistencyError + require.ErrorAs(t, err, &consistency) +} + func assertOperationFinishIncludesContextTag(t *testing.T, snapshot tally.Snapshot, operation string, success bool) { t.Helper() wantResult := "error" diff --git a/stovepipe/entity/request_history.go b/stovepipe/entity/request_history.go index e3c6b7d9d..3b80291f7 100644 --- a/stovepipe/entity/request_history.go +++ b/stovepipe/entity/request_history.go @@ -30,10 +30,12 @@ type GetRequestHistoryByURIRequest struct { Queue string } -// RequestHistory groups retained events for one request. +// RequestHistory groups retained events and the current materialized summary for one request. type RequestHistory struct { // RequestID is the queue-scoped identifier of the request. RequestID string // Events are retained request-log events ordered by occurrence time and stable identity. Events []RequestLog + // RequestSummary is the current lifecycle projection for this request. + RequestSummary RequestSummary } diff --git a/test/integration/stovepipe/BUILD.bazel b/test/integration/stovepipe/BUILD.bazel index 3eefca4a6..11017b9b2 100644 --- a/test/integration/stovepipe/BUILD.bazel +++ b/test/integration/stovepipe/BUILD.bazel @@ -22,5 +22,6 @@ go_test( "@com_github_stretchr_testify//require:go_default_library", "@com_github_stretchr_testify//suite:go_default_library", "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", ], ) diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go index e9402df70..5568cf0a8 100644 --- a/test/integration/stovepipe/suite_test.go +++ b/test/integration/stovepipe/suite_test.go @@ -39,6 +39,7 @@ import ( pb "github.com/uber/submitqueue/api/stovepipe/protopb" "github.com/uber/submitqueue/test/testutil" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) type StovepipeIntegrationSuite struct { @@ -186,13 +187,32 @@ func (s *StovepipeIntegrationSuite) TestRequestHistoryAPIs() { require.Zero(t, requestRows) byID, err := s.client.GetRequestHistoryByID(s.ctx, &pb.GetRequestHistoryByIDRequest{Queue: queue, RequestId: requestID}) - require.NoError(t, err) - assertHistoryEvents(t, byID.Events) + require.Error(t, err) + assert.Nil(t, byID) byURI, err := s.client.GetRequestHistoryByURI(s.ctx, &pb.GetRequestHistoryByURIRequest{Queue: queue, Uri: uri}) + require.Error(t, err) + assert.Nil(t, byURI) + + const baseURI = "git://history-api/main/base" + _, err = s.db.Exec(`INSERT INTO request_summary + (queue, request_id, uri, base_uri, state, request_version, state_timestamp_ms, version, accepted_at_ms, outcome_reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + queue, requestID, uri, baseURI, "succeeded", 3, 2000, 1, 1000, "build_succeeded") + require.NoError(t, err) + byID, err = s.client.GetRequestHistoryByID(s.ctx, &pb.GetRequestHistoryByIDRequest{Queue: queue, RequestId: requestID}) + require.NoError(t, err) + byURI, err = s.client.GetRequestHistoryByURI(s.ctx, &pb.GetRequestHistoryByURIRequest{Queue: queue, Uri: uri}) require.NoError(t, err) require.Len(t, byURI.Histories, 1) + wantSummary := &pb.RequestSummary{ + RequestId: requestID, Queue: queue, ChangeUri: uri, BaseUri: baseURI, RequestState: "succeeded", + StateUpdatedAtMs: 2000, AcceptedAtMs: 1000, OutcomeReason: "build_succeeded", + } + assert.True(t, proto.Equal(wantSummary, byID.RequestSummary)) + assert.True(t, proto.Equal(wantSummary, byURI.Histories[0].RequestSummary)) assert.Equal(t, requestID, byURI.Histories[0].RequestId) + assertHistoryEvents(t, byID.Events) assertHistoryEvents(t, byURI.Histories[0].Events) } @@ -222,6 +242,11 @@ func (s *StovepipeIntegrationSuite) TestRequestHistoryAbsence() { ) require.NoError(t, err) + _, err = s.db.Exec(`INSERT INTO request_summary + (queue, request_id, uri, base_uri, state, request_version, state_timestamp_ms, version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, queue, missingID, missingURI, "", "accepted", 1, 1000, 1) + require.NoError(t, err) + tests := []struct { name string call func() error